Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: pass Qwen3 enable_thinking via chat_template_kwargs (#17592)

JH_up76edf7f3e286
diff --git a/internal/entity/models/openai_api_compatible.go b/internal/entity/models/openai_api_compatible.go
index 5e48e06f50..764b45be6c 100644
--- a/internal/entity/models/openai_api_compatible.go
+++ b/internal/entity/models/openai_api_compatible.go
@@ -97,18 +97,7 @@ func (m *OpenAIAPICompatibleModel) ChatWithMessages(ctx context.Context, modelNa
// Build request body
reqBody := buildRequestBody(chatModelConfig, modelName, messages, false)
-
- if chatModelConfig != nil && chatModelConfig.Thinking != nil {
- if *chatModelConfig.Thinking {
- reqBody["thinking"] = map[string]interface{}{
- "type": "enabled",
- }
- } else {
- reqBody["thinking"] = map[string]interface{}{
- "type": "disabled",
- }
- }
- }
+ applyVllmCompatibleThinking(reqBody, modelName, chatModelConfig)
body, err := m.baseModel.doRequest(ctx, url, apiConfig, reqBody, nonStreamCallTimeout)
if err != nil {
@@ -143,17 +132,7 @@ func (m *OpenAIAPICompatibleModel) ChatStreamlyWithSender(ctx context.Context, m
"include_usage": true,
}
- if chatModelConfig != nil && chatModelConfig.Thinking != nil {
- if *chatModelConfig.Thinking {
- reqBody["thinking"] = map[string]interface{}{
- "type": "enabled",
- }
- } else {
- reqBody["thinking"] = map[string]interface{}{
- "type": "disabled",
- }
- }
- }
+ applyVllmCompatibleThinking(reqBody, modelName, chatModelConfig)
return m.baseModel.doStreamRequest(ctx, url, apiConfig, reqBody, streamCallTimeout, func(body io.ReadCloser) error {
return HandleStreamingResponse(body, modelUsage, chatModelConfig, OpenAIParserConfig, sender)
diff --git a/internal/entity/models/qwen3_thinking_payload_test.go b/internal/entity/models/qwen3_thinking_payload_test.go
new file mode 100644
index 0000000000..c13492356f
--- /dev/null
+++ b/internal/entity/models/qwen3_thinking_payload_test.go
@@ -0,0 +1,197 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package models
+
+import (
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+func newThinkingPayloadServer(t *testing.T, requestBody chan<- map[string]interface{}) *httptest.Server {
+ t.Helper()
+
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var body map[string]interface{}
+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+ t.Errorf("decode request: %v", err)
+ return
+ }
+ requestBody <- body
+
+ if body["stream"] == true {
+ w.Header().Set("Content-Type", "text/event-stream")
+ _, _ = io.WriteString(w, "data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n\n")
+ _, _ = io.WriteString(w, "data: [DONE]\n\n")
+ return
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = io.WriteString(w, `{"choices":[{"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}`)
+ }))
+}
+
+func boolPointer(value bool) *bool {
+ return &value
+}
+
+func runThinkingPayloadRequest(t *testing.T, driver ModelDriver, modelName string, stream bool, config *ChatConfig) {
+ t.Helper()
+
+ apiKey := "test-key"
+ messages := []Message{{Role: "user", Content: "hello"}}
+ apiConfig := &APIConfig{ApiKey: &apiKey}
+ if stream {
+ if err := driver.ChatStreamlyWithSender(t.Context(), modelName, messages, apiConfig, config, nil, func(*string, *string) error { return nil }); err != nil {
+ t.Fatalf("ChatStreamlyWithSender: %v", err)
+ }
+ return
+ }
+
+ if _, err := driver.ChatWithMessages(t.Context(), modelName, messages, apiConfig, config, nil); err != nil {
+ t.Fatalf("ChatWithMessages: %v", err)
+ }
+}
+
+func TestVllmCompatibleQwen3ThinkingPayload(t *testing.T) {
+ withSSRFBypass(t)
+
+ drivers := []struct {
+ name string
+ new func(string) ModelDriver
+ }{
+ {
+ name: "vllm",
+ new: func(baseURL string) ModelDriver {
+ return NewVllmModel(map[string]string{"default": baseURL}, URLSuffix{Chat: "chat/completions", AsyncChat: "chat/completions"})
+ },
+ },
+ {
+ name: "openai-compatible",
+ new: func(baseURL string) ModelDriver {
+ return NewOpenAIAPICompatibleModel(map[string]string{"default": baseURL}, URLSuffix{Chat: "chat/completions"})
+ },
+ },
+ }
+ cases := []struct {
+ name string
+ model string
+ thinking *bool
+ want bool
+ }{
+ {name: "default disabled", model: "qwen3-8b", want: false},
+ {name: "explicit enabled", model: "Qwen/Qwen3-8B", thinking: boolPointer(true), want: true},
+ {name: "explicit disabled", model: "qwen3-8b", thinking: boolPointer(false), want: false},
+ {name: "preview forced enabled", model: "qwen3.8-max-preview", thinking: boolPointer(false), want: true},
+ {name: "flagship forced enabled", model: "qwen3.8-2.4t-a95b", thinking: boolPointer(false), want: true},
+ }
+
+ for _, driverCase := range drivers {
+ for _, stream := range []bool{false, true} {
+ for _, testCase := range cases {
+ name := driverCase.name + "/" + testCase.name
+ if stream {
+ name = driverCase.name + "/stream/" + testCase.name
+ }
+ t.Run(name, func(t *testing.T) {
+ requestBody := make(chan map[string]interface{}, 1)
+ server := newThinkingPayloadServer(t, requestBody)
+ defer server.Close()
+
+ var config *ChatConfig
+ if testCase.thinking != nil {
+ config = &ChatConfig{Thinking: testCase.thinking}
+ }
+ runThinkingPayloadRequest(t, driverCase.new(server.URL), testCase.model, stream, config)
+
+ body := <-requestBody
+ templateKwargs, ok := body["chat_template_kwargs"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("chat_template_kwargs=%#v, want object", body["chat_template_kwargs"])
+ }
+ if got := templateKwargs["enable_thinking"]; got != testCase.want {
+ t.Errorf("enable_thinking=%#v, want %v", got, testCase.want)
+ }
+ for _, field := range []string{"thinking", "enable_thinking", "extra_body"} {
+ if _, exists := body[field]; exists {
+ t.Errorf("unexpected root-level %s in request: %#v", field, body[field])
+ }
+ }
+ })
+ }
+ }
+ }
+}
+
+func TestVllmCompatibleNonQwenThinkingPayloadUnchanged(t *testing.T) {
+ withSSRFBypass(t)
+
+ drivers := []struct {
+ name string
+ new func(string) ModelDriver
+ }{
+ {name: "vllm", new: func(baseURL string) ModelDriver {
+ return NewVllmModel(map[string]string{"default": baseURL}, URLSuffix{Chat: "chat/completions"})
+ }},
+ {name: "openai-compatible", new: func(baseURL string) ModelDriver {
+ return NewOpenAIAPICompatibleModel(map[string]string{"default": baseURL}, URLSuffix{Chat: "chat/completions"})
+ }},
+ }
+
+ for _, driverCase := range drivers {
+ t.Run(driverCase.name, func(t *testing.T) {
+ requestBody := make(chan map[string]interface{}, 1)
+ server := newThinkingPayloadServer(t, requestBody)
+ defer server.Close()
+
+ runThinkingPayloadRequest(t, driverCase.new(server.URL), "deepseek-r1", false, &ChatConfig{Thinking: boolPointer(true)})
+
+ body := <-requestBody
+ thinking, ok := body["thinking"].(map[string]interface{})
+ if !ok || thinking["type"] != "enabled" {
+ t.Fatalf("thinking=%#v, want {type: enabled}", body["thinking"])
+ }
+ if _, exists := body["chat_template_kwargs"]; exists {
+ t.Errorf("chat_template_kwargs should be absent for non-Qwen3 model")
+ }
+ })
+ }
+}
+
+func TestAliyunQwen3ThinkingPayloadRemainsProviderNative(t *testing.T) {
+ withSSRFBypass(t)
+
+ requestBody := make(chan map[string]interface{}, 1)
+ server := newThinkingPayloadServer(t, requestBody)
+ defer server.Close()
+
+ driver := NewAliyunModel(map[string]string{"default": server.URL}, URLSuffix{Chat: "chat/completions"})
+ runThinkingPayloadRequest(t, driver, "qwen3-8b", false, nil)
+
+ body := <-requestBody
+ if got := body["enable_thinking"]; got != false {
+ t.Fatalf("enable_thinking=%#v, want false", got)
+ }
+ if _, exists := body["chat_template_kwargs"]; exists {
+ t.Errorf("chat_template_kwargs should be absent for native Aliyun")
+ }
+ if _, exists := body["thinking"]; exists {
+ t.Errorf("thinking should be absent for native Aliyun Qwen3")
+ }
+}
diff --git a/internal/entity/models/vllm.go b/internal/entity/models/vllm.go
index 8db20d9fbc..514e3f0c40 100644
--- a/internal/entity/models/vllm.go
+++ b/internal/entity/models/vllm.go
@@ -52,6 +52,40 @@ func (v *VllmModel) Name() string {
return "VLLM"
}
+// applyVllmCompatibleThinking maps Qwen3 controls to the chat-template payload
+// expected by vLLM while preserving the generic thinking payload for other models.
+func applyVllmCompatibleThinking(reqBody map[string]interface{}, modelName string, config *ChatConfig) {
+ modelNameLower := strings.ToLower(modelName)
+ if strings.Contains(modelNameLower, "qwen3") {
+ enableThinking := false
+ if config != nil && config.Thinking != nil {
+ enableThinking = *config.Thinking
+ }
+ if strings.Contains(modelNameLower, "-preview") || strings.Contains(modelNameLower, "2.4t-a95b") {
+ enableThinking = true
+ }
+
+ chatTemplateKwargs, ok := reqBody["chat_template_kwargs"].(map[string]interface{})
+ if !ok {
+ chatTemplateKwargs = map[string]interface{}{}
+ }
+ chatTemplateKwargs["enable_thinking"] = enableThinking
+ reqBody["chat_template_kwargs"] = chatTemplateKwargs
+ delete(reqBody, "thinking")
+ delete(reqBody, "enable_thinking")
+ return
+ }
+
+ if config == nil || config.Thinking == nil {
+ return
+ }
+ thinkingType := "disabled"
+ if *config.Thinking {
+ thinkingType = "enabled"
+ }
+ reqBody["thinking"] = map[string]interface{}{"type": thinkingType}
+}
+
// ChatWithMessages sends multiple messages with roles and returns response
func (v *VllmModel) ChatWithMessages(ctx context.Context, modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, modelUsage *common.ModelUsage) (*ChatResponse, error) {
if err := v.baseModel.APIConfigCheck(apiConfig); err != nil {
@@ -76,20 +110,7 @@ func (v *VllmModel) ChatWithMessages(ctx context.Context, modelName string, mess
// Build request body
reqBody := buildRequestBody(chatModelConfig, modelName, messages, false)
-
- if chatModelConfig != nil {
- if chatModelConfig.Thinking != nil {
- if *chatModelConfig.Thinking {
- reqBody["thinking"] = map[string]interface{}{
- "type": "enabled",
- }
- } else {
- reqBody["thinking"] = map[string]interface{}{
- "type": "disabled",
- }
- }
- }
- }
+ applyVllmCompatibleThinking(reqBody, modelName, chatModelConfig)
body, err := v.baseModel.doRequest(ctx, url, apiConfig, reqBody, nonStreamCallTimeout)
if err != nil {
@@ -121,18 +142,7 @@ func (v *VllmModel) ChatStreamlyWithSender(ctx context.Context, modelName string
// Build request body with streaming enabled
reqBody := buildRequestBody(modelConfig, modelName, messages, true)
-
- if modelConfig.Thinking != nil {
- if *modelConfig.Thinking {
- reqBody["thinking"] = map[string]interface{}{
- "type": "enabled",
- }
- } else {
- reqBody["thinking"] = map[string]interface{}{
- "type": "disabled",
- }
- }
- }
+ applyVllmCompatibleThinking(reqBody, modelName, modelConfig)
reqBody["stream_options"] = map[string]interface{}{"include_usage": true}
diff --git a/rag/llm/chat_model.py b/rag/llm/chat_model.py
index 07468d295f..3603cd1a12 100644
--- a/rag/llm/chat_model.py
+++ b/rag/llm/chat_model.py
@@ -42,6 +42,8 @@ from rag.llm.tool_decorator import FunctionToolSession, is_tool
from rag.nlp import is_chinese, is_english
from rag.utils.url_utils import ensure_v1
+logger = logging.getLogger(__name__)
+
class LLMErrorCode(StrEnum):
ERROR_RATE_LIMIT = "RATE_LIMIT_EXCEEDED"
@@ -97,10 +99,9 @@ ALLOWED_GEN_CONF_KEYS = frozenset(
}
)
-# LiteLLM additionally understands reasoning-control parameters that the
-# model-family policies may inject into `gen_conf` (e.g. `thinking` for
-# Anthropic / Kimi reasoning models, `enable_thinking` for Qwen models,
-# `reasoning_effort` for OpenAI o-series).
+# LiteLLM additionally understands reasoning-control parameters that must
+# survive configuration cleaning until model-family policies are applied at
+# the final request-construction boundary.
LITELLM_ALLOWED_GEN_CONF_KEYS = ALLOWED_GEN_CONF_KEYS | frozenset(
{
"thinking",
@@ -119,11 +120,13 @@ def _apply_model_family_policies(
gen_conf: dict | None = None,
request_kwargs: dict | None = None,
):
+ """Normalize reasoning controls for a model/provider without mutating inputs."""
model_name_lower = (model_name or "").lower()
sanitized_gen_conf = deepcopy(gen_conf) if gen_conf else {}
sanitized_kwargs = dict(request_kwargs) if request_kwargs else {}
def _thinking_type():
+ """Return the normalized explicit thinking mode, if one was supplied."""
val = sanitized_gen_conf.get("thinking")
if isinstance(val, dict):
val = val.get("type")
@@ -137,16 +140,28 @@ def _apply_model_family_policies(
return None
def _pop_thinking_controls():
+ """Remove generic controls after translating them to provider payloads."""
sanitized_gen_conf.pop("thinking", None)
sanitized_gen_conf.pop("enable_thinking", None)
def _merge_extra_body(target: dict, extra: dict) -> None:
+ """Merge top-level request body fields."""
body = target.get("extra_body")
if not isinstance(body, dict):
body = {}
body.update(extra)
target["extra_body"] = body
+ def _merge_qwen_chat_template_kwargs(target: dict, enable_thinking: bool) -> None:
+ """Set Qwen thinking without replacing other chat-template options."""
+ body = target.get("extra_body")
+ body = dict(body) if isinstance(body, dict) else {}
+ template_kwargs = body.get("chat_template_kwargs")
+ template_kwargs = dict(template_kwargs) if isinstance(template_kwargs, dict) else {}
+ template_kwargs["enable_thinking"] = enable_thinking
+ body["chat_template_kwargs"] = template_kwargs
+ target["extra_body"] = body
+
thinking_type = _thinking_type()
# Qwen3 keeps RAGFlow's system default of disabling thinking unless explicitly overridden.
@@ -165,7 +180,15 @@ def _apply_model_family_policies(
}:
sanitized_gen_conf["enable_thinking"] = enable_thinking
else:
- _merge_extra_body(sanitized_kwargs, {"enable_thinking": enable_thinking})
+ target = sanitized_gen_conf if backend == "litellm" else sanitized_kwargs
+ _merge_qwen_chat_template_kwargs(target, enable_thinking)
+ logger.debug(
+ "Applied Qwen3 thinking policy: backend=%s provider=%s enable_thinking=%s payload_path=%s",
+ backend,
+ provider,
+ enable_thinking,
+ "extra_body.chat_template_kwargs.enable_thinking",
+ )
if backend == "base":
return sanitized_gen_conf, sanitized_kwargs
@@ -1915,12 +1938,8 @@ class LiteLLMBase(ABC):
return LLMErrorCode.ERROR_GENERIC
def _clean_conf(self, gen_conf):
- gen_conf, _ = _apply_model_family_policies(
- self.model_name,
- backend="litellm",
- provider=self.provider,
- gen_conf=gen_conf,
- )
+ """Copy and filter generation settings before final request construction."""
+ gen_conf = deepcopy(gen_conf) if gen_conf else {}
deepseek_max_tokens = None
if self.provider == SupportedLiteLLMProvider.DeepSeek:
@@ -1964,6 +1983,7 @@ class LiteLLMBase(ABC):
return text
async def async_chat(self, system, history, gen_conf, **kwargs):
+ """Send one non-streaming LiteLLM chat request with normalized settings."""
hist = list(history) if history else []
if system:
if not hist or hist[0].get("role") != "system":
@@ -1971,12 +1991,6 @@ class LiteLLMBase(ABC):
logging.info("[HISTORY]" + json.dumps(hist, ensure_ascii=False, indent=2))
gen_conf = self._clean_conf(gen_conf)
- _, kwargs = _apply_model_family_policies(
- self.model_name,
- backend="litellm",
- provider=self.provider,
- request_kwargs=kwargs,
- )
completion_args = self._construct_completion_args(history=hist, stream=False, tools=False, **{**gen_conf, **kwargs})
@@ -2514,11 +2528,19 @@ class LiteLLMBase(ABC):
assert False, "Shouldn't be here."
def _construct_completion_args(self, history, stream: bool, tools: bool, **kwargs):
+ """Build the final LiteLLM arguments and apply model policies exactly once."""
+ kwargs, policy_request_kwargs = _apply_model_family_policies(
+ self.model_name,
+ backend="litellm",
+ provider=self.provider,
+ gen_conf=kwargs,
+ )
completion_args = {
"model": self.model_name,
"messages": history,
"api_key": self.api_key,
**kwargs,
+ **policy_request_kwargs,
}
if self.provider == SupportedLiteLLMProvider.Nvidia:
completion_args["num_retries"] = 0
@@ -2546,7 +2568,9 @@ class LiteLLMBase(ABC):
"tool_choice": "auto",
}
)
- if self.provider in FACTORY_DEFAULT_BASE_URL:
+ # OpenRouter is handled separately below because it may add routing
+ # metadata without replacing model-specific fields in extra_body.
+ if self.provider in FACTORY_DEFAULT_BASE_URL and self.provider != SupportedLiteLLMProvider.OpenRouter:
completion_args.update({"api_base": self.base_url})
elif self.provider == SupportedLiteLLMProvider.Bedrock:
import boto3
@@ -2595,6 +2619,7 @@ class LiteLLMBase(ABC):
raise ValueError(f"Unsupported Bedrock auth_mode: {mode}")
elif self.provider == SupportedLiteLLMProvider.OpenRouter:
+ completion_args["api_base"] = self.base_url
if self.provider_order:
def _to_order_list(x):
@@ -2606,13 +2631,13 @@ class LiteLLMBase(ABC):
return [str(s).strip() for s in x if str(s).strip()]
return []
- extra_body = {}
- provider_cfg = {}
- provider_order = _to_order_list(self.provider_order)
- provider_cfg["order"] = provider_order
- provider_cfg["allow_fallbacks"] = False
- extra_body["provider"] = provider_cfg
- completion_args.update({"extra_body": extra_body})
+ extra_body = completion_args.get("extra_body")
+ extra_body = dict(extra_body) if isinstance(extra_body, dict) else {}
+ extra_body["provider"] = {
+ "order": _to_order_list(self.provider_order),
+ "allow_fallbacks": False,
+ }
+ completion_args["extra_body"] = extra_body
elif self.provider == SupportedLiteLLMProvider.GPUStack:
completion_args.update(
{
diff --git a/test/unit_test/rag/llm/conftest.py b/test/unit_test/rag/llm/conftest.py
index 7ac4c5e9bf..165978dce4 100644
--- a/test/unit_test/rag/llm/conftest.py
+++ b/test/unit_test/rag/llm/conftest.py
@@ -61,10 +61,13 @@ def _install_rag_llm_stub():
Tongyi_Qianwen = "Tongyi-Qianwen"
Dashscope = "Dashscope"
Moonshot = "Moonshot"
+ DeepSeek = "DeepSeek"
Nvidia = "NVIDIA"
+ Anthropic = "Anthropic"
Bedrock = "Bedrock"
Ollama = "Ollama"
OpenRouter = "OpenRouter"
+ GPUStack = "GPUStack"
MiniMax = "MiniMax"
ZHIPU_AI = "ZHIPU-AI"
OpenAI = "OpenAI"
diff --git a/test/unit_test/rag/llm/test_chat_model_qwen3_transport.py b/test/unit_test/rag/llm/test_chat_model_qwen3_transport.py
new file mode 100644
index 0000000000..1062a95b45
--- /dev/null
+++ b/test/unit_test/rag/llm/test_chat_model_qwen3_transport.py
@@ -0,0 +1,242 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import json
+from types import SimpleNamespace
+
+import httpx
+import pytest
+from openai import AsyncOpenAI
+
+from rag.llm import SupportedLiteLLMProvider, chat_model
+from rag.llm.chat_model import Base, LiteLLMBase
+
+pytestmark = pytest.mark.p1
+
+
+def _completion_response():
+ """Build the minimal non-stream LiteLLM response used by call-path tests."""
+ return SimpleNamespace(
+ choices=[
+ SimpleNamespace(
+ message=SimpleNamespace(content="ok", tool_calls=[]),
+ finish_reason="stop",
+ )
+ ],
+ usage=SimpleNamespace(prompt_tokens=1, completion_tokens=1, total_tokens=2),
+ )
+
+
+def _stream_chunk():
+ """Build the minimal streaming LiteLLM response chunk."""
+ return SimpleNamespace(
+ choices=[
+ SimpleNamespace(
+ delta=SimpleNamespace(content="ok", reasoning_content=None, reasoning=None, tool_calls=None),
+ finish_reason="stop",
+ )
+ ],
+ usage=None,
+ )
+
+
+async def _stream_response():
+ """Yield one response chunk through LiteLLM's async-stream contract."""
+ yield _stream_chunk()
+
+
+def _make_litellm_qwen(provider=SupportedLiteLLMProvider.OpenAI):
+ """Create a lightweight Qwen model without initializing external clients."""
+ model = LiteLLMBase.__new__(LiteLLMBase)
+ model.model_name = "qwen3-8b"
+ model.provider = provider
+ model.api_key = "test-key"
+ model.base_url = "https://example.test/v1"
+ model.max_retries = 0
+ model.max_rounds = 0
+ model.timeout = 1
+ model.tools = [
+ {
+ "type": "function",
+ "function": {
+ "name": "lookup",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ }
+ ]
+ model.last_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
+ return model
+
+
+async def test_qwen3_openai_sdk_sends_nested_transport_json():
+ """The OpenAI SDK must flatten extra_body into vLLM's final JSON shape."""
+ sent_body = {}
+
+ def handle_request(request):
+ """Capture the exact JSON produced by the OpenAI SDK transport."""
+ sent_body.update(json.loads(request.content))
+ return httpx.Response(
+ 200,
+ json={
+ "id": "chatcmpl-test",
+ "object": "chat.completion",
+ "created": 0,
+ "model": "qwen3-8b",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+ },
+ )
+
+ http_client = httpx.AsyncClient(transport=httpx.MockTransport(handle_request))
+ model = Base.__new__(Base)
+ model.model_name = "qwen3-8b"
+ model.async_client = AsyncOpenAI(
+ api_key="test-key",
+ base_url="https://example.test/v1",
+ http_client=http_client,
+ max_retries=0,
+ )
+ model.last_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
+
+ try:
+ answer, _ = await model._async_chat(
+ [{"role": "user", "content": "hello"}],
+ {"thinking": "disabled"},
+ extra_body={
+ "seed": 1,
+ "chat_template_kwargs": {"custom_template_flag": "keep"},
+ },
+ )
+ finally:
+ await model.async_client.close()
+
+ assert answer == "ok"
+ assert sent_body["seed"] == 1
+ assert sent_body["chat_template_kwargs"] == {
+ "custom_template_flag": "keep",
+ "enable_thinking": False,
+ }
+ assert "enable_thinking" not in sent_body
+ assert "thinking" not in sent_body
+ assert "extra_body" not in sent_body
+
+
[email protected]("path", ["chat", "stream", "tools", "stream_tools"])
+async def test_litellm_qwen3_payload_reaches_every_chat_path(monkeypatch, path):
+ """Every LiteLLM chat entry point must retain the normalized Qwen payload."""
+ calls = []
+
+ async def fake_acompletion(**kwargs):
+ """Capture the final arguments passed through each LiteLLM entry point."""
+ calls.append(kwargs)
+ if kwargs.get("stream"):
+ return _stream_response()
+ return _completion_response()
+
+ monkeypatch.setattr(chat_model.litellm, "acompletion", fake_acompletion)
+ model = _make_litellm_qwen()
+ history = [{"role": "user", "content": "hello"}]
+ gen_conf = {
+ "thinking": "enabled",
+ "extra_body": {
+ "seed": 1,
+ "chat_template_kwargs": {"custom_template_flag": "keep"},
+ },
+ }
+
+ if path == "chat":
+ await model.async_chat("", history, gen_conf)
+ elif path == "stream":
+ _ = [item async for item in model.async_chat_streamly("", history, gen_conf)]
+ elif path == "tools":
+ await model.async_chat_with_tools("", history, gen_conf)
+ else:
+ _ = [item async for item in model.async_chat_streamly_with_tools("", history, gen_conf)]
+
+ assert len(calls) == 1
+ completion_args = calls[0]
+ assert completion_args["extra_body"] == {
+ "seed": 1,
+ "chat_template_kwargs": {
+ "custom_template_flag": "keep",
+ "enable_thinking": True,
+ },
+ }
+ assert "thinking" not in completion_args
+ assert "enable_thinking" not in completion_args
+ assert gen_conf == {
+ "thinking": "enabled",
+ "extra_body": {
+ "seed": 1,
+ "chat_template_kwargs": {"custom_template_flag": "keep"},
+ },
+ }
+
+
+def test_litellm_dashscope_qwen3_keeps_provider_native_payload():
+ """DashScope must keep its provider-native root body field."""
+ model = _make_litellm_qwen(SupportedLiteLLMProvider.Dashscope)
+
+ completion_args = model._construct_completion_args(
+ history=[],
+ stream=False,
+ tools=False,
+ thinking="disabled",
+ )
+
+ assert completion_args["extra_body"] == {"enable_thinking": False}
+ assert "enable_thinking" not in completion_args
+ assert "chat_template_kwargs" not in completion_args["extra_body"]
+
+
+def test_litellm_openrouter_qwen3_preserves_routing_and_thinking_payload():
+ """OpenRouter routing must not replace Qwen chat-template settings."""
+ model = _make_litellm_qwen(SupportedLiteLLMProvider.OpenRouter)
+ model.provider_order = "provider-a, provider-b"
+
+ completion_args = model._construct_completion_args(
+ history=[],
+ stream=False,
+ tools=False,
+ thinking="disabled",
+ extra_body={
+ "seed": 1,
+ "chat_template_kwargs": {"custom_template_flag": "keep"},
+ },
+ )
+
+ assert completion_args["api_base"] == model.base_url
+ assert completion_args["extra_body"] == {
+ "seed": 1,
+ "chat_template_kwargs": {
+ "custom_template_flag": "keep",
+ "enable_thinking": False,
+ },
+ "provider": {
+ "order": ["provider-a", "provider-b"],
+ "allow_fallbacks": False,
+ },
+ }
+ assert "thinking" not in completion_args
+ assert "enable_thinking" not in completion_args
diff --git a/test/unit_test/rag/llm/test_chat_model_thinking_policy.py b/test/unit_test/rag/llm/test_chat_model_thinking_policy.py
index 4318fc05ae..3931c3a847 100644
--- a/test/unit_test/rag/llm/test_chat_model_thinking_policy.py
+++ b/test/unit_test/rag/llm/test_chat_model_thinking_policy.py
@@ -23,6 +23,7 @@ pytestmark = pytest.mark.p1
def test_qwen3_uses_system_disabled_default():
+ """Base-compatible Qwen3 requests disable thinking by default."""
gen_conf, kwargs = _apply_model_family_policies(
"qwen3-plus",
backend="base",
@@ -31,10 +32,11 @@ def test_qwen3_uses_system_disabled_default():
)
assert gen_conf == {}
- assert kwargs["extra_body"]["enable_thinking"] is False
+ assert kwargs["extra_body"] == {"chat_template_kwargs": {"enable_thinking": False}}
def test_qwen3_can_enable_thinking_explicitly():
+ """An explicit Qwen3 thinking choice reaches chat_template_kwargs."""
gen_conf, kwargs = _apply_model_family_policies(
"qwen3-plus",
backend="base",
@@ -43,7 +45,28 @@ def test_qwen3_can_enable_thinking_explicitly():
)
assert gen_conf == {"temperature": 0.2}
- assert kwargs["extra_body"] == {"seed": 1, "enable_thinking": True}
+ assert kwargs["extra_body"] == {"seed": 1, "chat_template_kwargs": {"enable_thinking": True}}
+
+
+def test_qwen3_preserves_existing_chat_template_kwargs():
+ """Qwen policy updates its field without dropping other template options."""
+ gen_conf, kwargs = _apply_model_family_policies(
+ "qwen3-plus",
+ backend="base",
+ gen_conf={"thinking": "disabled"},
+ request_kwargs={
+ "extra_body": {
+ "seed": 1,
+ "chat_template_kwargs": {"enable_thinking": True, "custom_template_flag": "keep"},
+ }
+ },
+ )
+
+ assert gen_conf == {}
+ assert kwargs["extra_body"] == {
+ "seed": 1,
+ "chat_template_kwargs": {"enable_thinking": False, "custom_template_flag": "keep"},
+ }
def test_qwen3_preview_variant_forces_thinking_true():
@@ -56,7 +79,7 @@ def test_qwen3_preview_variant_forces_thinking_true():
)
assert gen_conf == {}
- assert kwargs["extra_body"]["enable_thinking"] is True
+ assert kwargs["extra_body"]["chat_template_kwargs"]["enable_thinking"] is True
def test_qwen3_preview_ignores_disabled_thinking():
@@ -70,7 +93,7 @@ def test_qwen3_preview_ignores_disabled_thinking():
assert "thinking" not in gen_conf
assert gen_conf == {"temperature": 0.2}
- assert kwargs["extra_body"]["enable_thinking"] is True
+ assert kwargs["extra_body"]["chat_template_kwargs"]["enable_thinking"] is True
def test_qwen3_24t_a95b_forces_thinking_true():
@@ -83,7 +106,7 @@ def test_qwen3_24t_a95b_forces_thinking_true():
)
assert gen_conf == {}
- assert kwargs["extra_body"]["enable_thinking"] is True
+ assert kwargs["extra_body"]["chat_template_kwargs"]["enable_thinking"] is True
def test_qwen3_24t_a95b_ignores_disabled_thinking():
@@ -97,7 +120,7 @@ def test_qwen3_24t_a95b_ignores_disabled_thinking():
assert "thinking" not in gen_conf
assert gen_conf == {"temperature": 0.2}
- assert kwargs["extra_body"]["enable_thinking"] is True
+ assert kwargs["extra_body"]["chat_template_kwargs"]["enable_thinking"] is True
@pytest.mark.parametrize(
@@ -105,6 +128,7 @@ def test_qwen3_24t_a95b_ignores_disabled_thinking():
[SupportedLiteLLMProvider.Tongyi_Qianwen, SupportedLiteLLMProvider.Dashscope],
)
def test_qwen3_litellm_provider_uses_provider_field(provider):
+ """Native DashScope providers keep their provider-specific body field."""
gen_conf, kwargs = _apply_model_family_policies(
"qwen3-max",
backend="litellm",
@@ -117,6 +141,34 @@ def test_qwen3_litellm_provider_uses_provider_field(provider):
assert gen_conf["enable_thinking"] is False
+def test_qwen3_litellm_openai_uses_nested_extra_body():
+ """Non-DashScope LiteLLM providers carry Qwen controls in extra_body."""
+ gen_conf, kwargs = _apply_model_family_policies(
+ "qwen3-8b",
+ backend="litellm",
+ provider=SupportedLiteLLMProvider.OpenAI,
+ gen_conf={
+ "thinking": "enabled",
+ "extra_body": {
+ "seed": 1,
+ "chat_template_kwargs": {"custom_template_flag": "keep"},
+ },
+ },
+ request_kwargs={},
+ )
+
+ assert kwargs == {}
+ assert gen_conf == {
+ "extra_body": {
+ "seed": 1,
+ "chat_template_kwargs": {
+ "custom_template_flag": "keep",
+ "enable_thinking": True,
+ },
+ }
+ }
+
+
def test_kimi_thinking_maps_to_moonshot_payload():
gen_conf, kwargs = _apply_model_family_policies(
"kimi-k2.6-preview",
@@ -225,6 +277,29 @@ def test_deepseek_thinking_enabled_via_extra_body():
assert gen_conf["extra_body"]["thinking"] == {"type": "enabled"}
+def test_deepseek_extra_body_keeps_shallow_merge_semantics():
+ """The Qwen fix must not recursively merge unrelated provider payloads."""
+ gen_conf, kwargs = _apply_model_family_policies(
+ "deepseek-v4-flash",
+ backend="litellm",
+ provider=SupportedLiteLLMProvider.DeepSeek,
+ gen_conf={
+ "thinking": "disabled",
+ "extra_body": {
+ "seed": 1,
+ "thinking": {"budget_tokens": 128},
+ },
+ },
+ request_kwargs={},
+ )
+
+ assert kwargs == {}
+ assert gen_conf["extra_body"] == {
+ "seed": 1,
+ "thinking": {"type": "disabled"},
+ }
+
+
def test_litellm_provider_body_fields_move_to_extra_body_before_drop_params():
completion_args = {
"model": "kimi-latest",