Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: give API clients the same Qwen thinking sampling as Chat (#12791)

Nilay1150a7de77ee
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index e04ffc8e44f..29f8f16506c 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -28501,12 +28501,16 @@ async def _npu_chat_completions(payload, request: Request, current_subject: str)
),
)
- _fill_recommended_sampling_openai(payload, upstream.model)
_normalize_chat_reasoning_controls(payload)
if not upstream.supports_reasoning:
payload.enable_thinking = False
elif payload.enable_thinking is None:
payload.enable_thinking = payload.reasoning_effort != "none"
+ _fill_recommended_sampling_openai(
+ payload,
+ upstream.model,
+ thinking = payload.enable_thinking if upstream.supports_reasoning else None,
+ )
wants_stream = bool(payload.stream)
relay = _NpuStreamRelay(upstream.public_model, _normalize_stop_sequences(payload.stop))
@@ -28896,22 +28900,40 @@ def _normalize_chat_reasoning_controls(payload) -> None:
payload.preserve_thinking = nested["preserve_thinking"]
-def _fill_recommended_sampling_openai(payload, model_id) -> None:
+def _sampling_thinking_mode(llama_backend, payload) -> Optional[bool]:
+ if not getattr(llama_backend, "supports_reasoning", False):
+ return None
+ return _think_parsing_expected(llama_backend, payload)
+
+
+def _client_sampling(payload) -> dict:
+ from utils.inference.inference_config import SAMPLING_FIELD_NAMES
+ return {
+ f: (getattr(payload, f) if f in payload.model_fields_set else None)
+ for f in SAMPLING_FIELD_NAMES
+ }
+
+
+def _fill_recommended_sampling_openai(
+ payload,
+ model_id,
+ thinking = None,
+ explicit = None,
+) -> None:
"""Apply per-model recommended sampling (and any operator UNSLOTH_SAMPLING_* pin) to a
ChatCompletionRequest in place.
Only the sampling fields the client did NOT explicitly send (tracked via
``model_fields_set``) are overwritten, so a client that sets a field stays byte-identical
unless an operator pins it. Fields with neither a recommendation nor a pin keep their
- existing (schema-default) value.
+ existing (schema-default) value. A second fill must pass the first fill's ``explicit``:
+ setattr marks every field as set.
"""
- from utils.inference.inference_config import resolve_effective_sampling, SAMPLING_FIELD_NAMES
+ from utils.inference.inference_config import resolve_effective_sampling
- explicit = {
- f: (getattr(payload, f) if f in payload.model_fields_set else None)
- for f in SAMPLING_FIELD_NAMES
- }
- effective = resolve_effective_sampling(model_id, explicit)
+ if explicit is None:
+ explicit = _client_sampling(payload)
+ effective = resolve_effective_sampling(model_id, explicit, thinking = thinking)
for field, value in effective.items():
setattr(payload, field, value)
@@ -29923,7 +29945,13 @@ async def produce_openai_chat_completions(
if using_gguf
else getattr(backend, "active_model_name", None)
) or model_name
- _fill_recommended_sampling_openai(payload, _reco_model_id)
+ _client_sampling_fields = _client_sampling(payload)
+ _fill_recommended_sampling_openai(
+ payload,
+ _reco_model_id,
+ thinking = _sampling_thinking_mode(llama_backend, payload) if using_gguf else None,
+ explicit = _client_sampling_fields,
+ )
# ── Standard OpenAI function-calling pass-through (GGUF only) ────
# When a client (opencode / Claude Code via OpenAI compat / Cursor /
@@ -32341,6 +32369,15 @@ async def produce_openai_chat_completions(
prefer_tool_use = False,
)
+ # The safetensors thinking mode is only known from the template classified above.
+ if _sf_features.get("supports_reasoning"):
+ _fill_recommended_sampling_openai(
+ payload,
+ _reco_model_id,
+ thinking = bool(_sf_parse_think),
+ explicit = _client_sampling_fields,
+ )
+
# A continued turn renders no generation prompt, so nothing is prefilled and the
# resumed text is the visible answer. Only the first turn continues; later tool-loop
# turns render a fresh generation prompt and prefill as usual. MLX also resumes a thought.
@@ -37436,7 +37473,11 @@ async def _responses_stream(
# Streaming /v1/responses builds the passthrough body directly (bypassing
# openai_chat_completions), so apply recommended sampling here too.
- _fill_recommended_sampling_openai(chat_req, getattr(llama_backend, "model_identifier", None))
+ _fill_recommended_sampling_openai(
+ chat_req,
+ getattr(llama_backend, "model_identifier", None),
+ thinking = _sampling_thinking_mode(llama_backend, chat_req),
+ )
body = await _build_openai_passthrough_body_async(
chat_req, backend_ctx = llama_backend.context_length, llama_backend = llama_backend
)
@@ -40387,6 +40428,7 @@ async def anthropic_messages(
"repetition_penalty": payload.repetition_penalty,
"presence_penalty": payload.presence_penalty,
},
+ thinking = _sampling_thinking_mode(llama_backend, payload),
)
temperature = _anthropic_sampling["temperature"]
top_p = _anthropic_sampling["top_p"]
diff --git a/studio/backend/tests/test_npu_chat_route.py b/studio/backend/tests/test_npu_chat_route.py
index 83ba5506950..922b9c85570 100644
--- a/studio/backend/tests/test_npu_chat_route.py
+++ b/studio/backend/tests/test_npu_chat_route.py
@@ -342,6 +342,22 @@ def test_thinking_off_and_non_reasoning_models(flm):
assert recorded[-1]["body"]["think"] is False
+ "fields, expected",
+ [
+ ({}, (0.6, 0.95)),
+ ({"enable_thinking": False}, (0.7, 0.8)),
+ ({"reasoning_effort": "none"}, (0.7, 0.8)),
+ ({"enable_thinking": False, "temperature": 0.3}, (0.3, 0.8)),
+ ],
+)
+def test_qwen_sampling_follows_the_thinking_mode(flm, fields, expected):
+ recorded = flm(reasoning = True)
+ _call(stream = False, **fields)
+ body = recorded[-1]["body"]
+ assert (body["temperature"], body["top_p"]) == expected
+
+
def test_non_streaming_reply_is_collected_from_the_stream(flm):
flm()
status, body = _call(stream = False)
diff --git a/studio/backend/tests/test_sampling_resolution.py b/studio/backend/tests/test_sampling_resolution.py
index 87e16c5a140..07f12b9d7ff 100644
--- a/studio/backend/tests/test_sampling_resolution.py
+++ b/studio/backend/tests/test_sampling_resolution.py
@@ -305,9 +305,9 @@ def test_chat_route_lifts_harness_template_kwargs_before_sampling(monkeypatch, t
real_fill = inference_route._fill_recommended_sampling_openai
- def _capture_after_sampling(route_payload, model_id):
+ def _capture_after_sampling(route_payload, model_id, **kwargs):
assert route_payload.enable_thinking is thinking_mode
- real_fill(route_payload, model_id)
+ real_fill(route_payload, model_id, **kwargs)
raise _StopAfterSampling
monkeypatch.setattr(inference_route, "_automatic_model_load_may_run", lambda: False)
@@ -433,6 +433,214 @@ def test_chat_route_normalizes_reasoning_effort_before_generation(
assert captured["preserve_thinking"] is expected_preserve
+def _qwen_backend(
+ model_identifier,
+ captured,
+ *,
+ supports_reasoning = True,
+ reasoning_default = True,
+):
+ from types import SimpleNamespace
+ def _generate(**kwargs):
+ captured.update(kwargs)
+ yield "done"
+
+ return SimpleNamespace(
+ is_loaded = True,
+ is_vision = False,
+ supports_tools = False,
+ supports_tool_passthrough = False,
+ supports_reasoning = supports_reasoning,
+ reasoning_always_on = False,
+ reasoning_default = reasoning_default,
+ _is_audio = False,
+ model_identifier = model_identifier,
+ context_length = 4096,
+ count_chat_tokens = lambda *a, **k: 2,
+ generate_chat_completion = _generate,
+ )
+
+
+ "model_identifier, supports_reasoning, reasoning_default, request_kwargs, expected",
+ [
+ ("unsloth/Qwen3.6-27B-MTP-GGUF", True, True, {}, (0.6, 0.95, 1.5)),
+ ("unsloth/Qwen3.6-27B-MTP-GGUF", True, True, {"enable_thinking": False}, (0.7, 0.8, 1.5)),
+ ("unsloth/Qwen3.5-4B-GGUF", True, False, {}, (0.7, 0.8, 1.5)),
+ ("unsloth/Qwen3.5-4B-GGUF", True, False, {"reasoning_effort": "high"}, (0.6, 0.95, 1.5)),
+ ("unsloth/Qwen3-8B-GGUF", True, True, {"enable_thinking": False}, (0.7, 0.8, 0.0)),
+ ("unsloth/Qwen3-4B-Instruct-2507-GGUF", False, True, {}, (0.6, 0.95, 0.0)),
+ ],
+)
+def test_chat_route_recommends_the_qwen_sampling_for_the_active_thinking_mode(
+ monkeypatch, model_identifier, supports_reasoning, reasoning_default, request_kwargs, expected
+):
+ import asyncio
+ from types import SimpleNamespace
+
+ from models.inference import ChatCompletionRequest
+ from routes import inference as inference_route
+ from state.tool_policy import reset_tool_policy
+
+ async def _no_auto_switch(*_args, **_kwargs):
+ return None
+
+ class _Request:
+ state = SimpleNamespace(skip_api_monitor = True)
+ url = SimpleNamespace(path = "/v1/chat/completions")
+ method = "POST"
+ scope = {}
+
+ async def is_disconnected(self):
+ return False
+
+ captured = {}
+ reset_tool_policy()
+ llama_backend = _qwen_backend(
+ model_identifier,
+ captured,
+ supports_reasoning = supports_reasoning,
+ reasoning_default = reasoning_default,
+ )
+ monkeypatch.setattr(inference_route, "_automatic_model_load_may_run", lambda: False)
+ monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _no_auto_switch)
+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: llama_backend)
+
+ payload = ChatCompletionRequest(
+ model = "local-model", messages = [{"role": "user", "content": "hi"}], **request_kwargs
+ )
+ response = asyncio.run(
+ inference_route.openai_chat_completions(payload, _Request(), "test-user")
+ )
+
+ assert response.status_code == 200
+ temperature, top_p, presence_penalty = expected
+ assert captured["temperature"] == temperature
+ assert captured["top_p"] == top_p
+ assert captured["top_k"] == 20
+ assert captured["min_p"] == 0.0
+ assert captured["presence_penalty"] == presence_penalty
+
+
+ "thinking, expected", [(None, (0.6, 0.95)), ({"type": "disabled"}, (0.7, 0.8))]
+)
+def test_messages_route_recommends_the_qwen_sampling_for_the_active_thinking_mode(
+ monkeypatch, thinking, expected
+):
+ import asyncio
+ from types import SimpleNamespace
+
+ from models.inference import AnthropicMessagesRequest
+ from routes import inference as inference_route
+
+ class _Request:
+ state = SimpleNamespace()
+ url = SimpleNamespace(path = "/v1/messages")
+ method = "POST"
+
+ async def is_disconnected(self):
+ return False
+
+ captured = {}
+ llama_backend = _qwen_backend("unsloth/Qwen3.6-35B-A3B-GGUF", captured)
+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: llama_backend)
+
+ payload = AnthropicMessagesRequest(
+ max_tokens = 16,
+ messages = [{"role": "user", "content": "hi"}],
+ **({"thinking": thinking} if thinking else {}),
+ )
+ response = asyncio.run(
+ inference_route.anthropic_messages(payload, request = _Request(), current_subject = "t")
+ )
+
+ assert response.status_code == 200
+ assert (captured["temperature"], captured["top_p"]) == expected
+ assert captured["presence_penalty"] == 1.5
+
+
+_QWEN_THINKING_TEMPLATE = (
+ "{% for m in messages %}<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n{% endfor %}"
+ "<|im_start|>assistant\n{% if enable_thinking is defined and enable_thinking is false %}"
+ "<think>\n\n</think>\n\n{% endif %}"
+)
+_PLAIN_TEMPLATE = (
+ "{% for m in messages %}<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n{% endfor %}"
+)
+
+
+ "template, request_kwargs, expected",
+ [
+ (_QWEN_THINKING_TEMPLATE, {}, (0.6, 0.95)),
+ (_QWEN_THINKING_TEMPLATE, {"enable_thinking": True}, (0.6, 0.95)),
+ (_QWEN_THINKING_TEMPLATE, {"enable_thinking": False}, (0.7, 0.8)),
+ (_QWEN_THINKING_TEMPLATE, {"enable_thinking": True, "temperature": 0.3}, (0.3, 0.95)),
+ (_PLAIN_TEMPLATE, {"enable_thinking": True}, (0.7, 0.8)),
+ ],
+)
+def test_safetensors_chat_route_recommends_the_qwen_sampling_for_the_active_thinking_mode(
+ monkeypatch, template, request_kwargs, expected
+):
+ import asyncio
+ from types import SimpleNamespace
+
+ from models.inference import ChatCompletionRequest
+ from routes import inference as inference_route
+ from state.tool_policy import reset_tool_policy
+
+ class _Request:
+ state = SimpleNamespace(skip_api_monitor = True)
+ url = SimpleNamespace(path = "/v1/chat/completions")
+ method = "POST"
+ scope = {}
+ headers = {}
+
+ async def is_disconnected(self):
+ return False
+
+ captured = {}
+
+ class _Backend:
+ active_model_name = "unsloth/Qwen3.6-27B"
+ models = {
+ "unsloth/Qwen3.6-27B": {
+ "chat_template_info": {"template": template},
+ "context_length": 2048,
+ }
+ }
+
+ def generate_chat_response(self, **kwargs):
+ captured.update(kwargs)
+ yield "done"
+
+ def reset_generation_state(self, caller_cancel_event = None):
+ pass
+
+ reset_tool_policy()
+ monkeypatch.setattr(inference_route, "_automatic_model_load_may_run", lambda: False)
+ monkeypatch.setattr(
+ inference_route,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(
+ is_loaded = False, supports_tools = False, is_vision = False, context_length = None
+ ),
+ )
+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: _Backend())
+
+ payload = ChatCompletionRequest(
+ model = "default",
+ messages = [{"role": "user", "content": "hi"}],
+ stream = False,
+ **request_kwargs,
+ )
+ asyncio.run(inference_route.openai_chat_completions(payload, _Request(), "test-user"))
+
+ assert (captured["temperature"], captured["top_p"]) == expected
+ assert captured["presence_penalty"] == 1.5
+
+
@pytest.mark.parametrize(
"reasoning_style, enable_thinking, reasoning_effort, expected",
[
diff --git a/studio/backend/utils/inference/inference_config.py b/studio/backend/utils/inference/inference_config.py
index c91919db76b..3633e8e1109 100644
--- a/studio/backend/utils/inference/inference_config.py
+++ b/studio/backend/utils/inference/inference_config.py
@@ -149,6 +149,12 @@ SAMPLING_FIELD_NAMES = tuple(_SAMPLING_FIELDS)
# The five fields the Chat UI's mergeBackendRecommendedInference (presets/preset-policy.ts) seeds, auto-recommended here for request parity. repetition_penalty stays manual-only (client-sent or the UNSLOTH_SAMPLING_REPETITION_PENALTY pin), matching the UI where it is never auto-filled per model.
_UI_RECOMMENDED_FIELDS = ("temperature", "top_p", "top_k", "min_p", "presence_penalty")
+# Mirrors resolveQwenThinkingParams in the Chat UI (qwen-sampling-table.ts).
+_QWEN3_THINKING_SAMPLING = {
+ True: {"temperature": 0.6, "top_p": 0.95, "top_k": 20, "min_p": 0.0},
+ False: {"temperature": 0.7, "top_p": 0.8, "top_k": 20, "min_p": 0.0},
+}
+
def _clean_sampling_value(field: str, val: Any):
"""Coerce ``val`` to the field's numeric type when it is a finite, in-range number, else None. Rejects bool, non-numeric, NaN/inf and out-of-range values so neither a bad operator env var nor a malformed model recommendation can reach llama-server. NaN matters because ``nan < lo`` and ``nan > hi`` are both False, so a plain range check would let it through. Coerce before the finiteness check: ``math.isfinite`` and ``float()`` raise ``OverflowError`` on an int too big for a C double (an oversized UNSLOTH_SAMPLING_TOP_K would otherwise 500 the request), while an in-range int is range-checked exactly and ``int()`` rejects a NaN/inf that reached an int field."""
@@ -204,9 +210,12 @@ def resolve_effective_sampling(
explicit: Dict[str, Any],
*,
fill_defaults: bool = True,
+ thinking: Optional[bool] = None,
) -> Dict[str, Any]:
"""Resolve the effective sampling params for a request. ``explicit`` maps each field in :data:`SAMPLING_FIELD_NAMES` to the client-sent value, or ``None`` when the client omitted it. Precedence, highest first: an operator ``UNSLOTH_SAMPLING_*`` pin, the client's explicit value, the per-model recommendation, then the static schema default. When ``fill_defaults`` is False a field with none of the first three is omitted rather than set to the static default, so a raw proxy body (``/v1/completions``) keeps llama-server's own default for that field."""
recommended = _recommended_sampling(model_id or "")
+ if thinking is not None and "qwen3" in (model_id or "").lower():
+ recommended = {**recommended, **_QWEN3_THINKING_SAMPLING[thinking]}
effective: Dict[str, Any] = {}
for field, (_env, default, _lo, _hi, _int) in _SAMPLING_FIELDS.items():
override = _operator_sampling_override(field)