Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: give API clients the same Qwen thinking sampling as Chat (#12791)
Nilay1150a7de77ee
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex e04ffc8e44f..29f8f16506c 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -28501,12 +28501,16 @@ async def _npu_chat_completions(payload, request: Request, current_subject: str)),)- _fill_recommended_sampling_openai(payload, upstream.model)_normalize_chat_reasoning_controls(payload)if not upstream.supports_reasoning:payload.enable_thinking = Falseelif payload.enable_thinking is None:payload.enable_thinking = payload.reasoning_effort != "none"+ _fill_recommended_sampling_openai(+ payload,+ upstream.model,+ thinking = payload.enable_thinking if upstream.supports_reasoning else None,+ )wants_stream = bool(payload.stream)relay = _NpuStreamRelay(upstream.public_model, _normalize_stop_sequences(payload.stop))@@ -28896,22 +28900,40 @@ def _normalize_chat_reasoning_controls(payload) -> None:payload.preserve_thinking = nested["preserve_thinking"]-def _fill_recommended_sampling_openai(payload, model_id) -> None:+def _sampling_thinking_mode(llama_backend, payload) -> Optional[bool]:+ if not getattr(llama_backend, "supports_reasoning", False):+ return None+ return _think_parsing_expected(llama_backend, payload)+++def _client_sampling(payload) -> dict:+ from utils.inference.inference_config import SAMPLING_FIELD_NAMES+ return {+ f: (getattr(payload, f) if f in payload.model_fields_set else None)+ for f in SAMPLING_FIELD_NAMES+ }+++def _fill_recommended_sampling_openai(+ payload,+ model_id,+ thinking = None,+ explicit = None,+) -> None:"""Apply per-model recommended sampling (and any operator UNSLOTH_SAMPLING_* pin) to aChatCompletionRequest in place.Only the sampling fields the client did NOT explicitly send (tracked via``model_fields_set``) are overwritten, so a client that sets a field stays byte-identicalunless an operator pins it. Fields with neither a recommendation nor a pin keep their- existing (schema-default) value.+ existing (schema-default) value. A second fill must pass the first fill's ``explicit``:+ setattr marks every field as set."""- from utils.inference.inference_config import resolve_effective_sampling, SAMPLING_FIELD_NAMES+ from utils.inference.inference_config import resolve_effective_sampling- explicit = {- f: (getattr(payload, f) if f in payload.model_fields_set else None)- for f in SAMPLING_FIELD_NAMES- }- effective = resolve_effective_sampling(model_id, explicit)+ if explicit is None:+ explicit = _client_sampling(payload)+ effective = resolve_effective_sampling(model_id, explicit, thinking = thinking)for field, value in effective.items():setattr(payload, field, value)@@ -29923,7 +29945,13 @@ async def produce_openai_chat_completions(if using_ggufelse getattr(backend, "active_model_name", None)) or model_name- _fill_recommended_sampling_openai(payload, _reco_model_id)+ _client_sampling_fields = _client_sampling(payload)+ _fill_recommended_sampling_openai(+ payload,+ _reco_model_id,+ thinking = _sampling_thinking_mode(llama_backend, payload) if using_gguf else None,+ explicit = _client_sampling_fields,+ )# ── Standard OpenAI function-calling pass-through (GGUF only) ────# When a client (opencode / Claude Code via OpenAI compat / Cursor /@@ -32341,6 +32369,15 @@ async def produce_openai_chat_completions(prefer_tool_use = False,)+ # The safetensors thinking mode is only known from the template classified above.+ if _sf_features.get("supports_reasoning"):+ _fill_recommended_sampling_openai(+ payload,+ _reco_model_id,+ thinking = bool(_sf_parse_think),+ explicit = _client_sampling_fields,+ )+# A continued turn renders no generation prompt, so nothing is prefilled and the# resumed text is the visible answer. Only the first turn continues; later tool-loop# turns render a fresh generation prompt and prefill as usual. MLX also resumes a thought.@@ -37436,7 +37473,11 @@ async def _responses_stream(# Streaming /v1/responses builds the passthrough body directly (bypassing# openai_chat_completions), so apply recommended sampling here too.- _fill_recommended_sampling_openai(chat_req, getattr(llama_backend, "model_identifier", None))+ _fill_recommended_sampling_openai(+ chat_req,+ getattr(llama_backend, "model_identifier", None),+ thinking = _sampling_thinking_mode(llama_backend, chat_req),+ )body = await _build_openai_passthrough_body_async(chat_req, backend_ctx = llama_backend.context_length, llama_backend = llama_backend)@@ -40387,6 +40428,7 @@ async def anthropic_messages("repetition_penalty": payload.repetition_penalty,"presence_penalty": payload.presence_penalty,},+ thinking = _sampling_thinking_mode(llama_backend, payload),)temperature = _anthropic_sampling["temperature"]top_p = _anthropic_sampling["top_p"]diff --git a/studio/backend/tests/test_npu_chat_route.py b/studio/backend/tests/test_npu_chat_route.pyindex 83ba5506950..922b9c85570 100644--- a/studio/backend/tests/test_npu_chat_route.py+++ b/studio/backend/tests/test_npu_chat_route.py@@ -342,6 +342,22 @@ def test_thinking_off_and_non_reasoning_models(flm):assert recorded[-1]["body"]["think"] is False+ "fields, expected",+ [+ ({}, (0.6, 0.95)),+ ({"enable_thinking": False}, (0.7, 0.8)),+ ({"reasoning_effort": "none"}, (0.7, 0.8)),+ ({"enable_thinking": False, "temperature": 0.3}, (0.3, 0.8)),+ ],+)+def test_qwen_sampling_follows_the_thinking_mode(flm, fields, expected):+ recorded = flm(reasoning = True)+ _call(stream = False, **fields)+ body = recorded[-1]["body"]+ assert (body["temperature"], body["top_p"]) == expected++def test_non_streaming_reply_is_collected_from_the_stream(flm):flm()status, body = _call(stream = False)diff --git a/studio/backend/tests/test_sampling_resolution.py b/studio/backend/tests/test_sampling_resolution.pyindex 87e16c5a140..07f12b9d7ff 100644--- a/studio/backend/tests/test_sampling_resolution.py+++ b/studio/backend/tests/test_sampling_resolution.py@@ -305,9 +305,9 @@ def test_chat_route_lifts_harness_template_kwargs_before_sampling(monkeypatch, treal_fill = inference_route._fill_recommended_sampling_openai- def _capture_after_sampling(route_payload, model_id):+ def _capture_after_sampling(route_payload, model_id, **kwargs):assert route_payload.enable_thinking is thinking_mode- real_fill(route_payload, model_id)+ real_fill(route_payload, model_id, **kwargs)raise _StopAfterSamplingmonkeypatch.setattr(inference_route, "_automatic_model_load_may_run", lambda: False)@@ -433,6 +433,214 @@ def test_chat_route_normalizes_reasoning_effort_before_generation(assert captured["preserve_thinking"] is expected_preserve+def _qwen_backend(+ model_identifier,+ captured,+ *,+ supports_reasoning = True,+ reasoning_default = True,+):+ from types import SimpleNamespace+ def _generate(**kwargs):+ captured.update(kwargs)+ yield "done"++ return SimpleNamespace(+ is_loaded = True,+ is_vision = False,+ supports_tools = False,+ supports_tool_passthrough = False,+ supports_reasoning = supports_reasoning,+ reasoning_always_on = False,+ reasoning_default = reasoning_default,+ _is_audio = False,+ model_identifier = model_identifier,+ context_length = 4096,+ count_chat_tokens = lambda *a, **k: 2,+ generate_chat_completion = _generate,+ )+++ "model_identifier, supports_reasoning, reasoning_default, request_kwargs, expected",+ [+ ("unsloth/Qwen3.6-27B-MTP-GGUF", True, True, {}, (0.6, 0.95, 1.5)),+ ("unsloth/Qwen3.6-27B-MTP-GGUF", True, True, {"enable_thinking": False}, (0.7, 0.8, 1.5)),+ ("unsloth/Qwen3.5-4B-GGUF", True, False, {}, (0.7, 0.8, 1.5)),+ ("unsloth/Qwen3.5-4B-GGUF", True, False, {"reasoning_effort": "high"}, (0.6, 0.95, 1.5)),+ ("unsloth/Qwen3-8B-GGUF", True, True, {"enable_thinking": False}, (0.7, 0.8, 0.0)),+ ("unsloth/Qwen3-4B-Instruct-2507-GGUF", False, True, {}, (0.6, 0.95, 0.0)),+ ],+)+def test_chat_route_recommends_the_qwen_sampling_for_the_active_thinking_mode(+ monkeypatch, model_identifier, supports_reasoning, reasoning_default, request_kwargs, expected+):+ import asyncio+ from types import SimpleNamespace++ from models.inference import ChatCompletionRequest+ from routes import inference as inference_route+ from state.tool_policy import reset_tool_policy++ async def _no_auto_switch(*_args, **_kwargs):+ return None++ class _Request:+ state = SimpleNamespace(skip_api_monitor = True)+ url = SimpleNamespace(path = "/v1/chat/completions")+ method = "POST"+ scope = {}++ async def is_disconnected(self):+ return False++ captured = {}+ reset_tool_policy()+ llama_backend = _qwen_backend(+ model_identifier,+ captured,+ supports_reasoning = supports_reasoning,+ reasoning_default = reasoning_default,+ )+ monkeypatch.setattr(inference_route, "_automatic_model_load_may_run", lambda: False)+ monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _no_auto_switch)+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: llama_backend)++ payload = ChatCompletionRequest(+ model = "local-model", messages = [{"role": "user", "content": "hi"}], **request_kwargs+ )+ response = asyncio.run(+ inference_route.openai_chat_completions(payload, _Request(), "test-user")+ )++ assert response.status_code == 200+ temperature, top_p, presence_penalty = expected+ assert captured["temperature"] == temperature+ assert captured["top_p"] == top_p+ assert captured["top_k"] == 20+ assert captured["min_p"] == 0.0+ assert captured["presence_penalty"] == presence_penalty+++ "thinking, expected", [(None, (0.6, 0.95)), ({"type": "disabled"}, (0.7, 0.8))]+)+def test_messages_route_recommends_the_qwen_sampling_for_the_active_thinking_mode(+ monkeypatch, thinking, expected+):+ import asyncio+ from types import SimpleNamespace++ from models.inference import AnthropicMessagesRequest+ from routes import inference as inference_route++ class _Request:+ state = SimpleNamespace()+ url = SimpleNamespace(path = "/v1/messages")+ method = "POST"++ async def is_disconnected(self):+ return False++ captured = {}+ llama_backend = _qwen_backend("unsloth/Qwen3.6-35B-A3B-GGUF", captured)+ monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: llama_backend)++ payload = AnthropicMessagesRequest(+ max_tokens = 16,+ messages = [{"role": "user", "content": "hi"}],+ **({"thinking": thinking} if thinking else {}),+ )+ response = asyncio.run(+ inference_route.anthropic_messages(payload, request = _Request(), current_subject = "t")+ )++ assert response.status_code == 200+ assert (captured["temperature"], captured["top_p"]) == expected+ assert captured["presence_penalty"] == 1.5+++_QWEN_THINKING_TEMPLATE = (+ "{% for m in messages %}<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n{% endfor %}"+ "<|im_start|>assistant\n{% if enable_thinking is defined and enable_thinking is false %}"+ "<think>\n\n</think>\n\n{% endif %}"+)+_PLAIN_TEMPLATE = (+ "{% for m in messages %}<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n{% endfor %}"+)+++ "template, request_kwargs, expected",+ [+ (_QWEN_THINKING_TEMPLATE, {}, (0.6, 0.95)),+ (_QWEN_THINKING_TEMPLATE, {"enable_thinking": True}, (0.6, 0.95)),+ (_QWEN_THINKING_TEMPLATE, {"enable_thinking": False}, (0.7, 0.8)),+ (_QWEN_THINKING_TEMPLATE, {"enable_thinking": True, "temperature": 0.3}, (0.3, 0.95)),+ (_PLAIN_TEMPLATE, {"enable_thinking": True}, (0.7, 0.8)),+ ],+)+def test_safetensors_chat_route_recommends_the_qwen_sampling_for_the_active_thinking_mode(+ monkeypatch, template, request_kwargs, expected+):+ import asyncio+ from types import SimpleNamespace++ from models.inference import ChatCompletionRequest+ from routes import inference as inference_route+ from state.tool_policy import reset_tool_policy++ class _Request:+ state = SimpleNamespace(skip_api_monitor = True)+ url = SimpleNamespace(path = "/v1/chat/completions")+ method = "POST"+ scope = {}+ headers = {}++ async def is_disconnected(self):+ return False++ captured = {}++ class _Backend:+ active_model_name = "unsloth/Qwen3.6-27B"+ models = {+ "unsloth/Qwen3.6-27B": {+ "chat_template_info": {"template": template},+ "context_length": 2048,+ }+ }++ def generate_chat_response(self, **kwargs):+ captured.update(kwargs)+ yield "done"++ def reset_generation_state(self, caller_cancel_event = None):+ pass++ reset_tool_policy()+ monkeypatch.setattr(inference_route, "_automatic_model_load_may_run", lambda: False)+ monkeypatch.setattr(+ inference_route,+ "get_llama_cpp_backend",+ lambda: SimpleNamespace(+ is_loaded = False, supports_tools = False, is_vision = False, context_length = None+ ),+ )+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: _Backend())++ payload = ChatCompletionRequest(+ model = "default",+ messages = [{"role": "user", "content": "hi"}],+ stream = False,+ **request_kwargs,+ )+ asyncio.run(inference_route.openai_chat_completions(payload, _Request(), "test-user"))++ assert (captured["temperature"], captured["top_p"]) == expected+ assert captured["presence_penalty"] == 1.5++@pytest.mark.parametrize("reasoning_style, enable_thinking, reasoning_effort, expected",[diff --git a/studio/backend/utils/inference/inference_config.py b/studio/backend/utils/inference/inference_config.pyindex c91919db76b..3633e8e1109 100644--- a/studio/backend/utils/inference/inference_config.py+++ b/studio/backend/utils/inference/inference_config.py@@ -149,6 +149,12 @@ SAMPLING_FIELD_NAMES = tuple(_SAMPLING_FIELDS)# The five fields the Chat UI's mergeBackendRecommendedInference (presets/preset-policy.ts) seeds, auto-recommended here for request parity. repetition_penalty stays manual-only (client-sent or the UNSLOTH_SAMPLING_REPETITION_PENALTY pin), matching the UI where it is never auto-filled per model._UI_RECOMMENDED_FIELDS = ("temperature", "top_p", "top_k", "min_p", "presence_penalty")+# Mirrors resolveQwenThinkingParams in the Chat UI (qwen-sampling-table.ts).+_QWEN3_THINKING_SAMPLING = {+ True: {"temperature": 0.6, "top_p": 0.95, "top_k": 20, "min_p": 0.0},+ False: {"temperature": 0.7, "top_p": 0.8, "top_k": 20, "min_p": 0.0},+}+def _clean_sampling_value(field: str, val: Any):"""Coerce ``val`` to the field's numeric type when it is a finite, in-range number, else None. Rejects bool, non-numeric, NaN/inf and out-of-range values so neither a bad operator env var nor a malformed model recommendation can reach llama-server. NaN matters because ``nan < lo`` and ``nan > hi`` are both False, so a plain range check would let it through. Coerce before the finiteness check: ``math.isfinite`` and ``float()`` raise ``OverflowError`` on an int too big for a C double (an oversized UNSLOTH_SAMPLING_TOP_K would otherwise 500 the request), while an in-range int is range-checked exactly and ``int()`` rejects a NaN/inf that reached an int field."""@@ -204,9 +210,12 @@ def resolve_effective_sampling(explicit: Dict[str, Any],*,fill_defaults: bool = True,+ thinking: Optional[bool] = None,) -> Dict[str, Any]:"""Resolve the effective sampling params for a request. ``explicit`` maps each field in :data:`SAMPLING_FIELD_NAMES` to the client-sent value, or ``None`` when the client omitted it. Precedence, highest first: an operator ``UNSLOTH_SAMPLING_*`` pin, the client's explicit value, the per-model recommendation, then the static schema default. When ``fill_defaults`` is False a field with none of the first three is omitted rather than set to the static default, so a raw proxy body (``/v1/completions``) keeps llama-server's own default for that field."""recommended = _recommended_sampling(model_id or "")+ if thinking is not None and "qwen3" in (model_id or "").lower():+ recommended = {**recommended, **_QWEN3_THINKING_SAMPLING[thinking]}effective: Dict[str, Any] = {}for field, (_env, default, _lo, _hi, _int) in _SAMPLING_FIELDS.items():override = _operator_sampling_override(field)