Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Precedence: UNSLOTH_SAMPLING_* pin, client value, model recommendation, then schema default."""
from pathlib import Path
import pytest
import yaml
from utils.inference.inference_config import resolve_effective_sampling, SAMPLING_FIELD_NAMES
from utils.inference import inference_config as ic
_SCHEMA_DEFAULTS = {
"temperature": 0.6,
"top_p": 0.95,
"top_k": 20,
"min_p": 0.01,
"repetition_penalty": 1.0,
"presence_penalty": 0.0,
}
@pytest.fixture(autouse = True)
def _isolate(monkeypatch):
ic._recommended_sampling.cache_clear()
for field in SAMPLING_FIELD_NAMES:
monkeypatch.delenv(ic._SAMPLING_FIELDS[field][0], raising = False)
yield
ic._recommended_sampling.cache_clear()
def _all_omitted():
return {f: None for f in SAMPLING_FIELD_NAMES}
def _set_recommended(monkeypatch, mapping):
# Patch load_inference_config directly: _recommended_sampling sources from it, as the Chat UI does.
monkeypatch.setattr(ic, "load_inference_config", lambda mid: dict(mapping))
ic._recommended_sampling.cache_clear()
def test_recommended_applies_when_client_omits(monkeypatch):
_set_recommended(monkeypatch, {"temperature": 1.0, "top_k": 64, "min_p": 0.0})
eff = resolve_effective_sampling("some/model", _all_omitted())
assert eff["temperature"] == 1.0
assert eff["top_k"] == 64
assert eff["min_p"] == 0.0
assert eff["top_p"] == 0.95
def test_client_explicit_beats_recommended(monkeypatch):
_set_recommended(monkeypatch, {"temperature": 1.0})
eff = resolve_effective_sampling("some/model", {**_all_omitted(), "temperature": 0.2})
assert eff["temperature"] == 0.2
def test_operator_pin_beats_client_and_recommended(monkeypatch):
_set_recommended(monkeypatch, {"temperature": 1.0})
monkeypatch.setenv("UNSLOTH_SAMPLING_TEMPERATURE", "0.9")
eff = resolve_effective_sampling("some/model", {**_all_omitted(), "temperature": 0.2})
assert eff["temperature"] == 0.9
def test_unknown_model_matches_ui_inference_block(monkeypatch):
ui_block = {
"temperature": 0.7,
"top_p": 0.95,
"top_k": -1,
"min_p": 0.01,
"presence_penalty": 0.0,
"repetition_penalty": 1.0,
}
monkeypatch.setattr(ic, "load_inference_config", lambda mid: dict(ui_block))
ic._recommended_sampling.cache_clear()
eff = resolve_effective_sampling("some/unknown-model", _all_omitted())
assert eff["temperature"] == 0.7
assert eff["top_k"] == -1
assert eff["min_p"] == 0.01
def test_empty_recommendation_falls_back_to_schema_defaults(monkeypatch):
monkeypatch.setattr(ic, "load_inference_config", lambda mid: {})
ic._recommended_sampling.cache_clear()
eff = resolve_effective_sampling("some/model", _all_omitted())
assert eff == _SCHEMA_DEFAULTS
@pytest.mark.parametrize(
"model",
["unsloth/gemma-4-E4B", "unsloth/Qwen3-4B", "unsloth/Qwen3.5-9B", "someorg/unknown-xyz"],
)
def test_recommendation_matches_ui_source(model):
ic._recommended_sampling.cache_clear()
ui = ic.load_inference_config(model)
rec = ic._recommended_sampling(model)
for f in ic._UI_RECOMMENDED_FIELDS:
cleaned = ic._clean_sampling_value(f, ui.get(f))
if cleaned is not None:
assert rec.get(f) == cleaned, f"{model}:{f} rec={rec.get(f)} ui={ui.get(f)}"
def test_model_recommended_sampling_values_are_in_range():
defaults_dir = Path(ic.__file__).resolve().parents[2] / "assets" / "configs" / "model_defaults"
invalid = []
for path in sorted(defaults_dir.rglob("*.yaml")):
inference = (yaml.safe_load(path.read_text(encoding = "utf-8")) or {}).get(
"inference", {}
) or {}
for field in ic._UI_RECOMMENDED_FIELDS:
if field in inference and ic._clean_sampling_value(field, inference[field]) is None:
invalid.append(f"{path.relative_to(defaults_dir)}:{field}={inference[field]!r}")
assert not invalid, "Out-of-range model sampling defaults: " + ", ".join(invalid)
def test_qwen38_reuses_qwen36_sampling_defaults():
qwen36 = ic.load_inference_config("unsloth/Qwen3.6-27B-GGUF")
qwen38 = ic.load_inference_config("unsloth/Qwen3.8-27B-GGUF")
assert qwen38 == qwen36
assert qwen38 == {
"temperature": 0.7,
"top_p": 0.8,
"top_k": 20,
"min_p": 0.0,
"presence_penalty": 1.5,
"trust_remote_code": False,
}
def test_repetition_penalty_not_auto_recommended(monkeypatch):
# The Chat UI never adopts a backend repetition_penalty, so the server must not auto-apply one.
monkeypatch.setattr(
ic, "load_inference_config", lambda mid: {"temperature": 0.7, "repetition_penalty": 1.05}
)
ic._recommended_sampling.cache_clear()
eff = resolve_effective_sampling("some/lfm2-model", _all_omitted())
assert eff["temperature"] == 0.7
assert eff["repetition_penalty"] == 1.0
monkeypatch.setenv("UNSLOTH_SAMPLING_REPETITION_PENALTY", "1.05")
eff2 = resolve_effective_sampling("some/lfm2-model", _all_omitted())
assert eff2["repetition_penalty"] == 1.05
@pytest.mark.parametrize(
"raw, expected",
[
("0.5", 0.5),
("abc", None),
("9.0", None),
("-1", None),
(" ", None),
("nan", None), # NaN would pass a naive range check
("inf", None),
("-inf", None),
],
)
def test_operator_override_parsing(monkeypatch, raw, expected):
monkeypatch.setenv("UNSLOTH_SAMPLING_TEMPERATURE", raw)
assert ic._operator_sampling_override("temperature") == expected
def test_out_of_range_recommendation_is_dropped(monkeypatch):
_set_recommended(monkeypatch, {"temperature": 5.0, "top_k": 64})
eff = resolve_effective_sampling("some/model", _all_omitted())
assert eff["temperature"] == 0.6
assert eff["top_k"] == 64
def test_operator_override_top_k_int_and_range(monkeypatch):
monkeypatch.setenv("UNSLOTH_SAMPLING_TOP_K", "40")
assert ic._operator_sampling_override("top_k") == 40
monkeypatch.setenv("UNSLOTH_SAMPLING_TOP_K", "200")
assert ic._operator_sampling_override("top_k") is None
monkeypatch.setenv("UNSLOTH_SAMPLING_TOP_K", "-1")
assert ic._operator_sampling_override("top_k") == -1
@pytest.mark.parametrize(
"field, val",
[
("top_k", 10**400),
("top_k", float("nan")),
("top_k", float("inf")),
(
"temperature",
10**400,
),
],
)
def test_clean_sampling_value_rejects_unrepresentable(field, val):
assert ic._clean_sampling_value(field, val) is None
def test_oversized_operator_override_ignored(monkeypatch):
monkeypatch.setenv("UNSLOTH_SAMPLING_TOP_K", "9" * 400)
assert ic._operator_sampling_override("top_k") is None
_set_recommended(monkeypatch, {})
eff = resolve_effective_sampling("some/model", _all_omitted())
assert eff["top_k"] == 20
def test_oversized_recommendation_ignored(monkeypatch):
_set_recommended(monkeypatch, {"temperature": 10**400, "top_k": 64})
eff = resolve_effective_sampling("some/model", _all_omitted())
assert eff["temperature"] == 0.6
assert eff["top_k"] == 64
def test_fill_recommended_sampling_openai_payload(monkeypatch):
from models.inference import ChatCompletionRequest
from routes.inference import _fill_recommended_sampling_openai
_set_recommended(monkeypatch, {"temperature": 1.0, "top_k": 64, "min_p": 0.0})
payload = ChatCompletionRequest(
model = "m", messages = [{"role": "user", "content": "hi"}], temperature = 0.2
)
_fill_recommended_sampling_openai(payload, "some/model")
assert payload.temperature == 0.2
assert payload.top_k == 64
assert payload.min_p == 0.0
assert payload.top_p == 0.95
def test_fill_recommended_sampling_openai_operator_pin_overrides_client(monkeypatch):
from models.inference import ChatCompletionRequest
from routes.inference import _fill_recommended_sampling_openai
monkeypatch.setattr(ic, "load_model_defaults", lambda mid: {})
monkeypatch.setattr(ic, "get_family_inference_params", lambda mid: {})
ic._recommended_sampling.cache_clear()
monkeypatch.setenv("UNSLOTH_SAMPLING_TEMPERATURE", "0.9")
payload = ChatCompletionRequest(
model = "m", messages = [{"role": "user", "content": "hi"}], temperature = 0.2
)
_fill_recommended_sampling_openai(payload, "some/model")
assert payload.temperature == 0.9
@pytest.mark.parametrize("thinking_mode", [True, False])
def test_chat_route_lifts_harness_template_kwargs_before_sampling(monkeypatch, thinking_mode):
"""The chat route must lift chat_template_kwargs onto the typed field before sampling is filled."""
import asyncio
from types import SimpleNamespace
from models.inference import ChatCompletionRequest
from routes import inference as inference_route
class _StopAfterSampling(Exception):
pass
async def _no_auto_switch(*_args, **_kwargs):
return None
llama_backend = SimpleNamespace(
is_loaded = True,
model_identifier = "unsloth/Qwen3.8-27B-GGUF",
_is_audio = False,
)
request = SimpleNamespace(
state = SimpleNamespace(skip_api_monitor = True),
url = SimpleNamespace(path = "/v1/chat/completions"),
method = "POST",
scope = {},
)
payload = ChatCompletionRequest(
model = "deepseek-harness-model",
messages = [{"role": "user", "content": "hi"}],
chat_template_kwargs = {"enable_thinking": thinking_mode},
)
assert payload.enable_thinking is None
real_fill = inference_route._fill_recommended_sampling_openai
def _capture_after_sampling(route_payload, model_id, **kwargs):
assert route_payload.enable_thinking is thinking_mode
real_fill(route_payload, model_id, **kwargs)
raise _StopAfterSampling
monkeypatch.setattr(inference_route, "_automatic_model_load_may_run", lambda: False)
monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _no_auto_switch)
monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: llama_backend)
monkeypatch.setattr(
inference_route, "_fill_recommended_sampling_openai", _capture_after_sampling
)
with pytest.raises(_StopAfterSampling):
asyncio.run(inference_route.openai_chat_completions(payload, request, "test-user"))
@pytest.mark.parametrize(
"request_kwargs, expected_thinking, expected_effort, expected_preserve",
[
({"reasoning_effort": "none"}, False, "none", None),
({"enable_thinking": True, "reasoning_effort": "none"}, True, None, None),
({"enable_thinking": False, "reasoning_effort": "high"}, False, None, None),
({"thinking": {"type": "enabled"}, "reasoning_effort": "none"}, False, "none", None),
(
{
"chat_template_kwargs": {
"enable_thinking": True,
"reasoning_effort": "medium",
"preserve_thinking": True,
}
},
True,
"medium",
True,
),
(
{
"reasoning_effort": "high",
"preserve_thinking": False,
"chat_template_kwargs": {
"enable_thinking": False,
"reasoning_effort": "xhigh",
"preserve_thinking": True,
},
},
True,
"high",
False,
),
(
{"chat_template_kwargs": {"enable_thinking": True, "reasoning_effort": "none"}},
True,
None,
None,
),
# bool("false") is True, so an invalid nested type is ignored, not coerced.
({"chat_template_kwargs": {"enable_thinking": "false"}}, None, None, None),
(
{"chat_template_kwargs": {"enable_thinking": False, "reasoning_effort": {}}},
False,
None,
None,
),
],
)
def test_chat_route_normalizes_reasoning_effort_before_generation(
monkeypatch, request_kwargs, expected_thinking, expected_effort, expected_preserve
):
import asyncio
from types import SimpleNamespace
from models.inference import ChatCompletionRequest
from routes import inference as inference_route
from state.tool_policy import reset_tool_policy
captured = {}
def _generate(**kwargs):
captured.update(kwargs)
yield "done"
async def _no_auto_switch(*_args, **_kwargs):
return None
reset_tool_policy()
llama_backend = SimpleNamespace(
is_loaded = True,
is_vision = False,
supports_tools = False,
supports_reasoning = True,
reasoning_always_on = False,
_is_audio = False,
model_identifier = "unsloth/Qwen3.8-27B-GGUF",
context_length = 4096,
generate_chat_completion = _generate,
)
class _Request:
state = SimpleNamespace(skip_api_monitor = True)
url = SimpleNamespace(path = "/v1/chat/completions")
method = "POST"
scope = {}
async def is_disconnected(self):
return False
payload = ChatCompletionRequest(
model = "local-model",
messages = [{"role": "user", "content": "hi"}],
**request_kwargs,
)
monkeypatch.setattr(inference_route, "_automatic_model_load_may_run", lambda: False)
monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _no_auto_switch)
monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: llama_backend)
response = asyncio.run(
inference_route.openai_chat_completions(payload, _Request(), "test-user")
)
assert response.status_code == 200
assert captured["enable_thinking"] is expected_thinking
assert captured["reasoning_effort"] == expected_effort
assert captured["preserve_thinking"] is expected_preserve
def _qwen_backend(
model_identifier,
captured,
*,
supports_reasoning = True,
reasoning_default = True,
):
from types import SimpleNamespace
def _generate(**kwargs):
captured.update(kwargs)
yield "done"
return SimpleNamespace(
is_loaded = True,
is_vision = False,
supports_tools = False,
supports_tool_passthrough = False,
supports_reasoning = supports_reasoning,
reasoning_always_on = False,
reasoning_default = reasoning_default,
_is_audio = False,
model_identifier = model_identifier,
context_length = 4096,
count_chat_tokens = lambda *a, **k: 2,
generate_chat_completion = _generate,
)
@pytest.mark.parametrize(
"model_identifier, supports_reasoning, reasoning_default, request_kwargs, expected",
[
("unsloth/Qwen3.6-27B-MTP-GGUF", True, True, {}, (0.6, 0.95, 1.5)),
("unsloth/Qwen3.6-27B-MTP-GGUF", True, True, {"enable_thinking": False}, (0.7, 0.8, 1.5)),
("unsloth/Qwen3.5-4B-GGUF", True, False, {}, (0.7, 0.8, 1.5)),
("unsloth/Qwen3.5-4B-GGUF", True, False, {"reasoning_effort": "high"}, (0.6, 0.95, 1.5)),
("unsloth/Qwen3-8B-GGUF", True, True, {"enable_thinking": False}, (0.7, 0.8, 0.0)),
("unsloth/Qwen3-4B-Instruct-2507-GGUF", False, True, {}, (0.6, 0.95, 0.0)),
],
)
def test_chat_route_recommends_the_qwen_sampling_for_the_active_thinking_mode(
monkeypatch, model_identifier, supports_reasoning, reasoning_default, request_kwargs, expected
):
import asyncio
from types import SimpleNamespace
from models.inference import ChatCompletionRequest
from routes import inference as inference_route
from state.tool_policy import reset_tool_policy
async def _no_auto_switch(*_args, **_kwargs):
return None
class _Request:
state = SimpleNamespace(skip_api_monitor = True)
url = SimpleNamespace(path = "/v1/chat/completions")
method = "POST"
scope = {}
async def is_disconnected(self):
return False
captured = {}
reset_tool_policy()
llama_backend = _qwen_backend(
model_identifier,
captured,
supports_reasoning = supports_reasoning,
reasoning_default = reasoning_default,
)
monkeypatch.setattr(inference_route, "_automatic_model_load_may_run", lambda: False)
monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _no_auto_switch)
monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: llama_backend)
payload = ChatCompletionRequest(
model = "local-model", messages = [{"role": "user", "content": "hi"}], **request_kwargs
)
response = asyncio.run(
inference_route.openai_chat_completions(payload, _Request(), "test-user")
)
assert response.status_code == 200
temperature, top_p, presence_penalty = expected
assert captured["temperature"] == temperature
assert captured["top_p"] == top_p
assert captured["top_k"] == 20
assert captured["min_p"] == 0.0
assert captured["presence_penalty"] == presence_penalty
@pytest.mark.parametrize(
"thinking, expected", [(None, (0.6, 0.95)), ({"type": "disabled"}, (0.7, 0.8))]
)
def test_messages_route_recommends_the_qwen_sampling_for_the_active_thinking_mode(
monkeypatch, thinking, expected
):
import asyncio
from types import SimpleNamespace
from models.inference import AnthropicMessagesRequest
from routes import inference as inference_route
class _Request:
state = SimpleNamespace()
url = SimpleNamespace(path = "/v1/messages")
method = "POST"
async def is_disconnected(self):
return False
captured = {}
llama_backend = _qwen_backend("unsloth/Qwen3.6-35B-A3B-GGUF", captured)
monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: llama_backend)
payload = AnthropicMessagesRequest(
max_tokens = 16,
messages = [{"role": "user", "content": "hi"}],
**({"thinking": thinking} if thinking else {}),
)
response = asyncio.run(
inference_route.anthropic_messages(payload, request = _Request(), current_subject = "t")
)
assert response.status_code == 200
assert (captured["temperature"], captured["top_p"]) == expected
assert captured["presence_penalty"] == 1.5
_QWEN_THINKING_TEMPLATE = (
"{% for m in messages %}<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n{% endfor %}"
"<|im_start|>assistant\n{% if enable_thinking is defined and enable_thinking is false %}"
"<think>\n\n</think>\n\n{% endif %}"
)
_PLAIN_TEMPLATE = (
"{% for m in messages %}<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n{% endfor %}"
)
@pytest.mark.parametrize(
"template, request_kwargs, expected",
[
(_QWEN_THINKING_TEMPLATE, {}, (0.6, 0.95)),
(_QWEN_THINKING_TEMPLATE, {"enable_thinking": True}, (0.6, 0.95)),
(_QWEN_THINKING_TEMPLATE, {"enable_thinking": False}, (0.7, 0.8)),
(_QWEN_THINKING_TEMPLATE, {"enable_thinking": True, "temperature": 0.3}, (0.3, 0.95)),
(_PLAIN_TEMPLATE, {"enable_thinking": True}, (0.7, 0.8)),
],
)
def test_safetensors_chat_route_recommends_the_qwen_sampling_for_the_active_thinking_mode(
monkeypatch, template, request_kwargs, expected
):
import asyncio
from types import SimpleNamespace
from models.inference import ChatCompletionRequest
from routes import inference as inference_route
from state.tool_policy import reset_tool_policy
class _Request:
state = SimpleNamespace(skip_api_monitor = True)
url = SimpleNamespace(path = "/v1/chat/completions")
method = "POST"
scope = {}
headers = {}
async def is_disconnected(self):
return False
captured = {}
class _Backend:
active_model_name = "unsloth/Qwen3.6-27B"
models = {
"unsloth/Qwen3.6-27B": {
"chat_template_info": {"template": template},
"context_length": 2048,
}
}
def generate_chat_response(self, **kwargs):
captured.update(kwargs)
yield "done"
def reset_generation_state(self, caller_cancel_event = None):
pass
reset_tool_policy()
monkeypatch.setattr(inference_route, "_automatic_model_load_may_run", lambda: False)
monkeypatch.setattr(
inference_route,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = False, supports_tools = False, is_vision = False, context_length = None
),
)
monkeypatch.setattr(inference_route, "get_inference_backend", lambda: _Backend())
payload = ChatCompletionRequest(
model = "default",
messages = [{"role": "user", "content": "hi"}],
stream = False,
**request_kwargs,
)
asyncio.run(inference_route.openai_chat_completions(payload, _Request(), "test-user"))
assert (captured["temperature"], captured["top_p"]) == expected
assert captured["presence_penalty"] == 1.5
@pytest.mark.parametrize(
"reasoning_style, enable_thinking, reasoning_effort, expected",
[
("enable_thinking", True, "none", {"enable_thinking": True}),
("enable_thinking", False, "high", {"enable_thinking": False}),
("reasoning_effort", True, "none", {"reasoning_effort": "high"}),
("reasoning_effort", False, "high", {"reasoning_effort": "low"}),
("enable_thinking_effort", True, "none", {"enable_thinking": True}),
("enable_thinking_effort", False, "high", {"enable_thinking": False}),
],
)
def test_conflicting_controls_resolve_before_model_specific_translation(
reasoning_style, enable_thinking, reasoning_effort, expected
):
from core.inference.llama_cpp import LlamaCppBackend
from routes.inference import _resolve_reasoning_controls
backend = LlamaCppBackend.__new__(LlamaCppBackend)
backend._supports_reasoning = True
backend._reasoning_always_on = False
backend._reasoning_style = reasoning_style
backend._reasoning_effort_levels = ["high", "max"]
backend._supports_preserve_thinking = False
backend._architecture = None
resolved = _resolve_reasoning_controls(enable_thinking, reasoning_effort)
assert backend._request_reasoning_kwargs(*resolved) == expected
def test_fill_recommended_sampling_completions_body(monkeypatch):
# /v1/completions is a raw proxy: unset fields stay absent so llama-server keeps its defaults.
from routes.inference import _fill_recommended_sampling_completions
_set_recommended(monkeypatch, {"temperature": 1.0, "top_k": 64, "min_p": 0.0})
body = {"prompt": "hi", "temperature": 0.2}
_fill_recommended_sampling_completions(body, "some/model")
assert body["temperature"] == 0.2
assert body["top_k"] == 64
assert body["min_p"] == 0.0
assert "top_p" not in body
assert "presence_penalty" not in body
assert "repeat_penalty" not in body
def test_fill_recommended_sampling_completions_operator_pin(monkeypatch):
from routes.inference import _fill_recommended_sampling_completions
monkeypatch.setattr(ic, "load_inference_config", lambda mid: {})
ic._recommended_sampling.cache_clear()
monkeypatch.setenv("UNSLOTH_SAMPLING_TEMPERATURE", "0.9")
monkeypatch.setenv("UNSLOTH_SAMPLING_REPETITION_PENALTY", "1.2")
body = {"prompt": "hi", "temperature": 0.2, "repeat_penalty": 1.05}
_fill_recommended_sampling_completions(body, "some/model")
assert body["temperature"] == 0.9
assert body["repeat_penalty"] == 1.2 # repetition pin lands on llama-server's key
assert "repetition_penalty" not in body
@pytest.mark.parametrize("effort", ["hihg", ""])
def test_count_tokens_rejects_an_effort_the_chat_endpoint_would_reject(effort):
from models.inference import ChatCountTokensRequest
from pydantic import ValidationError
with pytest.raises(ValidationError):
ChatCountTokensRequest.model_validate(
{"messages": [{"role": "user", "content": "hi"}], "reasoning_effort": effort}
)
@pytest.mark.parametrize(
"reasoning_style, request_kwargs, expected_kwargs",
[
(
"reasoning_effort",
{"enable_thinking": False, "reasoning_effort": "high"},
{"reasoning_effort": "low"},
),
(
"reasoning_effort",
{"enable_thinking": True, "reasoning_effort": "none"},
{"reasoning_effort": "high"},
),
(
"enable_thinking",
{"enable_thinking": False, "reasoning_effort": "high"},
{"enable_thinking": False},
),
(
"enable_thinking",
{"enable_thinking": True, "reasoning_effort": "none"},
{"enable_thinking": True},
),
(
"reasoning_effort",
{"enable_thinking": True, "reasoning_effort": "high"},
{"reasoning_effort": "high"},
),
],
)
def test_contradictory_controls_are_resolved_for_every_family(
reasoning_style, request_kwargs, expected_kwargs
):
"""Conflict rule runs before family translation, so effort-dial models like gpt-oss change too."""
from types import SimpleNamespace
from core.inference.llama_cpp import LlamaCppBackend
from models.inference import ChatCompletionRequest
from routes import inference as inference_route
backend = SimpleNamespace(
_supports_reasoning = True,
_reasoning_always_on = False,
_reasoning_style = reasoning_style,
_reasoning_effort_levels = ("none", "low", "medium", "high"),
_supports_preserve_thinking = False,
_architecture = "gpt-oss",
)
payload = ChatCompletionRequest(
model = "unsloth/gpt-oss-20b-GGUF",
messages = [{"role": "user", "content": "hi"}],
**request_kwargs,
)
inference_route._normalize_chat_reasoning_controls(payload)
assert (
LlamaCppBackend._request_reasoning_kwargs(
backend, payload.enable_thinking, payload.reasoning_effort, None
)
== expected_kwargs
)
@pytest.mark.parametrize(
"typed, nested",
[
({"enable_thinking": True, "reasoning_effort": "none"}, {"reasoning_effort": "high"}),
({"enable_thinking": False, "reasoning_effort": "high"}, {"reasoning_effort": "none"}),
({"reasoning_effort": "none"}, {"enable_thinking": True, "reasoning_effort": "xhigh"}),
({}, {"enable_thinking": False}),
({}, {"reasoning_effort": "medium"}),
({"enable_thinking": True}, {"reasoning_effort": "medium"}),
],
)
def test_normalizing_twice_cannot_change_the_answer(typed, nested):
"""Normalizing twice must not change the result, since the first pass writes onto the typed fields."""
from models.inference import ChatCompletionRequest
from routes import inference as inference_route
payload = ChatCompletionRequest.model_validate(
{
"model": "unsloth/Qwen3.8-27B-GGUF",
"messages": [{"role": "user", "content": "hi"}],
**typed,
"chat_template_kwargs": dict(nested),
}
)
inference_route._normalize_chat_reasoning_controls(payload)
once = (payload.enable_thinking, payload.reasoning_effort, payload.preserve_thinking)
inference_route._normalize_chat_reasoning_controls(payload)
twice = (payload.enable_thinking, payload.reasoning_effort, payload.preserve_thinking)
inference_route._normalize_chat_reasoning_controls(payload)
thrice = (payload.enable_thinking, payload.reasoning_effort, payload.preserve_thinking)
assert once == twice == thrice
def test_normalizing_does_not_mutate_the_clients_nested_dict():
"""The lift consumes the reasoning keys without reaching through to the parsed body."""
from models.inference import ChatCompletionRequest
from routes import inference as inference_route
client_dict = {"enable_thinking": False, "reasoning_effort": "none", "keep_me": 1}
payload = ChatCompletionRequest.model_validate(
{
"model": "unsloth/Qwen3.8-27B-GGUF",
"messages": [{"role": "user", "content": "hi"}],
"chat_template_kwargs": client_dict,
}
)
inference_route._normalize_chat_reasoning_controls(payload)
assert client_dict == {"enable_thinking": False, "reasoning_effort": "none", "keep_me": 1}
assert payload.model_extra["chat_template_kwargs"] == {"keep_me": 1}
@pytest.mark.parametrize(
"nested_enable_thinking",
[
"false",
"true",
0,
1,
None,
[],
{},
"",
],
)
def test_a_nested_enable_thinking_that_is_not_a_json_boolean_is_ignored(nested_enable_thinking):
"""Only a real JSON boolean controls thinking; a string like "false" must not read as ON via bool()."""
from models.inference import ChatCompletionRequest
from routes import inference as inference_route
payload = ChatCompletionRequest.model_validate(
{
"model": "unsloth/Qwen3.8-27B-GGUF",
"messages": [{"role": "user", "content": "hi"}],
"chat_template_kwargs": {"enable_thinking": nested_enable_thinking},
}
)
inference_route._normalize_chat_reasoning_controls(payload)
assert payload.enable_thinking is None
assert payload.reasoning_effort is None
assert payload.model_extra["chat_template_kwargs"] == {}
@pytest.mark.parametrize("nested_preserve_thinking", ["true", "false", 0, 1, None, [], {}])
def test_a_nested_preserve_thinking_that_is_not_a_json_boolean_is_ignored(nested_preserve_thinking):
from models.inference import ChatCompletionRequest
from routes import inference as inference_route
payload = ChatCompletionRequest.model_validate(
{
"model": "unsloth/Qwen3.8-27B-GGUF",
"messages": [{"role": "user", "content": "hi"}],
"chat_template_kwargs": {"preserve_thinking": nested_preserve_thinking},
}
)
inference_route._normalize_chat_reasoning_controls(payload)
assert payload.preserve_thinking is None
assert payload.model_extra["chat_template_kwargs"] == {}
def test_an_ignored_nested_control_resolves_exactly_like_omitting_it():
"""An invalid nested control must resolve exactly like omitting the key, on every template surface."""
from models.inference import ChatCompletionRequest
from routes import inference as inference_route
def _resolved(nested):
payload = ChatCompletionRequest.model_validate(
{
"model": "unsloth/Qwen3.8-27B-GGUF",
"messages": [{"role": "user", "content": "hi"}],
**({"chat_template_kwargs": nested} if nested is not None else {}),
}
)
inference_route._normalize_chat_reasoning_controls(payload)
return payload.enable_thinking, payload.reasoning_effort, payload.preserve_thinking
omitted = _resolved(None)
for nested in (
{"enable_thinking": "false"},
{"enable_thinking": 0},
{"reasoning_effort": "hihg"},
{"reasoning_effort": "HIGH"},
{"reasoning_effort": 5},
{"preserve_thinking": "true"},
{"enable_thinking": "false", "reasoning_effort": "hihg"},
):
assert _resolved(nested) == omitted, nested
assert _resolved({"enable_thinking": "false", "reasoning_effort": "none"}) == (
False,
"none",
None,
)
def test_an_anthropic_derived_boolean_stays_out_of_the_explicit_field_set():
"""The discard needs the live model_fields_set, else the derived boolean outranks nested controls."""
from models.inference import ChatCompletionRequest
payload = ChatCompletionRequest.model_validate(
{
"model": "unsloth/Qwen3.8-27B-GGUF",
"messages": [{"role": "user", "content": "hi"}],
"thinking": {"type": "enabled"},
"chat_template_kwargs": {"enable_thinking": False},
}
)
assert payload.enable_thinking is True
assert "enable_thinking" not in payload.model_fields_set
from routes import inference as inference_route
inference_route._normalize_chat_reasoning_controls(payload)
assert payload.enable_thinking is False