Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Stop unsloth chat and unsloth inference ignoring the model's recommended settings (#11478)
Nilay5708db9d9b68
diff --git a/unsloth_cli/_inference.py b/unsloth_cli/_inference.pyindex 76e576e0873..999f0f9f1b5 100644--- a/unsloth_cli/_inference.py+++ b/unsloth_cli/_inference.py@@ -403,15 +403,30 @@ class ChatBackend:messages: list,*,system_prompt: str,- temperature: float,- top_p: float,- top_k: int,+ temperature: Optional[float],+ top_p: Optional[float],+ top_k: Optional[int],max_new_tokens: Optional[int],- repetition_penalty: float,+ repetition_penalty: Optional[float],enable_thinking: bool,use_adapter: Optional[bool] = None,):self.reply_hit_token_limit = False+ ensure_studio_backend_path(seed_cache_env = False)+ from utils.inference.inference_config import resolve_effective_sampling++ model_id = getattr(+ self._backend, "model_identifier" if self._kind == "gguf" else "active_model_name", None+ )+ sampling = resolve_effective_sampling(+ model_id,+ dict(+ temperature = temperature,+ top_p = top_p,+ top_k = top_k,+ repetition_penalty = repetition_penalty,+ ),+ )if self._kind == "gguf":# llama-server takes the system prompt as the first message.msgs = list(messages)@@ -420,25 +435,19 @@ class ChatBackend:return self._watch_metadata(self._backend.generate_chat_completion(messages = msgs,- temperature = temperature,- top_p = top_p,- top_k = top_k,max_tokens = max_new_tokens,- repetition_penalty = repetition_penalty,enable_thinking = enable_thinking,+ **sampling,))holder: dict = {}gen_kwargs = dict(messages = messages,system_prompt = system_prompt,- temperature = temperature,- top_p = top_p,- top_k = top_k,max_new_tokens = max_new_tokens,- repetition_penalty = repetition_penalty,enable_thinking = enable_thinking,stats_holder = holder,+ **sampling,)if use_adapter is not None:stream = self._backend.generate_with_adapter_control(@@ -915,11 +924,11 @@ class HttpChatBackend:messages: list,*,system_prompt: str,- temperature: float,- top_p: float,- top_k: int,+ temperature: Optional[float],+ top_p: Optional[float],+ top_k: Optional[int],max_new_tokens: Optional[int],- repetition_penalty: float,+ repetition_penalty: Optional[float],enable_thinking: bool,use_adapter: Optional[bool] = None,):@@ -932,12 +941,15 @@ class HttpChatBackend:"model": "default","messages": msgs,"stream": True,- "temperature": temperature,- "top_p": top_p,- "top_k": top_k,- "repetition_penalty": repetition_penalty,"enable_thinking": enable_thinking,}+ sampling = dict(+ temperature = temperature,+ top_p = top_p,+ top_k = top_k,+ repetition_penalty = repetition_penalty,+ )+ body.update({key: value for key, value in sampling.items() if value is not None})if max_new_tokens is not None:body["max_tokens"] = max_new_tokensresp = self._request("POST", "/v1/chat/completions", body)diff --git a/unsloth_cli/commands/chat.py b/unsloth_cli/commands/chat.pyindex 37b39332b84..6f32b0c6cab 100644--- a/unsloth_cli/commands/chat.py+++ b/unsloth_cli/commands/chat.py@@ -164,16 +164,24 @@ def chat(hf_token: Optional[str] = typer.Option(None, "--hf-token", envvar = "HF_TOKEN", help = "Hugging Face token if needed."),- temperature: float = typer.Option(0.7, "--temperature"),- top_p: float = typer.Option(0.9, "--top-p"),- top_k: int = typer.Option(40, "--top-k"),+ temperature: Optional[float] = typer.Option(+ None, "--temperature", help = "Unset uses the model's recommended value."+ ),+ top_p: Optional[float] = typer.Option(+ None, "--top-p", help = "Unset uses the model's recommended value."+ ),+ top_k: Optional[int] = typer.Option(+ None, "--top-k", help = "Unset uses the model's recommended value."+ ),max_new_tokens: Optional[int] = typer.Option(None,"--max-new-tokens",help = "Cap on generated tokens. Unset lets a reply use whatever the ""model's context window leaves free after the conversation.",),- repetition_penalty: float = typer.Option(1.1, "--repetition-penalty"),+ repetition_penalty: Optional[float] = typer.Option(+ None, "--repetition-penalty", help = "Unset leaves it off (1.0)."+ ),system_prompt: str = typer.Option("", "--system-prompt", help = "Optional system prompt for the conversation."),diff --git a/unsloth_cli/commands/inference.py b/unsloth_cli/commands/inference.pyindex e40678611b6..53372ad1a07 100644--- a/unsloth_cli/commands/inference.py+++ b/unsloth_cli/commands/inference.py@@ -24,16 +24,24 @@ def inference(hf_token: Optional[str] = typer.Option(None, "--hf-token", envvar = "HF_TOKEN", help = "Hugging Face token if needed."),- temperature: float = typer.Option(0.7, "--temperature"),- top_p: float = typer.Option(0.9, "--top-p"),- top_k: int = typer.Option(40, "--top-k"),+ temperature: Optional[float] = typer.Option(+ None, "--temperature", help = "Unset uses the model's recommended value."+ ),+ top_p: Optional[float] = typer.Option(+ None, "--top-p", help = "Unset uses the model's recommended value."+ ),+ top_k: Optional[int] = typer.Option(+ None, "--top-k", help = "Unset uses the model's recommended value."+ ),max_new_tokens: Optional[int] = typer.Option(None,"--max-new-tokens",help = "Cap on generated tokens. Unset lets a reply use whatever the ""model's context window leaves free after the conversation.",),- repetition_penalty: float = typer.Option(1.1, "--repetition-penalty"),+ repetition_penalty: Optional[float] = typer.Option(+ None, "--repetition-penalty", help = "Unset leaves it off (1.0)."+ ),system_prompt: str = typer.Option("","--system-prompt",diff --git a/unsloth_cli/tests/test_inference_chat.py b/unsloth_cli/tests/test_inference_chat.pyindex 7e6b1517a41..f045f2ce43f 100644--- a/unsloth_cli/tests/test_inference_chat.py+++ b/unsloth_cli/tests/test_inference_chat.py@@ -318,6 +318,49 @@ def test_chatbackend_gguf_leaves_max_tokens_unset_for_llama_server():assert [call["max_tokens"] for call in fake.calls] == [None, 8]+_UNSET_SAMPLING = dict(temperature = None, top_p = None, top_k = None, repetition_penalty = None)+_GEMMA3_SAMPLING = dict(temperature = 1.0, top_p = 0.95, top_k = 64, repetition_penalty = 1.0)+++def _clear_sampling_pins(monkeypatch):+ for field in _GEMMA3_SAMPLING:+ monkeypatch.delenv(f"UNSLOTH_SAMPLING_{field.upper()}", raising = False)+++def test_chatbackend_resolves_unset_sampling_to_the_model_recommendation(monkeypatch):+ _clear_sampling_pins(monkeypatch)+ fake = _FakeBackend()+ fake.active_model_name = "unsloth/gemma-3-270m-it"+ backend = ChatBackend("unsloth", fake)++ list(+ backend.stream([{"role": "user", "content": "x"}], **{**_STREAM_KWARGS, **_UNSET_SAMPLING})+ )++ kwargs = fake.calls[0][2]+ assert {field: kwargs[field] for field in _GEMMA3_SAMPLING} == _GEMMA3_SAMPLING+++def test_chatbackend_gguf_resolves_unset_sampling_but_keeps_explicit_values(monkeypatch):+ _clear_sampling_pins(monkeypatch)+ fake = _FakeGgufBackend()+ fake.model_identifier = "unsloth/gemma-3-4b-it-GGUF"+ backend = ChatBackend("gguf", fake)++ list(+ backend.stream(+ [{"role": "user", "content": "x"}],+ **{**_STREAM_KWARGS, **_UNSET_SAMPLING, "temperature": 0.3},+ )+ )++ call = fake.calls[0]+ assert {field: call[field] for field in _GEMMA3_SAMPLING} == {+ **_GEMMA3_SAMPLING,+ "temperature": 0.3,+ }++class _FakeStatsBackend:def __init__(self, stats):self._stats = stats@@ -1177,6 +1220,44 @@ def test_http_backend_sends_an_explicit_max_tokens(monkeypatch):assert _http_stream_body(monkeypatch, 128)["max_tokens"] == 128[email protected]("command", ["chat", "inference"])+def test_cli_leaves_unset_sampling_to_the_server(monkeypatch, command):+ from unsloth_cli.commands import inference as infermod++ module, app, argv = {+ "chat": (chatmod, _chat_app(), ["fake-model"]),+ "inference": (infermod, _inference_app(), ["fake-model", "hello"]),+ }[command]+ backend = HttpChatBackend("http://localhost:8888", "token")+ bodies = []++ def fake_request(+ method,+ path,+ payload = None,+ timeout = None,+ ):+ bodies.append(payload)+ return _FakeSSEResponse([b"data: [DONE]\n"])++ monkeypatch.setattr(backend, "_request", fake_request)+ monkeypatch.setattr(backend, "close", lambda: None)+ monkeypatch.setattr(module, "connect_studio_server", lambda *a, **k: backend)+ if command == "chat":+ monkeypatch.setattr(chatmod, "resolve_model_config", lambda *a, **k: _FakeConfig())+ monkeypatch.setattr(chatmod, "_compare_needs_second_model", lambda: False)++ for extra in ([], ["--temperature", "0.3"]):+ result = CliRunner().invoke(app, [*argv, *extra], input = "hi\n/exit\n")+ assert result.exit_code == 0, result.output++ sampling = ("temperature", "top_p", "top_k", "repetition_penalty")+ assert [{k: body[k] for k in sampling if k in body} for body in bodies] == [+ {},+ {"temperature": 0.3},+ ]++def _http_finish(monkeypatch, finish_reason):backend = HttpChatBackend("http://localhost:8888", "token")chunk = json.dumps({"choices": [{"delta": {"content": "hi"}, "finish_reason": finish_reason}]})