Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Stop unsloth chat and unsloth inference ignoring the model's recommended settings (#11478)

Nilay5708db9d9b68
diff --git a/unsloth_cli/_inference.py b/unsloth_cli/_inference.py
index 76e576e0873..999f0f9f1b5 100644
--- a/unsloth_cli/_inference.py
+++ b/unsloth_cli/_inference.py
@@ -403,15 +403,30 @@ class ChatBackend:
messages: list,
*,
system_prompt: str,
- temperature: float,
- top_p: float,
- top_k: int,
+ temperature: Optional[float],
+ top_p: Optional[float],
+ top_k: Optional[int],
max_new_tokens: Optional[int],
- repetition_penalty: float,
+ repetition_penalty: Optional[float],
enable_thinking: bool,
use_adapter: Optional[bool] = None,
):
self.reply_hit_token_limit = False
+ ensure_studio_backend_path(seed_cache_env = False)
+ from utils.inference.inference_config import resolve_effective_sampling
+
+ model_id = getattr(
+ self._backend, "model_identifier" if self._kind == "gguf" else "active_model_name", None
+ )
+ sampling = resolve_effective_sampling(
+ model_id,
+ dict(
+ temperature = temperature,
+ top_p = top_p,
+ top_k = top_k,
+ repetition_penalty = repetition_penalty,
+ ),
+ )
if self._kind == "gguf":
# llama-server takes the system prompt as the first message.
msgs = list(messages)
@@ -420,25 +435,19 @@ class ChatBackend:
return self._watch_metadata(
self._backend.generate_chat_completion(
messages = msgs,
- temperature = temperature,
- top_p = top_p,
- top_k = top_k,
max_tokens = max_new_tokens,
- repetition_penalty = repetition_penalty,
enable_thinking = enable_thinking,
+ **sampling,
)
)
holder: dict = {}
gen_kwargs = dict(
messages = messages,
system_prompt = system_prompt,
- temperature = temperature,
- top_p = top_p,
- top_k = top_k,
max_new_tokens = max_new_tokens,
- repetition_penalty = repetition_penalty,
enable_thinking = enable_thinking,
stats_holder = holder,
+ **sampling,
)
if use_adapter is not None:
stream = self._backend.generate_with_adapter_control(
@@ -915,11 +924,11 @@ class HttpChatBackend:
messages: list,
*,
system_prompt: str,
- temperature: float,
- top_p: float,
- top_k: int,
+ temperature: Optional[float],
+ top_p: Optional[float],
+ top_k: Optional[int],
max_new_tokens: Optional[int],
- repetition_penalty: float,
+ repetition_penalty: Optional[float],
enable_thinking: bool,
use_adapter: Optional[bool] = None,
):
@@ -932,12 +941,15 @@ class HttpChatBackend:
"model": "default",
"messages": msgs,
"stream": True,
- "temperature": temperature,
- "top_p": top_p,
- "top_k": top_k,
- "repetition_penalty": repetition_penalty,
"enable_thinking": enable_thinking,
}
+ sampling = dict(
+ temperature = temperature,
+ top_p = top_p,
+ top_k = top_k,
+ repetition_penalty = repetition_penalty,
+ )
+ body.update({key: value for key, value in sampling.items() if value is not None})
if max_new_tokens is not None:
body["max_tokens"] = max_new_tokens
resp = self._request("POST", "/v1/chat/completions", body)
diff --git a/unsloth_cli/commands/chat.py b/unsloth_cli/commands/chat.py
index 37b39332b84..6f32b0c6cab 100644
--- a/unsloth_cli/commands/chat.py
+++ b/unsloth_cli/commands/chat.py
@@ -164,16 +164,24 @@ def chat(
hf_token: Optional[str] = typer.Option(
None, "--hf-token", envvar = "HF_TOKEN", help = "Hugging Face token if needed."
),
- temperature: float = typer.Option(0.7, "--temperature"),
- top_p: float = typer.Option(0.9, "--top-p"),
- top_k: int = typer.Option(40, "--top-k"),
+ temperature: Optional[float] = typer.Option(
+ None, "--temperature", help = "Unset uses the model's recommended value."
+ ),
+ top_p: Optional[float] = typer.Option(
+ None, "--top-p", help = "Unset uses the model's recommended value."
+ ),
+ top_k: Optional[int] = typer.Option(
+ None, "--top-k", help = "Unset uses the model's recommended value."
+ ),
max_new_tokens: Optional[int] = typer.Option(
None,
"--max-new-tokens",
help = "Cap on generated tokens. Unset lets a reply use whatever the "
"model's context window leaves free after the conversation.",
),
- repetition_penalty: float = typer.Option(1.1, "--repetition-penalty"),
+ repetition_penalty: Optional[float] = typer.Option(
+ None, "--repetition-penalty", help = "Unset leaves it off (1.0)."
+ ),
system_prompt: str = typer.Option(
"", "--system-prompt", help = "Optional system prompt for the conversation."
),
diff --git a/unsloth_cli/commands/inference.py b/unsloth_cli/commands/inference.py
index e40678611b6..53372ad1a07 100644
--- a/unsloth_cli/commands/inference.py
+++ b/unsloth_cli/commands/inference.py
@@ -24,16 +24,24 @@ def inference(
hf_token: Optional[str] = typer.Option(
None, "--hf-token", envvar = "HF_TOKEN", help = "Hugging Face token if needed."
),
- temperature: float = typer.Option(0.7, "--temperature"),
- top_p: float = typer.Option(0.9, "--top-p"),
- top_k: int = typer.Option(40, "--top-k"),
+ temperature: Optional[float] = typer.Option(
+ None, "--temperature", help = "Unset uses the model's recommended value."
+ ),
+ top_p: Optional[float] = typer.Option(
+ None, "--top-p", help = "Unset uses the model's recommended value."
+ ),
+ top_k: Optional[int] = typer.Option(
+ None, "--top-k", help = "Unset uses the model's recommended value."
+ ),
max_new_tokens: Optional[int] = typer.Option(
None,
"--max-new-tokens",
help = "Cap on generated tokens. Unset lets a reply use whatever the "
"model's context window leaves free after the conversation.",
),
- repetition_penalty: float = typer.Option(1.1, "--repetition-penalty"),
+ repetition_penalty: Optional[float] = typer.Option(
+ None, "--repetition-penalty", help = "Unset leaves it off (1.0)."
+ ),
system_prompt: str = typer.Option(
"",
"--system-prompt",
diff --git a/unsloth_cli/tests/test_inference_chat.py b/unsloth_cli/tests/test_inference_chat.py
index 7e6b1517a41..f045f2ce43f 100644
--- a/unsloth_cli/tests/test_inference_chat.py
+++ b/unsloth_cli/tests/test_inference_chat.py
@@ -318,6 +318,49 @@ def test_chatbackend_gguf_leaves_max_tokens_unset_for_llama_server():
assert [call["max_tokens"] for call in fake.calls] == [None, 8]
+_UNSET_SAMPLING = dict(temperature = None, top_p = None, top_k = None, repetition_penalty = None)
+_GEMMA3_SAMPLING = dict(temperature = 1.0, top_p = 0.95, top_k = 64, repetition_penalty = 1.0)
+
+
+def _clear_sampling_pins(monkeypatch):
+ for field in _GEMMA3_SAMPLING:
+ monkeypatch.delenv(f"UNSLOTH_SAMPLING_{field.upper()}", raising = False)
+
+
+def test_chatbackend_resolves_unset_sampling_to_the_model_recommendation(monkeypatch):
+ _clear_sampling_pins(monkeypatch)
+ fake = _FakeBackend()
+ fake.active_model_name = "unsloth/gemma-3-270m-it"
+ backend = ChatBackend("unsloth", fake)
+
+ list(
+ backend.stream([{"role": "user", "content": "x"}], **{**_STREAM_KWARGS, **_UNSET_SAMPLING})
+ )
+
+ kwargs = fake.calls[0][2]
+ assert {field: kwargs[field] for field in _GEMMA3_SAMPLING} == _GEMMA3_SAMPLING
+
+
+def test_chatbackend_gguf_resolves_unset_sampling_but_keeps_explicit_values(monkeypatch):
+ _clear_sampling_pins(monkeypatch)
+ fake = _FakeGgufBackend()
+ fake.model_identifier = "unsloth/gemma-3-4b-it-GGUF"
+ backend = ChatBackend("gguf", fake)
+
+ list(
+ backend.stream(
+ [{"role": "user", "content": "x"}],
+ **{**_STREAM_KWARGS, **_UNSET_SAMPLING, "temperature": 0.3},
+ )
+ )
+
+ call = fake.calls[0]
+ assert {field: call[field] for field in _GEMMA3_SAMPLING} == {
+ **_GEMMA3_SAMPLING,
+ "temperature": 0.3,
+ }
+
+
class _FakeStatsBackend:
def __init__(self, stats):
self._stats = stats
@@ -1177,6 +1220,44 @@ def test_http_backend_sends_an_explicit_max_tokens(monkeypatch):
assert _http_stream_body(monkeypatch, 128)["max_tokens"] == 128
[email protected]("command", ["chat", "inference"])
+def test_cli_leaves_unset_sampling_to_the_server(monkeypatch, command):
+ from unsloth_cli.commands import inference as infermod
+
+ module, app, argv = {
+ "chat": (chatmod, _chat_app(), ["fake-model"]),
+ "inference": (infermod, _inference_app(), ["fake-model", "hello"]),
+ }[command]
+ backend = HttpChatBackend("http://localhost:8888", "token")
+ bodies = []
+
+ def fake_request(
+ method,
+ path,
+ payload = None,
+ timeout = None,
+ ):
+ bodies.append(payload)
+ return _FakeSSEResponse([b"data: [DONE]\n"])
+
+ monkeypatch.setattr(backend, "_request", fake_request)
+ monkeypatch.setattr(backend, "close", lambda: None)
+ monkeypatch.setattr(module, "connect_studio_server", lambda *a, **k: backend)
+ if command == "chat":
+ monkeypatch.setattr(chatmod, "resolve_model_config", lambda *a, **k: _FakeConfig())
+ monkeypatch.setattr(chatmod, "_compare_needs_second_model", lambda: False)
+
+ for extra in ([], ["--temperature", "0.3"]):
+ result = CliRunner().invoke(app, [*argv, *extra], input = "hi\n/exit\n")
+ assert result.exit_code == 0, result.output
+
+ sampling = ("temperature", "top_p", "top_k", "repetition_penalty")
+ assert [{k: body[k] for k in sampling if k in body} for body in bodies] == [
+ {},
+ {"temperature": 0.3},
+ ]
+
+
def _http_finish(monkeypatch, finish_reason):
backend = HttpChatBackend("http://localhost:8888", "token")
chunk = json.dumps({"choices": [{"delta": {"content": "hi"}, "finish_reason": finish_reason}]})