Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Stop unsloth chat from reloading a model the server already has (#10257)
Nilay0ee6cefcfe45
diff --git a/unsloth_cli/commands/chat.py b/unsloth_cli/commands/chat.pyindex 2f1bc0866dc..dc96ba410b2 100644--- a/unsloth_cli/commands/chat.py+++ b/unsloth_cli/commands/chat.py@@ -172,7 +172,13 @@ def chat(system_prompt: str = typer.Option("", "--system-prompt", help = "Optional system prompt for the conversation."),- max_seq_length: int = typer.Option(4096, "--max-seq-length"),+ max_seq_length: int = typer.Option(+ 0,+ "--max-seq-length",+ help = "Context length in tokens. 0 takes the checkpoint's trained window on GGUF "+ "and MLX, and 2048 on the transformers backend. A value that differs from a "+ "running Unsloth server's reloads the model.",+ ),load_in_4bit: bool = typer.Option(True, "--load-in-4bit/--no-load-in-4bit"),tensor_parallel: bool = typer.Option(False,diff --git a/unsloth_cli/commands/inference.py b/unsloth_cli/commands/inference.pyindex 08797f08a88..ca15d79bdeb 100644--- a/unsloth_cli/commands/inference.py+++ b/unsloth_cli/commands/inference.py@@ -34,7 +34,13 @@ def inference("--system-prompt",help = "Optional system prompt to prepend.",),- max_seq_length: int = typer.Option(2048, "--max-seq-length"),+ max_seq_length: int = typer.Option(+ 0,+ "--max-seq-length",+ help = "Context length in tokens. 0 takes the checkpoint's trained window on GGUF "+ "and MLX, and 2048 on the transformers backend. A value that differs from a "+ "running Unsloth server's reloads the model.",+ ),load_in_4bit: bool = typer.Option(True, "--load-in-4bit/--no-load-in-4bit"),tensor_parallel: bool = typer.Option(False,diff --git a/unsloth_cli/tests/test_inference_chat.py b/unsloth_cli/tests/test_inference_chat.pyindex 7c7b1824df9..51106078d04 100644--- a/unsloth_cli/tests/test_inference_chat.py+++ b/unsloth_cli/tests/test_inference_chat.py@@ -1459,12 +1459,14 @@ def test_chat_forwards_gguf_runtime_options_to_loader(monkeypatch):)assert result.exit_code == 0, result.output+ # 0 is the "no context requested" sentinel: a server that also asked for nothing+ # reuses the resident model instead of relaunching it at the CLI's own number.assert loads == [("fake-model",{"hf_token": None,- "max_seq_length": 4096,+ "max_seq_length": 0,"load_in_4bit": True,"tensor_parallel": True,"speculative_type": "dspark",@@ -1517,7 +1519,7 @@ def test_inference_forwards_gguf_runtime_options_to_loader(monkeypatch):"fake-model",{"hf_token": None,- "max_seq_length": 2048,+ "max_seq_length": 0,"load_in_4bit": True,"tensor_parallel": True,"speculative_type": "dspark",@@ -1530,6 +1532,50 @@ def test_inference_forwards_gguf_runtime_options_to_loader(monkeypatch):assert closed == [True][email protected]("command", ["chat", "inference"])+def test_local_load_forwards_the_unrequested_context_sentinel(monkeypatch, command):+ """No server: the sentinel reaches the loader, so each backend picks its own window.++ GGUF and MLX read the trained window from the checkpoint; the transformers backend has+ no window to read and falls back to 2048.+ """+ from unsloth_cli.commands import inference as infermod++ module, app, argv = {+ "chat": (chatmod, _chat_app(), ["fake-model"]),+ "inference": (infermod, _inference_app(), ["fake-model", "hello"]),+ }[command]+ loads = []++ class _FakeBackend:+ def stream(self, *a, **k):+ return iter(["answer"])++ def close(self):+ pass++ monkeypatch.setattr(module, "connect_studio_server", lambda *a, **k: None)+ monkeypatch.setattr(+ module,+ "load_chat_backend",+ lambda model, **kwargs: (loads.append(kwargs["max_seq_length"]), _FakeBackend())[1],+ )+ if command == "chat":+ monkeypatch.setattr(chatmod, "resolve_model_config", lambda *a, **k: _FakeConfig())+ monkeypatch.setattr(chatmod, "_compare_needs_second_model", lambda: False)++ result = CliRunner().invoke(app, argv, input = "/exit\n")++ assert result.exit_code == 0, result.output+ assert loads == [0]++ loads.clear()+ result = CliRunner().invoke(app, [*argv, "--max-seq-length", "8192"], input = "/exit\n")++ assert result.exit_code == 0, result.output+ assert loads == [8192]++def test_chat_server_mode_compare_loads_base_locally(monkeypatch):streamed, closed, base_loads = [], [], []