Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Stop unsloth chat from reloading a model the server already has (#10257)

Nilay0ee6cefcfe45
diff --git a/unsloth_cli/commands/chat.py b/unsloth_cli/commands/chat.py
index 2f1bc0866dc..dc96ba410b2 100644
--- a/unsloth_cli/commands/chat.py
+++ b/unsloth_cli/commands/chat.py
@@ -172,7 +172,13 @@ def chat(
system_prompt: str = typer.Option(
"", "--system-prompt", help = "Optional system prompt for the conversation."
),
- max_seq_length: int = typer.Option(4096, "--max-seq-length"),
+ max_seq_length: int = typer.Option(
+ 0,
+ "--max-seq-length",
+ help = "Context length in tokens. 0 takes the checkpoint's trained window on GGUF "
+ "and MLX, and 2048 on the transformers backend. A value that differs from a "
+ "running Unsloth server's reloads the model.",
+ ),
load_in_4bit: bool = typer.Option(True, "--load-in-4bit/--no-load-in-4bit"),
tensor_parallel: bool = typer.Option(
False,
diff --git a/unsloth_cli/commands/inference.py b/unsloth_cli/commands/inference.py
index 08797f08a88..ca15d79bdeb 100644
--- a/unsloth_cli/commands/inference.py
+++ b/unsloth_cli/commands/inference.py
@@ -34,7 +34,13 @@ def inference(
"--system-prompt",
help = "Optional system prompt to prepend.",
),
- max_seq_length: int = typer.Option(2048, "--max-seq-length"),
+ max_seq_length: int = typer.Option(
+ 0,
+ "--max-seq-length",
+ help = "Context length in tokens. 0 takes the checkpoint's trained window on GGUF "
+ "and MLX, and 2048 on the transformers backend. A value that differs from a "
+ "running Unsloth server's reloads the model.",
+ ),
load_in_4bit: bool = typer.Option(True, "--load-in-4bit/--no-load-in-4bit"),
tensor_parallel: bool = typer.Option(
False,
diff --git a/unsloth_cli/tests/test_inference_chat.py b/unsloth_cli/tests/test_inference_chat.py
index 7c7b1824df9..51106078d04 100644
--- a/unsloth_cli/tests/test_inference_chat.py
+++ b/unsloth_cli/tests/test_inference_chat.py
@@ -1459,12 +1459,14 @@ def test_chat_forwards_gguf_runtime_options_to_loader(monkeypatch):
)
assert result.exit_code == 0, result.output
+ # 0 is the "no context requested" sentinel: a server that also asked for nothing
+ # reuses the resident model instead of relaunching it at the CLI's own number.
assert loads == [
(
"fake-model",
{
"hf_token": None,
- "max_seq_length": 4096,
+ "max_seq_length": 0,
"load_in_4bit": True,
"tensor_parallel": True,
"speculative_type": "dspark",
@@ -1517,7 +1519,7 @@ def test_inference_forwards_gguf_runtime_options_to_loader(monkeypatch):
"fake-model",
{
"hf_token": None,
- "max_seq_length": 2048,
+ "max_seq_length": 0,
"load_in_4bit": True,
"tensor_parallel": True,
"speculative_type": "dspark",
@@ -1530,6 +1532,50 @@ def test_inference_forwards_gguf_runtime_options_to_loader(monkeypatch):
assert closed == [True]
[email protected]("command", ["chat", "inference"])
+def test_local_load_forwards_the_unrequested_context_sentinel(monkeypatch, command):
+ """No server: the sentinel reaches the loader, so each backend picks its own window.
+
+ GGUF and MLX read the trained window from the checkpoint; the transformers backend has
+ no window to read and falls back to 2048.
+ """
+ from unsloth_cli.commands import inference as infermod
+
+ module, app, argv = {
+ "chat": (chatmod, _chat_app(), ["fake-model"]),
+ "inference": (infermod, _inference_app(), ["fake-model", "hello"]),
+ }[command]
+ loads = []
+
+ class _FakeBackend:
+ def stream(self, *a, **k):
+ return iter(["answer"])
+
+ def close(self):
+ pass
+
+ monkeypatch.setattr(module, "connect_studio_server", lambda *a, **k: None)
+ monkeypatch.setattr(
+ module,
+ "load_chat_backend",
+ lambda model, **kwargs: (loads.append(kwargs["max_seq_length"]), _FakeBackend())[1],
+ )
+ if command == "chat":
+ monkeypatch.setattr(chatmod, "resolve_model_config", lambda *a, **k: _FakeConfig())
+ monkeypatch.setattr(chatmod, "_compare_needs_second_model", lambda: False)
+
+ result = CliRunner().invoke(app, argv, input = "/exit\n")
+
+ assert result.exit_code == 0, result.output
+ assert loads == [0]
+
+ loads.clear()
+ result = CliRunner().invoke(app, [*argv, "--max-seq-length", "8192"], input = "/exit\n")
+
+ assert result.exit_code == 0, result.output
+ assert loads == [8192]
+
+
def test_chat_server_mode_compare_loads_base_locally(monkeypatch):
streamed, closed, base_loads = [], [], []