Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: fit the context again when MTP is forced on a reload (#12231)
Nilayf160cf598695
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 55643f02562..74ab448c85c 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -574,6 +574,7 @@ class GgufLoadIntent:# for the session.disable_vision: bool = Falsen_ctx: int = 4096+ max_seq_length_auto_derived: bool = Falsechat_template_override: Optional[str] = Nonecache_type_kv: Optional[str] = Nonespeculative_type: Optional[str] = None@@ -23125,6 +23126,7 @@ class LlamaCppBackend:is_vision = intent.is_visiondisable_vision = intent.disable_visionn_ctx = intent.n_ctx+ _replayed_ctx_refit = Falsechat_template_override = intent.chat_template_overridecache_type_kv = intent.cache_type_kvspeculative_type = intent.speculative_type@@ -25039,6 +25041,24 @@ class LlamaCppBackend:_draft_cpu_no_embedded = _draft_on_cpu and (_separate_draft_launches or not self._nextn_predict_layers)+ # A replay was fitted without the drafter: re-fit it like a fresh MTP load, before any explicit_ctx reader.+ if (+ explicit_ctx+ and ctx_override is None+ and intent.max_seq_length_auto_derived+ # CPU-pinned drafters too: the target still pays rollback state for them.+ and _mtp_will_engage+ # Forced = anything that bypasses the Auto drop probe, advanced arguments included.+ and (+ (_canonicalize_spec_mode(speculative_type) or "auto") != "auto"+ or _user_mtp_via_extras+ or _user_draft_via_extras+ or _extra_args_set_spec_type(extra_args)+ or _extra_args_mtp_draft_path(extra_args, env = _spec_env)+ )+ ):+ explicit_ctx = False+ _replayed_ctx_refit = True# The two tensor -> layer downgrades that need nothing the probe# decides run BEFORE it: the probe is gated on `not tensor_parallel`@@ -31237,7 +31257,8 @@ class LlamaCppBackend:else list(_pv_requested))self._extra_args_source = (model_identifier, hf_variant)- self._requested_n_ctx = int(n_ctx)+ # A re-fit replay records what launched, which is what the client replays next.+ self._requested_n_ctx = int(effective_ctx if _replayed_ctx_refit else n_ctx)# Local n_parallel may have been reduced above; the snapshot has the ask.self._requested_n_parallel = max(1, int(intent.n_parallel))# Commit with the rest of the known-good state: only a launch that gotdiff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex d61bac1507b..5bbe063c51c 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -84,6 +84,10 @@ class LoadRequest(BaseModel):"(llama.cpp or MLX) choose the context itself."),)+ max_seq_length_auto_derived: bool = Field(+ False,+ description = "max_seq_length is the previous load's fitted context, not a user choice",+ )load_in_4bit: bool = Field(True, description = "Load model in 4-bit quantization")is_lora: bool = Field(False, description = "Whether this is a LoRA adapter")gguf_variant: Optional[str] = Field(diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.pyindex ef367fc7854..017943c9a92 100644--- a/studio/backend/tests/test_llama_cpp_placement.py+++ b/studio/backend/tests/test_llama_cpp_placement.py@@ -1404,6 +1404,155 @@ def test_the_drop_actually_releases_the_reserve_the_fit_charges(tmp_path):assert cmd[cmd.index("--fit") + 1] == "off"+def _replayed_context_mtp_backend(tmp_path: Path):+ gb = 1024**3+ backend, gguf = _backend(+ tmp_path, vulkan = False, memory = [(0, 45_914, 46_080), (1, 8_032, 8_176)]+ )++ def read_metadata(_path):+ backend._nextn_predict_layers = 1+ backend._context_length = 262_144++ backend._read_gguf_metadata = read_metadata+ backend._get_gguf_size_bytes = lambda _path: 31 * gb+ backend._can_estimate_kv = lambda: True+ backend._estimate_kv_cache_bytes = lambda n_ctx, *args, **kwargs: n_ctx * 106_000+ backend._compute_buffer_ctx_bytes = lambda *args, **kwargs: 0+ backend._estimate_compute_buffer_bytes = lambda **kwargs: 1+ backend._mtp_draft_kv_bytes = lambda *args, **kwargs: 0+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, *args, **kwargs: n_ctx * 20_000+ backend.probe_server_capabilities = lambda _binary = None: {+ "mtp_token": "draft-mtp",+ "supports_ngram_mod": True,+ "spec_draft_n_max_flag": "--spec-draft-n-max",+ }+ return backend, gguf+++def _launched_ctx(result) -> int:+ return int(result["cmd"][result["cmd"].index("-c") + 1])+++def _replayed_auto_context(tmp_path: Path) -> int:+ backend, gguf = _replayed_context_mtp_backend(tmp_path)+ auto = _launch(backend, gguf, n_ctx = 0, n_parallel = 4, speculative_type = "auto")+ assert backend.spec_fallback_reason == "drafter_no_vram"+ assert auto["env"]["CUDA_VISIBLE_DEVICES"] == "0"+ return _launched_ctx(auto)+++def test_forcing_the_drafter_refits_a_context_replayed_from_auto(tmp_path):+ replayed = _replayed_auto_context(tmp_path)+ backend, gguf = _replayed_context_mtp_backend(tmp_path)+ fresh = _launch(backend, gguf, n_ctx = 0, n_parallel = 4, speculative_type = "mtp")++ backend, gguf = _replayed_context_mtp_backend(tmp_path)+ result = _launch(+ backend,+ gguf,+ n_ctx = replayed,+ max_seq_length_auto_derived = True,+ n_parallel = 4,+ speculative_type = "mtp",+ )++ assert _launched_ctx(fresh) < replayed+ assert _launched_ctx(result) == _launched_ctx(fresh)+ assert result["env"]["CUDA_VISIBLE_DEVICES"] == "0"+ assert result["cmd"][result["cmd"].index("--spec-type") + 1] == "draft-mtp"+ assert backend._requested_n_ctx == _launched_ctx(result)+++def test_a_replayed_context_gets_the_slot_refit_of_a_fresh_drafter_load(tmp_path):+ def slot_bound_backend():+ backend, gguf = _replayed_context_mtp_backend(tmp_path)+ backend._get_gpu_memory = lambda _binary = None, **_kw: [(0, 45_914, 46_080)]+ backend._get_gpu_free_memory = lambda _binary = None, **_kw: [(0, 45_914)]+ backend._estimate_kv_cache_bytes = lambda n_ctx, *args, **kwargs: n_ctx * 16_000+ backend._estimate_mtp_overhead_bytes = (+ lambda n_ctx, *args, _np = None, n_parallel = None, **kwargs: n_ctx * 5_000+ + int(_np or n_parallel or 4) * 3 * 1024**3+ )+ caps = backend.probe_server_capabilities()+ backend.probe_server_capabilities = lambda _binary = None: {+ **caps,+ "supports_kv_unified": True,+ }+ return backend, gguf++ backend, gguf = slot_bound_backend()+ auto = _launch(backend, gguf, n_ctx = 0, n_parallel = 4, speculative_type = "auto")+ assert backend.spec_fallback_reason == "drafter_no_vram"+ backend, gguf = slot_bound_backend()+ fresh = _launch(backend, gguf, n_ctx = 0, n_parallel = 4, speculative_type = "mtp")++ backend, gguf = slot_bound_backend()+ result = _launch(+ backend,+ gguf,+ n_ctx = _launched_ctx(auto),+ max_seq_length_auto_derived = True,+ n_parallel = 4,+ speculative_type = "mtp",+ )++ assert fresh["cmd"][fresh["cmd"].index("--parallel") + 1] != "4"+ assert _launched_ctx(fresh) > 8192+ assert _launched_ctx(result) == _launched_ctx(fresh)+ assert (+ result["cmd"][result["cmd"].index("--parallel") + 1]+ == (fresh["cmd"][fresh["cmd"].index("--parallel") + 1])+ )+++def test_a_drafter_forced_through_extra_args_also_refits(tmp_path):+ replayed = _replayed_auto_context(tmp_path)+ backend, gguf = _replayed_context_mtp_backend(tmp_path)+ fresh = _launch(backend, gguf, n_ctx = 0, n_parallel = 4, speculative_type = "mtp")++ backend, gguf = _replayed_context_mtp_backend(tmp_path)+ result = _launch(+ backend,+ gguf,+ n_ctx = replayed,+ max_seq_length_auto_derived = True,+ n_parallel = 4,+ speculative_type = "auto",+ extra_args = ["--spec-type", "draft-mtp"],+ )++ assert _launched_ctx(result) == _launched_ctx(fresh) < replayed+ assert result["env"]["CUDA_VISIBLE_DEVICES"] == "0"+++def test_forcing_the_drafter_keeps_a_ctx_size_passed_through_extra_args(tmp_path):+ replayed = _replayed_auto_context(tmp_path)+ backend, gguf = _replayed_context_mtp_backend(tmp_path)++ result = _launch(+ backend,+ gguf,+ n_ctx = replayed,+ max_seq_length_auto_derived = True,+ n_parallel = 4,+ speculative_type = "mtp",+ extra_args = ["-c", str(replayed)],+ )++ assert _launched_ctx(result) == replayed+++def test_forcing_the_drafter_keeps_a_typed_context(tmp_path):+ replayed = _replayed_auto_context(tmp_path)+ backend, gguf = _replayed_context_mtp_backend(tmp_path)++ result = _launch(backend, gguf, n_ctx = replayed, n_parallel = 4, speculative_type = "mtp")++ assert _launched_ctx(result) == replayed+ assert backend._requested_n_ctx == replayed++def test_a_cpu_offloaded_sidecar_is_not_probed_because_a_head_also_exists(tmp_path):"""The drafter that launches decides, not one that merely exists.diff --git a/studio/backend/tests/test_mmproj_placement_policy.py b/studio/backend/tests/test_mmproj_placement_policy.pyindex df4f00ee198..c9d77e68796 100644--- a/studio/backend/tests/test_mmproj_placement_policy.py+++ b/studio/backend/tests/test_mmproj_placement_policy.py@@ -1631,3 +1631,52 @@ def test_an_unloadable_drafter_is_not_charged_before_it_is_dropped(tmp_path):assert "--model-draft" not in cmdassert "--no-mmproj-offload" not in cmdassert backend.mtp_draft_suppressed_path == str(tmp_path / "mtp.gguf")+++def test_a_replayed_context_places_the_projector_like_a_fresh_forced_drafter(tmp_path):+ # Classified after the projector probe, the replay priced it at 65536 and pinned it to CPU.+ memory = [(0, 13_500, 24_000)]++ def load(**kwargs):+ backend, gguf = _backend(tmp_path, memory = memory, drafter_bytes = 2 * GIB, native_ctx = 65536)+ cmd = _launch(backend, gguf, mtp_draft_path = str(tmp_path / "mtp.gguf"), **kwargs)["cmd"]+ return backend, cmd, int(cmd[cmd.index("-c") + 1])++ _, auto, replayed = load(speculative_type = "auto", n_ctx = 0)+ assert "--model-draft" not in auto+ _, fresh, fresh_ctx = load(speculative_type = "mtp", n_ctx = 0)+ backend, replay, replay_ctx = load(+ speculative_type = "mtp", n_ctx = replayed, max_seq_length_auto_derived = True+ )++ assert "--no-mmproj-offload" not in fresh+ assert "--no-mmproj-offload" not in replay+ assert replay_ctx == fresh_ctx < replayed+ assert backend._requested_n_ctx == replay_ctx+++def test_a_replayed_context_refits_for_a_cpu_pinned_drafter_too(tmp_path):+ # --spec-draft-ngl 0 keeps the drafter off the GPU; the hybrid target still pays rollback state.+ memory = [(0, 10_000, 24_000)]+ extras = ["--spec-draft-ngl", "0"]++ def load(**kwargs):+ backend, gguf = _backend(tmp_path, memory = memory, drafter_bytes = 2 * GIB, native_ctx = 65536)+ backend._rollback_state_bytes = lambda n_parallel = 1, *_a, **_kw: n_parallel * 256 * MIB+ cmd = _launch(+ backend,+ gguf,+ mtp_draft_path = str(tmp_path / "mtp.gguf"),+ extra_args = extras,+ **kwargs,+ )["cmd"]+ return cmd, int(cmd[cmd.index("-c") + 1])++ _, replayed = load(speculative_type = "off", n_ctx = 0)+ fresh, fresh_ctx = load(speculative_type = "mtp", n_ctx = 0)+ replay, replay_ctx = load(+ speculative_type = "mtp", n_ctx = replayed, max_seq_length_auto_derived = True+ )++ assert replay_ctx == fresh_ctx < replayed+ assert ("--no-mmproj-offload" in replay) == ("--no-mmproj-offload" in fresh)diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.tsindex 9e5cf0ba510..de0d1f8efe6 100644--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts@@ -133,6 +133,7 @@ import {loadedContextForParams,mergeBackendRecommendedInference,resolveFitMaxSeqLength,+ isReplayedLoadContext,unpinnedDefaultRequest,unpinnedLoadContext,resolveLoadMaxSeqLength,@@ -2472,6 +2473,11 @@ export function useChatModelRuntime() {nativePathLease: loadNativePathLease,hf_token: hfToken,max_seq_length: loadMaxSeqLength,+ max_seq_length_auto_derived: isReplayedLoadContext(+ isGguf,+ loadCustomContextLength,+ loadMaxSeqLength,+ ),load_in_4bit: true,is_lora: isLora,gguf_variant: ggufVariant ?? null,diff --git a/studio/frontend/src/features/chat/presets/preset-policy.ts b/studio/frontend/src/features/chat/presets/preset-policy.tsindex cc86c14905e..e9af04b1e19 100644--- a/studio/frontend/src/features/chat/presets/preset-policy.ts+++ b/studio/frontend/src/features/chat/presets/preset-policy.ts@@ -530,6 +530,14 @@ export function resolveFitMaxSeqLength(return customContextLength && customContextLength > 0 ? customContextLength : 0;}+export function isReplayedLoadContext(+ isGguf: boolean | null | undefined,+ customContextLength: number | null,+ maxSeqLength: number,+): boolean {+ return isGguf === true && customContextLength == null && maxSeqLength > 0;+}+/** The context pin a completed load leaves behind: the Context Length the user EXPLICITLY set, or* null for Auto. Takes the user's setting, never the n_ctx that went on the wire, which cannot* answer this: `resolveLoadMaxSeqLength` sends the resolved context on a same-model reload, sodiff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex e4d818129ac..12d60960417 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -57,6 +57,7 @@ export interface LoadModelRequest {nativePathLease?: string | null;hf_token: string | null;max_seq_length: number;+ max_seq_length_auto_derived?: boolean;load_in_4bit: boolean;is_lora: boolean;gguf_variant?: string | null;diff --git a/studio/frontend/tests/replayed-load-context.test.ts b/studio/frontend/tests/replayed-load-context.test.tsnew file mode 100644index 00000000000..24ac0a6f40a--- /dev/null+++ b/studio/frontend/tests/replayed-load-context.test.ts@@ -0,0 +1,93 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";++import { readSrc, registerBundlerResolver } from "./helpers/kit.ts";++registerBundlerResolver();++const RUNTIME = readSrc("features/chat/hooks/use-chat-model-runtime.ts");+const policy = await import("../src/features/chat/presets/preset-policy.ts");++const MODEL = "unsloth/Qwen3.6-27B-MTP-GGUF";+const VARIANT = "UD-Q8_K_XL";+const LOADED = 120064;++function replayed({+ isGguf = true,+ modelId = MODEL,+ customContextLength = null,+ presetSource = "custom",+ gpuMemoryMode = "auto",+ gpuLayers = -1,+}: {+ isGguf?: boolean;+ modelId?: string;+ customContextLength?: number | null;+ presetSource?: "builtin-default" | "custom" | "modified";+ gpuMemoryMode?: "auto" | "manual";+ gpuLayers?: number;+}) {+ const sent = policy.resolveFitMaxSeqLength(+ isGguf,+ gpuMemoryMode,+ gpuLayers,+ customContextLength,+ policy.resolveLoadMaxSeqLength({+ modelId,+ ggufVariant: isGguf ? VARIANT : null,+ isGguf,+ customContextLength,+ loadedContextLength: LOADED,+ currentCheckpoint: MODEL,+ activeGgufVariant: VARIANT,+ isMlx: false,+ pinnedMaxSeqLength: null,+ defaultMaxSeqLength: 4096,+ presetSource,+ }),+ );+ return {+ sent,+ replayed: policy.isReplayedLoadContext(isGguf, customContextLength, sent),+ };+}++test("a same-model reload on a saved preset replays the fitted context", () => {+ for (const presetSource of ["custom", "modified"] as const) {+ assert.deepEqual(replayed({ presetSource }), {+ sent: LOADED,+ replayed: true,+ });+ }+});++test("a typed context is the user's own", () => {+ assert.deepEqual(replayed({ customContextLength: 65536 }), {+ sent: 65536,+ replayed: false,+ });+ assert.deepEqual(replayed({ customContextLength: LOADED }), {+ sent: LOADED,+ replayed: false,+ });+});++test("loads that send no context are not replays", () => {+ assert.equal(replayed({ presetSource: "builtin-default" }).replayed, false);+ assert.equal(replayed({ modelId: "unsloth/Other-GGUF" }).replayed, false);+ assert.equal(replayed({ gpuMemoryMode: "manual" }).replayed, false);+});++test("non-GGUF loads never flag a replay", () => {+ assert.equal(replayed({ isGguf: false }).replayed, false);+});++test("the load request carries the flag", () => {+ assert.match(+ RUNTIME,+ /max_seq_length_auto_derived: isReplayedLoadContext\(\s*isGguf,\s*loadCustomContextLength,\s*loadMaxSeqLength,\s*\)/,+ );+});