Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fit the context again when MTP is forced on a reload (#12231)

Nilayf160cf598695
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 55643f02562..74ab448c85c 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -574,6 +574,7 @@ class GgufLoadIntent:
# for the session.
disable_vision: bool = False
n_ctx: int = 4096
+ max_seq_length_auto_derived: bool = False
chat_template_override: Optional[str] = None
cache_type_kv: Optional[str] = None
speculative_type: Optional[str] = None
@@ -23125,6 +23126,7 @@ class LlamaCppBackend:
is_vision = intent.is_vision
disable_vision = intent.disable_vision
n_ctx = intent.n_ctx
+ _replayed_ctx_refit = False
chat_template_override = intent.chat_template_override
cache_type_kv = intent.cache_type_kv
speculative_type = intent.speculative_type
@@ -25039,6 +25041,24 @@ class LlamaCppBackend:
_draft_cpu_no_embedded = _draft_on_cpu and (
_separate_draft_launches or not self._nextn_predict_layers
)
+ # A replay was fitted without the drafter: re-fit it like a fresh MTP load, before any explicit_ctx reader.
+ if (
+ explicit_ctx
+ and ctx_override is None
+ and intent.max_seq_length_auto_derived
+ # CPU-pinned drafters too: the target still pays rollback state for them.
+ and _mtp_will_engage
+ # Forced = anything that bypasses the Auto drop probe, advanced arguments included.
+ and (
+ (_canonicalize_spec_mode(speculative_type) or "auto") != "auto"
+ or _user_mtp_via_extras
+ or _user_draft_via_extras
+ or _extra_args_set_spec_type(extra_args)
+ or _extra_args_mtp_draft_path(extra_args, env = _spec_env)
+ )
+ ):
+ explicit_ctx = False
+ _replayed_ctx_refit = True
# The two tensor -> layer downgrades that need nothing the probe
# decides run BEFORE it: the probe is gated on `not tensor_parallel`
@@ -31237,7 +31257,8 @@ class LlamaCppBackend:
else list(_pv_requested)
)
self._extra_args_source = (model_identifier, hf_variant)
- self._requested_n_ctx = int(n_ctx)
+ # A re-fit replay records what launched, which is what the client replays next.
+ self._requested_n_ctx = int(effective_ctx if _replayed_ctx_refit else n_ctx)
# Local n_parallel may have been reduced above; the snapshot has the ask.
self._requested_n_parallel = max(1, int(intent.n_parallel))
# Commit with the rest of the known-good state: only a launch that got
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index d61bac1507b..5bbe063c51c 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -84,6 +84,10 @@ class LoadRequest(BaseModel):
"(llama.cpp or MLX) choose the context itself."
),
)
+ max_seq_length_auto_derived: bool = Field(
+ False,
+ description = "max_seq_length is the previous load's fitted context, not a user choice",
+ )
load_in_4bit: bool = Field(True, description = "Load model in 4-bit quantization")
is_lora: bool = Field(False, description = "Whether this is a LoRA adapter")
gguf_variant: Optional[str] = Field(
diff --git a/studio/backend/tests/test_llama_cpp_placement.py b/studio/backend/tests/test_llama_cpp_placement.py
index ef367fc7854..017943c9a92 100644
--- a/studio/backend/tests/test_llama_cpp_placement.py
+++ b/studio/backend/tests/test_llama_cpp_placement.py
@@ -1404,6 +1404,155 @@ def test_the_drop_actually_releases_the_reserve_the_fit_charges(tmp_path):
assert cmd[cmd.index("--fit") + 1] == "off"
+def _replayed_context_mtp_backend(tmp_path: Path):
+ gb = 1024**3
+ backend, gguf = _backend(
+ tmp_path, vulkan = False, memory = [(0, 45_914, 46_080), (1, 8_032, 8_176)]
+ )
+
+ def read_metadata(_path):
+ backend._nextn_predict_layers = 1
+ backend._context_length = 262_144
+
+ backend._read_gguf_metadata = read_metadata
+ backend._get_gguf_size_bytes = lambda _path: 31 * gb
+ backend._can_estimate_kv = lambda: True
+ backend._estimate_kv_cache_bytes = lambda n_ctx, *args, **kwargs: n_ctx * 106_000
+ backend._compute_buffer_ctx_bytes = lambda *args, **kwargs: 0
+ backend._estimate_compute_buffer_bytes = lambda **kwargs: 1
+ backend._mtp_draft_kv_bytes = lambda *args, **kwargs: 0
+ backend._estimate_mtp_overhead_bytes = lambda n_ctx, *args, **kwargs: n_ctx * 20_000
+ backend.probe_server_capabilities = lambda _binary = None: {
+ "mtp_token": "draft-mtp",
+ "supports_ngram_mod": True,
+ "spec_draft_n_max_flag": "--spec-draft-n-max",
+ }
+ return backend, gguf
+
+
+def _launched_ctx(result) -> int:
+ return int(result["cmd"][result["cmd"].index("-c") + 1])
+
+
+def _replayed_auto_context(tmp_path: Path) -> int:
+ backend, gguf = _replayed_context_mtp_backend(tmp_path)
+ auto = _launch(backend, gguf, n_ctx = 0, n_parallel = 4, speculative_type = "auto")
+ assert backend.spec_fallback_reason == "drafter_no_vram"
+ assert auto["env"]["CUDA_VISIBLE_DEVICES"] == "0"
+ return _launched_ctx(auto)
+
+
+def test_forcing_the_drafter_refits_a_context_replayed_from_auto(tmp_path):
+ replayed = _replayed_auto_context(tmp_path)
+ backend, gguf = _replayed_context_mtp_backend(tmp_path)
+ fresh = _launch(backend, gguf, n_ctx = 0, n_parallel = 4, speculative_type = "mtp")
+
+ backend, gguf = _replayed_context_mtp_backend(tmp_path)
+ result = _launch(
+ backend,
+ gguf,
+ n_ctx = replayed,
+ max_seq_length_auto_derived = True,
+ n_parallel = 4,
+ speculative_type = "mtp",
+ )
+
+ assert _launched_ctx(fresh) < replayed
+ assert _launched_ctx(result) == _launched_ctx(fresh)
+ assert result["env"]["CUDA_VISIBLE_DEVICES"] == "0"
+ assert result["cmd"][result["cmd"].index("--spec-type") + 1] == "draft-mtp"
+ assert backend._requested_n_ctx == _launched_ctx(result)
+
+
+def test_a_replayed_context_gets_the_slot_refit_of_a_fresh_drafter_load(tmp_path):
+ def slot_bound_backend():
+ backend, gguf = _replayed_context_mtp_backend(tmp_path)
+ backend._get_gpu_memory = lambda _binary = None, **_kw: [(0, 45_914, 46_080)]
+ backend._get_gpu_free_memory = lambda _binary = None, **_kw: [(0, 45_914)]
+ backend._estimate_kv_cache_bytes = lambda n_ctx, *args, **kwargs: n_ctx * 16_000
+ backend._estimate_mtp_overhead_bytes = (
+ lambda n_ctx, *args, _np = None, n_parallel = None, **kwargs: n_ctx * 5_000
+ + int(_np or n_parallel or 4) * 3 * 1024**3
+ )
+ caps = backend.probe_server_capabilities()
+ backend.probe_server_capabilities = lambda _binary = None: {
+ **caps,
+ "supports_kv_unified": True,
+ }
+ return backend, gguf
+
+ backend, gguf = slot_bound_backend()
+ auto = _launch(backend, gguf, n_ctx = 0, n_parallel = 4, speculative_type = "auto")
+ assert backend.spec_fallback_reason == "drafter_no_vram"
+ backend, gguf = slot_bound_backend()
+ fresh = _launch(backend, gguf, n_ctx = 0, n_parallel = 4, speculative_type = "mtp")
+
+ backend, gguf = slot_bound_backend()
+ result = _launch(
+ backend,
+ gguf,
+ n_ctx = _launched_ctx(auto),
+ max_seq_length_auto_derived = True,
+ n_parallel = 4,
+ speculative_type = "mtp",
+ )
+
+ assert fresh["cmd"][fresh["cmd"].index("--parallel") + 1] != "4"
+ assert _launched_ctx(fresh) > 8192
+ assert _launched_ctx(result) == _launched_ctx(fresh)
+ assert (
+ result["cmd"][result["cmd"].index("--parallel") + 1]
+ == (fresh["cmd"][fresh["cmd"].index("--parallel") + 1])
+ )
+
+
+def test_a_drafter_forced_through_extra_args_also_refits(tmp_path):
+ replayed = _replayed_auto_context(tmp_path)
+ backend, gguf = _replayed_context_mtp_backend(tmp_path)
+ fresh = _launch(backend, gguf, n_ctx = 0, n_parallel = 4, speculative_type = "mtp")
+
+ backend, gguf = _replayed_context_mtp_backend(tmp_path)
+ result = _launch(
+ backend,
+ gguf,
+ n_ctx = replayed,
+ max_seq_length_auto_derived = True,
+ n_parallel = 4,
+ speculative_type = "auto",
+ extra_args = ["--spec-type", "draft-mtp"],
+ )
+
+ assert _launched_ctx(result) == _launched_ctx(fresh) < replayed
+ assert result["env"]["CUDA_VISIBLE_DEVICES"] == "0"
+
+
+def test_forcing_the_drafter_keeps_a_ctx_size_passed_through_extra_args(tmp_path):
+ replayed = _replayed_auto_context(tmp_path)
+ backend, gguf = _replayed_context_mtp_backend(tmp_path)
+
+ result = _launch(
+ backend,
+ gguf,
+ n_ctx = replayed,
+ max_seq_length_auto_derived = True,
+ n_parallel = 4,
+ speculative_type = "mtp",
+ extra_args = ["-c", str(replayed)],
+ )
+
+ assert _launched_ctx(result) == replayed
+
+
+def test_forcing_the_drafter_keeps_a_typed_context(tmp_path):
+ replayed = _replayed_auto_context(tmp_path)
+ backend, gguf = _replayed_context_mtp_backend(tmp_path)
+
+ result = _launch(backend, gguf, n_ctx = replayed, n_parallel = 4, speculative_type = "mtp")
+
+ assert _launched_ctx(result) == replayed
+ assert backend._requested_n_ctx == replayed
+
+
def test_a_cpu_offloaded_sidecar_is_not_probed_because_a_head_also_exists(tmp_path):
"""The drafter that launches decides, not one that merely exists.
diff --git a/studio/backend/tests/test_mmproj_placement_policy.py b/studio/backend/tests/test_mmproj_placement_policy.py
index df4f00ee198..c9d77e68796 100644
--- a/studio/backend/tests/test_mmproj_placement_policy.py
+++ b/studio/backend/tests/test_mmproj_placement_policy.py
@@ -1631,3 +1631,52 @@ def test_an_unloadable_drafter_is_not_charged_before_it_is_dropped(tmp_path):
assert "--model-draft" not in cmd
assert "--no-mmproj-offload" not in cmd
assert backend.mtp_draft_suppressed_path == str(tmp_path / "mtp.gguf")
+
+
+def test_a_replayed_context_places_the_projector_like_a_fresh_forced_drafter(tmp_path):
+ # Classified after the projector probe, the replay priced it at 65536 and pinned it to CPU.
+ memory = [(0, 13_500, 24_000)]
+
+ def load(**kwargs):
+ backend, gguf = _backend(tmp_path, memory = memory, drafter_bytes = 2 * GIB, native_ctx = 65536)
+ cmd = _launch(backend, gguf, mtp_draft_path = str(tmp_path / "mtp.gguf"), **kwargs)["cmd"]
+ return backend, cmd, int(cmd[cmd.index("-c") + 1])
+
+ _, auto, replayed = load(speculative_type = "auto", n_ctx = 0)
+ assert "--model-draft" not in auto
+ _, fresh, fresh_ctx = load(speculative_type = "mtp", n_ctx = 0)
+ backend, replay, replay_ctx = load(
+ speculative_type = "mtp", n_ctx = replayed, max_seq_length_auto_derived = True
+ )
+
+ assert "--no-mmproj-offload" not in fresh
+ assert "--no-mmproj-offload" not in replay
+ assert replay_ctx == fresh_ctx < replayed
+ assert backend._requested_n_ctx == replay_ctx
+
+
+def test_a_replayed_context_refits_for_a_cpu_pinned_drafter_too(tmp_path):
+ # --spec-draft-ngl 0 keeps the drafter off the GPU; the hybrid target still pays rollback state.
+ memory = [(0, 10_000, 24_000)]
+ extras = ["--spec-draft-ngl", "0"]
+
+ def load(**kwargs):
+ backend, gguf = _backend(tmp_path, memory = memory, drafter_bytes = 2 * GIB, native_ctx = 65536)
+ backend._rollback_state_bytes = lambda n_parallel = 1, *_a, **_kw: n_parallel * 256 * MIB
+ cmd = _launch(
+ backend,
+ gguf,
+ mtp_draft_path = str(tmp_path / "mtp.gguf"),
+ extra_args = extras,
+ **kwargs,
+ )["cmd"]
+ return cmd, int(cmd[cmd.index("-c") + 1])
+
+ _, replayed = load(speculative_type = "off", n_ctx = 0)
+ fresh, fresh_ctx = load(speculative_type = "mtp", n_ctx = 0)
+ replay, replay_ctx = load(
+ speculative_type = "mtp", n_ctx = replayed, max_seq_length_auto_derived = True
+ )
+
+ assert replay_ctx == fresh_ctx < replayed
+ assert ("--no-mmproj-offload" in replay) == ("--no-mmproj-offload" in fresh)
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index 9e5cf0ba510..de0d1f8efe6 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -133,6 +133,7 @@ import {
loadedContextForParams,
mergeBackendRecommendedInference,
resolveFitMaxSeqLength,
+ isReplayedLoadContext,
unpinnedDefaultRequest,
unpinnedLoadContext,
resolveLoadMaxSeqLength,
@@ -2472,6 +2473,11 @@ export function useChatModelRuntime() {
nativePathLease: loadNativePathLease,
hf_token: hfToken,
max_seq_length: loadMaxSeqLength,
+ max_seq_length_auto_derived: isReplayedLoadContext(
+ isGguf,
+ loadCustomContextLength,
+ loadMaxSeqLength,
+ ),
load_in_4bit: true,
is_lora: isLora,
gguf_variant: ggufVariant ?? null,
diff --git a/studio/frontend/src/features/chat/presets/preset-policy.ts b/studio/frontend/src/features/chat/presets/preset-policy.ts
index cc86c14905e..e9af04b1e19 100644
--- a/studio/frontend/src/features/chat/presets/preset-policy.ts
+++ b/studio/frontend/src/features/chat/presets/preset-policy.ts
@@ -530,6 +530,14 @@ export function resolveFitMaxSeqLength(
return customContextLength && customContextLength > 0 ? customContextLength : 0;
}
+export function isReplayedLoadContext(
+ isGguf: boolean | null | undefined,
+ customContextLength: number | null,
+ maxSeqLength: number,
+): boolean {
+ return isGguf === true && customContextLength == null && maxSeqLength > 0;
+}
+
/** The context pin a completed load leaves behind: the Context Length the user EXPLICITLY set, or
* null for Auto. Takes the user's setting, never the n_ctx that went on the wire, which cannot
* answer this: `resolveLoadMaxSeqLength` sends the resolved context on a same-model reload, so
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index e4d818129ac..12d60960417 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -57,6 +57,7 @@ export interface LoadModelRequest {
nativePathLease?: string | null;
hf_token: string | null;
max_seq_length: number;
+ max_seq_length_auto_derived?: boolean;
load_in_4bit: boolean;
is_lora: boolean;
gguf_variant?: string | null;
diff --git a/studio/frontend/tests/replayed-load-context.test.ts b/studio/frontend/tests/replayed-load-context.test.ts
new file mode 100644
index 00000000000..24ac0a6f40a
--- /dev/null
+++ b/studio/frontend/tests/replayed-load-context.test.ts
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { readSrc, registerBundlerResolver } from "./helpers/kit.ts";
+
+registerBundlerResolver();
+
+const RUNTIME = readSrc("features/chat/hooks/use-chat-model-runtime.ts");
+const policy = await import("../src/features/chat/presets/preset-policy.ts");
+
+const MODEL = "unsloth/Qwen3.6-27B-MTP-GGUF";
+const VARIANT = "UD-Q8_K_XL";
+const LOADED = 120064;
+
+function replayed({
+ isGguf = true,
+ modelId = MODEL,
+ customContextLength = null,
+ presetSource = "custom",
+ gpuMemoryMode = "auto",
+ gpuLayers = -1,
+}: {
+ isGguf?: boolean;
+ modelId?: string;
+ customContextLength?: number | null;
+ presetSource?: "builtin-default" | "custom" | "modified";
+ gpuMemoryMode?: "auto" | "manual";
+ gpuLayers?: number;
+}) {
+ const sent = policy.resolveFitMaxSeqLength(
+ isGguf,
+ gpuMemoryMode,
+ gpuLayers,
+ customContextLength,
+ policy.resolveLoadMaxSeqLength({
+ modelId,
+ ggufVariant: isGguf ? VARIANT : null,
+ isGguf,
+ customContextLength,
+ loadedContextLength: LOADED,
+ currentCheckpoint: MODEL,
+ activeGgufVariant: VARIANT,
+ isMlx: false,
+ pinnedMaxSeqLength: null,
+ defaultMaxSeqLength: 4096,
+ presetSource,
+ }),
+ );
+ return {
+ sent,
+ replayed: policy.isReplayedLoadContext(isGguf, customContextLength, sent),
+ };
+}
+
+test("a same-model reload on a saved preset replays the fitted context", () => {
+ for (const presetSource of ["custom", "modified"] as const) {
+ assert.deepEqual(replayed({ presetSource }), {
+ sent: LOADED,
+ replayed: true,
+ });
+ }
+});
+
+test("a typed context is the user's own", () => {
+ assert.deepEqual(replayed({ customContextLength: 65536 }), {
+ sent: 65536,
+ replayed: false,
+ });
+ assert.deepEqual(replayed({ customContextLength: LOADED }), {
+ sent: LOADED,
+ replayed: false,
+ });
+});
+
+test("loads that send no context are not replays", () => {
+ assert.equal(replayed({ presetSource: "builtin-default" }).replayed, false);
+ assert.equal(replayed({ modelId: "unsloth/Other-GGUF" }).replayed, false);
+ assert.equal(replayed({ gpuMemoryMode: "manual" }).replayed, false);
+});
+
+test("non-GGUF loads never flag a replay", () => {
+ assert.equal(replayed({ isGguf: false }).replayed, false);
+});
+
+test("the load request carries the flag", () => {
+ assert.match(
+ RUNTIME,
+ /max_seq_length_auto_derived: isReplayedLoadContext\(\s*isGguf,\s*loadCustomContextLength,\s*loadMaxSeqLength,\s*\)/,
+ );
+});