Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio video: map MiniMax-H3's hosted int8 checkpoint instead of reading it whole (faster load, same weights) (#15563)

Daniel Han739bdfab2316
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index a6fcbeec3af..b2edccf9ebe 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -316,13 +316,17 @@ def _install_fp8_unit_scale() -> None:
_PREQUANT_MMAP_ENV = "UNSLOTH_DIFFUSION_PREQUANT_MMAP"
-def prequant_mmap_enabled(destination: Any) -> bool:
- """Map only for a non-CPU destination: a host-placed module would keep the file open (Windows then cannot delete it)."""
+def prequant_mmap_enabled(destination: Any, *, host_copy_follows: bool = False) -> bool:
+ """Map only for a non-CPU destination: a host-placed module would keep the file open (Windows then cannot delete it).
+ ``host_copy_follows``: host-placed but copied off before first render, so Linux maps (avoids a 20 GB copy)."""
import os
+ import sys
raw = (os.environ.get(_PREQUANT_MMAP_ENV) or "").strip().lower()
if raw in ("0", "false", "no", "off"):
return False
+ if host_copy_follows and sys.platform.startswith("linux"):
+ return True
dest = str(destination or "").strip().lower()
return bool(dest) and not dest.startswith("cpu") and dest != "meta"
@@ -331,9 +335,11 @@ def _read_prequant_for(
path: str,
destination: Any,
logger: Any = None,
+ *,
+ host_copy_follows: bool = False,
) -> Any:
"""``_load_prequant_checkpoint`` mapped when enabled; anything the mapping cannot open is re-read in full."""
- if prequant_mmap_enabled(destination):
+ if prequant_mmap_enabled(destination, host_copy_follows = host_copy_follows):
try:
return _load_prequant_checkpoint(path, map_location = "cpu", mmap = True)
except Exception as exc: # noqa: BLE001 - retried unmapped; the real error resurfaces there
@@ -2165,6 +2171,7 @@ def load_prequantized_transformer(
placement_device: Optional[str] = None,
family: Optional[str] = None,
into: Optional[Any] = None,
+ host_copy_follows: bool = False,
) -> Optional[Any]:
"""Load the pre-quantized transformer described by ``source`` onto ``device``.
@@ -2283,7 +2290,9 @@ def load_prequantized_transformer(
else:
# Pickles load only under the ALLOWLIST, never free-running: a mutated file must fail,
# not run.
- ckpt = _read_prequant_for(path, placement_device or device, logger)
+ ckpt = _read_prequant_for(
+ path, placement_device or device, logger, host_copy_follows = host_copy_follows
+ )
if not _validate_checkpoint(
ckpt,
scheme,
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 4a02e5a9f1c..7c37c327455 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -2685,6 +2685,17 @@ def _request_available_bytes(device_obj: Any, module: Any) -> int:
return int(free_bytes) + int(reserved_bytes) - _user_reserve_bytes(device_obj, total_bytes)
+def _h3_host_copy_follows(device: Any) -> bool:
+ """True on a discrete GPU (pinning allowed): H3's prequant denoiser is copied off its host load."""
+ if str(device or "").strip().lower() in ("", "cpu", "meta"):
+ return False
+ try:
+ from .host_pinning import host_pinning_allowed
+ return bool(host_pinning_allowed(device))
+ except Exception: # noqa: BLE001
+ return False
+
+
def _h3_free_device_bytes(device: str) -> Optional[int]:
"""Live free VRAM, or None when it cannot be read. None means "cannot tell", never zero."""
free = _h3_signed_free_device_bytes(device)
@@ -10489,6 +10500,7 @@ class VideoBackend:
# Config from this partition: reference loads stage transformer_ref/; else offline loads hit Hub.
config_subfolder = denoiser_component,
logger = logger,
+ host_copy_follows = _h3_host_copy_follows(device),
)
if transformer is None:
# Best-effort: missing, mismatched or unreadable checkpoints fall back to dense, not after teardown.
diff --git a/studio/backend/tests/test_h3_prequant_mapped_read.py b/studio/backend/tests/test_h3_prequant_mapped_read.py
new file mode 100644
index 00000000000..0ff8606bffa
--- /dev/null
+++ b/studio/backend/tests/test_h3_prequant_mapped_read.py
@@ -0,0 +1,98 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""H3's host-placed prequant pickle is mapped on Linux; ``UNSLOTH_DIFFUSION_PREQUANT_MMAP=0`` restores the full read."""
+
+import ast
+import inspect
+from pathlib import Path
+
+import torch
+
+BACKEND = Path(__file__).resolve().parents[1]
+
+
+def test_host_destination_maps_only_when_a_host_copy_follows_on_linux(monkeypatch):
+ from core.inference import diffusion_prequant as pq
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PREQUANT_MMAP", raising = False)
+ monkeypatch.setattr("sys.platform", "linux")
+ assert not pq.prequant_mmap_enabled("cpu")
+ assert pq.prequant_mmap_enabled("cpu", host_copy_follows = True)
+ assert pq.prequant_mmap_enabled("cuda", host_copy_follows = False)
+ monkeypatch.setattr("sys.platform", "win32")
+ assert not pq.prequant_mmap_enabled("cpu", host_copy_follows = True)
+ monkeypatch.setattr("sys.platform", "linux")
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_PREQUANT_MMAP", "0")
+ assert not pq.prequant_mmap_enabled("cpu", host_copy_follows = True)
+
+
+def test_mapped_host_read_matches_the_full_read(tmp_path, monkeypatch):
+ from core.inference import diffusion_prequant as pq
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PREQUANT_MMAP", raising = False)
+ monkeypatch.setattr("sys.platform", "linux")
+ ckpt = {
+ "format": pq.PREQUANT_FORMAT,
+ "state_dict": {"w": torch.arange(64, dtype = torch.int8).reshape(8, 8), "s": torch.rand(8)},
+ "metadata": {"scheme": "int8"},
+ }
+ path = tmp_path / "x.pt"
+ torch.save(ckpt, path)
+ seen = []
+ real = pq._load_prequant_checkpoint
+
+ def spy(p, **kwargs):
+ seen.append(kwargs.get("mmap"))
+ return real(p, **kwargs)
+
+ monkeypatch.setattr(pq, "_load_prequant_checkpoint", spy)
+ mapped = pq._read_prequant_for(str(path), "cpu", host_copy_follows = True)
+ full = pq._read_prequant_for(str(path), "cpu")
+ assert seen == [True, None]
+ for key, value in ckpt["state_dict"].items():
+ assert torch.equal(mapped["state_dict"][key], value)
+ assert torch.equal(full["state_dict"][key], value)
+
+
+def test_loader_forwards_the_hint():
+ from core.inference import diffusion_prequant as pq
+
+ params = inspect.signature(pq.load_prequantized_transformer).parameters
+ assert params["host_copy_follows"].default is False
+ src = inspect.getsource(pq.load_prequantized_transformer)
+ assert "host_copy_follows = host_copy_follows" in src
+
+
+def _h3_prequant_call() -> ast.Call:
+ tree = ast.parse((BACKEND / "core/inference/video.py").read_text(encoding = "utf-8"))
+ for node in ast.walk(tree):
+ if isinstance(node, ast.FunctionDef) and node.name == "_load_h3_modular_pipeline":
+ for call in ast.walk(node):
+ if (
+ isinstance(call, ast.Call)
+ and isinstance(call.func, ast.Name)
+ and call.func.id == "load_prequantized_transformer"
+ ):
+ return call
+ raise AssertionError("no load_prequantized_transformer call in _load_h3_modular_pipeline")
+
+
+def test_h3_load_passes_the_hint_from_its_render_device():
+ call = _h3_prequant_call()
+ kw = {k.arg: k.value for k in call.keywords}
+ assert ast.unparse(kw["device"]) == "'cpu'"
+ assert ast.unparse(kw["host_copy_follows"]) == "_h3_host_copy_follows(device)"
+
+
+def test_h3_host_copy_follows_only_on_a_discrete_gpu(monkeypatch):
+ from core.inference import host_pinning
+ from core.inference import video as V
+
+ monkeypatch.setattr(host_pinning, "device_is_integrated", lambda device = None: False)
+ assert V._h3_host_copy_follows("cuda")
+ assert V._h3_host_copy_follows("cuda:1")
+ assert not V._h3_host_copy_follows("cpu")
+ assert not V._h3_host_copy_follows(None)
+ monkeypatch.setattr(host_pinning, "device_is_integrated", lambda device = None: True)
+ assert not V._h3_host_copy_follows("cuda")