Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio video: map MiniMax-H3's hosted int8 checkpoint instead of reading it whole (faster load, same weights) (#15563)
Daniel Han739bdfab2316
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.pyindex a6fcbeec3af..b2edccf9ebe 100644--- a/studio/backend/core/inference/diffusion_prequant.py+++ b/studio/backend/core/inference/diffusion_prequant.py@@ -316,13 +316,17 @@ def _install_fp8_unit_scale() -> None:_PREQUANT_MMAP_ENV = "UNSLOTH_DIFFUSION_PREQUANT_MMAP"-def prequant_mmap_enabled(destination: Any) -> bool:- """Map only for a non-CPU destination: a host-placed module would keep the file open (Windows then cannot delete it)."""+def prequant_mmap_enabled(destination: Any, *, host_copy_follows: bool = False) -> bool:+ """Map only for a non-CPU destination: a host-placed module would keep the file open (Windows then cannot delete it).+ ``host_copy_follows``: host-placed but copied off before first render, so Linux maps (avoids a 20 GB copy)."""import os+ import sysraw = (os.environ.get(_PREQUANT_MMAP_ENV) or "").strip().lower()if raw in ("0", "false", "no", "off"):return False+ if host_copy_follows and sys.platform.startswith("linux"):+ return Truedest = str(destination or "").strip().lower()return bool(dest) and not dest.startswith("cpu") and dest != "meta"@@ -331,9 +335,11 @@ def _read_prequant_for(path: str,destination: Any,logger: Any = None,+ *,+ host_copy_follows: bool = False,) -> Any:"""``_load_prequant_checkpoint`` mapped when enabled; anything the mapping cannot open is re-read in full."""- if prequant_mmap_enabled(destination):+ if prequant_mmap_enabled(destination, host_copy_follows = host_copy_follows):try:return _load_prequant_checkpoint(path, map_location = "cpu", mmap = True)except Exception as exc: # noqa: BLE001 - retried unmapped; the real error resurfaces there@@ -2165,6 +2171,7 @@ def load_prequantized_transformer(placement_device: Optional[str] = None,family: Optional[str] = None,into: Optional[Any] = None,+ host_copy_follows: bool = False,) -> Optional[Any]:"""Load the pre-quantized transformer described by ``source`` onto ``device``.@@ -2283,7 +2290,9 @@ def load_prequantized_transformer(else:# Pickles load only under the ALLOWLIST, never free-running: a mutated file must fail,# not run.- ckpt = _read_prequant_for(path, placement_device or device, logger)+ ckpt = _read_prequant_for(+ path, placement_device or device, logger, host_copy_follows = host_copy_follows+ )if not _validate_checkpoint(ckpt,scheme,diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 4a02e5a9f1c..7c37c327455 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -2685,6 +2685,17 @@ def _request_available_bytes(device_obj: Any, module: Any) -> int:return int(free_bytes) + int(reserved_bytes) - _user_reserve_bytes(device_obj, total_bytes)+def _h3_host_copy_follows(device: Any) -> bool:+ """True on a discrete GPU (pinning allowed): H3's prequant denoiser is copied off its host load."""+ if str(device or "").strip().lower() in ("", "cpu", "meta"):+ return False+ try:+ from .host_pinning import host_pinning_allowed+ return bool(host_pinning_allowed(device))+ except Exception: # noqa: BLE001+ return False++def _h3_free_device_bytes(device: str) -> Optional[int]:"""Live free VRAM, or None when it cannot be read. None means "cannot tell", never zero."""free = _h3_signed_free_device_bytes(device)@@ -10489,6 +10500,7 @@ class VideoBackend:# Config from this partition: reference loads stage transformer_ref/; else offline loads hit Hub.config_subfolder = denoiser_component,logger = logger,+ host_copy_follows = _h3_host_copy_follows(device),)if transformer is None:# Best-effort: missing, mismatched or unreadable checkpoints fall back to dense, not after teardown.diff --git a/studio/backend/tests/test_h3_prequant_mapped_read.py b/studio/backend/tests/test_h3_prequant_mapped_read.pynew file mode 100644index 00000000000..0ff8606bffa--- /dev/null+++ b/studio/backend/tests/test_h3_prequant_mapped_read.py@@ -0,0 +1,98 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""H3's host-placed prequant pickle is mapped on Linux; ``UNSLOTH_DIFFUSION_PREQUANT_MMAP=0`` restores the full read."""++import ast+import inspect+from pathlib import Path++import torch++BACKEND = Path(__file__).resolve().parents[1]+++def test_host_destination_maps_only_when_a_host_copy_follows_on_linux(monkeypatch):+ from core.inference import diffusion_prequant as pq++ monkeypatch.delenv("UNSLOTH_DIFFUSION_PREQUANT_MMAP", raising = False)+ monkeypatch.setattr("sys.platform", "linux")+ assert not pq.prequant_mmap_enabled("cpu")+ assert pq.prequant_mmap_enabled("cpu", host_copy_follows = True)+ assert pq.prequant_mmap_enabled("cuda", host_copy_follows = False)+ monkeypatch.setattr("sys.platform", "win32")+ assert not pq.prequant_mmap_enabled("cpu", host_copy_follows = True)+ monkeypatch.setattr("sys.platform", "linux")+ monkeypatch.setenv("UNSLOTH_DIFFUSION_PREQUANT_MMAP", "0")+ assert not pq.prequant_mmap_enabled("cpu", host_copy_follows = True)+++def test_mapped_host_read_matches_the_full_read(tmp_path, monkeypatch):+ from core.inference import diffusion_prequant as pq++ monkeypatch.delenv("UNSLOTH_DIFFUSION_PREQUANT_MMAP", raising = False)+ monkeypatch.setattr("sys.platform", "linux")+ ckpt = {+ "format": pq.PREQUANT_FORMAT,+ "state_dict": {"w": torch.arange(64, dtype = torch.int8).reshape(8, 8), "s": torch.rand(8)},+ "metadata": {"scheme": "int8"},+ }+ path = tmp_path / "x.pt"+ torch.save(ckpt, path)+ seen = []+ real = pq._load_prequant_checkpoint++ def spy(p, **kwargs):+ seen.append(kwargs.get("mmap"))+ return real(p, **kwargs)++ monkeypatch.setattr(pq, "_load_prequant_checkpoint", spy)+ mapped = pq._read_prequant_for(str(path), "cpu", host_copy_follows = True)+ full = pq._read_prequant_for(str(path), "cpu")+ assert seen == [True, None]+ for key, value in ckpt["state_dict"].items():+ assert torch.equal(mapped["state_dict"][key], value)+ assert torch.equal(full["state_dict"][key], value)+++def test_loader_forwards_the_hint():+ from core.inference import diffusion_prequant as pq++ params = inspect.signature(pq.load_prequantized_transformer).parameters+ assert params["host_copy_follows"].default is False+ src = inspect.getsource(pq.load_prequantized_transformer)+ assert "host_copy_follows = host_copy_follows" in src+++def _h3_prequant_call() -> ast.Call:+ tree = ast.parse((BACKEND / "core/inference/video.py").read_text(encoding = "utf-8"))+ for node in ast.walk(tree):+ if isinstance(node, ast.FunctionDef) and node.name == "_load_h3_modular_pipeline":+ for call in ast.walk(node):+ if (+ isinstance(call, ast.Call)+ and isinstance(call.func, ast.Name)+ and call.func.id == "load_prequantized_transformer"+ ):+ return call+ raise AssertionError("no load_prequantized_transformer call in _load_h3_modular_pipeline")+++def test_h3_load_passes_the_hint_from_its_render_device():+ call = _h3_prequant_call()+ kw = {k.arg: k.value for k in call.keywords}+ assert ast.unparse(kw["device"]) == "'cpu'"+ assert ast.unparse(kw["host_copy_follows"]) == "_h3_host_copy_follows(device)"+++def test_h3_host_copy_follows_only_on_a_discrete_gpu(monkeypatch):+ from core.inference import host_pinning+ from core.inference import video as V++ monkeypatch.setattr(host_pinning, "device_is_integrated", lambda device = None: False)+ assert V._h3_host_copy_follows("cuda")+ assert V._h3_host_copy_follows("cuda:1")+ assert not V._h3_host_copy_follows("cpu")+ assert not V._h3_host_copy_follows(None)+ monkeypatch.setattr(host_pinning, "device_is_integrated", lambda device = None: True)+ assert not V._h3_host_copy_follows("cuda")