Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep fp32 timestep and RoPE tables in fp32 on single-file loads (#17142)
Daniel Han6705f4865a99
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex c0d777ee415..7d4cfa32c9e 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -56,6 +56,7 @@ from .diffusion_content import (whole_pipeline_gguf_family,)from .diffusion_gguf_pipeline import load_whole_pipeline_gguf, whole_pipeline_gguf_resident_mib+from .diffusion_init_buffers import install_single_file_fp32_buffersfrom . import hub_reachabilityfrom .diffusion_families import (SMALL_DIT_BF16_WHEN_RESIDENT,@@ -8299,6 +8300,7 @@ class DiffusionBackend:model_kind = model_kind,)_refuse_lora_without_loader(fam, loras)+ install_single_file_fp32_buffers(logger)kind = resolve_model_kind(gguf_filename, model_kind)# Validate every mode string that can raise BEFORE this load evicts the previous pipeline.normalize_transformer_quant(transformer_quant)diff --git a/studio/backend/core/inference/diffusion_comfy_quant.py b/studio/backend/core/inference/diffusion_comfy_quant.pyindex 339c89fa3aa..54fce04c8e0 100644--- a/studio/backend/core/inference/diffusion_comfy_quant.py+++ b/studio/backend/core/inference/diffusion_comfy_quant.py@@ -13,6 +13,7 @@ from typing import Any, Optionalfrom .diffusion_comfy_block import BLOCK_SIZES, MXFP8, NVFP4, tiled_scale_numelfrom .diffusion_fp8_dequant import chunk_rows, dequant_rows, fused_dequant_enabled+from .diffusion_init_buffers import is_fp32_init_tablefrom .diffusion_single_file_converters import host_ram_budget_scopefrom .diffusion_small_host import FP8_STORAGE_HOST_RESERVE_MIB@@ -54,8 +55,6 @@ _CONSUMED = {MXFP8: (".weight_scale", ".input_scale"),}_LEGACY_RENAME = {".scale_weight": ".weight_scale", ".scale_input": ".input_scale"}-# __init__ buffers kept as built: Qwen-Image-2.1's fp32 timestep table in bf16 moved renders (LPIPS 0.106 -> 0.174).-FP32_INIT_BUFFER_MODULES = frozenset({"QwenImage21TemporalTimesteps"})# Container prefixes a declaration may omit while its weight carries one._DECLARATION_PREFIXES = ("model.diffusion_model.", "diffusion_model.")@@ -1782,15 +1781,15 @@ def load_comfy_quant_transformer(model.load_state_dict(converted, strict = False, assign = True)loaded = set(converted)del converted- # __init__ buffers (Wan's float64 rope) follow the compute dtype; checkpoint buffers keep theirs (H3 curve table)+ # __init__ buffers follow the compute dtype except fp32 tables; checkpoint buffers keep theirs (H3 curve)for module_name, module in model.named_modules():if module_name in block_native or module_name in block_packed:continue # fp32 scales / alpha the block kernels need; codes are not floating- if type(module).__name__ in FP32_INIT_BUFFER_MODULES:- continuefor buffer_name, buffer in list(module._buffers.items()):if (f"{module_name}.{buffer_name}" if module_name else buffer_name) in loaded:continue+ if is_fp32_init_table(module, buffer_name, buffer):+ continueif buffer is not None and buffer.is_floating_point():want = _dtype_for(f"{module_name}.{buffer_name}")if buffer.dtype != want:diff --git a/studio/backend/core/inference/diffusion_init_buffers.py b/studio/backend/core/inference/diffusion_init_buffers.pynew file mode 100644index 00000000000..198ba9f434a--- /dev/null+++ b/studio/backend/core/inference/diffusion_init_buffers.py@@ -0,0 +1,92 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""fp32 tables built in ``__init__`` (Qwen-Image-2.1 timestep ``freqs``, MiniMax-H3 RoPE ``inv_freq``) stay fp32.+``from_pretrained`` leaves them as built; ``from_single_file`` ends with ``model.to(torch_dtype)``, rounding them."""++from __future__ import annotations++import functools+from typing import Any, Optional++_MARK = "_unsloth_fp32_init_buffers"+++def is_fp32_init_table(module: Any, name: str, buffer: Any) -> bool:+ import torch+ return (+ buffer is not None+ and buffer.dtype == torch.float32+ and name in getattr(module, "_non_persistent_buffers_set", ())+ )+++def _rounded(model: Any) -> bool:+ import torch+ low = (torch.bfloat16, torch.float16)+ return any(+ b is not None and b.dtype in low and n in m._non_persistent_buffers_set+ for m in model.modules()+ for n, b in m._buffers.items()+ )+++def restore_fp32_init_buffers(model: Any, logger: Any = None) -> list:+ """Rebuilds the fp32 tables a cast rounded; call before offload hooks, which hold their own tensor refs."""+ if not callable(getattr(type(model), "from_config", None)) or not _rounded(model):+ return []+ from accelerate import init_empty_weights++ with init_empty_weights(): # parameters on meta, buffers built for real+ fresh = type(model).from_config(model.config)+ fixed = []+ for module_name, module in fresh.named_modules():+ for name, buffer in module._buffers.items():+ if not is_fp32_init_table(module, name, buffer) or buffer.device.type == "meta":+ continue+ try:+ target = model.get_submodule(module_name) if module_name else model+ except AttributeError:+ continue+ old = target._buffers.get(name)+ if old is None or old.dtype == buffer.dtype or old.shape != buffer.shape:+ continue+ if not old.is_floating_point() or old.device.type == "meta":+ continue+ target._buffers[name] = buffer.to(old.device)+ fixed.append(f"{module_name}.{name}" if module_name else name)+ if fixed and logger is not None:+ logger.info(+ "diffusion.load: %d fp32 __init__ table(s) kept in fp32 (%s)",+ len(fixed),+ ", ".join(fixed[:4]),+ )+ return fixed+++def install_single_file_fp32_buffers(logger: Optional[Any] = None) -> bool:+ """Wraps ``FromOriginalModelMixin.from_single_file`` (every model / pipeline single-file load) once."""+ try:+ from diffusers.loaders.single_file_model import FromOriginalModelMixin+ except Exception: # noqa: BLE001+ return False+ current = FromOriginalModelMixin.__dict__.get("from_single_file")+ func = getattr(current, "__func__", None)+ if func is None:+ return False+ if getattr(func, _MARK, False):+ return True++ @functools.wraps(func)+ def from_single_file(cls, *args: Any, **kwargs: Any) -> Any:+ model = func(cls, *args, **kwargs)+ try:+ restore_fp32_init_buffers(model, logger)+ except Exception as exc: # noqa: BLE001 - the as-loaded model still renders+ if logger is not None:+ logger.warning("diffusion.load: fp32 __init__ tables not restored: %s", exc)+ return model++ setattr(from_single_file, _MARK, True)+ FromOriginalModelMixin.from_single_file = classmethod(from_single_file)+ return Truediff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 5acf9bfdeb4..2a82a3ec040 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -61,6 +61,7 @@ from .diffusion_attention import (select_attention_backend,)from .diffusion_flow_shift import apply_comfy_flow_shift, apply_scheduler_overrides+from .diffusion_init_buffers import install_single_file_fp32_buffersfrom .diffusion_component_sizes import video_family_has_no_text_encoderfrom . import hub_reachabilityfrom .diffusion_comfy_quant import (@@ -8966,6 +8967,7 @@ class VideoBackend:text_encoder_quant = text_encoder_quant,h3_task = h3_task,)+ install_single_file_fp32_buffers(logger)kind = resolve_video_model_kind(gguf_filename, model_kind)from .video_minimax_h3 import is_h3_nativediff --git a/studio/backend/tests/test_diffusion_comfy_quant.py b/studio/backend/tests/test_diffusion_comfy_quant.pyindex e2eda1103ac..16acd69a93c 100644--- a/studio/backend/tests/test_diffusion_comfy_quant.py+++ b/studio/backend/tests/test_diffusion_comfy_quant.py@@ -1026,7 +1026,8 @@ def test_buffers_built_in_init_follow_the_compute_dtype(comfy_file):def test_a_timestep_table_built_in_fp32_stays_fp32(comfy_file):- """Qwen-Image-2.1's sinusoidal timestep table is computed in fp32; bf16 frequencies shift every timestep."""+ """fp32 tables built in __init__ (Qwen-Image-2.1's timesteps, MiniMax-H3's RoPE) stay fp32, as from_pretrained+ leaves them; a float64 one still follows the compute dtype."""path, _, _ = comfy_fileclass QwenImage21TemporalTimesteps(nn.Module):@@ -1040,6 +1041,7 @@ def test_a_timestep_table_built_in_fp32_stays_fp32(comfy_file):super().__init__(**kwargs)self.time_proj = QwenImage21TemporalTimesteps()self.register_buffer("rope", torch.ones(4, dtype = torch.float32), persistent = False)+ self.register_buffer("wide", torch.ones(4, dtype = torch.float64), persistent = False)model = cq.load_comfy_quant_transformer(_Timed,@@ -1051,9 +1053,8 @@ def test_a_timestep_table_built_in_fp32_stays_fp32(comfy_file):)assert model.time_proj.freqs.dtype == torch.float32assert torch.equal(model.time_proj.freqs, QwenImage21TemporalTimesteps().freqs)- assert (- model.rope.dtype == torch.bfloat16- ) # every other __init__ buffer still follows the compute dtype+ assert model.rope.dtype == torch.float32+ assert model.wide.dtype == torch.bfloat16def test_a_lora_on_stored_fp8_layers_is_refused_with_the_way_out(tmp_path, monkeypatch):diff --git a/studio/backend/tests/test_diffusion_init_buffers.py b/studio/backend/tests/test_diffusion_init_buffers.pynew file mode 100644index 00000000000..eea94156dc4--- /dev/null+++ b/studio/backend/tests/test_diffusion_init_buffers.py@@ -0,0 +1,120 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""fp32 __init__ tables survive diffusers' from_single_file, as they do from_pretrained."""++from __future__ import annotations++import pytest++torch = pytest.importorskip("torch")+diffusers = pytest.importorskip("diffusers")+safetensors_torch = pytest.importorskip("safetensors.torch")++from core.inference import diffusion_init_buffers as ib # noqa: E402++QWEN21 = {+ "attention_head_dim": 16,+ "axes_dims_rope": [4, 6, 6],+ "causal_condition": True,+ "context_in_dim": 32,+ "in_channels": 16,+ "mlp_ratio": 2,+ "num_attention_heads": 2,+ "num_layers": 2,+ "out_channels": 16,+ "patch_size": 1,+}+++def qwen21_files(tmp_path):+ cls = getattr(diffusers, "QwenImage21Transformer2DModel", None)+ if cls is None:+ pytest.skip("this diffusers has no QwenImage21Transformer2DModel")+ torch.manual_seed(0)+ model = cls(**QWEN21)+ repo = tmp_path / "repo"+ model.save_pretrained(repo / "transformer")+ single = tmp_path / "qwen21.safetensors"+ safetensors_torch.save_file(+ {+ f"model.diffusion_model.{k}": v.to(torch.bfloat16).contiguous()+ for k, v in model.state_dict().items()+ },+ str(single),+ )+ return cls, repo, single, model.time_text_embed.time_proj.freqs.clone()+++def _load(cls, repo, single):+ return cls.from_single_file(+ str(single), config = str(repo), subfolder = "transformer", torch_dtype = torch.bfloat16+ )+++def test_single_file_rounds_the_timestep_table_without_the_wrapper(qwen21_files, monkeypatch):+ from diffusers.loaders.single_file_model import FromOriginalModelMixin++ cls, repo, single, _ = qwen21_files+ stock = FromOriginalModelMixin.__dict__["from_single_file"].__func__+ monkeypatch.setattr(+ FromOriginalModelMixin,+ "from_single_file",+ classmethod(getattr(stock, "__wrapped__", stock)),+ )+ assert _load(cls, repo, single).time_text_embed.time_proj.freqs.dtype == torch.bfloat16+++def test_single_file_keeps_the_timestep_table_fp32(qwen21_files, monkeypatch):+ from diffusers.loaders.single_file_model import FromOriginalModelMixin++ cls, repo, single, freqs = qwen21_files+ monkeypatch.setattr(+ FromOriginalModelMixin,+ "from_single_file",+ FromOriginalModelMixin.__dict__["from_single_file"],+ )+ assert ib.install_single_file_fp32_buffers() and ib.install_single_file_fp32_buffers()+ model = _load(cls, repo, single)+ table = model.time_text_embed.time_proj.freqs+ assert table.dtype == torch.float32 and torch.equal(table, freqs)+ assert {p.dtype for p in model.parameters()} == {torch.bfloat16}+ ref = cls.from_pretrained(str(repo), subfolder = "transformer", torch_dtype = torch.bfloat16)+ ours = {n: b.dtype for n, b in model.named_buffers()}+ assert ours == {n: b.dtype for n, b in ref.named_buffers()}+++def test_only_rounded_fp32_init_tables_are_rebuilt():+ class _Tables(torch.nn.Module):+ def __init__(self) -> None:+ super().__init__()+ self.register_buffer(+ "table", torch.arange(4, dtype = torch.float32) / 3, persistent = False+ )+ self.register_buffer("wide", torch.arange(4, dtype = torch.float64) / 3, persistent = False)+ self.register_buffer("kept", torch.arange(4, dtype = torch.float32) / 3)+ self.config = {}++ @classmethod+ def from_config(cls, config):+ return cls()++ model = _Tables().to(torch.bfloat16)+ assert ib.restore_fp32_init_buffers(model) == ["table"]+ assert model.table.dtype == torch.float32 and torch.equal(model.table, _Tables().table)+ assert model.wide.dtype == torch.bfloat16 and model.kept.dtype == torch.bfloat16+ assert ib.restore_fp32_init_buffers(model) == [] # nothing rounded left: no rebuild+ assert ib.restore_fp32_init_buffers(torch.nn.Linear(2, 2)) == [] # no from_config: untouched+++def test_both_backends_install_the_wrapper_before_loading():+ import ast+ import inspect++ from core.inference import diffusion, video++ for module, cls in ((diffusion, diffusion.DiffusionBackend), (video, video.VideoBackend)):+ body = ast.parse(inspect.getsource(cls)).body[0]+ load = next(n for n in body.body if getattr(n, "name", None) == "load_pipeline")+ assert "install_single_file_fp32_buffers" in ast.unparse(load), module.__name__