Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio FLUX.3: keep bf16 weights as stored on fp16-only GPUs and compute in float32 per layer (#17173)

Daniel Hanf81dad0a977c
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 9e5dfe7978e..bb6e7b1c590 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -175,6 +175,7 @@ from .diffusion_flux3 import (
FLUX3_COMPONENTS,
FLUX3_COMPONENT_FOLDERS,
FLUX3_INDEX,
+ flux3_storage_dtype,
is_flux3_family,
load_flux3_pipeline,
)
@@ -9685,6 +9686,11 @@ class DiffusionBackend:
text_encoder = pipe_kwargs.get("text_encoder"),
transformer = pipe_kwargs.get("transformer"),
family_name = fam.name,
+ storage_dtype = flux3_storage_dtype(target, dtype),
+ # A transformer quant rebuilds the dense Linears it reads.
+ transformer_storage = transformer_quant_pinned is None
+ and native_scheme is None
+ and native_offload_scheme is None,
)
finally:
self._stop_load_prefetch()
@@ -10047,6 +10053,7 @@ class DiffusionBackend:
config_source = _base_local_dir,
).get("text_encoder"),
family_name = fam.name,
+ storage_dtype = flux3_storage_dtype(target, dtype),
)
elif fam.name == MOSS_IMAGE_FAMILY_NAME:
from .diffusion_moss_image import load_moss_image_pipeline
@@ -12181,6 +12188,7 @@ class DiffusionBackend:
check_cancelled = check_cancelled,
local_files_only = local_files_only,
family_name = getattr(fam, "name", None),
+ storage_dtype = flux3_storage_dtype(target, dtype),
)
check_cancelled()
pipe.to(device)
diff --git a/studio/backend/core/inference/diffusion_flux3.py b/studio/backend/core/inference/diffusion_flux3.py
index 534693d5326..175463757f6 100644
--- a/studio/backend/core/inference/diffusion_flux3.py
+++ b/studio/backend/core/inference/diffusion_flux3.py
@@ -582,6 +582,118 @@ class _Flux3KleinPipelineMixin:
return _hooks()
+def flux3_storage_dtype(target: Any, dtype: Any) -> Any:
+ """bf16 when a CUDA load computes FLUX.3 in float32: keep bf16 weights and upcast per op instead of doubling
+ the load (about 40 GB for klein). None otherwise."""
+ from .diffusion_manual_cast import manual_cast_disabled
+
+ if manual_cast_disabled() or getattr(target, "device", None) != "cuda":
+ return None
+ try:
+ import torch
+ except Exception: # noqa: BLE001 - no torch, no load
+ return None
+ return torch.bfloat16 if dtype is torch.float32 else None
+
+
+def _component_files(
+ repo_id: str, subfolder: str, hf_token: Optional[str], local_files_only: bool
+) -> list[Path]:
+ """Safetensors files from_pretrained reads for ``subfolder``; empty when unresolvable."""
+ from .diffusion_small_host import _weight_files
+
+ root = Path(str(repo_id)).expanduser()
+ if root.is_dir():
+ sub = root / subfolder
+ return _weight_files(sub) if sub.is_dir() else []
+ from huggingface_hub import hf_hub_download
+
+ kwargs: dict[str, Any] = {
+ "token": hf_token or None,
+ "local_files_only": local_files_only,
+ "cache_dir": _live_cache_dir(),
+ }
+ stem = "model" if subfolder.startswith("text_encoder") else "diffusion_pytorch_model"
+ try:
+ index = Path(
+ hf_hub_download(repo_id, f"{subfolder}/{stem}.safetensors.index.json", **kwargs)
+ )
+ shards = sorted(
+ set(json.loads(index.read_text(encoding = "utf-8")).get("weight_map", {}).values())
+ )
+ return [Path(hf_hub_download(repo_id, f"{subfolder}/{s}", **kwargs)) for s in shards]
+ except Exception: # noqa: BLE001 - unsharded, or unreachable: try the single file
+ pass
+ try:
+ return [Path(hf_hub_download(repo_id, f"{subfolder}/{stem}.safetensors", **kwargs))]
+ except Exception: # noqa: BLE001 - no claim: the caller keeps the widened load
+ return []
+
+
+def _stored_as(
+ repo_id: str,
+ subfolder: str,
+ storage_dtype: Any,
+ hf_token: Optional[str],
+ local_files_only: bool,
+) -> bool:
+ """True when every float tensor of ``subfolder`` is stored as ``storage_dtype`` (lossless load)."""
+ from .diffusion_small_host import _safetensors_bytes_by_dtype
+
+ want = str(storage_dtype).replace("torch.", "")
+ files = _component_files(repo_id, subfolder, hf_token, local_files_only)
+ if not files:
+ return False
+ seen = False
+ for f in files:
+ sizes = _safetensors_bytes_by_dtype(f)
+ if not sizes:
+ return False
+ for dtype in sizes:
+ if dtype in ("i64", "i32", "i16", "i8", "u8", "bool"):
+ continue
+ if dtype != want:
+ return False
+ seen = True
+ return seen
+
+
+def _manual_cast_load(
+ loader: Callable[[Any], Any],
+ *,
+ name: str,
+ repo_id: str,
+ dtype: Any,
+ storage_dtype: Any,
+ hf_token: Optional[str],
+ local_files_only: bool,
+) -> Any:
+ """Load at ``storage_dtype`` with per-op upcast to ``dtype`` if stored exactly so; else the widened load."""
+ if storage_dtype is None or not _stored_as(
+ repo_id, name, storage_dtype, hf_token, local_files_only
+ ):
+ if storage_dtype is not None:
+ logger.info(
+ "diffusion.flux3: %s loads widened to %s (its stored weights are not all %s)",
+ name,
+ dtype,
+ storage_dtype,
+ )
+ return loader(dtype)
+ from .diffusion_manual_cast import apply_manual_cast_
+
+ module = loader(storage_dtype)
+ kept = apply_manual_cast_(module, dtype)
+ logger.info(
+ "diffusion.flux3: %s keeps %.2f GiB of %s weights as stored and computes in %s (manual cast)",
+ name,
+ kept / (1 << 30),
+ str(storage_dtype).replace("torch.", ""),
+ str(dtype).replace("torch.", ""),
+ )
+ return module
+
+
def _pin_compute_dtype(transformer: Any, dtype: Any) -> None:
"""Make ``transformer.dtype`` answer the compute dtype for a GGUF denoiser.
@@ -615,9 +727,14 @@ def load_flux3_pipeline(
local_files_only: bool = False,
check_cancelled: Optional[Callable[[], None]] = None,
family_name: Optional[str] = None,
+ storage_dtype: Any = None,
+ transformer_storage: bool = True,
) -> Any:
"""A ready ``Flux3KleinPipeline`` for ``repo_id`` (on CPU; the caller places it).
+ ``storage_dtype`` (``flux3_storage_dtype``): the text encoder, and the transformer unless ``transformer_storage``
+ is False (a transformer quant will rebuild its Linears), load at that dtype and upcast per op to ``dtype``.
+
``family_name`` (the family the load resolved) must agree with the block graph the checkpoint runs: a base
pipeline loaded as the distilled family would run CFG while Studio suppresses its negative prompt, and the
reverse. The pre-eviction preflight refuses that from the manifest; this is the backstop.
@@ -643,14 +760,30 @@ def load_flux3_pipeline(
tokenizer = AutoProcessor.from_pretrained(repo_id, subfolder = "text_encoder", **common)
check_cancelled()
if text_encoder is None:
- text_encoder = Qwen3VLForConditionalGeneration.from_pretrained(
- repo_id, subfolder = "text_encoder", dtype = dtype, **common
+ text_encoder = _manual_cast_load(
+ lambda load_dtype: Qwen3VLForConditionalGeneration.from_pretrained(
+ repo_id, subfolder = "text_encoder", dtype = load_dtype, **common
+ ),
+ name = "text_encoder",
+ repo_id = repo_id,
+ dtype = dtype,
+ storage_dtype = storage_dtype,
+ hf_token = token,
+ local_files_only = local_files_only,
)
text_encoder.eval()
check_cancelled()
if transformer is None:
- transformer = diffusers.Flux3Transformer2DModel.from_pretrained(
- repo_id, subfolder = "transformer", torch_dtype = dtype, **common
+ transformer = _manual_cast_load(
+ lambda load_dtype: diffusers.Flux3Transformer2DModel.from_pretrained(
+ repo_id, subfolder = "transformer", torch_dtype = load_dtype, **common
+ ),
+ name = "transformer",
+ repo_id = repo_id,
+ dtype = dtype,
+ storage_dtype = storage_dtype if transformer_storage else None,
+ hf_token = token,
+ local_files_only = local_files_only,
)
check_cancelled()
vae = diffusers.AutoencoderKLFlux2.from_pretrained(
diff --git a/studio/backend/core/inference/diffusion_manual_cast.py b/studio/backend/core/inference/diffusion_manual_cast.py
new file mode 100644
index 00000000000..a803d5ba937
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_manual_cast.py
@@ -0,0 +1,117 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Linear / Embedding weights stay at stored half dtype and upcast exactly per call. UNSLOTH_STUDIO_MANUAL_CAST=0
+disables."""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Optional
+
+MANUAL_CAST_ENV = "UNSLOTH_STUDIO_MANUAL_CAST"
+MANUAL_CAST_ATTR = "_unsloth_manual_cast"
+
+_CLASSES: dict[str, Any] = {}
+
+
+def manual_cast_disabled() -> bool:
+ return (os.environ.get(MANUAL_CAST_ENV) or "").strip().lower() in ("0", "off", "false", "no")
+
+
+def _classes() -> tuple[Any, Any]:
+ if not _CLASSES:
+ import torch
+ import torch.nn.functional as F
+
+ class ManualCastLinear(torch.nn.Linear):
+ def forward(self, x):
+ dtype = self._unsloth_compute_dtype
+ weight = self.weight if self.weight.dtype == dtype else self.weight.to(dtype)
+ bias = self.bias
+ if bias is not None and bias.dtype != dtype:
+ bias = bias.to(dtype)
+ return F.linear(x if x.dtype == dtype else x.to(dtype), weight, bias)
+
+ class ManualCastEmbedding(torch.nn.Embedding):
+ def forward(self, ids):
+ out = F.embedding(
+ ids,
+ self.weight,
+ self.padding_idx,
+ self.max_norm,
+ self.norm_type,
+ self.scale_grad_by_freq,
+ self.sparse,
+ )
+ dtype = self._unsloth_compute_dtype
+ return out if out.dtype == dtype else out.to(dtype)
+
+ _CLASSES["linear"] = ManualCastLinear
+ _CLASSES["embedding"] = ManualCastEmbedding
+ return _CLASSES["linear"], _CLASSES["embedding"]
+
+
+def _keep_fp32_patterns(module: Any) -> tuple[str, ...]:
+ pats = getattr(module, "_keep_in_fp32_modules", None) or ()
+ strict = getattr(module, "_keep_in_fp32_modules_strict", None) or ()
+ return tuple(str(p) for p in (*pats, *strict))
+
+
+def _first_float_owner(module: Any, keep: tuple[str, ...]) -> Optional[str]:
+ """Owner of the param ``module.dtype`` reads; converted whole so pipelines cast latents to the compute dtype."""
+ for name, param in module.named_parameters():
+ if keep and any(k in name for k in keep):
+ continue
+ if param.is_floating_point():
+ return name.rpartition(".")[0]
+ return None
+
+
+def apply_manual_cast_(module: Any, compute_dtype: Any) -> int:
+ """Swap plain half Linear / Embedding for manual-cast twins, convert other half tensors; returns bytes kept.
+ Subclasses (quantised, GGUF, LoRA) are left alone: they do their own cast."""
+ import torch
+
+ linear_cls, embedding_cls = _classes()
+ half = (torch.bfloat16, torch.float16)
+ keep = _keep_fp32_patterns(module)
+ first_owner = _first_float_owner(module, keep)
+ kept_bytes = 0
+ with torch.no_grad():
+ for mname, sub in module.named_modules():
+ swap = None
+ if type(sub) is torch.nn.Linear:
+ swap = linear_cls
+ elif type(sub) is torch.nn.Embedding:
+ swap = embedding_cls
+ weight = getattr(sub, "weight", None)
+ if (
+ swap is not None
+ and mname != first_owner
+ and isinstance(weight, torch.Tensor)
+ and weight.dtype in half
+ and weight.device.type != "meta"
+ ):
+ sub.__class__ = swap
+ sub._unsloth_compute_dtype = compute_dtype
+ kept_bytes += weight.numel() * weight.element_size()
+ bias = getattr(sub, "bias", None)
+ if isinstance(bias, torch.Tensor):
+ kept_bytes += bias.numel() * bias.element_size()
+ continue
+ for _pname, param in list(sub.named_parameters(recurse = False)):
+ if param.dtype in half:
+ param.data = param.data.to(compute_dtype)
+ for bname, buf in list(sub.named_buffers(recurse = False)):
+ if buf is not None and buf.dtype in half:
+ sub._buffers[bname] = buf.to(compute_dtype)
+ setattr(
+ module, MANUAL_CAST_ATTR, {"compute_dtype": str(compute_dtype), "kept_bytes": kept_bytes}
+ )
+ return kept_bytes
+
+
+def manual_cast_info(module: Any) -> Optional[dict]:
+ info = getattr(module, MANUAL_CAST_ATTR, None)
+ return info if isinstance(info, dict) else None
diff --git a/studio/backend/core/inference/diffusion_speed.py b/studio/backend/core/inference/diffusion_speed.py
index fbc15e11417..f56c8964867 100644
--- a/studio/backend/core/inference/diffusion_speed.py
+++ b/studio/backend/core/inference/diffusion_speed.py
@@ -2094,7 +2094,13 @@ def _fuse_qkv(pipe: Any, logger: Any) -> bool:
logger.info("diffusion.speed: fuse_qkv skipped (quantised projections stay unfused)")
return False
from . import diffusion_lora_quant_merge as lora_quant_merge
+ from .diffusion_manual_cast import manual_cast_info
+ # A fused to_qkv is a plain Linear at the stored dtype, which a manual-cast (float32) forward cannot feed.
+ if any(manual_cast_info(m) for m in _denoiser_dits(pipe)):
+ if logger is not None:
+ logger.info("diffusion.speed: fuse_qkv skipped (manual-cast weights stay unfused)")
+ return False
# A fused to_qkv copy would miss in-place re-merges of merged adapter weights.
if lora_quant_merge.merged(pipe):
if logger is not None:
diff --git a/studio/backend/tests/test_diffusion_flux3_manual_cast.py b/studio/backend/tests/test_diffusion_flux3_manual_cast.py
new file mode 100644
index 00000000000..edc487d86a3
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_flux3_manual_cast.py
@@ -0,0 +1,294 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""FLUX.3 on fp16-only cards: bf16 weights kept as stored, float32 compute per op (diffusion_manual_cast)."""
+
+from __future__ import annotations
+
+import ast
+import inspect
+import json
+import struct
+from types import SimpleNamespace
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_flux3 as f3 # noqa: E402
+from core.inference import diffusion_manual_cast as mc # noqa: E402
+
+
+def _bf16_rounded_(model):
+ """Round every float tensor through bf16 so the float32 model equals the bf16-stored one exactly."""
+ with torch.no_grad():
+ for p in model.parameters():
+ p.data = p.data.to(torch.bfloat16).to(torch.float32)
+ return model
+
+
+def _stored_bf16(model, keep = ()):
+ """What from_pretrained(torch_dtype=bf16) yields: bf16 everywhere except the class's fp32-kept modules."""
+ with torch.no_grad():
+ for name, p in model.named_parameters():
+ if not any(k in name for k in keep):
+ p.data = p.data.to(torch.bfloat16)
+ return model
+
+
+def _tiny_flux3():
+ fmod = pytest.importorskip("diffusers.models.transformers.transformer_flux3")
+ torch.manual_seed(0)
+ model = fmod.Flux3Transformer2DModel(
+ in_channels = 16,
+ cond_in_channels = 16,
+ num_layers = 2,
+ num_single_layers = 2,
+ attention_head_dim = 32,
+ num_attention_heads = 4,
+ joint_attention_dim = 32,
+ timestep_channels = 32,
+ axes_dims_rope = (8, 8, 8, 8),
+ ).eval()
+ g = torch.Generator().manual_seed(1)
+ img, txt, ref = 64, 9, 32
+ kw = {
+ "hidden_states": torch.randn(2, img, 16, generator = g),
+ "encoder_hidden_states": torch.randn(2, txt, 32, generator = g),
+ "timestep": torch.tensor([0.7, 0.7]),
+ "img_ids": torch.randint(0, 16, (img, 4), generator = g).float(),
+ "txt_ids": torch.randint(0, 16, (txt, 4), generator = g).float(),
+ "image_cond_hidden_states": torch.randn(2, ref, 16, generator = g),
+ "image_cond_ids": torch.randint(0, 16, (ref, 4), generator = g).float(),
+ "return_dict": False,
+ }
+ if "target_timestep" in inspect.signature(model.forward).parameters and model.config.get(
+ "flow_map", False
+ ):
+ kw["target_timestep"] = torch.tensor([0.5, 0.5])
+ return model, kw
+
+
+def test_flux3_manual_cast_is_bit_identical_to_the_float32_load():
+ import copy
+
+ model, kw = _tiny_flux3()
+ ref_model = _bf16_rounded_(model)
+ cast_model = _stored_bf16(copy.deepcopy(ref_model), keep = ref_model._keep_in_fp32_modules or ())
+ kept = mc.apply_manual_cast_(cast_model, torch.float32)
+ assert kept > 0
+ # module.dtype drives the latents' dtype in the FLUX.3 denoise block
+ assert cast_model.dtype is torch.float32
+ linears = [m for m in cast_model.modules() if isinstance(m, torch.nn.Linear)]
+ stored = [m for m in linears if m.weight.dtype is torch.bfloat16]
+ assert stored and len(stored) >= len(linears) - 1
+ assert all(type(m).__name__ == "ManualCastLinear" for m in stored)
+ for m in cast_model.modules():
+ if type(m).__name__ == "ManualCastLinear":
+ continue
+ for p in m.parameters(recurse = False):
+ assert p.dtype is torch.float32
+ with torch.no_grad():
+ want = ref_model(**kw)[0]
+ got = cast_model(**kw)[0]
+ assert got.dtype is torch.float32
+ assert torch.equal(got, want)
+ info = mc.manual_cast_info(cast_model)
+ assert info["kept_bytes"] == kept and info["compute_dtype"] == "torch.float32"
+
+
+def test_qwen3vl_encoder_manual_cast_matches_float32():
+ import copy
+
+ transformers = pytest.importorskip("transformers")
+ Qwen3VLConfig = getattr(transformers, "Qwen3VLConfig", None)
+ if Qwen3VLConfig is None:
+ pytest.skip("transformers without Qwen3-VL")
+ cfg = Qwen3VLConfig(
+ text_config = dict(
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 8,
+ vocab_size = 128,
+ rope_scaling = {"rope_type": "default", "mrope_section": [1, 1, 2]},
+ ),
+ vision_config = dict(
+ depth = 1,
+ hidden_size = 16,
+ intermediate_size = 32,
+ num_heads = 2,
+ out_hidden_size = 32,
+ deepstack_visual_indexes = [0],
+ ),
+ )
+ torch.manual_seed(0)
+ ref = _bf16_rounded_(transformers.Qwen3VLForConditionalGeneration(cfg).eval())
+ cast = _stored_bf16(copy.deepcopy(ref))
+ assert mc.apply_manual_cast_(cast, torch.float32) > 0
+ assert cast.dtype is torch.float32
+ assert cast.get_input_embeddings().weight.dtype is torch.bfloat16
+ ids = torch.randint(0, 128, (1, 7), generator = torch.Generator().manual_seed(2))
+ with torch.no_grad():
+ want = ref(input_ids = ids, output_hidden_states = True).hidden_states
+ got = cast(input_ids = ids, output_hidden_states = True).hidden_states
+ assert len(got) == len(want)
+ for a, b in zip(got, want):
+ assert a.dtype is torch.float32
+ assert torch.equal(a, b)
+
+
+def test_manual_cast_linear_traces_without_graph_breaks():
+ lin = torch.nn.Linear(8, 4).to(torch.bfloat16)
+ holder = torch.nn.Sequential(torch.nn.LayerNorm(8), lin)
+ mc.apply_manual_cast_(holder, torch.float32)
+ assert type(holder[1]).__name__ == "ManualCastLinear"
+ x = torch.randn(3, 8)
+ compiled = torch.compile(holder, backend = "eager", fullgraph = True)
+ with torch.no_grad():
+ assert torch.equal(compiled(x), holder(x))
+
+
+def test_manual_cast_leaves_subclassed_linears_alone():
+ class Quantised(torch.nn.Linear):
+ pass
+
+ m = torch.nn.Sequential(torch.nn.Linear(4, 4), Quantised(4, 4), torch.nn.Linear(4, 4)).to(
+ torch.bfloat16
+ )
+ mc.apply_manual_cast_(m, torch.float32)
+ assert type(m[0]) is torch.nn.Linear and m[0].weight.dtype is torch.float32
+ assert type(m[1]) is Quantised and m[1].weight.dtype is torch.float32
+ assert type(m[2]).__name__ == "ManualCastLinear" and m[2].weight.dtype is torch.bfloat16
+
+
+def test_storage_dtype_only_for_cuda_float32_loads(monkeypatch):
+ cuda = SimpleNamespace(device = "cuda")
+ monkeypatch.delenv(mc.MANUAL_CAST_ENV, raising = False)
+ assert f3.flux3_storage_dtype(cuda, torch.float32) is torch.bfloat16
+ for dtype in (torch.float16, torch.bfloat16):
+ assert f3.flux3_storage_dtype(cuda, dtype) is None
+ for device in ("cpu", "mps", "xpu", None):
+ assert f3.flux3_storage_dtype(SimpleNamespace(device = device), torch.float32) is None
+ assert f3.flux3_storage_dtype(None, torch.float32) is None
+ for off in ("0", "off", "false", "no"):
+ monkeypatch.setenv(mc.MANUAL_CAST_ENV, off)
+ assert f3.flux3_storage_dtype(cuda, torch.float32) is None
+
+
+def _write_safetensors(path, dtypes):
+ header = {}
+ offset = 0
+ for i, dt in enumerate(dtypes):
+ header[f"t{i}.weight"] = {"dtype": dt, "shape": [2], "data_offsets": [offset, offset + 4]}
+ offset += 4
+ raw = json.dumps(header).encode()
+ path.write_bytes(struct.pack("<Q", len(raw)) + raw + b"\0" * offset)
+
+
+ "dtypes, stored",
+ [
+ (["BF16", "BF16"], True),
+ (["BF16", "I64"], True),
+ (["BF16", "F32"], False),
+ (["F16", "F16"], False),
+ (["F32"], False),
+ ],
+)
+def test_manual_cast_load_reads_the_stored_dtype(tmp_path, dtypes, stored):
+ sub = tmp_path / "transformer"
+ sub.mkdir()
+ _write_safetensors(sub / "diffusion_pytorch_model.safetensors", dtypes)
+ seen = []
+
+ def loader(load_dtype):
+ seen.append(load_dtype)
+ return torch.nn.Sequential(torch.nn.LayerNorm(4), torch.nn.Linear(4, 4)).to(load_dtype)
+
+ module = f3._manual_cast_load(
+ loader,
+ name = "transformer",
+ repo_id = str(tmp_path),
+ dtype = torch.float32,
+ storage_dtype = torch.bfloat16,
+ hf_token = None,
+ local_files_only = True,
+ )
+ if stored:
+ assert seen == [torch.bfloat16]
+ assert type(module[1]).__name__ == "ManualCastLinear"
+ assert module[1].weight.dtype is torch.bfloat16
+ else:
+ # not exactly bf16 on disk: widened load (bf16 would round it)
+ assert seen == [torch.float32]
+ assert mc.manual_cast_info(module) is None
+
+
+def test_manual_cast_load_without_storage_dtype_is_the_widened_load(tmp_path):
+ seen = []
+ module = f3._manual_cast_load(
+ lambda d: seen.append(d) or torch.nn.Linear(2, 2).to(d),
+ name = "text_encoder",
+ repo_id = str(tmp_path),
+ dtype = torch.float16,
+ storage_dtype = None,
+ hf_token = None,
+ local_files_only = True,
+ )
+ assert seen == [torch.float16] and type(module) is torch.nn.Linear
+
+
+def test_missing_component_keeps_the_widened_load(tmp_path):
+ assert not f3._stored_as(str(tmp_path), "transformer", torch.bfloat16, None, True)
+
+
+def test_every_flux3_load_site_passes_the_storage_dtype():
+ from core.inference import diffusion as d
+
+ tree = ast.parse(inspect.getsource(d))
+ calls = [
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "load_flux3_pipeline"
+ ]
+ assert len(calls) == 3
+ for call in calls:
+ kw = {k.arg: ast.unparse(k.value) for k in call.keywords}
+ assert kw.get("storage_dtype", "").startswith("flux3_storage_dtype(target"), kw
+ # a quantised transformer stays widened: its quantiser rebuilds the Linears
+ gated = [c for c in calls if any(k.arg == "transformer_storage" for k in c.keywords)]
+ assert len(gated) == 1
+ text = ast.unparse(next(k.value for k in gated[0].keywords if k.arg == "transformer_storage"))
+ for name in (
+ "transformer_quant_pinned is None",
+ "native_scheme is None",
+ "native_offload_scheme is None",
+ ):
+ assert name in text
+
+
+def test_load_flux3_pipeline_routes_both_components_through_the_cast():
+ src = inspect.getsource(f3.load_flux3_pipeline)
+ assert src.count("_manual_cast_load(") == 2
+ assert "storage_dtype if transformer_storage else None" in src
+
+
+def test_fuse_qkv_skips_a_manual_cast_denoiser():
+ from core.inference import diffusion_speed as ds
+
+ transformer = torch.nn.Sequential(torch.nn.Linear(4, 4).to(torch.bfloat16))
+ mc.apply_manual_cast_(transformer, torch.float32)
+ calls = []
+
+ class _Pipe:
+ def __init__(self):
+ self.transformer = transformer
+
+ def fuse_qkv_projections(self):
+ calls.append(1)
+
+ assert ds._fuse_qkv(_Pipe(), None) is False
+ assert calls == []