Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: harden VAE encode dtype for Qwen-Image-2.1 GGUF Edit/Reference (#16375) (#16376)

Indrajit swaindf59af398a02
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index bea9f497cf1..dbaa3b5974b 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -12844,6 +12844,48 @@ class DiffusionBackend:
self._cn_pipes[key] = pipe
return pipe
+ @staticmethod
+ def _pipeline_compute_dtype(pipe: Any) -> Any:
+ """Best-effort floating dtype for a GGUF pipe whose denoiser params are packed uint8 only."""
+ dtype = getattr(pipe, "dtype", None)
+ if dtype is not None and getattr(dtype, "is_floating_point", False):
+ return dtype
+ for name in ("text_encoder", "text_encoder_2", "transformer", "unet"):
+ mod = getattr(pipe, name, None)
+ if mod is None:
+ continue
+ try:
+ found = next(
+ (p.dtype for p in mod.parameters() if p.dtype.is_floating_point),
+ None,
+ )
+ except (AttributeError, RuntimeError, StopIteration, TypeError):
+ found = None
+ if found is not None:
+ return found
+ return None
+
+ @staticmethod
+ def _vae_running_dtype(vae: Any) -> Any:
+ """Dtype of the layer that first consumes the encode input (``encoder.conv_in``), else the first floating
+ encoder / VAE weight; None when it cannot be read, so the input is left as is (#16375)."""
+ encoder = getattr(vae, "encoder", None)
+ first = getattr(getattr(encoder, "conv_in", None), "weight", None)
+ dtype = getattr(first, "dtype", None)
+ if dtype is not None and getattr(dtype, "is_floating_point", False):
+ return dtype
+ for mod in (encoder, vae):
+ if mod is None:
+ continue
+ try:
+ found = next((p.dtype for p in mod.parameters() if p.dtype.is_floating_point), None)
+ except (AttributeError, RuntimeError, StopIteration, TypeError):
+ found = None
+ if found is not None:
+ return found
+ dtype = getattr(vae, "dtype", None)
+ return dtype if dtype is not None and getattr(dtype, "is_floating_point", False) else None
+
@staticmethod
def _align_vae_dtype(pipe: Any, denoiser_attr: str = "transformer") -> None:
"""A prior txt2img decode can leave the VAE upcast to fp32; re-align it to the denoiser's dtype."""
@@ -12857,6 +12899,9 @@ class DiffusionBackend:
(p.dtype for p in denoiser.parameters() if p.dtype.is_floating_point),
None,
)
+ # Packed-uint8 GGUF denoisers carry no float dtype (#16375).
+ if target_dtype is None:
+ target_dtype = DiffusionBackend._pipeline_compute_dtype(pipe)
if target_dtype is None:
return
if next(vae.parameters()).dtype != target_dtype:
@@ -12866,8 +12911,8 @@ class DiffusionBackend:
@staticmethod
def _make_vae_encode_dtype_safe(pipe: Any) -> None:
- """Casts encode inputs to the VAE's dtype; img2img uses the text encoder's dtype, which can
- mismatch."""
+ """Casts encode inputs to the VAE's dtype; img2img / Qwen Edit use the TE dtype, which can
+ mismatch (#16375)."""
vae = getattr(pipe, "vae", None)
if vae is None or getattr(vae, "_unsloth_dtype_safe_encode", False):
return
@@ -12878,16 +12923,18 @@ class DiffusionBackend:
@functools.wraps(original)
def _encode(x: Any, *args: Any, **kwargs: Any) -> Any:
- try:
- if isinstance(x, torch.Tensor):
- target = next(
- (p.dtype for p in vae.parameters() if p.dtype.is_floating_point),
- None,
- )
+ if isinstance(x, torch.Tensor):
+ target = None
+ try:
+ target = DiffusionBackend._vae_running_dtype(vae)
+ except (AttributeError, RuntimeError, StopIteration, TypeError):
+ target = None
+ try:
+ # An unreadable dtype leaves the input alone: guessing fp32 breaks a half encoder.
if target is not None and x.dtype != target:
x = x.to(dtype = target)
- except (AttributeError, RuntimeError, StopIteration, TypeError):
- pass
+ except (AttributeError, RuntimeError, TypeError):
+ pass
return original(x, *args, **kwargs)
vae.encode = _encode
diff --git a/studio/backend/tests/test_diffusion_img2img_dtype.py b/studio/backend/tests/test_diffusion_img2img_dtype.py
index f14587fca87..c351b615113 100644
--- a/studio/backend/tests/test_diffusion_img2img_dtype.py
+++ b/studio/backend/tests/test_diffusion_img2img_dtype.py
@@ -105,3 +105,65 @@ def test_missing_vae_and_broken_probe_are_no_ops():
DiffusionBackend._make_vae_encode_dtype_safe(types.SimpleNamespace(vae = vae))
vae.encode(torch.zeros(1, 3, 8, 8, dtype = torch.float32))
assert vae.seen_dtype == torch.float32
+
+
+def _encoder(conv_dtype, other_dtypes = ()):
+ weight = types.SimpleNamespace(dtype = conv_dtype)
+
+ class _Encoder:
+ conv_in = types.SimpleNamespace(weight = weight)
+
+ def parameters(self):
+ yield weight
+ for dtype in other_dtypes:
+ yield types.SimpleNamespace(dtype = dtype)
+
+ return _Encoder()
+
+
+def test_encode_follows_conv_in_when_the_vae_probe_fails():
+ # #16375: under offload the VAE parameter walk can fail; the layer that eats the image still says fp32.
+ import torch
+
+ class _BrokenVae(_RecordingVae):
+ def parameters(self):
+ raise RuntimeError("meta device")
+
+ vae = _BrokenVae(dtype = torch.float32)
+ vae.encoder = _encoder(torch.float32)
+ DiffusionBackend._make_vae_encode_dtype_safe(types.SimpleNamespace(vae = vae))
+ vae.encode(torch.zeros(1, 3, 8, 8, dtype = torch.float16))
+ assert vae.seen_dtype == torch.float32
+
+
+def test_encode_follows_conv_in_over_an_earlier_mismatched_vae_weight():
+ import torch
+
+ vae = _RecordingVae(dtype = torch.float16) # first VAE weight is half, the encoder runs fp32
+ vae.encoder = _encoder(torch.float32)
+ DiffusionBackend._make_vae_encode_dtype_safe(types.SimpleNamespace(vae = vae))
+ vae.encode(torch.zeros(1, 3, 8, 8, dtype = torch.float16))
+ assert vae.seen_dtype == torch.float32
+
+
+def test_a_half_conv_in_keeps_half_input_beside_fp32_norm_weights():
+ import torch
+
+ vae = _RecordingVae(dtype = torch.float16)
+ vae.encoder = _encoder(torch.float16, other_dtypes = (torch.float32,))
+ DiffusionBackend._make_vae_encode_dtype_safe(types.SimpleNamespace(vae = vae))
+ vae.encode(torch.zeros(1, 3, 8, 8, dtype = torch.float16))
+ assert vae.seen_dtype == torch.float16
+
+
+def test_an_unreadable_dtype_leaves_the_input_alone():
+ import torch
+
+ class _OpaqueVae(_RecordingVae):
+ def parameters(self):
+ raise RuntimeError("meta device")
+
+ vae = _OpaqueVae(dtype = torch.float16)
+ DiffusionBackend._make_vae_encode_dtype_safe(types.SimpleNamespace(vae = vae))
+ vae.encode(torch.zeros(1, 3, 8, 8, dtype = torch.float16))
+ assert vae.seen_dtype == torch.float16
diff --git a/studio/backend/tests/test_diffusion_sdxl.py b/studio/backend/tests/test_diffusion_sdxl.py
index 1ec1e09669e..e55f96d422d 100644
--- a/studio/backend/tests/test_diffusion_sdxl.py
+++ b/studio/backend/tests/test_diffusion_sdxl.py
@@ -125,6 +125,26 @@ def test_align_vae_dtype_skips_gguf_packed_uint8_params():
assert vae2.moved_to is None
+def test_align_vae_dtype_falls_back_to_pipe_dtype_when_denoiser_is_all_packed():
+ # Qwen-Image-2.1 GGUF Edit: an all-uint8 transformer falls back to the pipe dtype (#16375).
+ import torch
+
+ class _AllPacked:
+ def parameters(self):
+ yield types.SimpleNamespace(dtype = torch.uint8)
+
+ vae = _FakeVae(dtype = torch.float32)
+ te = _FakeVae(dtype = torch.float16)
+ pipe = types.SimpleNamespace(
+ transformer = _AllPacked(),
+ text_encoder = te,
+ dtype = torch.float16,
+ vae = vae,
+ )
+ DiffusionBackend._align_vae_dtype(pipe)
+ assert vae.moved_to == torch.float16
+
+
def test_sdxl_lora_supported_on_diffusers():
assert diffusion_lora.supports_lora(
engine = "diffusers", family = "sdxl", model_kind = "pipeline", transformer_quant = None