Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Fix MiniMax-H3 VAE rms_rope crash on offloaded qk_norm_scale (#16485)

Jialong(Bruce) Li912fca4f39b8
diff --git a/comfy/ldm/minimax/vae.py b/comfy/ldm/minimax/vae.py
index cc4c16de7..ffb358463 100644
--- a/comfy/ldm/minimax/vae.py
+++ b/comfy/ldm/minimax/vae.py
@@ -298,8 +298,10 @@ class Attention(nn.Module):
# fused per-head RMSNorm + partial split-half rope, in place when autograd is off
rms_rope = (comfy.quant_ops.ck.rms_rope_split_half if torch.is_grad_enabled()
else comfy.quant_ops.ck.rms_rope_split_half_)
+ # the eager fallback does no device alignment, so a CPU-resident (offloaded)
+ # buffer must be moved to the input's device before the fused call
query, key = rms_rope(
- query, key, rotary_pos_emb, self.qk_norm_scale,
+ query, key, rotary_pos_emb, self.qk_norm_scale.to(query.device),
epsilon=self.norm_q.eps, rot_dim=rotary_pos_emb.shape[-3] * 2)
else:
query = comfy.rmsnorm.rms_norm(query, self.norm_q.weight, self.norm_q.eps)
diff --git a/tests-unit/comfy_test/test_minimax_h3_vae.py b/tests-unit/comfy_test/test_minimax_h3_vae.py
new file mode 100644
index 000000000..77ed67ede
--- /dev/null
+++ b/tests-unit/comfy_test/test_minimax_h3_vae.py
@@ -0,0 +1,65 @@
+import torch
+
+from comfy.cli_args import args
+
+if not torch.cuda.is_available():
+ args.cpu = True
+
+import comfy.quant_ops
+from comfy.ldm.minimax.vae import Attention
+
+
+class _OffloadedScale:
+ """Stands in for a buffer that comfy's dynamic-VRAM offload left on another
+ device: it only tracks its device label and can be moved with .to(), like
+ the real qk_norm_scale tensor the bug report is about."""
+
+ def __init__(self, device):
+ self.device = torch.device(device)
+
+ def to(self, device):
+ return _OffloadedScale(device)
+
+
+class _FakeCK:
+ def rms_rope_split_half(self, query, key, freqs, scale, epsilon, rot_dim):
+ return self._check(query, key, scale)
+
+ def rms_rope_split_half_(self, query, key, freqs, scale, epsilon, rot_dim):
+ return self._check(query, key, scale)
+
+ @staticmethod
+ def _check(query, key, scale):
+ if scale.device != query.device:
+ # mirrors comfy_kitchen's eager backend, which does no device
+ # alignment and refuses to run the fused kernel on mixed devices
+ raise RuntimeError(
+ f"Expected all tensors to be on the same device, but got weight is on "
+ f"{scale.device}, different from other tensors on {query.device}"
+ )
+ return query, key
+
+
+def test_attention_moves_offloaded_qk_norm_scale_to_input_device(monkeypatch):
+ monkeypatch.setattr(comfy.quant_ops, "ck", _FakeCK(), raising=False)
+
+ heads, dim_head = 2, 4
+ attn = Attention(heads=heads, dim_head=dim_head)
+ # simulate a partially-offloaded VAE: qk_norm_scale sits on a different
+ # device than the tensors flowing through the fused rope kernel
+ attn._buffers["qk_norm_scale"] = _OffloadedScale("meta")
+
+ batch_size, seq_len = 1, 3
+ x = torch.randn(batch_size, seq_len, heads * dim_head)
+ rotary_pos_emb = torch.randn(batch_size, 1, seq_len, dim_head // 2, 2, 2)
+
+ class _Norm:
+ weight = None
+ eps = 1e-5
+
+ with torch.no_grad():
+ out = attn.forward(
+ x, rotary_pos_emb, pre_norm=_Norm(), residual=None, residual_scale=None
+ )
+
+ assert out.device == x.device