The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Fix MiniMax-H3 VAE rms_rope crash on offloaded qk_norm_scale (#16485)
Jialong(Bruce) Li912fca4f39b8
diff --git a/comfy/ldm/minimax/vae.py b/comfy/ldm/minimax/vae.pyindex cc4c16de7..ffb358463 100644--- a/comfy/ldm/minimax/vae.py+++ b/comfy/ldm/minimax/vae.py@@ -298,8 +298,10 @@ class Attention(nn.Module):# fused per-head RMSNorm + partial split-half rope, in place when autograd is offrms_rope = (comfy.quant_ops.ck.rms_rope_split_half if torch.is_grad_enabled()else comfy.quant_ops.ck.rms_rope_split_half_)+ # the eager fallback does no device alignment, so a CPU-resident (offloaded)+ # buffer must be moved to the input's device before the fused callquery, key = rms_rope(- query, key, rotary_pos_emb, self.qk_norm_scale,+ query, key, rotary_pos_emb, self.qk_norm_scale.to(query.device),epsilon=self.norm_q.eps, rot_dim=rotary_pos_emb.shape[-3] * 2)else:query = comfy.rmsnorm.rms_norm(query, self.norm_q.weight, self.norm_q.eps)diff --git a/tests-unit/comfy_test/test_minimax_h3_vae.py b/tests-unit/comfy_test/test_minimax_h3_vae.pynew file mode 100644index 000000000..77ed67ede--- /dev/null+++ b/tests-unit/comfy_test/test_minimax_h3_vae.py@@ -0,0 +1,65 @@+import torch++from comfy.cli_args import args++if not torch.cuda.is_available():+ args.cpu = True++import comfy.quant_ops+from comfy.ldm.minimax.vae import Attention+++class _OffloadedScale:+ """Stands in for a buffer that comfy's dynamic-VRAM offload left on another+ device: it only tracks its device label and can be moved with .to(), like+ the real qk_norm_scale tensor the bug report is about."""++ def __init__(self, device):+ self.device = torch.device(device)++ def to(self, device):+ return _OffloadedScale(device)+++class _FakeCK:+ def rms_rope_split_half(self, query, key, freqs, scale, epsilon, rot_dim):+ return self._check(query, key, scale)++ def rms_rope_split_half_(self, query, key, freqs, scale, epsilon, rot_dim):+ return self._check(query, key, scale)++ @staticmethod+ def _check(query, key, scale):+ if scale.device != query.device:+ # mirrors comfy_kitchen's eager backend, which does no device+ # alignment and refuses to run the fused kernel on mixed devices+ raise RuntimeError(+ f"Expected all tensors to be on the same device, but got weight is on "+ f"{scale.device}, different from other tensors on {query.device}"+ )+ return query, key+++def test_attention_moves_offloaded_qk_norm_scale_to_input_device(monkeypatch):+ monkeypatch.setattr(comfy.quant_ops, "ck", _FakeCK(), raising=False)++ heads, dim_head = 2, 4+ attn = Attention(heads=heads, dim_head=dim_head)+ # simulate a partially-offloaded VAE: qk_norm_scale sits on a different+ # device than the tensors flowing through the fused rope kernel+ attn._buffers["qk_norm_scale"] = _OffloadedScale("meta")++ batch_size, seq_len = 1, 3+ x = torch.randn(batch_size, seq_len, heads * dim_head)+ rotary_pos_emb = torch.randn(batch_size, 1, seq_len, dim_head // 2, 2, 2)++ class _Norm:+ weight = None+ eps = 1e-5++ with torch.no_grad():+ out = attn.forward(+ x, rotary_pos_emb, pre_norm=_Norm(), residual=None, residual_scale=None+ )++ assert out.device == x.device