The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Make Ernie use comfy kitchen rms rope (#15055)
comfyanonymous0cb84e7e6e0b
diff --git a/comfy/ldm/ernie/model.py b/comfy/ldm/ernie/model.pyindex f158ca1d2..88a3775d0 100644--- a/comfy/ldm/ernie/model.py+++ b/comfy/ldm/ernie/model.py@@ -5,6 +5,7 @@ import torch.nn.functional as Ffrom comfy.ldm.modules.attention import optimized_attentionimport comfy.model_management+import comfy.opsimport comfy.quant_opsdef rope(pos: torch.Tensor, dim: int, theta: int) -> torch.Tensor:@@ -111,11 +112,17 @@ class ErnieImageAttention(nn.Module):query = q_flat.view(B, S, self.heads, self.head_dim)key = k_flat.view(B, S, self.heads, self.head_dim)- query = self.norm_q(query)- key = self.norm_k(key)-- if image_rotary_emb is not None:- query, key = comfy.quant_ops.ck.apply_rope_split_half(query, key, image_rotary_emb)+ if image_rotary_emb is not None and not comfy.model_management.in_training:+ q_scale, _, q_offload_stream = comfy.ops.cast_bias_weight(self.norm_q, query, offloadable=True)+ k_scale, _, k_offload_stream = comfy.ops.cast_bias_weight(self.norm_k, key, offloadable=True)+ query, key = comfy.quant_ops.ck.rms_rope_split_half(query, key, image_rotary_emb, q_scale, k_scale, self.norm_q.eps)+ comfy.ops.uncast_bias_weight(self.norm_q, q_scale, None, q_offload_stream)+ comfy.ops.uncast_bias_weight(self.norm_k, k_scale, None, k_offload_stream)+ else:+ query = self.norm_q(query)+ key = self.norm_k(key)+ if image_rotary_emb is not None:+ query, key = comfy.quant_ops.ck.apply_rope_split_half(query, key, image_rotary_emb)q_flat = query.reshape(B, S, -1)k_flat = key.reshape(B, S, -1)