The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Reduce Peak WAN inference VRAM usage (#9898)
rattus128e42682b24ef0
diff --git a/comfy/ldm/flux/math.py b/comfy/ldm/flux/math.pyindex 4d743cda2..fb7cd7586 100644--- a/comfy/ldm/flux/math.py+++ b/comfy/ldm/flux/math.py@@ -35,11 +35,10 @@ def rope(pos: Tensor, dim: int, theta: int) -> Tensor:out = rearrange(out, "b n d (i j) -> b n d i j", i=2, j=2)return out.to(dtype=torch.float32, device=pos.device)+def apply_rope1(x: Tensor, freqs_cis: Tensor):+ x_ = x.to(dtype=freqs_cis.dtype).reshape(*x.shape[:-1], -1, 1, 2)+ x_out = freqs_cis[..., 0] * x_[..., 0] + freqs_cis[..., 1] * x_[..., 1]+ return x_out.reshape(*x.shape).type_as(x)def apply_rope(xq: Tensor, xk: Tensor, freqs_cis: Tensor):- xq_ = xq.to(dtype=freqs_cis.dtype).reshape(*xq.shape[:-1], -1, 1, 2)- xk_ = xk.to(dtype=freqs_cis.dtype).reshape(*xk.shape[:-1], -1, 1, 2)- xq_out = freqs_cis[..., 0] * xq_[..., 0] + freqs_cis[..., 1] * xq_[..., 1]- xk_out = freqs_cis[..., 0] * xk_[..., 0] + freqs_cis[..., 1] * xk_[..., 1]- return xq_out.reshape(*xq.shape).type_as(xq), xk_out.reshape(*xk.shape).type_as(xk)-+ return apply_rope1(xq, freqs_cis), apply_rope1(xk, freqs_cis)diff --git a/comfy/ldm/wan/model.py b/comfy/ldm/wan/model.pyindex 63472ada2..67dcf8f1e 100644--- a/comfy/ldm/wan/model.py+++ b/comfy/ldm/wan/model.py@@ -8,7 +8,7 @@ from einops import rearrangefrom comfy.ldm.modules.attention import optimized_attentionfrom comfy.ldm.flux.layers import EmbedND-from comfy.ldm.flux.math import apply_rope+from comfy.ldm.flux.math import apply_rope1import comfy.ldm.common_ditimport comfy.model_managementimport comfy.patcher_extension@@ -60,20 +60,24 @@ class WanSelfAttention(nn.Module):"""b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim- # query, key, value function- def qkv_fn(x):+ def qkv_fn_q(x):q = self.norm_q(self.q(x)).view(b, s, n, d)+ return apply_rope1(q, freqs)++ def qkv_fn_k(x):k = self.norm_k(self.k(x)).view(b, s, n, d)- v = self.v(x).view(b, s, n * d)- return q, k, v+ return apply_rope1(k, freqs)- q, k, v = qkv_fn(x)- q, k = apply_rope(q, k, freqs)+ #These two are VRAM hogs, so we want to do all of q computation and+ #have pytorch garbage collect the intermediates on the sub function+ #return before we touch k+ q = qkv_fn_q(x)+ k = qkv_fn_k(x)x = optimized_attention(q.view(b, s, n * d),k.view(b, s, n * d),- v,+ self.v(x).view(b, s, n * d),heads=self.num_heads,transformer_options=transformer_options,)