Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

speed up and reduce VRAM of QWEN VAE and WAN (less so) (#12036)

rattus4e6a1b66a93e
diff --git a/comfy/ldm/wan/vae.py b/comfy/ldm/wan/vae.py
index 08315f1a8..40e767213 100644
--- a/comfy/ldm/wan/vae.py
+++ b/comfy/ldm/wan/vae.py
@@ -5,7 +5,7 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange
-from comfy.ldm.modules.diffusionmodules.model import vae_attention
+from comfy.ldm.modules.diffusionmodules.model import vae_attention, torch_cat_if_needed
import comfy.ops
ops = comfy.ops.disable_weight_init
@@ -20,22 +20,29 @@ class CausalConv3d(ops.Conv3d):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- self._padding = (self.padding[2], self.padding[2], self.padding[1],
- self.padding[1], 2 * self.padding[0], 0)
- self.padding = (0, 0, 0)
+ self._padding = 2 * self.padding[0]
+ self.padding = (0, self.padding[1], self.padding[2])
def forward(self, x, cache_x=None, cache_list=None, cache_idx=None):
if cache_list is not None:
cache_x = cache_list[cache_idx]
cache_list[cache_idx] = None
- padding = list(self._padding)
- if cache_x is not None and self._padding[4] > 0:
- cache_x = cache_x.to(x.device)
- x = torch.cat([cache_x, x], dim=2)
- padding[4] -= cache_x.shape[2]
+ if cache_x is None and x.shape[2] == 1:
+ #Fast path - the op will pad for use by truncating the weight
+ #and save math on a pile of zeros.
+ return super().forward(x, autopad="causal_zero")
+
+ if self._padding > 0:
+ padding_needed = self._padding
+ if cache_x is not None:
+ cache_x = cache_x.to(x.device)
+ padding_needed = max(0, padding_needed - cache_x.shape[2])
+ padding_shape = list(x.shape)
+ padding_shape[2] = padding_needed
+ padding = torch.zeros(padding_shape, device=x.device, dtype=x.dtype)
+ x = torch_cat_if_needed([padding, cache_x, x], dim=2)
del cache_x
- x = F.pad(x, padding)
return super().forward(x)
diff --git a/comfy/ops.py b/comfy/ops.py
index 415c39e92..e406ba7ed 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -203,7 +203,9 @@ class disable_weight_init:
def reset_parameters(self):
return None
- def _conv_forward(self, input, weight, bias, *args, **kwargs):
+ def _conv_forward(self, input, weight, bias, autopad=None, *args, **kwargs):
+ if autopad == "causal_zero":
+ weight = weight[:, :, -input.shape[2]:, :, :]
if NVIDIA_MEMORY_CONV_BUG_WORKAROUND and weight.dtype in (torch.float16, torch.bfloat16):
out = torch.cudnn_convolution(input, weight, self.padding, self.stride, self.dilation, self.groups, benchmark=False, deterministic=False, allow_tf32=True)
if bias is not None:
@@ -212,15 +214,15 @@ class disable_weight_init:
else:
return super()._conv_forward(input, weight, bias, *args, **kwargs)
- def forward_comfy_cast_weights(self, input):
+ def forward_comfy_cast_weights(self, input, autopad=None):
weight, bias, offload_stream = cast_bias_weight(self, input, offloadable=True)
- x = self._conv_forward(input, weight, bias)
+ x = self._conv_forward(input, weight, bias, autopad=autopad)
uncast_bias_weight(self, weight, bias, offload_stream)
return x
def forward(self, *args, **kwargs):
run_every_op()
- if self.comfy_cast_weights or len(self.weight_function) > 0 or len(self.bias_function) > 0:
+ if self.comfy_cast_weights or len(self.weight_function) > 0 or len(self.bias_function) > 0 or "autopad" in kwargs:
return self.forward_comfy_cast_weights(*args, **kwargs)
else:
return super().forward(*args, **kwargs)