The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Remove code to support RMSNorm on old pytorch. (#12499)
comfyanonymous4454fab7f003
diff --git a/comfy/ops.py b/comfy/ops.pyindex 688937e43..026062f56 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -21,7 +21,6 @@ import loggingimport comfy.model_managementfrom comfy.cli_args import args, PerformanceFeature, enables_dynamic_vramimport comfy.float-import comfy.rmsnormimport jsonimport comfy.memory_managementimport comfy.pinned_memory@@ -463,7 +462,7 @@ class disable_weight_init:else:return super().forward(*args, **kwargs)- class RMSNorm(comfy.rmsnorm.RMSNorm, CastWeightBiasOp):+ class RMSNorm(torch.nn.RMSNorm, CastWeightBiasOp):def reset_parameters(self):self.bias = Nonereturn None@@ -475,8 +474,7 @@ class disable_weight_init:weight = Nonebias = Noneoffload_stream = None- x = comfy.rmsnorm.rms_norm(input, weight, self.eps) # TODO: switch to commented out line when old torch is deprecated- # x = torch.nn.functional.rms_norm(input, self.normalized_shape, weight, self.eps)+ x = torch.nn.functional.rms_norm(input, self.normalized_shape, weight, self.eps)uncast_bias_weight(self, weight, bias, offload_stream)return xdiff --git a/comfy/rmsnorm.py b/comfy/rmsnorm.pyindex 555542a46..ab7cf14fa 100644--- a/comfy/rmsnorm.py+++ b/comfy/rmsnorm.py@@ -1,57 +1,10 @@import torchimport comfy.model_management-import numbers-import logging--RMSNorm = None--try:- rms_norm_torch = torch.nn.functional.rms_norm- RMSNorm = torch.nn.RMSNorm-except:- rms_norm_torch = None- logging.warning("Please update pytorch to use native RMSNorm")+RMSNorm = torch.nn.RMSNormdef rms_norm(x, weight=None, eps=1e-6):- if rms_norm_torch is not None and not (torch.jit.is_tracing() or torch.jit.is_scripting()):- if weight is None:- return rms_norm_torch(x, (x.shape[-1],), eps=eps)- else:- return rms_norm_torch(x, weight.shape, weight=comfy.model_management.cast_to(weight, dtype=x.dtype, device=x.device), eps=eps)+ if weight is None:+ return torch.nn.functional.rms_norm(x, (x.shape[-1],), eps=eps)else:- r = x * torch.rsqrt(torch.mean(x**2, dim=-1, keepdim=True) + eps)- if weight is None:- return r- else:- return r * comfy.model_management.cast_to(weight, dtype=x.dtype, device=x.device)---if RMSNorm is None:- class RMSNorm(torch.nn.Module):- def __init__(- self,- normalized_shape,- eps=1e-6,- elementwise_affine=True,- device=None,- dtype=None,- ):- factory_kwargs = {"device": device, "dtype": dtype}- super().__init__()- if isinstance(normalized_shape, numbers.Integral):- # mypy error: incompatible types in assignment- normalized_shape = (normalized_shape,) # type: ignore[assignment]- self.normalized_shape = tuple(normalized_shape) # type: ignore[arg-type]- self.eps = eps- self.elementwise_affine = elementwise_affine- if self.elementwise_affine:- self.weight = torch.nn.Parameter(- torch.empty(self.normalized_shape, **factory_kwargs)- )- else:- self.register_parameter("weight", None)- self.bias = None-- def forward(self, x):- return rms_norm(x, self.weight, self.eps)+ return torch.nn.functional.rms_norm(x, weight.shape, weight=comfy.model_management.cast_to(weight, dtype=x.dtype, device=x.device), eps=eps)