The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Change model code to use function wrappers for Linear calls (#16861)
rattus62c49c4d4fe5
diff --git a/comfy/ldm/lightricks/model.py b/comfy/ldm/lightricks/model.pyindex 0d6f457ee..b011f5d81 100644--- a/comfy/ldm/lightricks/model.py+++ b/comfy/ldm/lightricks/model.py@@ -327,7 +327,7 @@ class FeedForward(nn.Module):# Dropout, so leave it to the stock path whenever it could be active.if comfy.model_management.in_training:return self.net(x)- return self.net[2](self.net[0].proj(x), input_act="gelu_tanh")+ return comfy.ops.linear_input_act(self.net[2], self.net[0].proj(x), "gelu_tanh")def apply_rotary_emb(input_tensor, freqs_cis):rotation_matrix, split_pe = freqs_cisdiff --git a/comfy/ldm/minimax/model.py b/comfy/ldm/minimax/model.pyindex ca5010ffa..440d47ef3 100644--- a/comfy/ldm/minimax/model.py+++ b/comfy/ldm/minimax/model.py@@ -208,7 +208,7 @@ class MLP(nn.Module):self.fc2 = operations.Linear(ffn, hidden, bias=False, dtype=dtype, device=device)def forward(self, x):- return self.fc2(self.fc1(x), input_act="swiglu")+ return comfy.ops.linear_input_act(self.fc2, self.fc1(x), "swiglu")class AdalnProj(nn.Module):diff --git a/comfy/ldm/minimax/vae.py b/comfy/ldm/minimax/vae.pyindex 9768f6700..84962b6a3 100644--- a/comfy/ldm/minimax/vae.py+++ b/comfy/ldm/minimax/vae.py@@ -269,8 +269,8 @@ class FeedForward(nn.Module):def forward(self, x, pre_norm, residual, residual_scale):# norm, gated silu and residual addcmul fold into the INT8 kernels- h = self.w1(x, input_act="rms_norm", act_weight=pre_norm)- return self.w2(h, input_act="swiglu", residual=residual, residual_scale=residual_scale)+ h = comfy.ops.linear_input_act(self.w1, x, "rms_norm", act_weight=pre_norm)+ return comfy.ops.linear_input_act(self.w2, h, "swiglu", residual=residual, residual_scale=residual_scale)class Attention(nn.Module):@@ -289,7 +289,7 @@ class Attention(nn.Module):def forward(self, x, rotary_pos_emb, pre_norm, residual, residual_scale):batch_size, seq_len, _ = x.shape- qkv = self.to_qkv(x, input_act="rms_norm", act_weight=pre_norm)+ qkv = comfy.ops.linear_input_act(self.to_qkv, x, "rms_norm", act_weight=pre_norm)qkv = qkv.view(batch_size, seq_len, -1, 3 * self.dim_head)query, key, value = torch.chunk(qkv, 3, dim=-1)@@ -313,7 +313,7 @@ class Attention(nn.Module):out = out.transpose(1, 2).reshape(batch_size, seq_len, -1)else:out = optimized_attention(query, key, value, self.heads, skip_reshape=True)- return self.to_out(torch.nan_to_num(out), residual=residual, residual_scale=residual_scale)+ return comfy.ops.linear_input_act(self.to_out, torch.nan_to_num(out), None, residual=residual, residual_scale=residual_scale)class TransformerBlock(nn.Module):diff --git a/comfy/ldm/minimax_music/ar.py b/comfy/ldm/minimax_music/ar.pyindex 250822d7a..2a702c527 100644--- a/comfy/ldm/minimax_music/ar.py+++ b/comfy/ldm/minimax_music/ar.py@@ -95,7 +95,7 @@ class RVQMLP(nn.Module):def forward(self, x):if self.merged_mlp:- return self.down_proj(self.gate_up_proj(x), input_act="swiglu")+ return comfy.ops.linear_input_act(self.down_proj, self.gate_up_proj(x), "swiglu")return self.down_proj(torch.nn.functional.silu(self.gate_proj(x)) * self.up_proj(x))diff --git a/comfy/ldm/qwen_image21/model.py b/comfy/ldm/qwen_image21/model.pyindex 14c75e07d..e50d6c3cb 100644--- a/comfy/ldm/qwen_image21/model.py+++ b/comfy/ldm/qwen_image21/model.py@@ -62,7 +62,7 @@ class SwiGLUFeedForward(nn.Module):def forward(self, x):if self.fused:- return self.out(self.gate_up(x), input_act="swiglu")+ return comfy.ops.linear_input_act(self.out, self.gate_up(x), "swiglu")return self.out(F.silu(self.gate_layer(x)) * self.proj(x))diff --git a/comfy/ldm/wan/model.py b/comfy/ldm/wan/model.pyindex 642758f94..0925ac70c 100644--- a/comfy/ldm/wan/model.py+++ b/comfy/ldm/wan/model.py@@ -196,7 +196,7 @@ class WanFeedForward(nn.Sequential):"""[Linear, GELU(tanh), Linear], with the GELU folded into the down-projection."""def forward(self, x):- return self[2](self[0](x), input_act="gelu_tanh")+ return comfy.ops.linear_input_act(self[2], self[0](x), "gelu_tanh")class WanAttentionBlock(nn.Module):diff --git a/comfy/ops.py b/comfy/ops.pyindex c45cdf7d1..3ae8722b2 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -582,20 +582,16 @@ class disable_weight_init:def reset_parameters(self):return None- def forward_comfy_cast_weights(self, input, input_act=None, act_weight=None, act_eps=0.0,- residual=None, residual_scale=None):+ def forward_comfy_cast_weights(self, input):with CastBiasWeightContext(self, input, offloadable=True) as (weight, bias):- return linear_input_act_(input, weight, bias, input_act, act_weight, act_eps,- residual, residual_scale)+ return torch.nn.functional.linear(input, weight, bias)- def forward(self, input, input_act=None, act_weight=None, residual=None, residual_scale=None):+ def forward(self, *args, **kwargs):run_every_op()if self.comfy_cast_weights or len(self.weight_function) > 0 or len(self.bias_function) > 0:- return self.forward_comfy_cast_weights(input, input_act=input_act, act_weight=act_weight,- residual=residual, residual_scale=residual_scale)+ return self.forward_comfy_cast_weights(*args, **kwargs)else:- return linear_input_act_(input, self.weight, self.bias, input_act, act_weight,- residual=residual, residual_scale=residual_scale)+ return super().forward(*args, **kwargs)class Conv1d(torch.nn.Conv1d, CastWeightBiasOp):def reset_parameters(self):@@ -927,19 +923,17 @@ class fp8_ops(manual_cast):self.scale_input = Nonereturn None- def forward_comfy_cast_weights(self, input, input_act=None, act_weight=None, act_eps=0.0,- residual=None, residual_scale=None):- input = _eager_input_act(input, input_act, act_weight, act_eps)+ def forward_comfy_cast_weights(self, input):if len(self.weight_function) == 0 and len(self.bias_function) == 0:try:out = fp8_linear(self, input)if out is not None:- return _linear_residual(out, residual, residual_scale)+ return outexcept Exception as e:logging.info("Exception during fp8 op: {}".format(e))with CastBiasWeightContext(self, input, offloadable=True) as (weight, bias):- return _linear_residual(torch.nn.functional.linear(input, weight, bias), residual, residual_scale)+ return torch.nn.functional.linear(input, weight, bias)CUBLAS_IS_AVAILABLE = Falsetry:@@ -958,14 +952,12 @@ if CUBLAS_IS_AVAILABLE:with CastBiasWeightContext(self, input, offloadable=True) as (weight, bias):return cublas_half_matmul(input, weight, bias, self._epilogue_str, self.has_bias)- def forward(self, input, input_act=None, act_weight=None, residual=None, residual_scale=None):+ def forward(self, *args, **kwargs):run_every_op()- input = _eager_input_act(input, input_act, act_weight)if self.comfy_cast_weights or len(self.weight_function) > 0 or len(self.bias_function) > 0:- out = self.forward_comfy_cast_weights(input)+ return self.forward_comfy_cast_weights(*args, **kwargs)else:- out = super().forward(input)- return _linear_residual(out, residual, residual_scale)+ return super().forward(*args, **kwargs)# ==============================================================================# Mixed Precision Operations@@ -1451,11 +1443,6 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_preccompute_dtype=None,want_requant=False,weight_only_quant=False,- *,- input_act=None,- act_weight=None,- residual=None,- residual_scale=None,):if not weight_only_quant:with CastBiasWeightContext(@@ -1467,10 +1454,6 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec) as (weight, bias):if self._full_precision_mm and isinstance(weight, QuantizedTensor):weight = weight.dequantize()- if input_act is not None or residual is not None:- return linear_input_act_(input, weight, bias, input_act, act_weight,- residual=residual, residual_scale=residual_scale,- fp16_accumulation=not isinstance(self.weight, QuantizedTensor))return self._forward(input, weight, bias)with CastBiasWeightContext(@@ -1486,18 +1469,8 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precweight = weight.to(dtype=input.dtype)return self._forward(input, weight, bias)- def forward(self, input, input_act=None, act_weight=None, residual=None, residual_scale=None):+ def forward(self, input, *args, **kwargs):run_every_op()- if input_act is not None or residual is not None:- quantized = isinstance(self.weight, QuantizedTensor)- if (not comfy.model_management.in_training and not self._full_precision_mm- and ((quantized and self.weight._layout_cls == "TensorWiseINT8Layout"- and not getattr(self.weight._params, "transposed", False))- or (not quantized and _fp16_linear_wanted(input)))):- return self.forward_comfy_cast_weights(input, compute_dtype=input.dtype, want_requant=quantized,- input_act=input_act, act_weight=act_weight,- residual=residual, residual_scale=residual_scale)- input = _eager_input_act(input, input_act, act_weight)# ModelOpt AWQ-style smoothingpre_quant_scale = getattr(self, 'pre_quant_scale', None)@@ -1530,10 +1503,9 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precif scale is not None:scale = comfy.model_management.cast_to_device(scale, input.device, None)- output = QuantLinearFunc.apply(+ return QuantLinearFunc.apply(input, weight, bias, self.layout_type, scale, compute_dtype)- return _linear_residual(output, residual, residual_scale)# Inference path (unchanged)if _use_quantized and quantize_input:@@ -1562,7 +1534,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precif reshaped_nd:output = output.reshape((*input_shape[:-1], self.weight.shape[0]))- return _linear_residual(output, residual, residual_scale)+ return outputdef convert_weight(self, weight, inplace=False, **kwargs):if isinstance(weight, QuantizedTensor):diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.pyindex 35ef22961..161e65411 100644--- a/comfy/text_encoders/llama.py+++ b/comfy/text_encoders/llama.py@@ -754,7 +754,7 @@ class MLP(nn.Module):if self.merged_mlp:x = self.gate_up_proj(x)if self.merged_input_act is not None:- return self.down_proj(x, input_act=self.merged_input_act)+ return comfy.ops.linear_input_act(self.down_proj, x, self.merged_input_act)gate, up = x.chunk(2, dim=-1)return self.down_proj(self.activation(gate) * up)return self.down_proj(self.activation(self.gate_proj(x)) * self.up_proj(x))