Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Change model code to use function wrappers for Linear calls (#16861)

rattus62c49c4d4fe5
diff --git a/comfy/ldm/lightricks/model.py b/comfy/ldm/lightricks/model.py
index 0d6f457ee..b011f5d81 100644
--- a/comfy/ldm/lightricks/model.py
+++ b/comfy/ldm/lightricks/model.py
@@ -327,7 +327,7 @@ class FeedForward(nn.Module):
# Dropout, so leave it to the stock path whenever it could be active.
if comfy.model_management.in_training:
return self.net(x)
- return self.net[2](self.net[0].proj(x), input_act="gelu_tanh")
+ return comfy.ops.linear_input_act(self.net[2], self.net[0].proj(x), "gelu_tanh")
def apply_rotary_emb(input_tensor, freqs_cis):
rotation_matrix, split_pe = freqs_cis
diff --git a/comfy/ldm/minimax/model.py b/comfy/ldm/minimax/model.py
index ca5010ffa..440d47ef3 100644
--- a/comfy/ldm/minimax/model.py
+++ b/comfy/ldm/minimax/model.py
@@ -208,7 +208,7 @@ class MLP(nn.Module):
self.fc2 = operations.Linear(ffn, hidden, bias=False, dtype=dtype, device=device)
def forward(self, x):
- return self.fc2(self.fc1(x), input_act="swiglu")
+ return comfy.ops.linear_input_act(self.fc2, self.fc1(x), "swiglu")
class AdalnProj(nn.Module):
diff --git a/comfy/ldm/minimax/vae.py b/comfy/ldm/minimax/vae.py
index 9768f6700..84962b6a3 100644
--- a/comfy/ldm/minimax/vae.py
+++ b/comfy/ldm/minimax/vae.py
@@ -269,8 +269,8 @@ class FeedForward(nn.Module):
def forward(self, x, pre_norm, residual, residual_scale):
# norm, gated silu and residual addcmul fold into the INT8 kernels
- h = self.w1(x, input_act="rms_norm", act_weight=pre_norm)
- return self.w2(h, input_act="swiglu", residual=residual, residual_scale=residual_scale)
+ h = comfy.ops.linear_input_act(self.w1, x, "rms_norm", act_weight=pre_norm)
+ return comfy.ops.linear_input_act(self.w2, h, "swiglu", residual=residual, residual_scale=residual_scale)
class Attention(nn.Module):
@@ -289,7 +289,7 @@ class Attention(nn.Module):
def forward(self, x, rotary_pos_emb, pre_norm, residual, residual_scale):
batch_size, seq_len, _ = x.shape
- qkv = self.to_qkv(x, input_act="rms_norm", act_weight=pre_norm)
+ qkv = comfy.ops.linear_input_act(self.to_qkv, x, "rms_norm", act_weight=pre_norm)
qkv = qkv.view(batch_size, seq_len, -1, 3 * self.dim_head)
query, key, value = torch.chunk(qkv, 3, dim=-1)
@@ -313,7 +313,7 @@ class Attention(nn.Module):
out = out.transpose(1, 2).reshape(batch_size, seq_len, -1)
else:
out = optimized_attention(query, key, value, self.heads, skip_reshape=True)
- return self.to_out(torch.nan_to_num(out), residual=residual, residual_scale=residual_scale)
+ return comfy.ops.linear_input_act(self.to_out, torch.nan_to_num(out), None, residual=residual, residual_scale=residual_scale)
class TransformerBlock(nn.Module):
diff --git a/comfy/ldm/minimax_music/ar.py b/comfy/ldm/minimax_music/ar.py
index 250822d7a..2a702c527 100644
--- a/comfy/ldm/minimax_music/ar.py
+++ b/comfy/ldm/minimax_music/ar.py
@@ -95,7 +95,7 @@ class RVQMLP(nn.Module):
def forward(self, x):
if self.merged_mlp:
- return self.down_proj(self.gate_up_proj(x), input_act="swiglu")
+ return comfy.ops.linear_input_act(self.down_proj, self.gate_up_proj(x), "swiglu")
return self.down_proj(torch.nn.functional.silu(self.gate_proj(x)) * self.up_proj(x))
diff --git a/comfy/ldm/qwen_image21/model.py b/comfy/ldm/qwen_image21/model.py
index 14c75e07d..e50d6c3cb 100644
--- a/comfy/ldm/qwen_image21/model.py
+++ b/comfy/ldm/qwen_image21/model.py
@@ -62,7 +62,7 @@ class SwiGLUFeedForward(nn.Module):
def forward(self, x):
if self.fused:
- return self.out(self.gate_up(x), input_act="swiglu")
+ return comfy.ops.linear_input_act(self.out, self.gate_up(x), "swiglu")
return self.out(F.silu(self.gate_layer(x)) * self.proj(x))
diff --git a/comfy/ldm/wan/model.py b/comfy/ldm/wan/model.py
index 642758f94..0925ac70c 100644
--- a/comfy/ldm/wan/model.py
+++ b/comfy/ldm/wan/model.py
@@ -196,7 +196,7 @@ class WanFeedForward(nn.Sequential):
"""[Linear, GELU(tanh), Linear], with the GELU folded into the down-projection."""
def forward(self, x):
- return self[2](self[0](x), input_act="gelu_tanh")
+ return comfy.ops.linear_input_act(self[2], self[0](x), "gelu_tanh")
class WanAttentionBlock(nn.Module):
diff --git a/comfy/ops.py b/comfy/ops.py
index c45cdf7d1..3ae8722b2 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -582,20 +582,16 @@ class disable_weight_init:
def reset_parameters(self):
return None
- def forward_comfy_cast_weights(self, input, input_act=None, act_weight=None, act_eps=0.0,
- residual=None, residual_scale=None):
+ def forward_comfy_cast_weights(self, input):
with CastBiasWeightContext(self, input, offloadable=True) as (weight, bias):
- return linear_input_act_(input, weight, bias, input_act, act_weight, act_eps,
- residual, residual_scale)
+ return torch.nn.functional.linear(input, weight, bias)
- def forward(self, input, input_act=None, act_weight=None, residual=None, residual_scale=None):
+ def forward(self, *args, **kwargs):
run_every_op()
if self.comfy_cast_weights or len(self.weight_function) > 0 or len(self.bias_function) > 0:
- return self.forward_comfy_cast_weights(input, input_act=input_act, act_weight=act_weight,
- residual=residual, residual_scale=residual_scale)
+ return self.forward_comfy_cast_weights(*args, **kwargs)
else:
- return linear_input_act_(input, self.weight, self.bias, input_act, act_weight,
- residual=residual, residual_scale=residual_scale)
+ return super().forward(*args, **kwargs)
class Conv1d(torch.nn.Conv1d, CastWeightBiasOp):
def reset_parameters(self):
@@ -927,19 +923,17 @@ class fp8_ops(manual_cast):
self.scale_input = None
return None
- def forward_comfy_cast_weights(self, input, input_act=None, act_weight=None, act_eps=0.0,
- residual=None, residual_scale=None):
- input = _eager_input_act(input, input_act, act_weight, act_eps)
+ def forward_comfy_cast_weights(self, input):
if len(self.weight_function) == 0 and len(self.bias_function) == 0:
try:
out = fp8_linear(self, input)
if out is not None:
- return _linear_residual(out, residual, residual_scale)
+ return out
except Exception as e:
logging.info("Exception during fp8 op: {}".format(e))
with CastBiasWeightContext(self, input, offloadable=True) as (weight, bias):
- return _linear_residual(torch.nn.functional.linear(input, weight, bias), residual, residual_scale)
+ return torch.nn.functional.linear(input, weight, bias)
CUBLAS_IS_AVAILABLE = False
try:
@@ -958,14 +952,12 @@ if CUBLAS_IS_AVAILABLE:
with CastBiasWeightContext(self, input, offloadable=True) as (weight, bias):
return cublas_half_matmul(input, weight, bias, self._epilogue_str, self.has_bias)
- def forward(self, input, input_act=None, act_weight=None, residual=None, residual_scale=None):
+ def forward(self, *args, **kwargs):
run_every_op()
- input = _eager_input_act(input, input_act, act_weight)
if self.comfy_cast_weights or len(self.weight_function) > 0 or len(self.bias_function) > 0:
- out = self.forward_comfy_cast_weights(input)
+ return self.forward_comfy_cast_weights(*args, **kwargs)
else:
- out = super().forward(input)
- return _linear_residual(out, residual, residual_scale)
+ return super().forward(*args, **kwargs)
# ==============================================================================
# Mixed Precision Operations
@@ -1451,11 +1443,6 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
compute_dtype=None,
want_requant=False,
weight_only_quant=False,
- *,
- input_act=None,
- act_weight=None,
- residual=None,
- residual_scale=None,
):
if not weight_only_quant:
with CastBiasWeightContext(
@@ -1467,10 +1454,6 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
) as (weight, bias):
if self._full_precision_mm and isinstance(weight, QuantizedTensor):
weight = weight.dequantize()
- if input_act is not None or residual is not None:
- return linear_input_act_(input, weight, bias, input_act, act_weight,
- residual=residual, residual_scale=residual_scale,
- fp16_accumulation=not isinstance(self.weight, QuantizedTensor))
return self._forward(input, weight, bias)
with CastBiasWeightContext(
@@ -1486,18 +1469,8 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
weight = weight.to(dtype=input.dtype)
return self._forward(input, weight, bias)
- def forward(self, input, input_act=None, act_weight=None, residual=None, residual_scale=None):
+ def forward(self, input, *args, **kwargs):
run_every_op()
- if input_act is not None or residual is not None:
- quantized = isinstance(self.weight, QuantizedTensor)
- if (not comfy.model_management.in_training and not self._full_precision_mm
- and ((quantized and self.weight._layout_cls == "TensorWiseINT8Layout"
- and not getattr(self.weight._params, "transposed", False))
- or (not quantized and _fp16_linear_wanted(input)))):
- return self.forward_comfy_cast_weights(input, compute_dtype=input.dtype, want_requant=quantized,
- input_act=input_act, act_weight=act_weight,
- residual=residual, residual_scale=residual_scale)
- input = _eager_input_act(input, input_act, act_weight)
# ModelOpt AWQ-style smoothing
pre_quant_scale = getattr(self, 'pre_quant_scale', None)
@@ -1530,10 +1503,9 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
if scale is not None:
scale = comfy.model_management.cast_to_device(scale, input.device, None)
- output = QuantLinearFunc.apply(
+ return QuantLinearFunc.apply(
input, weight, bias, self.layout_type, scale, compute_dtype
)
- return _linear_residual(output, residual, residual_scale)
# Inference path (unchanged)
if _use_quantized and quantize_input:
@@ -1562,7 +1534,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
if reshaped_nd:
output = output.reshape((*input_shape[:-1], self.weight.shape[0]))
- return _linear_residual(output, residual, residual_scale)
+ return output
def convert_weight(self, weight, inplace=False, **kwargs):
if isinstance(weight, QuantizedTensor):
diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py
index 35ef22961..161e65411 100644
--- a/comfy/text_encoders/llama.py
+++ b/comfy/text_encoders/llama.py
@@ -754,7 +754,7 @@ class MLP(nn.Module):
if self.merged_mlp:
x = self.gate_up_proj(x)
if self.merged_input_act is not None:
- return self.down_proj(x, input_act=self.merged_input_act)
+ return comfy.ops.linear_input_act(self.down_proj, x, self.merged_input_act)
gate, up = x.chunk(2, dim=-1)
return self.down_proj(self.activation(gate) * up)
return self.down_proj(self.activation(self.gate_proj(x)) * self.up_proj(x))