The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Consolidate Linear forward API with prologue/epilogue fusion paths (#16816)
rattus0752bcb28cd3
diff --git a/comfy/ldm/lightricks/model.py b/comfy/ldm/lightricks/model.pyindex b011f5d81..0d6f457ee 100644--- a/comfy/ldm/lightricks/model.py+++ b/comfy/ldm/lightricks/model.py@@ -327,7 +327,7 @@ class FeedForward(nn.Module):# Dropout, so leave it to the stock path whenever it could be active.if comfy.model_management.in_training:return self.net(x)- return comfy.ops.linear_input_act(self.net[2], self.net[0].proj(x), "gelu_tanh")+ return self.net[2](self.net[0].proj(x), input_act="gelu_tanh")def apply_rotary_emb(input_tensor, freqs_cis):rotation_matrix, split_pe = freqs_cisdiff --git a/comfy/ldm/minimax/model.py b/comfy/ldm/minimax/model.pyindex 440d47ef3..ca5010ffa 100644--- a/comfy/ldm/minimax/model.py+++ b/comfy/ldm/minimax/model.py@@ -208,7 +208,7 @@ class MLP(nn.Module):self.fc2 = operations.Linear(ffn, hidden, bias=False, dtype=dtype, device=device)def forward(self, x):- return comfy.ops.linear_input_act(self.fc2, self.fc1(x), "swiglu")+ return self.fc2(self.fc1(x), input_act="swiglu")class AdalnProj(nn.Module):diff --git a/comfy/ldm/minimax/vae.py b/comfy/ldm/minimax/vae.pyindex f4dd9ae89..9768f6700 100644--- a/comfy/ldm/minimax/vae.py+++ b/comfy/ldm/minimax/vae.py@@ -269,10 +269,8 @@ class FeedForward(nn.Module):def forward(self, x, pre_norm, residual, residual_scale):# norm, gated silu and residual addcmul fold into the INT8 kernels- with comfy.ops.CastBiasWeightContext(pre_norm if pre_norm.weight is not None else None, x, offloadable=True) as (weight, bias):- h = comfy.ops.linear_input_act(self.w1, x, "rms_norm", weight, pre_norm.eps)- return comfy.ops.linear_input_act(- self.w2, h, "swiglu", residual=residual, residual_scale=residual_scale)+ h = self.w1(x, input_act="rms_norm", act_weight=pre_norm)+ return self.w2(h, input_act="swiglu", residual=residual, residual_scale=residual_scale)class Attention(nn.Module):@@ -291,8 +289,7 @@ class Attention(nn.Module):def forward(self, x, rotary_pos_emb, pre_norm, residual, residual_scale):batch_size, seq_len, _ = x.shape- with comfy.ops.CastBiasWeightContext(pre_norm if pre_norm.weight is not None else None, x, offloadable=True) as (weight, bias):- qkv = comfy.ops.linear_input_act(self.to_qkv, x, "rms_norm", weight, pre_norm.eps)+ qkv = self.to_qkv(x, input_act="rms_norm", act_weight=pre_norm)qkv = qkv.view(batch_size, seq_len, -1, 3 * self.dim_head)query, key, value = torch.chunk(qkv, 3, dim=-1)@@ -316,9 +313,7 @@ class Attention(nn.Module):out = out.transpose(1, 2).reshape(batch_size, seq_len, -1)else:out = optimized_attention(query, key, value, self.heads, skip_reshape=True)- return comfy.ops.linear_input_act(- self.to_out, torch.nan_to_num(out), None,- residual=residual, residual_scale=residual_scale)+ return self.to_out(torch.nan_to_num(out), residual=residual, residual_scale=residual_scale)class TransformerBlock(nn.Module):diff --git a/comfy/ldm/minimax_music/ar.py b/comfy/ldm/minimax_music/ar.pyindex 2a702c527..250822d7a 100644--- a/comfy/ldm/minimax_music/ar.py+++ b/comfy/ldm/minimax_music/ar.py@@ -95,7 +95,7 @@ class RVQMLP(nn.Module):def forward(self, x):if self.merged_mlp:- return comfy.ops.linear_input_act(self.down_proj, self.gate_up_proj(x), "swiglu")+ return self.down_proj(self.gate_up_proj(x), input_act="swiglu")return self.down_proj(torch.nn.functional.silu(self.gate_proj(x)) * self.up_proj(x))diff --git a/comfy/ldm/qwen_image21/model.py b/comfy/ldm/qwen_image21/model.pyindex e50d6c3cb..14c75e07d 100644--- a/comfy/ldm/qwen_image21/model.py+++ b/comfy/ldm/qwen_image21/model.py@@ -62,7 +62,7 @@ class SwiGLUFeedForward(nn.Module):def forward(self, x):if self.fused:- return comfy.ops.linear_input_act(self.out, self.gate_up(x), "swiglu")+ return self.out(self.gate_up(x), input_act="swiglu")return self.out(F.silu(self.gate_layer(x)) * self.proj(x))diff --git a/comfy/ldm/wan/model.py b/comfy/ldm/wan/model.pyindex 0925ac70c..642758f94 100644--- a/comfy/ldm/wan/model.py+++ b/comfy/ldm/wan/model.py@@ -196,7 +196,7 @@ class WanFeedForward(nn.Sequential):"""[Linear, GELU(tanh), Linear], with the GELU folded into the down-projection."""def forward(self, x):- return comfy.ops.linear_input_act(self[2], self[0](x), "gelu_tanh")+ return self[2](self[0](x), input_act="gelu_tanh")class WanAttentionBlock(nn.Module):diff --git a/comfy/ops.py b/comfy/ops.pyindex d5568af7f..c45cdf7d1 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -582,16 +582,20 @@ class disable_weight_init:def reset_parameters(self):return None- def forward_comfy_cast_weights(self, input):+ def forward_comfy_cast_weights(self, input, input_act=None, act_weight=None, act_eps=0.0,+ residual=None, residual_scale=None):with CastBiasWeightContext(self, input, offloadable=True) as (weight, bias):- return torch.nn.functional.linear(input, weight, bias)+ return linear_input_act_(input, weight, bias, input_act, act_weight, act_eps,+ residual, residual_scale)- def forward(self, *args, **kwargs):+ def forward(self, input, input_act=None, act_weight=None, residual=None, residual_scale=None):run_every_op()if self.comfy_cast_weights or len(self.weight_function) > 0 or len(self.bias_function) > 0:- return self.forward_comfy_cast_weights(*args, **kwargs)+ return self.forward_comfy_cast_weights(input, input_act=input_act, act_weight=act_weight,+ residual=residual, residual_scale=residual_scale)else:- return super().forward(*args, **kwargs)+ return linear_input_act_(input, self.weight, self.bias, input_act, act_weight,+ residual=residual, residual_scale=residual_scale)class Conv1d(torch.nn.Conv1d, CastWeightBiasOp):def reset_parameters(self):@@ -701,8 +705,11 @@ class disable_weight_init:self.bias = Nonereturn None+ def cast_weight(self, input):+ return CastBiasWeightContext(self if self.weight is not None else None, input, offloadable=True)+def forward_comfy_cast_weights(self, input):- with CastBiasWeightContext(self if self.weight is not None else None, input, offloadable=True) as (weight, bias):+ with self.cast_weight(input) as (weight, bias):return torch.nn.functional.rms_norm(input, self.normalized_shape, weight, self.eps)def forward(self, *args, **kwargs):@@ -920,17 +927,19 @@ class fp8_ops(manual_cast):self.scale_input = Nonereturn None- def forward_comfy_cast_weights(self, input):+ def forward_comfy_cast_weights(self, input, input_act=None, act_weight=None, act_eps=0.0,+ residual=None, residual_scale=None):+ input = _eager_input_act(input, input_act, act_weight, act_eps)if len(self.weight_function) == 0 and len(self.bias_function) == 0:try:out = fp8_linear(self, input)if out is not None:- return out+ return _linear_residual(out, residual, residual_scale)except Exception as e:logging.info("Exception during fp8 op: {}".format(e))with CastBiasWeightContext(self, input, offloadable=True) as (weight, bias):- return torch.nn.functional.linear(input, weight, bias)+ return _linear_residual(torch.nn.functional.linear(input, weight, bias), residual, residual_scale)CUBLAS_IS_AVAILABLE = Falsetry:@@ -949,12 +958,14 @@ if CUBLAS_IS_AVAILABLE:with CastBiasWeightContext(self, input, offloadable=True) as (weight, bias):return cublas_half_matmul(input, weight, bias, self._epilogue_str, self.has_bias)- def forward(self, *args, **kwargs):+ def forward(self, input, input_act=None, act_weight=None, residual=None, residual_scale=None):run_every_op()+ input = _eager_input_act(input, input_act, act_weight)if self.comfy_cast_weights or len(self.weight_function) > 0 or len(self.bias_function) > 0:- return self.forward_comfy_cast_weights(*args, **kwargs)+ out = self.forward_comfy_cast_weights(input)else:- return super().forward(*args, **kwargs)+ out = super().forward(input)+ return _linear_residual(out, residual, residual_scale)# ==============================================================================# Mixed Precision Operations@@ -979,14 +990,31 @@ INPUT_ACT_EAGER = {}+def _input_act_weight(x, input_act, act_weight, act_eps):+ if input_act == "rms_norm" and isinstance(act_weight, torch.nn.Module):+ with act_weight.cast_weight(x) as (weight, _):+ eps = act_weight.eps+ if eps is None:+ eps = torch.finfo(torch.float64 if x.dtype == torch.float64 else torch.float32).eps+ yield weight, eps+ else:+ yield act_weight, act_eps++def _eager_input_act(x, input_act, act_weight=None, act_eps=0.0):if input_act is None:return xif input_act == "rms_norm":- return comfy.rmsnorm.rms_norm(x, act_weight, act_eps)+ with _input_act_weight(x, input_act, act_weight, act_eps) as (weight, eps):+ return comfy.rmsnorm.rms_norm(x, weight, eps)return INPUT_ACT_EAGER[input_act](x)+def _linear_residual(out, residual, residual_scale):+ return out if residual is None else torch.addcmul(residual, out, residual_scale)++def _fp16_linear_wanted(x):"""kitchen's fp16-accumulate GEMM replaces a plain linear when the user opted intofp16 accumulation and the activation is fp16 on CUDA; weights come through cast_bias_weight."""@@ -996,6 +1024,25 @@ def _fp16_linear_wanted(x):def linear_input_act(linear, x, input_act, act_weight=None, act_eps=0.0,residual=None, residual_scale=None):+ run_every_op()+ weight = linear.weight+ quantized = isinstance(weight, QuantizedTensor)+ if (comfy.model_management.in_training or getattr(linear, "_full_precision_mm", False)+ or not ((quantized and weight._layout_cls == "TensorWiseINT8Layout"+ and not getattr(weight._params, "transposed", False))+ or (not quantized and _fp16_linear_wanted(x)))):+ out = linear(_eager_input_act(x, input_act, act_weight, act_eps))+ return _linear_residual(out, residual, residual_scale)++ with CastBiasWeightContext(linear, x, offloadable=True,+ compute_dtype=x.dtype if quantized else None,+ want_requant=quantized) as (weight, bias):+ return linear_input_act_(x, weight, bias, input_act, act_weight, act_eps,+ residual, residual_scale, fp16_accumulation=not quantized)+++def linear_input_act_(x, weight, bias, input_act=None, act_weight=None, act_eps=0.0,+ residual=None, residual_scale=None, *, fp16_accumulation=True):"""``linear(act(x))``, with ``act`` folded into an INT8 activation quantizer.An INT8 linear quantizes its input anyway, so an elementwise activation can@@ -1009,43 +1056,25 @@ def linear_input_act(linear, x, input_act, act_weight=None, act_eps=0.0,addcmul, ``residual + residual_scale * linear(act(x))``, fused into theINT8 GEMM epilogue where supported.+ Linear weights are already prepared. An RMSNorm module in act_weight owns+ its norm-weight cast context; a tensor uses the supplied act_eps directly."""- def _residual_out(out):- if residual is None:- return out- return torch.addcmul(residual, out, residual_scale)+ if input_act is None and residual is None:+ return torch.nn.functional.linear(x, weight, bias)- weight = linear.weight- full_precision_mm = getattr(linear, "_full_precision_mm", False)if (comfy.model_management.in_trainingor not isinstance(weight, QuantizedTensor)or weight._layout_cls != "TensorWiseINT8Layout"- or getattr(weight._params, "transposed", False)- or full_precision_mm):+ or getattr(weight._params, "transposed", False)):+ x = _eager_input_act(x, input_act, act_weight, act_eps)if (not comfy.model_management.in_trainingand not isinstance(weight, QuantizedTensor)- and not full_precision_mm- and _fp16_linear_wanted(x)):- weight, bias, offload_stream = cast_bias_weight(linear, x, offloadable=True)- try:- return quant_ops.ck.fp16_linear(- _eager_input_act(x, input_act, act_weight, act_eps),- weight, bias, residual=residual, residual_scale=residual_scale)- finally:- uncast_bias_weight(linear, weight, bias, offload_stream)- return _residual_out(linear(_eager_input_act(x, input_act, act_weight, act_eps)))-- # want_requant keeps a vbar-streamed layer on the INT8 path when a LoRA is- # patched in on the fly; without it the cast hands back a dequantized weight.- weight, bias, offload_stream = cast_bias_weight(- linear, x, offloadable=True, compute_dtype=x.dtype, want_requant=True)- try:- if not isinstance(weight, QuantizedTensor):- # A LoRA weight_function, or activations whose dtype differs from the- # weight's, make the cast hand back a dequantized tensor.- return _residual_out(torch.nn.functional.linear(- _eager_input_act(x, input_act, act_weight, act_eps), weight, bias))- qdata, scale = TensorWiseINT8Layout.get_plain_tensors(weight)+ and fp16_accumulation and _fp16_linear_wanted(x)):+ return quant_ops.ck.fp16_linear(x, weight, bias, residual=residual, residual_scale=residual_scale)+ return _linear_residual(torch.nn.functional.linear(x, weight, bias), residual, residual_scale)++ qdata, scale = TensorWiseINT8Layout.get_plain_tensors(weight)+ with _input_act_weight(x, input_act, act_weight, act_eps) as (act_weight, act_eps):return quant_ops.ck.int8_linear(x, qdata, scale, bias, x.dtype,convrot=getattr(weight._params, "convrot", False),@@ -1056,8 +1085,6 @@ def linear_input_act(linear, x, input_act, act_weight=None, act_eps=0.0,residual=residual,residual_scale=residual_scale,)- finally:- uncast_bias_weight(linear, weight, bias, offload_stream)# Grouped-scale integer weights that run through the shared INT8 GEMM (comfy_kitchen@@ -1424,6 +1451,11 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_preccompute_dtype=None,want_requant=False,weight_only_quant=False,+ *,+ input_act=None,+ act_weight=None,+ residual=None,+ residual_scale=None,):if not weight_only_quant:with CastBiasWeightContext(@@ -1435,6 +1467,10 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec) as (weight, bias):if self._full_precision_mm and isinstance(weight, QuantizedTensor):weight = weight.dequantize()+ if input_act is not None or residual is not None:+ return linear_input_act_(input, weight, bias, input_act, act_weight,+ residual=residual, residual_scale=residual_scale,+ fp16_accumulation=not isinstance(self.weight, QuantizedTensor))return self._forward(input, weight, bias)with CastBiasWeightContext(@@ -1450,8 +1486,18 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precweight = weight.to(dtype=input.dtype)return self._forward(input, weight, bias)- def forward(self, input, *args, **kwargs):+ def forward(self, input, input_act=None, act_weight=None, residual=None, residual_scale=None):run_every_op()+ if input_act is not None or residual is not None:+ quantized = isinstance(self.weight, QuantizedTensor)+ if (not comfy.model_management.in_training and not self._full_precision_mm+ and ((quantized and self.weight._layout_cls == "TensorWiseINT8Layout"+ and not getattr(self.weight._params, "transposed", False))+ or (not quantized and _fp16_linear_wanted(input)))):+ return self.forward_comfy_cast_weights(input, compute_dtype=input.dtype, want_requant=quantized,+ input_act=input_act, act_weight=act_weight,+ residual=residual, residual_scale=residual_scale)+ input = _eager_input_act(input, input_act, act_weight)# ModelOpt AWQ-style smoothingpre_quant_scale = getattr(self, 'pre_quant_scale', None)@@ -1484,9 +1530,10 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precif scale is not None:scale = comfy.model_management.cast_to_device(scale, input.device, None)- return QuantLinearFunc.apply(+ output = QuantLinearFunc.apply(input, weight, bias, self.layout_type, scale, compute_dtype)+ return _linear_residual(output, residual, residual_scale)# Inference path (unchanged)if _use_quantized and quantize_input:@@ -1515,7 +1562,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precif reshaped_nd:output = output.reshape((*input_shape[:-1], self.weight.shape[0]))- return output+ return _linear_residual(output, residual, residual_scale)def convert_weight(self, weight, inplace=False, **kwargs):if isinstance(weight, QuantizedTensor):diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.pyindex 161e65411..35ef22961 100644--- a/comfy/text_encoders/llama.py+++ b/comfy/text_encoders/llama.py@@ -754,7 +754,7 @@ class MLP(nn.Module):if self.merged_mlp:x = self.gate_up_proj(x)if self.merged_input_act is not None:- return comfy.ops.linear_input_act(self.down_proj, x, self.merged_input_act)+ return self.down_proj(x, input_act=self.merged_input_act)gate, up = x.chunk(2, dim=-1)return self.down_proj(self.activation(gate) * up)return self.down_proj(self.activation(self.gate_proj(x)) * self.up_proj(x))diff --git a/tests-unit/comfy_quant/test_mixed_precision.py b/tests-unit/comfy_quant/test_mixed_precision.pyindex 03e1b0e6e..700daa92c 100644--- a/tests-unit/comfy_quant/test_mixed_precision.py+++ b/tests-unit/comfy_quant/test_mixed_precision.py@@ -195,7 +195,7 @@ class TestMixedPrecisionOps(unittest.TestCase):lora_delta = torch.randn_like(weight) * 0.01return weight + lora_delta- model.layer1.weight_function.append(apply_lora)+ model.layer1.weight_function = [apply_lora]# Forward pass should work with LoRA (triggers weight_function path)input_tensor = torch.randn(5, 10, dtype=torch.bfloat16)diff --git a/tests-unit/comfy_test/test_minimax_h3_vae.py b/tests-unit/comfy_test/test_minimax_h3_vae.pyindex 77ed67ede..515d43476 100644--- a/tests-unit/comfy_test/test_minimax_h3_vae.py+++ b/tests-unit/comfy_test/test_minimax_h3_vae.py@@ -50,16 +50,26 @@ def test_attention_moves_offloaded_qk_norm_scale_to_input_device(monkeypatch):attn._buffers["qk_norm_scale"] = _OffloadedScale("meta")batch_size, seq_len = 1, 3- x = torch.randn(batch_size, seq_len, heads * dim_head)+ x = torch.arange(1, 25, dtype=torch.float32).reshape(batch_size, seq_len, heads * dim_head)rotary_pos_emb = torch.randn(batch_size, 1, seq_len, dim_head // 2, 2, 2)- class _Norm:- weight = None- eps = 1e-5-+ pre_norm = comfy.ops.manual_cast.RMSNorm(heads * dim_head, elementwise_affine=False, eps=1e-5)+ qkv_outputs = []+ hook = attn.to_qkv.register_forward_hook(lambda module, inputs, output: qkv_outputs.append(output))with torch.no_grad():+ attn.to_qkv.weight.copy_(torch.eye(heads * dim_head).repeat(3, 1))+ attn.to_qkv.bias.zero_()+ attn.to_out.weight.copy_(torch.eye(heads * dim_head))+ attn.to_out.bias.zero_()out = attn.forward(- x, rotary_pos_emb, pre_norm=_Norm(), residual=None, residual_scale=None+ x, rotary_pos_emb, pre_norm=pre_norm, residual=None, residual_scale=None)+ hook.remove()assert out.device == x.device+ normalized = torch.nn.functional.rms_norm(x, (heads * dim_head,), eps=pre_norm.eps)+ expected = torch.nn.functional.linear(normalized, attn.to_qkv.weight, attn.to_qkv.bias)+ unnormalized = torch.nn.functional.linear(x, attn.to_qkv.weight, attn.to_qkv.bias)+ assert len(qkv_outputs) == 1+ torch.testing.assert_close(qkv_outputs[0], expected)+ assert not torch.allclose(qkv_outputs[0], unnormalized)