Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Consolidate Linear forward API with prologue/epilogue fusion paths (#16816)

rattus0752bcb28cd3
diff --git a/comfy/ldm/lightricks/model.py b/comfy/ldm/lightricks/model.py
index b011f5d81..0d6f457ee 100644
--- a/comfy/ldm/lightricks/model.py
+++ b/comfy/ldm/lightricks/model.py
@@ -327,7 +327,7 @@ class FeedForward(nn.Module):
# Dropout, so leave it to the stock path whenever it could be active.
if comfy.model_management.in_training:
return self.net(x)
- return comfy.ops.linear_input_act(self.net[2], self.net[0].proj(x), "gelu_tanh")
+ return self.net[2](self.net[0].proj(x), input_act="gelu_tanh")
def apply_rotary_emb(input_tensor, freqs_cis):
rotation_matrix, split_pe = freqs_cis
diff --git a/comfy/ldm/minimax/model.py b/comfy/ldm/minimax/model.py
index 440d47ef3..ca5010ffa 100644
--- a/comfy/ldm/minimax/model.py
+++ b/comfy/ldm/minimax/model.py
@@ -208,7 +208,7 @@ class MLP(nn.Module):
self.fc2 = operations.Linear(ffn, hidden, bias=False, dtype=dtype, device=device)
def forward(self, x):
- return comfy.ops.linear_input_act(self.fc2, self.fc1(x), "swiglu")
+ return self.fc2(self.fc1(x), input_act="swiglu")
class AdalnProj(nn.Module):
diff --git a/comfy/ldm/minimax/vae.py b/comfy/ldm/minimax/vae.py
index f4dd9ae89..9768f6700 100644
--- a/comfy/ldm/minimax/vae.py
+++ b/comfy/ldm/minimax/vae.py
@@ -269,10 +269,8 @@ class FeedForward(nn.Module):
def forward(self, x, pre_norm, residual, residual_scale):
# norm, gated silu and residual addcmul fold into the INT8 kernels
- with comfy.ops.CastBiasWeightContext(pre_norm if pre_norm.weight is not None else None, x, offloadable=True) as (weight, bias):
- h = comfy.ops.linear_input_act(self.w1, x, "rms_norm", weight, pre_norm.eps)
- return comfy.ops.linear_input_act(
- self.w2, h, "swiglu", residual=residual, residual_scale=residual_scale)
+ h = self.w1(x, input_act="rms_norm", act_weight=pre_norm)
+ return self.w2(h, input_act="swiglu", residual=residual, residual_scale=residual_scale)
class Attention(nn.Module):
@@ -291,8 +289,7 @@ class Attention(nn.Module):
def forward(self, x, rotary_pos_emb, pre_norm, residual, residual_scale):
batch_size, seq_len, _ = x.shape
- with comfy.ops.CastBiasWeightContext(pre_norm if pre_norm.weight is not None else None, x, offloadable=True) as (weight, bias):
- qkv = comfy.ops.linear_input_act(self.to_qkv, x, "rms_norm", weight, pre_norm.eps)
+ qkv = self.to_qkv(x, input_act="rms_norm", act_weight=pre_norm)
qkv = qkv.view(batch_size, seq_len, -1, 3 * self.dim_head)
query, key, value = torch.chunk(qkv, 3, dim=-1)
@@ -316,9 +313,7 @@ class Attention(nn.Module):
out = out.transpose(1, 2).reshape(batch_size, seq_len, -1)
else:
out = optimized_attention(query, key, value, self.heads, skip_reshape=True)
- return comfy.ops.linear_input_act(
- self.to_out, torch.nan_to_num(out), None,
- residual=residual, residual_scale=residual_scale)
+ return self.to_out(torch.nan_to_num(out), residual=residual, residual_scale=residual_scale)
class TransformerBlock(nn.Module):
diff --git a/comfy/ldm/minimax_music/ar.py b/comfy/ldm/minimax_music/ar.py
index 2a702c527..250822d7a 100644
--- a/comfy/ldm/minimax_music/ar.py
+++ b/comfy/ldm/minimax_music/ar.py
@@ -95,7 +95,7 @@ class RVQMLP(nn.Module):
def forward(self, x):
if self.merged_mlp:
- return comfy.ops.linear_input_act(self.down_proj, self.gate_up_proj(x), "swiglu")
+ return self.down_proj(self.gate_up_proj(x), input_act="swiglu")
return self.down_proj(torch.nn.functional.silu(self.gate_proj(x)) * self.up_proj(x))
diff --git a/comfy/ldm/qwen_image21/model.py b/comfy/ldm/qwen_image21/model.py
index e50d6c3cb..14c75e07d 100644
--- a/comfy/ldm/qwen_image21/model.py
+++ b/comfy/ldm/qwen_image21/model.py
@@ -62,7 +62,7 @@ class SwiGLUFeedForward(nn.Module):
def forward(self, x):
if self.fused:
- return comfy.ops.linear_input_act(self.out, self.gate_up(x), "swiglu")
+ return self.out(self.gate_up(x), input_act="swiglu")
return self.out(F.silu(self.gate_layer(x)) * self.proj(x))
diff --git a/comfy/ldm/wan/model.py b/comfy/ldm/wan/model.py
index 0925ac70c..642758f94 100644
--- a/comfy/ldm/wan/model.py
+++ b/comfy/ldm/wan/model.py
@@ -196,7 +196,7 @@ class WanFeedForward(nn.Sequential):
"""[Linear, GELU(tanh), Linear], with the GELU folded into the down-projection."""
def forward(self, x):
- return comfy.ops.linear_input_act(self[2], self[0](x), "gelu_tanh")
+ return self[2](self[0](x), input_act="gelu_tanh")
class WanAttentionBlock(nn.Module):
diff --git a/comfy/ops.py b/comfy/ops.py
index d5568af7f..c45cdf7d1 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -582,16 +582,20 @@ class disable_weight_init:
def reset_parameters(self):
return None
- def forward_comfy_cast_weights(self, input):
+ def forward_comfy_cast_weights(self, input, input_act=None, act_weight=None, act_eps=0.0,
+ residual=None, residual_scale=None):
with CastBiasWeightContext(self, input, offloadable=True) as (weight, bias):
- return torch.nn.functional.linear(input, weight, bias)
+ return linear_input_act_(input, weight, bias, input_act, act_weight, act_eps,
+ residual, residual_scale)
- def forward(self, *args, **kwargs):
+ def forward(self, input, input_act=None, act_weight=None, residual=None, residual_scale=None):
run_every_op()
if self.comfy_cast_weights or len(self.weight_function) > 0 or len(self.bias_function) > 0:
- return self.forward_comfy_cast_weights(*args, **kwargs)
+ return self.forward_comfy_cast_weights(input, input_act=input_act, act_weight=act_weight,
+ residual=residual, residual_scale=residual_scale)
else:
- return super().forward(*args, **kwargs)
+ return linear_input_act_(input, self.weight, self.bias, input_act, act_weight,
+ residual=residual, residual_scale=residual_scale)
class Conv1d(torch.nn.Conv1d, CastWeightBiasOp):
def reset_parameters(self):
@@ -701,8 +705,11 @@ class disable_weight_init:
self.bias = None
return None
+ def cast_weight(self, input):
+ return CastBiasWeightContext(self if self.weight is not None else None, input, offloadable=True)
+
def forward_comfy_cast_weights(self, input):
- with CastBiasWeightContext(self if self.weight is not None else None, input, offloadable=True) as (weight, bias):
+ with self.cast_weight(input) as (weight, bias):
return torch.nn.functional.rms_norm(input, self.normalized_shape, weight, self.eps)
def forward(self, *args, **kwargs):
@@ -920,17 +927,19 @@ class fp8_ops(manual_cast):
self.scale_input = None
return None
- def forward_comfy_cast_weights(self, input):
+ def forward_comfy_cast_weights(self, input, input_act=None, act_weight=None, act_eps=0.0,
+ residual=None, residual_scale=None):
+ input = _eager_input_act(input, input_act, act_weight, act_eps)
if len(self.weight_function) == 0 and len(self.bias_function) == 0:
try:
out = fp8_linear(self, input)
if out is not None:
- return out
+ return _linear_residual(out, residual, residual_scale)
except Exception as e:
logging.info("Exception during fp8 op: {}".format(e))
with CastBiasWeightContext(self, input, offloadable=True) as (weight, bias):
- return torch.nn.functional.linear(input, weight, bias)
+ return _linear_residual(torch.nn.functional.linear(input, weight, bias), residual, residual_scale)
CUBLAS_IS_AVAILABLE = False
try:
@@ -949,12 +958,14 @@ if CUBLAS_IS_AVAILABLE:
with CastBiasWeightContext(self, input, offloadable=True) as (weight, bias):
return cublas_half_matmul(input, weight, bias, self._epilogue_str, self.has_bias)
- def forward(self, *args, **kwargs):
+ def forward(self, input, input_act=None, act_weight=None, residual=None, residual_scale=None):
run_every_op()
+ input = _eager_input_act(input, input_act, act_weight)
if self.comfy_cast_weights or len(self.weight_function) > 0 or len(self.bias_function) > 0:
- return self.forward_comfy_cast_weights(*args, **kwargs)
+ out = self.forward_comfy_cast_weights(input)
else:
- return super().forward(*args, **kwargs)
+ out = super().forward(input)
+ return _linear_residual(out, residual, residual_scale)
# ==============================================================================
# Mixed Precision Operations
@@ -979,14 +990,31 @@ INPUT_ACT_EAGER = {
}
+def _input_act_weight(x, input_act, act_weight, act_eps):
+ if input_act == "rms_norm" and isinstance(act_weight, torch.nn.Module):
+ with act_weight.cast_weight(x) as (weight, _):
+ eps = act_weight.eps
+ if eps is None:
+ eps = torch.finfo(torch.float64 if x.dtype == torch.float64 else torch.float32).eps
+ yield weight, eps
+ else:
+ yield act_weight, act_eps
+
+
def _eager_input_act(x, input_act, act_weight=None, act_eps=0.0):
if input_act is None:
return x
if input_act == "rms_norm":
- return comfy.rmsnorm.rms_norm(x, act_weight, act_eps)
+ with _input_act_weight(x, input_act, act_weight, act_eps) as (weight, eps):
+ return comfy.rmsnorm.rms_norm(x, weight, eps)
return INPUT_ACT_EAGER[input_act](x)
+def _linear_residual(out, residual, residual_scale):
+ return out if residual is None else torch.addcmul(residual, out, residual_scale)
+
+
def _fp16_linear_wanted(x):
"""kitchen's fp16-accumulate GEMM replaces a plain linear when the user opted into
fp16 accumulation and the activation is fp16 on CUDA; weights come through cast_bias_weight."""
@@ -996,6 +1024,25 @@ def _fp16_linear_wanted(x):
def linear_input_act(linear, x, input_act, act_weight=None, act_eps=0.0,
residual=None, residual_scale=None):
+ run_every_op()
+ weight = linear.weight
+ quantized = isinstance(weight, QuantizedTensor)
+ if (comfy.model_management.in_training or getattr(linear, "_full_precision_mm", False)
+ or not ((quantized and weight._layout_cls == "TensorWiseINT8Layout"
+ and not getattr(weight._params, "transposed", False))
+ or (not quantized and _fp16_linear_wanted(x)))):
+ out = linear(_eager_input_act(x, input_act, act_weight, act_eps))
+ return _linear_residual(out, residual, residual_scale)
+
+ with CastBiasWeightContext(linear, x, offloadable=True,
+ compute_dtype=x.dtype if quantized else None,
+ want_requant=quantized) as (weight, bias):
+ return linear_input_act_(x, weight, bias, input_act, act_weight, act_eps,
+ residual, residual_scale, fp16_accumulation=not quantized)
+
+
+def linear_input_act_(x, weight, bias, input_act=None, act_weight=None, act_eps=0.0,
+ residual=None, residual_scale=None, *, fp16_accumulation=True):
"""``linear(act(x))``, with ``act`` folded into an INT8 activation quantizer.
An INT8 linear quantizes its input anyway, so an elementwise activation can
@@ -1009,43 +1056,25 @@ def linear_input_act(linear, x, input_act, act_weight=None, act_eps=0.0,
addcmul, ``residual + residual_scale * linear(act(x))``, fused into the
INT8 GEMM epilogue where supported.
+ Linear weights are already prepared. An RMSNorm module in act_weight owns
+ its norm-weight cast context; a tensor uses the supplied act_eps directly.
"""
- def _residual_out(out):
- if residual is None:
- return out
- return torch.addcmul(residual, out, residual_scale)
+ if input_act is None and residual is None:
+ return torch.nn.functional.linear(x, weight, bias)
- weight = linear.weight
- full_precision_mm = getattr(linear, "_full_precision_mm", False)
if (comfy.model_management.in_training
or not isinstance(weight, QuantizedTensor)
or weight._layout_cls != "TensorWiseINT8Layout"
- or getattr(weight._params, "transposed", False)
- or full_precision_mm):
+ or getattr(weight._params, "transposed", False)):
+ x = _eager_input_act(x, input_act, act_weight, act_eps)
if (not comfy.model_management.in_training
and not isinstance(weight, QuantizedTensor)
- and not full_precision_mm
- and _fp16_linear_wanted(x)):
- weight, bias, offload_stream = cast_bias_weight(linear, x, offloadable=True)
- try:
- return quant_ops.ck.fp16_linear(
- _eager_input_act(x, input_act, act_weight, act_eps),
- weight, bias, residual=residual, residual_scale=residual_scale)
- finally:
- uncast_bias_weight(linear, weight, bias, offload_stream)
- return _residual_out(linear(_eager_input_act(x, input_act, act_weight, act_eps)))
-
- # want_requant keeps a vbar-streamed layer on the INT8 path when a LoRA is
- # patched in on the fly; without it the cast hands back a dequantized weight.
- weight, bias, offload_stream = cast_bias_weight(
- linear, x, offloadable=True, compute_dtype=x.dtype, want_requant=True)
- try:
- if not isinstance(weight, QuantizedTensor):
- # A LoRA weight_function, or activations whose dtype differs from the
- # weight's, make the cast hand back a dequantized tensor.
- return _residual_out(torch.nn.functional.linear(
- _eager_input_act(x, input_act, act_weight, act_eps), weight, bias))
- qdata, scale = TensorWiseINT8Layout.get_plain_tensors(weight)
+ and fp16_accumulation and _fp16_linear_wanted(x)):
+ return quant_ops.ck.fp16_linear(x, weight, bias, residual=residual, residual_scale=residual_scale)
+ return _linear_residual(torch.nn.functional.linear(x, weight, bias), residual, residual_scale)
+
+ qdata, scale = TensorWiseINT8Layout.get_plain_tensors(weight)
+ with _input_act_weight(x, input_act, act_weight, act_eps) as (act_weight, act_eps):
return quant_ops.ck.int8_linear(
x, qdata, scale, bias, x.dtype,
convrot=getattr(weight._params, "convrot", False),
@@ -1056,8 +1085,6 @@ def linear_input_act(linear, x, input_act, act_weight=None, act_eps=0.0,
residual=residual,
residual_scale=residual_scale,
)
- finally:
- uncast_bias_weight(linear, weight, bias, offload_stream)
# Grouped-scale integer weights that run through the shared INT8 GEMM (comfy_kitchen
@@ -1424,6 +1451,11 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
compute_dtype=None,
want_requant=False,
weight_only_quant=False,
+ *,
+ input_act=None,
+ act_weight=None,
+ residual=None,
+ residual_scale=None,
):
if not weight_only_quant:
with CastBiasWeightContext(
@@ -1435,6 +1467,10 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
) as (weight, bias):
if self._full_precision_mm and isinstance(weight, QuantizedTensor):
weight = weight.dequantize()
+ if input_act is not None or residual is not None:
+ return linear_input_act_(input, weight, bias, input_act, act_weight,
+ residual=residual, residual_scale=residual_scale,
+ fp16_accumulation=not isinstance(self.weight, QuantizedTensor))
return self._forward(input, weight, bias)
with CastBiasWeightContext(
@@ -1450,8 +1486,18 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
weight = weight.to(dtype=input.dtype)
return self._forward(input, weight, bias)
- def forward(self, input, *args, **kwargs):
+ def forward(self, input, input_act=None, act_weight=None, residual=None, residual_scale=None):
run_every_op()
+ if input_act is not None or residual is not None:
+ quantized = isinstance(self.weight, QuantizedTensor)
+ if (not comfy.model_management.in_training and not self._full_precision_mm
+ and ((quantized and self.weight._layout_cls == "TensorWiseINT8Layout"
+ and not getattr(self.weight._params, "transposed", False))
+ or (not quantized and _fp16_linear_wanted(input)))):
+ return self.forward_comfy_cast_weights(input, compute_dtype=input.dtype, want_requant=quantized,
+ input_act=input_act, act_weight=act_weight,
+ residual=residual, residual_scale=residual_scale)
+ input = _eager_input_act(input, input_act, act_weight)
# ModelOpt AWQ-style smoothing
pre_quant_scale = getattr(self, 'pre_quant_scale', None)
@@ -1484,9 +1530,10 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
if scale is not None:
scale = comfy.model_management.cast_to_device(scale, input.device, None)
- return QuantLinearFunc.apply(
+ output = QuantLinearFunc.apply(
input, weight, bias, self.layout_type, scale, compute_dtype
)
+ return _linear_residual(output, residual, residual_scale)
# Inference path (unchanged)
if _use_quantized and quantize_input:
@@ -1515,7 +1562,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
if reshaped_nd:
output = output.reshape((*input_shape[:-1], self.weight.shape[0]))
- return output
+ return _linear_residual(output, residual, residual_scale)
def convert_weight(self, weight, inplace=False, **kwargs):
if isinstance(weight, QuantizedTensor):
diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py
index 161e65411..35ef22961 100644
--- a/comfy/text_encoders/llama.py
+++ b/comfy/text_encoders/llama.py
@@ -754,7 +754,7 @@ class MLP(nn.Module):
if self.merged_mlp:
x = self.gate_up_proj(x)
if self.merged_input_act is not None:
- return comfy.ops.linear_input_act(self.down_proj, x, self.merged_input_act)
+ return self.down_proj(x, input_act=self.merged_input_act)
gate, up = x.chunk(2, dim=-1)
return self.down_proj(self.activation(gate) * up)
return self.down_proj(self.activation(self.gate_proj(x)) * self.up_proj(x))
diff --git a/tests-unit/comfy_quant/test_mixed_precision.py b/tests-unit/comfy_quant/test_mixed_precision.py
index 03e1b0e6e..700daa92c 100644
--- a/tests-unit/comfy_quant/test_mixed_precision.py
+++ b/tests-unit/comfy_quant/test_mixed_precision.py
@@ -195,7 +195,7 @@ class TestMixedPrecisionOps(unittest.TestCase):
lora_delta = torch.randn_like(weight) * 0.01
return weight + lora_delta
- model.layer1.weight_function.append(apply_lora)
+ model.layer1.weight_function = [apply_lora]
# Forward pass should work with LoRA (triggers weight_function path)
input_tensor = torch.randn(5, 10, dtype=torch.bfloat16)
diff --git a/tests-unit/comfy_test/test_minimax_h3_vae.py b/tests-unit/comfy_test/test_minimax_h3_vae.py
index 77ed67ede..515d43476 100644
--- a/tests-unit/comfy_test/test_minimax_h3_vae.py
+++ b/tests-unit/comfy_test/test_minimax_h3_vae.py
@@ -50,16 +50,26 @@ def test_attention_moves_offloaded_qk_norm_scale_to_input_device(monkeypatch):
attn._buffers["qk_norm_scale"] = _OffloadedScale("meta")
batch_size, seq_len = 1, 3
- x = torch.randn(batch_size, seq_len, heads * dim_head)
+ x = torch.arange(1, 25, dtype=torch.float32).reshape(batch_size, seq_len, heads * dim_head)
rotary_pos_emb = torch.randn(batch_size, 1, seq_len, dim_head // 2, 2, 2)
- class _Norm:
- weight = None
- eps = 1e-5
-
+ pre_norm = comfy.ops.manual_cast.RMSNorm(heads * dim_head, elementwise_affine=False, eps=1e-5)
+ qkv_outputs = []
+ hook = attn.to_qkv.register_forward_hook(lambda module, inputs, output: qkv_outputs.append(output))
with torch.no_grad():
+ attn.to_qkv.weight.copy_(torch.eye(heads * dim_head).repeat(3, 1))
+ attn.to_qkv.bias.zero_()
+ attn.to_out.weight.copy_(torch.eye(heads * dim_head))
+ attn.to_out.bias.zero_()
out = attn.forward(
- x, rotary_pos_emb, pre_norm=_Norm(), residual=None, residual_scale=None
+ x, rotary_pos_emb, pre_norm=pre_norm, residual=None, residual_scale=None
)
+ hook.remove()
assert out.device == x.device
+ normalized = torch.nn.functional.rms_norm(x, (heads * dim_head,), eps=pre_norm.eps)
+ expected = torch.nn.functional.linear(normalized, attn.to_qkv.weight, attn.to_qkv.bias)
+ unnormalized = torch.nn.functional.linear(x, attn.to_qkv.weight, attn.to_qkv.bias)
+ assert len(qkv_outputs) == 1
+ torch.testing.assert_close(qkv_outputs[0], expected)
+ assert not torch.allclose(qkv_outputs[0], unnormalized)