Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

More fp8 torch.compile regressions fixed. (#10625)

comfyanonymousaf4b7b5edb33
diff --git a/comfy/quant_ops.py b/comfy/quant_ops.py
index ed7b29963..c56e32a73 100644
--- a/comfy/quant_ops.py
+++ b/comfy/quant_ops.py
@@ -446,6 +446,25 @@ def fp8_linear(func, args, kwargs):
return torch.nn.functional.linear(input_tensor, weight, bias)
+def fp8_mm_(input_tensor, weight, bias=None, out_dtype=None):
+ if out_dtype is None:
+ out_dtype = input_tensor._layout_params['orig_dtype']
+
+ plain_input, scale_a = TensorCoreFP8Layout.get_plain_tensors(input_tensor)
+ plain_weight, scale_b = TensorCoreFP8Layout.get_plain_tensors(weight)
+
+ output = torch._scaled_mm(
+ plain_input.contiguous(),
+ plain_weight,
+ bias=bias,
+ scale_a=scale_a,
+ scale_b=scale_b,
+ out_dtype=out_dtype,
+ )
+
+ if isinstance(output, tuple): # TODO: remove when we drop support for torch 2.4
+ output = output[0]
+ return output
@register_layout_op(torch.ops.aten.addmm.default, "TensorCoreFP8Layout")
def fp8_addmm(func, args, kwargs):
@@ -454,25 +473,7 @@ def fp8_addmm(func, args, kwargs):
bias = args[0]
if isinstance(input_tensor, QuantizedTensor) and isinstance(weight, QuantizedTensor):
- out_dtype = kwargs.get("out_dtype")
- if out_dtype is None:
- out_dtype = input_tensor._layout_params['orig_dtype']
-
- plain_input, scale_a = TensorCoreFP8Layout.get_plain_tensors(input_tensor)
- plain_weight, scale_b = TensorCoreFP8Layout.get_plain_tensors(weight)
-
- output = torch._scaled_mm(
- plain_input.contiguous(),
- plain_weight,
- bias=bias,
- scale_a=scale_a,
- scale_b=scale_b,
- out_dtype=out_dtype,
- )
-
- if isinstance(output, tuple): # TODO: remove when we drop support for torch 2.4
- output = output[0]
- return output
+ return fp8_mm_(input_tensor, weight, bias=bias, out_dtype=kwargs.get("out_dtype", None))
a = list(args)
if isinstance(args[0], QuantizedTensor):
@@ -484,6 +485,21 @@ def fp8_addmm(func, args, kwargs):
return func(*a, **kwargs)
+@register_layout_op(torch.ops.aten.mm.default, "TensorCoreFP8Layout")
+def fp8_mm(func, args, kwargs):
+ input_tensor = args[0]
+ weight = args[1]
+
+ if isinstance(input_tensor, QuantizedTensor) and isinstance(weight, QuantizedTensor):
+ return fp8_mm_(input_tensor, weight, bias=None, out_dtype=kwargs.get("out_dtype", None))
+
+ a = list(args)
+ if isinstance(args[0], QuantizedTensor):
+ a[0] = args[0].dequantize()
+ if isinstance(args[1], QuantizedTensor):
+ a[1] = args[1].dequantize()
+ return func(*a, **kwargs)
+
@register_layout_op(torch.ops.aten.view.default, "TensorCoreFP8Layout")
@register_layout_op(torch.ops.aten.t.default, "TensorCoreFP8Layout")
def fp8_func(func, args, kwargs):