The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
FP8 bwd training (#13121)
Kohaku-Blueleaf5ebb0c2e0b72
diff --git a/comfy/model_management.py b/comfy/model_management.pyindex 2c250dacc..9617d8388 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -55,6 +55,7 @@ total_vram = 0# Training Related Statein_training = False+training_fp8_bwd = Falsedef get_supported_float8_types():diff --git a/comfy/ops.py b/comfy/ops.pyindex 1518ec9de..ca25693db 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -777,8 +777,16 @@ from .quant_ops import (class QuantLinearFunc(torch.autograd.Function):- """Custom autograd function for quantized linear: quantized forward, compute_dtype backward.- Handles any input rank by flattening to 2D for matmul and restoring shape after.+ """Custom autograd function for quantized linear: quantized forward, optionally FP8 backward.++ When training_fp8_bwd is enabled:+ - Forward: quantize input per layout (FP8/NVFP4), use quantized matmul+ - Backward: all matmuls use FP8 tensor cores via torch.mm dispatch+ - Cached input is FP8 (half the memory of bf16)++ When training_fp8_bwd is disabled:+ - Forward: quantize input per layout, use quantized matmul+ - Backward: dequantize weight to compute_dtype, use standard matmul"""@staticmethod@@ -786,7 +794,7 @@ class QuantLinearFunc(torch.autograd.Function):input_shape = input_float.shapeinp = input_float.detach().flatten(0, -2) # zero-cost view to 2D- # Quantize input (same as inference path)+ # Quantize input for forward (same layout as weight)if layout_type is not None:q_input = QuantizedTensor.from_float(inp, layout_type, scale=input_scale)else:@@ -797,43 +805,68 @@ class QuantLinearFunc(torch.autograd.Function):output = torch.nn.functional.linear(q_input, w, b)- # Restore original input shape+ # Unflatten output to match original input shapeif len(input_shape) > 2:output = output.unflatten(0, input_shape[:-1])- ctx.save_for_backward(input_float, weight)+ # Save for backwardctx.input_shape = input_shapectx.has_bias = bias is not Nonectx.compute_dtype = compute_dtypectx.weight_requires_grad = weight.requires_grad+ ctx.fp8_bwd = comfy.model_management.training_fp8_bwd++ if ctx.fp8_bwd:+ # Cache FP8 quantized input — half the memory of bf16+ if isinstance(q_input, QuantizedTensor) and layout_type.startswith('TensorCoreFP8'):+ ctx.q_input = q_input # already FP8, reuse+ else:+ # NVFP4 or other layout — quantize input to FP8 for backward+ ctx.q_input = QuantizedTensor.from_float(inp, "TensorCoreFP8E4M3Layout")+ ctx.save_for_backward(weight)+ else:+ ctx.q_input = None+ ctx.save_for_backward(input_float, weight)return output@staticmethod@torch.autograd.function.once_differentiabledef backward(ctx, grad_output):- input_float, weight = ctx.saved_tensorscompute_dtype = ctx.compute_dtypegrad_2d = grad_output.flatten(0, -2).to(compute_dtype)- # Dequantize weight to compute dtype for backward matmul- if isinstance(weight, QuantizedTensor):- weight_f = weight.dequantize().to(compute_dtype)+ # Value casting — only difference between fp8 and non-fp8 paths+ if ctx.fp8_bwd:+ weight, = ctx.saved_tensors+ # Wrap as FP8 QuantizedTensors → torch.mm dispatches to _scaled_mm+ grad_mm = QuantizedTensor.from_float(grad_2d, "TensorCoreFP8E5M2Layout")+ if isinstance(weight, QuantizedTensor) and weight._layout_cls.startswith("TensorCoreFP8"):+ weight_mm = weight+ elif isinstance(weight, QuantizedTensor):+ weight_mm = QuantizedTensor.from_float(weight.dequantize().to(compute_dtype), "TensorCoreFP8E4M3Layout")+ else:+ weight_mm = QuantizedTensor.from_float(weight.to(compute_dtype), "TensorCoreFP8E4M3Layout")+ input_mm = ctx.q_inputelse:- weight_f = weight.to(compute_dtype)+ input_float, weight = ctx.saved_tensors+ # Standard tensors → torch.mm does regular matmul+ grad_mm = grad_2d+ if isinstance(weight, QuantizedTensor):+ weight_mm = weight.dequantize().to(compute_dtype)+ else:+ weight_mm = weight.to(compute_dtype)+ input_mm = input_float.flatten(0, -2).to(compute_dtype) if ctx.weight_requires_grad else None- # grad_input = grad_output @ weight- grad_input = torch.mm(grad_2d, weight_f)+ # Computation — same for both paths, dispatch handles the rest+ grad_input = torch.mm(grad_mm, weight_mm)if len(ctx.input_shape) > 2:grad_input = grad_input.unflatten(0, ctx.input_shape[:-1])- # grad_weight (only if weight requires grad, typically frozen for quantized training)grad_weight = Noneif ctx.weight_requires_grad:- input_f = input_float.flatten(0, -2).to(compute_dtype)- grad_weight = torch.mm(grad_2d.t(), input_f)+ grad_weight = torch.mm(grad_mm.t(), input_mm)- # grad_biasgrad_bias = Noneif ctx.has_bias:grad_bias = grad_2d.sum(dim=0)diff --git a/comfy_extras/nodes_train.py b/comfy_extras/nodes_train.pyindex 0ad0acee6..df1b39fd5 100644--- a/comfy_extras/nodes_train.py+++ b/comfy_extras/nodes_train.py@@ -1030,6 +1030,11 @@ class TrainLoraNode(io.ComfyNode):default="bf16",tooltip="The dtype to use for lora.",),+ io.Boolean.Input(+ "quantized_backward",+ default=False,+ tooltip="When using training_dtype 'none' and training on quantized model, doing backward with quantized matmul when enabled.",+ ),io.Combo.Input("algorithm",options=list(adapter_maps.keys()),@@ -1097,6 +1102,7 @@ class TrainLoraNode(io.ComfyNode):seed,training_dtype,lora_dtype,+ quantized_backward,algorithm,gradient_checkpointing,checkpoint_depth,@@ -1117,6 +1123,7 @@ class TrainLoraNode(io.ComfyNode):seed = seed[0]training_dtype = training_dtype[0]lora_dtype = lora_dtype[0]+ quantized_backward = quantized_backward[0]algorithm = algorithm[0]gradient_checkpointing = gradient_checkpointing[0]offloading = offloading[0]@@ -1125,6 +1132,8 @@ class TrainLoraNode(io.ComfyNode):bucket_mode = bucket_mode[0]bypass_mode = bypass_mode[0]+ comfy.model_management.training_fp8_bwd = quantized_backward+# Process latents based on modeif bucket_mode:latents = _process_latents_bucket_mode(latents)