The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
[Trainer] training with proper offloading (#12189)
Kohaku-Blueleafcdcf4119b3e8
diff --git a/comfy/ldm/flux/math.py b/comfy/ldm/flux/math.pyindex f9597de5b..5e764bb46 100644--- a/comfy/ldm/flux/math.py+++ b/comfy/ldm/flux/math.py@@ -29,19 +29,34 @@ def rope(pos: Tensor, dim: int, theta: int) -> Tensor:return out.to(dtype=torch.float32, device=pos.device)+def _apply_rope1(x: Tensor, freqs_cis: Tensor):+ x_ = x.to(dtype=freqs_cis.dtype).reshape(*x.shape[:-1], -1, 1, 2)++ x_out = freqs_cis[..., 0] * x_[..., 0]+ x_out.addcmul_(freqs_cis[..., 1], x_[..., 1])++ return x_out.reshape(*x.shape).type_as(x)+++def _apply_rope(xq: Tensor, xk: Tensor, freqs_cis: Tensor):+ return apply_rope1(xq, freqs_cis), apply_rope1(xk, freqs_cis)++try:import comfy.quant_ops- apply_rope = comfy.quant_ops.ck.apply_rope- apply_rope1 = comfy.quant_ops.ck.apply_rope1+ q_apply_rope = comfy.quant_ops.ck.apply_rope+ q_apply_rope1 = comfy.quant_ops.ck.apply_rope1+ def apply_rope(xq, xk, freqs_cis):+ if comfy.model_management.in_training:+ return _apply_rope(xq, xk, freqs_cis)+ else:+ return apply_rope1(xq, freqs_cis), apply_rope1(xk, freqs_cis)+ def apply_rope1(x, freqs_cis):+ if comfy.model_management.in_training:+ return _apply_rope1(x, freqs_cis)+ else:+ return q_apply_rope1(x, freqs_cis)except:logging.warning("No comfy kitchen, using old apply_rope functions.")- def apply_rope1(x: Tensor, freqs_cis: Tensor):- x_ = x.to(dtype=freqs_cis.dtype).reshape(*x.shape[:-1], -1, 1, 2)-- x_out = freqs_cis[..., 0] * x_[..., 0]- x_out.addcmul_(freqs_cis[..., 1], x_[..., 1])-- return x_out.reshape(*x.shape).type_as(x)-- def apply_rope(xq: Tensor, xk: Tensor, freqs_cis: Tensor):- return apply_rope1(xq, freqs_cis), apply_rope1(xk, freqs_cis)+ apply_rope = _apply_rope+ apply_rope1 = _apply_rope1diff --git a/comfy/model_management.py b/comfy/model_management.pyindex 6018c1ab6..304931eb0 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -55,6 +55,11 @@ cpu_state = CPUState.GPUtotal_vram = 0++# Training Related State+in_training = False++def get_supported_float8_types():float8_types = []try:diff --git a/comfy/sampler_helpers.py b/comfy/sampler_helpers.pyindex 9134e6d71..1f75f2ba7 100644--- a/comfy/sampler_helpers.py+++ b/comfy/sampler_helpers.py@@ -122,20 +122,26 @@ def estimate_memory(model, noise_shape, conds):minimum_memory_required = model.model.memory_required([noise_shape[0]] + list(noise_shape[1:]), cond_shapes=cond_shapes_min)return memory_required, minimum_memory_required-def prepare_sampling(model: ModelPatcher, noise_shape, conds, model_options=None, force_full_load=False):+def prepare_sampling(model: ModelPatcher, noise_shape, conds, model_options=None, force_full_load=False, force_offload=False):executor = comfy.patcher_extension.WrapperExecutor.new_executor(_prepare_sampling,comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.PREPARE_SAMPLING, model_options, is_model_options=True))- return executor.execute(model, noise_shape, conds, model_options=model_options, force_full_load=force_full_load)+ return executor.execute(model, noise_shape, conds, model_options=model_options, force_full_load=force_full_load, force_offload=force_offload)-def _prepare_sampling(model: ModelPatcher, noise_shape, conds, model_options=None, force_full_load=False):+def _prepare_sampling(model: ModelPatcher, noise_shape, conds, model_options=None, force_full_load=False, force_offload=False):real_model: BaseModel = Nonemodels, inference_memory = get_additional_models(conds, model.model_dtype())models += get_additional_models_from_model_options(model_options)models += model.get_nested_additional_models() # TODO: does this require inference_memory update?- memory_required, minimum_memory_required = estimate_memory(model, noise_shape, conds)- comfy.model_management.load_models_gpu([model] + models, memory_required=memory_required + inference_memory, minimum_memory_required=minimum_memory_required + inference_memory, force_full_load=force_full_load)+ if force_offload: # In training + offload enabled, we want to force prepare sampling to trigger partial load+ memory_required = 1e20+ minimum_memory_required = None+ else:+ memory_required, minimum_memory_required = estimate_memory(model, noise_shape, conds)+ memory_required += inference_memory+ minimum_memory_required += inference_memory+ comfy.model_management.load_models_gpu([model] + models, memory_required=memory_required, minimum_memory_required=minimum_memory_required, force_full_load=force_full_load)real_model = model.modelreturn real_model, conds, modelsdiff --git a/comfy/weight_adapter/bypass.py b/comfy/weight_adapter/bypass.pyindex d4aaf98ca..b9d5ec7d9 100644--- a/comfy/weight_adapter/bypass.py+++ b/comfy/weight_adapter/bypass.py@@ -21,6 +21,7 @@ from typing import Optional, Unionimport torchimport torch.nn as nn+import comfy.model_managementfrom .base import WeightAdapterBase, WeightAdapterTrainBasefrom comfy.patcher_extension import PatcherInjection@@ -181,18 +182,21 @@ class BypassForwardHook:)return # Already injected- # Move adapter weights to module's device to avoid CPU-GPU transfer on every forward- device = None+ # Move adapter weights to compute device (GPU)+ # Use get_torch_device() instead of module.weight.device because+ # with offloading, module weights may be on CPU while compute happens on GPU+ device = comfy.model_management.get_torch_device()++ # Get dtype from module weight if availabledtype = Noneif hasattr(self.module, "weight") and self.module.weight is not None:- device = self.module.weight.devicedtype = self.module.weight.dtype- elif hasattr(self.module, "W_q"): # Quantized layers might use different attr- device = self.module.W_q.device- dtype = self.module.W_q.dtype- if device is not None:- self._move_adapter_weights_to_device(device, dtype)+ # Only use dtype if it's a standard float type, not quantized+ if dtype is not None and dtype not in (torch.float32, torch.float16, torch.bfloat16):+ dtype = None++ self._move_adapter_weights_to_device(device, dtype)self.original_forward = self.module.forwardself.module.forward = self._bypass_forwarddiff --git a/comfy_extras/nodes_train.py b/comfy_extras/nodes_train.pyindex 024a89391..630eedc9f 100644--- a/comfy_extras/nodes_train.py+++ b/comfy_extras/nodes_train.py@@ -4,6 +4,7 @@ import osimport numpy as npimport safetensorsimport torch+import torch.nn as nnimport torch.utils.checkpointfrom tqdm.auto import trangefrom PIL import Image, ImageDraw, ImageFont@@ -27,6 +28,11 @@ class TrainGuider(comfy_extras.nodes_custom_sampler.Guider_Basic):"""CFGGuider with modifications for training specific logic"""++ def __init__(self, *args, offloading=False, **kwargs):+ super().__init__(*args, **kwargs)+ self.offloading = offloading+def outer_sample(self,noise,@@ -45,9 +51,11 @@ class TrainGuider(comfy_extras.nodes_custom_sampler.Guider_Basic):noise.shape,self.conds,self.model_options,- force_full_load=True, # mirror behavior in TrainLoraNode.execute() to keep model loaded+ force_full_load=not self.offloading,+ force_offload=self.offloading,))+ torch.cuda.empty_cache()device = self.model_patcher.load_deviceif denoise_mask is not None:@@ -404,16 +412,97 @@ def find_all_highest_child_module_with_forward(return result-def patch(m):+def find_modules_at_depth(+ model: nn.Module, depth: int = 1, result=None, current_depth=0, name=None+) -> list[nn.Module]:+ """+ Find modules at a specific depth level for gradient checkpointing.++ Args:+ model: The model to search+ depth: Target depth level (1 = top-level blocks, 2 = their children, etc.)+ result: Accumulator for results+ current_depth: Current recursion depth+ name: Current module name for logging++ Returns:+ List of modules at the target depth+ """+ if result is None:+ result = []+ name = name or "root"++ # Skip container modules (they don't have meaningful forward)+ is_container = isinstance(model, (nn.ModuleList, nn.Sequential, nn.ModuleDict))+ has_forward = hasattr(model, "forward") and not is_container++ if has_forward:+ current_depth += 1+ if current_depth == depth:+ result.append(model)+ logging.debug(f"Found module at depth {depth}: {name} ({model.__class__.__name__})")+ return result++ # Recurse into children+ for next_name, child in model.named_children():+ find_modules_at_depth(child, depth, result, current_depth, f"{name}.{next_name}")++ return result+++class OffloadCheckpointFunction(torch.autograd.Function):+ """+ Gradient checkpointing that works with weight offloading.++ Forward: no_grad -> compute -> weights can be freed+ Backward: enable_grad -> recompute -> backward -> weights can be freed++ For single input, single output modules (Linear, Conv*).+ """++ @staticmethod+ def forward(ctx, x: torch.Tensor, forward_fn):+ ctx.save_for_backward(x)+ ctx.forward_fn = forward_fn+ with torch.no_grad():+ return forward_fn(x)++ @staticmethod+ def backward(ctx, grad_out: torch.Tensor):+ x, = ctx.saved_tensors+ forward_fn = ctx.forward_fn++ # Clear context early+ ctx.forward_fn = None++ with torch.enable_grad():+ x_detached = x.detach().requires_grad_(True)+ y = forward_fn(x_detached)+ y.backward(grad_out)+ grad_x = x_detached.grad++ # Explicit cleanup+ del y, x_detached, forward_fn++ return grad_x, None+++def patch(m, offloading=False):if not hasattr(m, "forward"):returnorg_forward = m.forward- def fwd(args, kwargs):- return org_forward(*args, **kwargs)+ # Branch 1: Linear/Conv* -> offload-compatible checkpoint (single input/output)+ if offloading and isinstance(m, (nn.Linear, nn.Conv1d, nn.Conv2d, nn.Conv3d)):+ def checkpointing_fwd(x):+ return OffloadCheckpointFunction.apply(x, org_forward)+ # Branch 2: Others -> standard checkpoint+ else:+ def fwd(args, kwargs):+ return org_forward(*args, **kwargs)- def checkpointing_fwd(*args, **kwargs):- return torch.utils.checkpoint.checkpoint(fwd, args, kwargs, use_reentrant=False)+ def checkpointing_fwd(*args, **kwargs):+ return torch.utils.checkpoint.checkpoint(fwd, args, kwargs, use_reentrant=False)m.org_forward = org_forwardm.forward = checkpointing_fwd@@ -936,6 +1025,18 @@ class TrainLoraNode(io.ComfyNode):default=True,tooltip="Use gradient checkpointing for training.",),+ io.Int.Input(+ "checkpoint_depth",+ default=1,+ min=1,+ max=5,+ tooltip="Depth level for gradient checkpointing.",+ ),+ io.Boolean.Input(+ "offloading",+ default=False,+ tooltip="Depth level for gradient checkpointing.",+ ),io.Combo.Input("existing_lora",options=folder_paths.get_filename_list("loras") + ["[None]"],@@ -982,6 +1083,8 @@ class TrainLoraNode(io.ComfyNode):lora_dtype,algorithm,gradient_checkpointing,+ checkpoint_depth,+ offloading,existing_lora,bucket_mode,bypass_mode,@@ -1000,6 +1103,8 @@ class TrainLoraNode(io.ComfyNode):lora_dtype = lora_dtype[0]algorithm = algorithm[0]gradient_checkpointing = gradient_checkpointing[0]+ offloading = offloading[0]+ checkpoint_depth = checkpoint_depth[0]existing_lora = existing_lora[0]bucket_mode = bucket_mode[0]bypass_mode = bypass_mode[0]@@ -1054,16 +1159,18 @@ class TrainLoraNode(io.ComfyNode):# Setup gradient checkpointingif gradient_checkpointing:- for m in find_all_highest_child_module_with_forward(- mp.model.diffusion_model- ):- patch(m)+ modules_to_patch = find_modules_at_depth(+ mp.model.diffusion_model, depth=checkpoint_depth+ )+ logging.info(f"Gradient checkpointing: patching {len(modules_to_patch)} modules at depth {checkpoint_depth}")+ for m in modules_to_patch:+ patch(m, offloading=offloading)torch.cuda.empty_cache()# With force_full_load=False we should be able to have offloading# But for offloading in training we need custom AutoGrad hooks for fwd/bwdcomfy.model_management.load_models_gpu(- [mp], memory_required=1e20, force_full_load=True+ [mp], memory_required=1e20, force_full_load=not offloading)torch.cuda.empty_cache()@@ -1100,7 +1207,7 @@ class TrainLoraNode(io.ComfyNode):)# Setup guider- guider = TrainGuider(mp)+ guider = TrainGuider(mp, offloading=offloading)guider.set_conds(positive)# Inject bypass hooks if bypass mode is enabled@@ -1113,6 +1220,7 @@ class TrainLoraNode(io.ComfyNode):# Run training looptry:+ comfy.model_management.in_training = True_run_training_loop(guider,train_sampler,@@ -1123,6 +1231,7 @@ class TrainLoraNode(io.ComfyNode):multi_res,)finally:+ comfy.model_management.in_training = False# Eject bypass hooks if they were injectedif bypass_injections is not None:for injection in bypass_injections:@@ -1132,19 +1241,20 @@ class TrainLoraNode(io.ComfyNode):unpatch(m)del train_sampler, optimizer- # Finalize adapters+ for param in lora_sd:+ lora_sd[param] = lora_sd[param].to(lora_dtype).detach()+for adapter in all_weight_adapters:adapter.requires_grad_(False)-- for param in lora_sd:- lora_sd[param] = lora_sd[param].to(lora_dtype)+ del adapter+ del all_weight_adapters# mp in train node is highly specialized for training# use it in inference will result in bad behavior so we don't return itreturn io.NodeOutput(lora_sd, loss_map, steps + existing_steps)-class LoraModelLoader(io.ComfyNode):#+class LoraModelLoader(io.ComfyNode):@classmethoddef define_schema(cls):return io.Schema(@@ -1166,6 +1276,11 @@ class LoraModelLoader(io.ComfyNode):#max=100.0,tooltip="How strongly to modify the diffusion model. This value can be negative.",),+ io.Boolean.Input(+ "bypass",+ default=False,+ tooltip="When enabled, applies LoRA in bypass mode without modifying base model weights. Useful for training and when model weights are offloaded.",+ ),],outputs=[io.Model.Output(@@ -1175,13 +1290,18 @@ class LoraModelLoader(io.ComfyNode):#)@classmethod- def execute(cls, model, lora, strength_model):+ def execute(cls, model, lora, strength_model, bypass=False):if strength_model == 0:return io.NodeOutput(model)- model_lora, _ = comfy.sd.load_lora_for_models(- model, None, lora, strength_model, 0- )+ if bypass:+ model_lora, _ = comfy.sd.load_bypass_lora_for_models(+ model, None, lora, strength_model, 0+ )+ else:+ model_lora, _ = comfy.sd.load_lora_for_models(+ model, None, lora, strength_model, 0+ )return io.NodeOutput(model_lora)