The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
[Trainer/bug] Ensure model is not inference mode (CORE-72) (#13400)
Kohaku-Blueleaff350acdf213a
diff --git a/comfy/model_patcher.py b/comfy/model_patcher.pyindex b716a69e2..d70b42bf8 100644--- a/comfy/model_patcher.py+++ b/comfy/model_patcher.py@@ -379,10 +379,11 @@ class ModelPatcher:def get_clone_model_override(self):return self.model, (self.backup, self.backup_buffers, self.object_patches_backup, self.pinned)- def clone(self, disable_dynamic=False, model_override=None):+ def clone(self, disable_dynamic=False, model_override=None, force_deepcopy=False):class_ = self.__class__- if self.is_dynamic() and disable_dynamic:- class_ = ModelPatcher+ if self.is_dynamic() and disable_dynamic or force_deepcopy:+ if self.is_dynamic() and disable_dynamic:+ class_ = ModelPatcherif model_override is None:if self.cached_patcher_init is None:raise RuntimeError("Cannot create non-dynamic delegate: cached_patcher_init is not initialized.")diff --git a/comfy_extras/nodes_train.py b/comfy_extras/nodes_train.pyindex 273f55e7c..bb68da6fa 100644--- a/comfy_extras/nodes_train.py+++ b/comfy_extras/nodes_train.py@@ -1149,45 +1149,45 @@ class TrainLoraNode(io.ComfyNode):# Process conditioningpositive = _process_conditioning(positive)- # Setup model and dtype- mp = model.clone()- use_grad_scaler = False- lora_dtype = node_helpers.string_to_torch_dtype(lora_dtype)- if training_dtype != "none":- dtype = node_helpers.string_to_torch_dtype(training_dtype)- mp.set_model_compute_dtype(dtype)- else:- # Detect model's native dtype for autocast- model_dtype = mp.model.get_dtype()- if model_dtype == torch.float16:- dtype = torch.float16- # GradScaler only supports float16 gradients, not bfloat16.- # Only enable it when lora params will also be in float16.- if lora_dtype != torch.bfloat16:- use_grad_scaler = True- # Warn about fp16 accumulation instability during training- if PerformanceFeature.Fp16Accumulation in args.fast:- logging.warning(- "WARNING: FP16 model detected with fp16_accumulation enabled. "- "This combination can be numerically unstable during training and may cause NaN values. "- "Suggested fixes: 1) Set training_dtype to 'bf16', or 2) Disable fp16_accumulation (remove from --fast flags)."- )+ with torch.inference_mode(False):+ # Setup model and dtype+ mp = model.clone(force_deepcopy=True)+ use_grad_scaler = False+ lora_dtype = node_helpers.string_to_torch_dtype(lora_dtype)+ if training_dtype != "none":+ dtype = node_helpers.string_to_torch_dtype(training_dtype)+ mp.set_model_compute_dtype(dtype)else:- # For fp8, bf16, or other dtypes, use bf16 autocast- dtype = torch.bfloat16+ # Detect model's native dtype for autocast+ model_dtype = mp.model.get_dtype()+ if model_dtype == torch.float16:+ dtype = torch.float16+ # GradScaler only supports float16 gradients, not bfloat16.+ # Only enable it when lora params will also be in float16.+ if lora_dtype != torch.bfloat16:+ use_grad_scaler = True+ # Warn about fp16 accumulation instability during training+ if PerformanceFeature.Fp16Accumulation in args.fast:+ logging.warning(+ "WARNING: FP16 model detected with fp16_accumulation enabled. "+ "This combination can be numerically unstable during training and may cause NaN values. "+ "Suggested fixes: 1) Set training_dtype to 'bf16', or 2) Disable fp16_accumulation (remove from --fast flags)."+ )+ else:+ # For fp8, bf16, or other dtypes, use bf16 autocast+ dtype = torch.bfloat16- # Prepare latents and compute counts- latents_dtype = dtype if dtype not in (None,) else torch.bfloat16- latents, num_images, multi_res = _prepare_latents_and_count(- latents, latents_dtype, bucket_mode- )+ # Prepare latents and compute counts+ latents_dtype = dtype if dtype not in (None,) else torch.bfloat16+ latents, num_images, multi_res = _prepare_latents_and_count(+ latents, latents_dtype, bucket_mode+ )- # Validate and expand conditioning- positive = _validate_and_expand_conditioning(positive, num_images, bucket_mode)+ # Validate and expand conditioning+ positive = _validate_and_expand_conditioning(positive, num_images, bucket_mode)- with torch.inference_mode(False):# Setup models for training- mp.model.requires_grad_(False)+ mp.model.requires_grad_(False).train()# Load existing LoRA weights if providedexisting_weights, existing_steps = _load_existing_lora(existing_lora)