The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Make highvram and normalvram shift the text encoders to vram and back.
comfyanonymous97ee230682c9
diff --git a/comfy/model_management.py b/comfy/model_management.pyindex 4f3f28571..f10d1ca87 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -327,12 +327,18 @@ def unload_if_low_vram(model):return model.cpu()return model-def text_encoder_device():+def text_encoder_offload_device():if args.gpu_only:return get_torch_device()else:return torch.device("cpu")+def text_encoder_device():+ if vram_state == VRAMState.HIGH_VRAM or vram_state == VRAMState.SHARED or vram_state == VRAMState.NORMAL_VRAM:+ return get_torch_device()+ else:+ return torch.device("cpu")+def get_autocast_device(dev):if hasattr(dev, 'type'):return dev.type@@ -422,10 +428,15 @@ def mps_mode():global cpu_statereturn cpu_state == CPUState.MPS-def should_use_fp16():+def should_use_fp16(device=None):global xpu_availableglobal directml_enabled+ if device is not None: #TODO+ if hasattr(device, 'type'):+ if (device.type == 'cpu' or device.type == 'mps'):+ return False+if FORCE_FP32:return Falsediff --git a/comfy/sd.py b/comfy/sd.pyindex 8eac1f8ed..320b0fb71 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -526,9 +526,10 @@ class CLIP:tokenizer = target.tokenizerself.device = model_management.text_encoder_device()- params["device"] = self.deviceself.cond_stage_model = clip(**(params))- self.cond_stage_model = self.cond_stage_model.to(self.device)+ if model_management.should_use_fp16(self.device):+ self.cond_stage_model.half()+ self.cond_stage_model = self.cond_stage_model.to(model_management.text_encoder_offload_device())self.tokenizer = tokenizer(embedding_directory=embedding_directory)self.patcher = ModelPatcher(self.cond_stage_model)@@ -559,11 +560,14 @@ class CLIP:if self.layer_idx is not None:self.cond_stage_model.clip_layer(self.layer_idx)try:+ self.cond_stage_model.to(self.device)self.patch_model()cond, pooled = self.cond_stage_model.encode_token_weights(tokens)self.unpatch_model()+ self.cond_stage_model.to(model_management.text_encoder_offload_device())except Exception as e:self.unpatch_model()+ self.cond_stage_model.to(model_management.text_encoder_offload_device())raise econd_out = conddiff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.pyindex 02a998e5b..5c627cb8c 100644--- a/comfy/sd1_clip.py+++ b/comfy/sd1_clip.py@@ -5,6 +5,8 @@ import comfy.opsimport torchimport tracebackimport zipfile+from . import model_management+import contextlibclass ClipTokenWeightEncoder:def encode_token_weights(self, token_weight_pairs):@@ -46,7 +48,6 @@ class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder):with modeling_utils.no_init_weights():self.transformer = CLIPTextModel(config)- self.device = deviceself.max_length = max_lengthif freeze:self.freeze()@@ -95,7 +96,7 @@ class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder):out_tokens += [tokens_temp]if len(embedding_weights) > 0:- new_embedding = torch.nn.Embedding(next_new_token, current_embeds.weight.shape[1], device=self.device)+ new_embedding = torch.nn.Embedding(next_new_token, current_embeds.weight.shape[1], device=current_embeds.weight.device, dtype=current_embeds.weight.dtype)new_embedding.weight[:token_dict_size] = current_embeds.weight[:]n = token_dict_sizefor x in embedding_weights:@@ -106,24 +107,34 @@ class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder):def forward(self, tokens):backup_embeds = self.transformer.get_input_embeddings()+ device = backup_embeds.weight.devicetokens = self.set_up_textual_embeddings(tokens, backup_embeds)- tokens = torch.LongTensor(tokens).to(self.device)- outputs = self.transformer(input_ids=tokens, output_hidden_states=self.layer=="hidden")- self.transformer.set_input_embeddings(backup_embeds)-- if self.layer == "last":- z = outputs.last_hidden_state- elif self.layer == "pooled":- z = outputs.pooler_output[:, None, :]+ tokens = torch.LongTensor(tokens).to(device)++ if backup_embeds.weight.dtype != torch.float32:+ print("autocast clip")+ precision_scope = torch.autocastelse:- z = outputs.hidden_states[self.layer_idx]- if self.layer_norm_hidden_state:- z = self.transformer.text_model.final_layer_norm(z)-- pooled_output = outputs.pooler_output- if self.text_projection is not None:- pooled_output = pooled_output @ self.text_projection- return z, pooled_output+ precision_scope = contextlib.nullcontext+ print("no autocast clip")++ with precision_scope(model_management.get_autocast_device(device)):+ outputs = self.transformer(input_ids=tokens, output_hidden_states=self.layer=="hidden")+ self.transformer.set_input_embeddings(backup_embeds)++ if self.layer == "last":+ z = outputs.last_hidden_state+ elif self.layer == "pooled":+ z = outputs.pooler_output[:, None, :]+ else:+ z = outputs.hidden_states[self.layer_idx]+ if self.layer_norm_hidden_state:+ z = self.transformer.text_model.final_layer_norm(z)++ pooled_output = outputs.pooler_output+ if self.text_projection is not None:+ pooled_output = pooled_output @ self.text_projection+ return z.float(), pooled_output.float()def encode(self, tokens):return self(tokens)