The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
SD1 and SD2 clip and tokenizer code is now more similar to the SDXL one.
comfyanonymouse60ca6929a99
diff --git a/comfy/lora.py b/comfy/lora.pyindex 3009a1c9e..d4cf94c95 100644--- a/comfy/lora.py+++ b/comfy/lora.py@@ -141,9 +141,9 @@ def model_lora_keys_clip(model, key_map={}):text_model_lora_key = "lora_te_text_model_encoder_layers_{}_{}"clip_l_present = False- for b in range(32):+ for b in range(32): #TODO: clean upfor c in LORA_CLIP_MAP:- k = "transformer.text_model.encoder.layers.{}.{}.weight".format(b, c)+ k = "clip_h.transformer.text_model.encoder.layers.{}.{}.weight".format(b, c)if k in sdk:lora_key = text_model_lora_key.format(b, LORA_CLIP_MAP[c])key_map[lora_key] = k@@ -154,6 +154,8 @@ def model_lora_keys_clip(model, key_map={}):k = "clip_l.transformer.text_model.encoder.layers.{}.{}.weight".format(b, c)if k in sdk:+ lora_key = text_model_lora_key.format(b, LORA_CLIP_MAP[c])+ key_map[lora_key] = klora_key = "lora_te1_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c]) #SDXL basekey_map[lora_key] = kclip_l_present = Truediff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.pyindex ffe2bd3bd..5368a45df 100644--- a/comfy/sd1_clip.py+++ b/comfy/sd1_clip.py@@ -35,7 +35,7 @@ class ClipTokenWeightEncoder:return z_empty.cpu(), first_pooled.cpu()return torch.cat(output, dim=-2).cpu(), first_pooled.cpu()-class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder):+class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):"""Uses the CLIP transformer encoder for text (from huggingface)"""LAYERS = ["last",@@ -342,7 +342,7 @@ def load_embed(embedding_name, embedding_directory, embedding_size, embed_key=Noembed_out = next(iter(values))return embed_out-class SD1Tokenizer:+class SDTokenizer:def __init__(self, tokenizer_path=None, max_length=77, pad_with_end=True, embedding_directory=None, embedding_size=768, embedding_key='clip_l'):if tokenizer_path is None:tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_tokenizer")@@ -454,3 +454,40 @@ class SD1Tokenizer:def untokenize(self, token_weight_pair):return list(map(lambda a: (a, self.inv_vocab[a[0]]), token_weight_pair))+++class SD1Tokenizer:+ def __init__(self, embedding_directory=None, clip_name="l", tokenizer=SDTokenizer):+ self.clip_name = clip_name+ self.clip = "clip_{}".format(self.clip_name)+ setattr(self, self.clip, tokenizer(embedding_directory=embedding_directory))++ def tokenize_with_weights(self, text:str, return_word_ids=False):+ out = {}+ out[self.clip_name] = getattr(self, self.clip).tokenize_with_weights(text, return_word_ids)+ return out++ def untokenize(self, token_weight_pair):+ return getattr(self, self.clip).untokenize(token_weight_pair)+++class SD1ClipModel(torch.nn.Module):+ def __init__(self, device="cpu", dtype=None, clip_name="l", clip_model=SDClipModel):+ super().__init__()+ self.clip_name = clip_name+ self.clip = "clip_{}".format(self.clip_name)+ setattr(self, self.clip, clip_model(device=device, dtype=dtype))++ def clip_layer(self, layer_idx):+ getattr(self, self.clip).clip_layer(layer_idx)++ def reset_clip_layer(self):+ getattr(self, self.clip).reset_clip_layer()++ def encode_token_weights(self, token_weight_pairs):+ token_weight_pairs = token_weight_pairs[self.clip_name]+ out, pooled = getattr(self, self.clip).encode_token_weights(token_weight_pairs)+ return out, pooled++ def load_sd(self, sd):+ return getattr(self, self.clip).load_sd(sd)diff --git a/comfy/sd2_clip.py b/comfy/sd2_clip.pyindex 05e50a005..9df868b76 100644--- a/comfy/sd2_clip.py+++ b/comfy/sd2_clip.py@@ -2,7 +2,7 @@ from comfy import sd1_clipimport torchimport os-class SD2ClipModel(sd1_clip.SD1ClipModel):+class SD2ClipHModel(sd1_clip.SDClipModel):def __init__(self, arch="ViT-H-14", device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, textmodel_path=None, dtype=None):if layer == "penultimate":layer="hidden"@@ -12,6 +12,14 @@ class SD2ClipModel(sd1_clip.SD1ClipModel):super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, textmodel_path=textmodel_path, dtype=dtype)self.empty_tokens = [[49406] + [49407] + [0] * 75]-class SD2Tokenizer(sd1_clip.SD1Tokenizer):+class SD2ClipHTokenizer(sd1_clip.SDTokenizer):def __init__(self, tokenizer_path=None, embedding_directory=None):super().__init__(tokenizer_path, pad_with_end=False, embedding_directory=embedding_directory, embedding_size=1024)++class SD2Tokenizer(sd1_clip.SD1Tokenizer):+ def __init__(self, embedding_directory=None):+ super().__init__(embedding_directory=embedding_directory, clip_name="h", tokenizer=SD2ClipHTokenizer)++class SD2ClipModel(sd1_clip.SD1ClipModel):+ def __init__(self, device="cpu", dtype=None):+ super().__init__(device=device, dtype=dtype, clip_name="h", clip_model=SD2ClipHModel)diff --git a/comfy/sdxl_clip.py b/comfy/sdxl_clip.pyindex e3ac2ee0b..4c508a0ea 100644--- a/comfy/sdxl_clip.py+++ b/comfy/sdxl_clip.py@@ -2,7 +2,7 @@ from comfy import sd1_clipimport torchimport os-class SDXLClipG(sd1_clip.SD1ClipModel):+class SDXLClipG(sd1_clip.SDClipModel):def __init__(self, device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, textmodel_path=None, dtype=None):if layer == "penultimate":layer="hidden"@@ -16,14 +16,14 @@ class SDXLClipG(sd1_clip.SD1ClipModel):def load_sd(self, sd):return super().load_sd(sd)-class SDXLClipGTokenizer(sd1_clip.SD1Tokenizer):+class SDXLClipGTokenizer(sd1_clip.SDTokenizer):def __init__(self, tokenizer_path=None, embedding_directory=None):super().__init__(tokenizer_path, pad_with_end=False, embedding_directory=embedding_directory, embedding_size=1280, embedding_key='clip_g')-class SDXLTokenizer(sd1_clip.SD1Tokenizer):+class SDXLTokenizer:def __init__(self, embedding_directory=None):- self.clip_l = sd1_clip.SD1Tokenizer(embedding_directory=embedding_directory)+ self.clip_l = sd1_clip.SDTokenizer(embedding_directory=embedding_directory)self.clip_g = SDXLClipGTokenizer(embedding_directory=embedding_directory)def tokenize_with_weights(self, text:str, return_word_ids=False):@@ -38,7 +38,7 @@ class SDXLTokenizer(sd1_clip.SD1Tokenizer):class SDXLClipModel(torch.nn.Module):def __init__(self, device="cpu", dtype=None):super().__init__()- self.clip_l = sd1_clip.SD1ClipModel(layer="hidden", layer_idx=11, device=device, dtype=dtype)+ self.clip_l = sd1_clip.SDClipModel(layer="hidden", layer_idx=11, device=device, dtype=dtype)self.clip_l.layer_norm_hidden_state = Falseself.clip_g = SDXLClipG(device=device, dtype=dtype)@@ -63,21 +63,6 @@ class SDXLClipModel(torch.nn.Module):else:return self.clip_l.load_sd(sd)-class SDXLRefinerClipModel(torch.nn.Module):+class SDXLRefinerClipModel(sd1_clip.SD1ClipModel):def __init__(self, device="cpu", dtype=None):- super().__init__()- self.clip_g = SDXLClipG(device=device, dtype=dtype)-- def clip_layer(self, layer_idx):- self.clip_g.clip_layer(layer_idx)-- def reset_clip_layer(self):- self.clip_g.reset_clip_layer()-- def encode_token_weights(self, token_weight_pairs):- token_weight_pairs_g = token_weight_pairs["g"]- g_out, g_pooled = self.clip_g.encode_token_weights(token_weight_pairs_g)- return g_out, g_pooled-- def load_sd(self, sd):- return self.clip_g.load_sd(sd)+ super().__init__(device=device, dtype=dtype, clip_name="g", clip_model=SDXLClipG)diff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex 820f2861c..fdd4ea4f5 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -38,8 +38,15 @@ class SD15(supported_models_base.BASE):if ids.dtype == torch.float32:state_dict['cond_stage_model.transformer.text_model.embeddings.position_ids'] = ids.round()+ replace_prefix = {}+ replace_prefix["cond_stage_model."] = "cond_stage_model.clip_l."+ state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix)return state_dict+ def process_clip_state_dict_for_saving(self, state_dict):+ replace_prefix = {"clip_l.": "cond_stage_model."}+ return utils.state_dict_prefix_replace(state_dict, replace_prefix)+def clip_target(self):return supported_models_base.ClipTarget(sd1_clip.SD1Tokenizer, sd1_clip.SD1ClipModel)@@ -62,12 +69,12 @@ class SD20(supported_models_base.BASE):return model_base.ModelType.EPSdef process_clip_state_dict(self, state_dict):- state_dict = utils.transformers_convert(state_dict, "cond_stage_model.model.", "cond_stage_model.transformer.text_model.", 24)+ state_dict = utils.transformers_convert(state_dict, "cond_stage_model.model.", "cond_stage_model.clip_h.transformer.text_model.", 24)return state_dictdef process_clip_state_dict_for_saving(self, state_dict):replace_prefix = {}- replace_prefix[""] = "cond_stage_model.model."+ replace_prefix["clip_h"] = "cond_stage_model.model"state_dict = utils.state_dict_prefix_replace(state_dict, replace_prefix)state_dict = diffusers_convert.convert_text_enc_state_dict_v20(state_dict)return state_dict