The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Move text_projection to base clip model.
comfyanonymousec96f6d03a0a
diff --git a/comfy/sd.py b/comfy/sd.pyindex 3568a2aa6..20d00952f 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -564,9 +564,6 @@ class CLIP:n.layer_idx = self.layer_idxreturn n- def load_from_state_dict(self, sd):- self.cond_stage_model.load_sd(sd)-def add_patches(self, patches, strength_patch=1.0, strength_model=1.0):return self.patcher.add_patches(patches, strength_patch, strength_model)diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.pyindex 4616ca4e9..477d5c309 100644--- a/comfy/sd1_clip.py+++ b/comfy/sd1_clip.py@@ -66,7 +66,9 @@ class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder):self.layer = layerself.layer_idx = Noneself.empty_tokens = [[49406] + [49407] * 76]- self.text_projection = None+ self.text_projection = torch.nn.Parameter(torch.eye(self.transformer.get_input_embeddings().weight.shape[1]))+ self.logit_scale = torch.nn.Parameter(torch.tensor(4.6055))+self.layer_norm_hidden_state = Trueif layer == "hidden":assert layer_idx is not None@@ -163,6 +165,10 @@ class SD1ClipModel(torch.nn.Module, ClipTokenWeightEncoder):return self(tokens)def load_sd(self, sd):+ if "text_projection" in sd:+ self.text_projection[:] = sd.pop("text_projection")+ if "text_projection.weight" in sd:+ self.text_projection[:] = sd.pop("text_projection.weight").transpose(0, 1)return self.transformer.load_state_dict(sd, strict=False)def parse_parentheses(string):diff --git a/comfy/sd2_clip_config.json b/comfy/sd2_clip_config.jsonindex ace6ef001..85cec832b 100644--- a/comfy/sd2_clip_config.json+++ b/comfy/sd2_clip_config.json@@ -17,7 +17,7 @@"num_attention_heads": 16,"num_hidden_layers": 24,"pad_token_id": 1,- "projection_dim": 512,+ "projection_dim": 1024,"torch_dtype": "float32","vocab_size": 49408}diff --git a/comfy/sdxl_clip.py b/comfy/sdxl_clip.pyindex d05c0a9b9..e3ac2ee0b 100644--- a/comfy/sdxl_clip.py+++ b/comfy/sdxl_clip.py@@ -11,15 +11,9 @@ class SDXLClipG(sd1_clip.SD1ClipModel):textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_config_bigg.json")super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, textmodel_path=textmodel_path, dtype=dtype)self.empty_tokens = [[49406] + [49407] + [0] * 75]- self.text_projection = torch.nn.Parameter(torch.empty(1280, 1280))- self.logit_scale = torch.nn.Parameter(torch.tensor(4.6055))self.layer_norm_hidden_state = Falsedef load_sd(self, sd):- if "text_projection" in sd:- self.text_projection[:] = sd.pop("text_projection")- if "text_projection.weight" in sd:- self.text_projection[:] = sd.pop("text_projection.weight").transpose(0, 1)return super().load_sd(sd)class SDXLClipGTokenizer(sd1_clip.SD1Tokenizer):