The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Cleaner CLIP text encoder implementation.
comfyanonymousfbdb14d4c4c3
diff --git a/comfy/clip_model.py b/comfy/clip_model.pynew file mode 100644index 000000000..e6a7bfa66--- /dev/null+++ b/comfy/clip_model.py@@ -0,0 +1,126 @@+import torch+from comfy.ldm.modules.attention import optimized_attention_for_device++class CLIPAttention(torch.nn.Module):+ def __init__(self, embed_dim, heads, dtype, device, operations):+ super().__init__()++ self.heads = heads+ self.q_proj = operations.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device)+ self.k_proj = operations.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device)+ self.v_proj = operations.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device)++ self.out_proj = operations.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device)++ def forward(self, x, mask=None, optimized_attention=None):+ q = self.q_proj(x)+ k = self.k_proj(x)+ v = self.v_proj(x)++ out = optimized_attention(q, k, v, self.heads, mask)+ return self.out_proj(out)++ACTIVATIONS = {"quick_gelu": lambda a: a * torch.sigmoid(1.702 * a),+ "gelu": torch.nn.functional.gelu,+}++class CLIPMLP(torch.nn.Module):+ def __init__(self, embed_dim, intermediate_size, activation, dtype, device, operations):+ super().__init__()+ self.fc1 = operations.Linear(embed_dim, intermediate_size, bias=True, dtype=dtype, device=device)+ self.activation = ACTIVATIONS[activation]+ self.fc2 = operations.Linear(intermediate_size, embed_dim, bias=True, dtype=dtype, device=device)++ def forward(self, x):+ x = self.fc1(x)+ x = self.activation(x)+ x = self.fc2(x)+ return x++class CLIPLayer(torch.nn.Module):+ def __init__(self, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations):+ super().__init__()+ self.layer_norm1 = operations.LayerNorm(embed_dim, dtype=dtype, device=device)+ self.self_attn = CLIPAttention(embed_dim, heads, dtype, device, operations)+ self.layer_norm2 = operations.LayerNorm(embed_dim, dtype=dtype, device=device)+ self.mlp = CLIPMLP(embed_dim, intermediate_size, intermediate_activation, dtype, device, operations)++ def forward(self, x, mask=None, optimized_attention=None):+ x += self.self_attn(self.layer_norm1(x), mask, optimized_attention)+ x += self.mlp(self.layer_norm2(x))+ return x+++class CLIPEncoder(torch.nn.Module):+ def __init__(self, num_layers, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations):+ super().__init__()+ self.layers = torch.nn.ModuleList([CLIPLayer(embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations) for i in range(num_layers)])++ def forward(self, x, mask=None, intermediate_output=None):+ optimized_attention = optimized_attention_for_device(x.device, mask=True)+ causal_mask = torch.empty(x.shape[1], x.shape[1], dtype=x.dtype, device=x.device).fill_(float("-inf")).triu_(1)+ if mask is not None:+ mask += causal_mask+ else:+ mask = causal_mask++ if intermediate_output is not None:+ if intermediate_output < 0:+ intermediate_output = len(self.layers) + intermediate_output++ intermediate = None+ for i, l in enumerate(self.layers):+ x = l(x, mask, optimized_attention)+ if i == intermediate_output:+ intermediate = x.clone()+ return x, intermediate++class CLIPEmbeddings(torch.nn.Module):+ def __init__(self, embed_dim, vocab_size=49408, num_positions=77, dtype=None, device=None):+ super().__init__()+ self.token_embedding = torch.nn.Embedding(vocab_size, embed_dim, dtype=dtype, device=device)+ self.position_embedding = torch.nn.Embedding(num_positions, embed_dim, dtype=dtype, device=device)++ def forward(self, input_tokens):+ return self.token_embedding(input_tokens) + self.position_embedding.weight+++class CLIPTextModel_(torch.nn.Module):+ def __init__(self, config_dict, dtype, device, operations):+ num_layers = config_dict["num_hidden_layers"]+ embed_dim = config_dict["hidden_size"]+ heads = config_dict["num_attention_heads"]+ intermediate_size = config_dict["intermediate_size"]+ intermediate_activation = config_dict["hidden_act"]++ super().__init__()+ self.embeddings = CLIPEmbeddings(embed_dim, dtype=torch.float32, device=device)+ self.encoder = CLIPEncoder(num_layers, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations)+ self.final_layer_norm = operations.LayerNorm(embed_dim, dtype=dtype, device=device)++ def forward(self, input_tokens, attention_mask=None, intermediate_output=None, final_layer_norm_intermediate=True):+ x = self.embeddings(input_tokens)+ #TODO: attention_mask+ x, i = self.encoder(x, intermediate_output=intermediate_output)+ x = self.final_layer_norm(x)+ if i is not None and final_layer_norm_intermediate:+ i = self.final_layer_norm(i)++ pooled_output = x[torch.arange(x.shape[0], device=x.device), input_tokens.to(dtype=torch.int, device=x.device).argmax(dim=-1),]+ return x, i, pooled_output++class CLIPTextModel(torch.nn.Module):+ def __init__(self, config_dict, dtype, device, operations):+ super().__init__()+ self.num_layers = config_dict["num_hidden_layers"]+ self.text_model = CLIPTextModel_(config_dict, dtype, device, operations)+ self.dtype = dtype++ def get_input_embeddings(self):+ return self.text_model.embeddings.token_embedding++ def set_input_embeddings(self, embeddings):+ self.text_model.embeddings.token_embedding = embeddings++ def forward(self, *args, **kwargs):+ return self.text_model(*args, **kwargs)diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.pyindex d3348c472..8299b1d94 100644--- a/comfy/ldm/modules/attention.py+++ b/comfy/ldm/modules/attention.py@@ -112,10 +112,13 @@ def attention_basic(q, k, v, heads, mask=None):del q, kif exists(mask):- mask = rearrange(mask, 'b ... -> b (...)')- max_neg_value = -torch.finfo(sim.dtype).max- mask = repeat(mask, 'b j -> (b h) () j', h=h)- sim.masked_fill_(~mask, max_neg_value)+ if mask.dtype == torch.bool:+ mask = rearrange(mask, 'b ... -> b (...)') #TODO: check if this bool part matches pytorch attention+ max_neg_value = -torch.finfo(sim.dtype).max+ mask = repeat(mask, 'b j -> (b h) () j', h=h)+ sim.masked_fill_(~mask, max_neg_value)+ else:+ sim += mask# attention, what we cannot get enough ofsim = sim.softmax(dim=-1)@@ -340,6 +343,18 @@ else:if model_management.pytorch_attention_enabled():optimized_attention_masked = attention_pytorch+def optimized_attention_for_device(device, mask=False):+ if device == torch.device("cpu"): #TODO+ if model_management.pytorch_attention_enabled():+ return attention_pytorch+ else:+ return attention_basic+ if mask:+ return optimized_attention_masked++ return optimized_attention++class CrossAttention(nn.Module):def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., dtype=None, device=None, operations=comfy.ops):super().__init__()diff --git a/comfy/sd1_clip.py b/comfy/sd1_clip.pyindex 4e9f6bffe..1acd972c4 100644--- a/comfy/sd1_clip.py+++ b/comfy/sd1_clip.py@@ -1,12 +1,14 @@import os-from transformers import CLIPTokenizer, CLIPTextModel, CLIPTextConfig, modeling_utils+from transformers import CLIPTokenizerimport comfy.opsimport torchimport tracebackimport zipfilefrom . import model_managementimport contextlib+import comfy.clip_model+import jsondef gen_empty_tokens(special_tokens, length):start_token = special_tokens.get("start", None)@@ -65,35 +67,19 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):"hidden"]def __init__(self, version="openai/clip-vit-large-patch14", device="cpu", max_length=77,- freeze=True, layer="last", layer_idx=None, textmodel_json_config=None, textmodel_path=None, dtype=None,- special_tokens={"start": 49406, "end": 49407, "pad": 49407},layer_norm_hidden_state=True, config_class=CLIPTextConfig,- model_class=CLIPTextModel, inner_name="text_model"): # clip-vit-base-patch32+ freeze=True, layer="last", layer_idx=None, textmodel_json_config=None, dtype=None, model_class=comfy.clip_model.CLIPTextModel,+ special_tokens={"start": 49406, "end": 49407, "pad": 49407}, layer_norm_hidden_state=True): # clip-vit-base-patch32super().__init__()assert layer in self.LAYERS- self.num_layers = 12- if textmodel_path is not None:- self.transformer = model_class.from_pretrained(textmodel_path)- else:- if textmodel_json_config is None:- textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_clip_config.json")- config = config_class.from_json_file(textmodel_json_config)- self.num_layers = config.num_hidden_layers- with comfy.ops.use_comfy_ops(device, dtype):- with modeling_utils.no_init_weights():- self.transformer = model_class(config)-- self.inner_name = inner_name- if dtype is not None:- inner_model = getattr(self.transformer, self.inner_name)- if hasattr(inner_model, "embeddings"):- embeddings_bak = inner_model.embeddings.to(torch.float32)- inner_model.embeddings = None- self.transformer.to(dtype)- inner_model.embeddings = embeddings_bak- else:- previous_inputs = self.transformer.get_input_embeddings().to(torch.float32, copy=True)- self.transformer.to(dtype)- self.transformer.set_input_embeddings(previous_inputs)++ if textmodel_json_config is None:+ textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd1_clip_config.json")++ with open(textmodel_json_config) as f:+ config = json.load(f)++ self.transformer = model_class(config, dtype, device, comfy.ops)+ self.num_layers = self.transformer.num_layersself.max_length = max_lengthif freeze:@@ -108,7 +94,7 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):self.layer_norm_hidden_state = layer_norm_hidden_stateif layer == "hidden":assert layer_idx is not None- assert abs(layer_idx) <= self.num_layers+ assert abs(layer_idx) < self.num_layersself.clip_layer(layer_idx)self.layer_default = (self.layer, self.layer_idx)@@ -119,7 +105,7 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):param.requires_grad = Falsedef clip_layer(self, layer_idx):- if abs(layer_idx) >= self.num_layers:+ if abs(layer_idx) > self.num_layers:self.layer = "last"else:self.layer = "hidden"@@ -174,7 +160,7 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):tokens = self.set_up_textual_embeddings(tokens, backup_embeds)tokens = torch.LongTensor(tokens).to(device)- if getattr(self.transformer, self.inner_name).final_layer_norm.weight.dtype != torch.float32:+ if self.transformer.dtype != torch.float32:precision_scope = torch.autocastelse:precision_scope = lambda a, dtype: contextlib.nullcontext(a)@@ -190,20 +176,16 @@ class SDClipModel(torch.nn.Module, ClipTokenWeightEncoder):if tokens[x, y] == max_token:break- outputs = self.transformer(input_ids=tokens, attention_mask=attention_mask, output_hidden_states=self.layer=="hidden")+ outputs = self.transformer(tokens, attention_mask, intermediate_output=self.layer_idx, final_layer_norm_intermediate=self.layer_norm_hidden_state)self.transformer.set_input_embeddings(backup_embeds)if self.layer == "last":- z = outputs.last_hidden_state- elif self.layer == "pooled":- z = outputs.pooler_output[:, None, :]+ z = outputs[0]else:- z = outputs.hidden_states[self.layer_idx]- if self.layer_norm_hidden_state:- z = getattr(self.transformer, self.inner_name).final_layer_norm(z)+ z = outputs[1]- if hasattr(outputs, "pooler_output"):- pooled_output = outputs.pooler_output.float()+ if outputs[2] is not None:+ pooled_output = outputs[2].float()else:pooled_output = Nonediff --git a/comfy/sd2_clip.py b/comfy/sd2_clip.pyindex 2ee0ca055..9c878d54a 100644--- a/comfy/sd2_clip.py+++ b/comfy/sd2_clip.py@@ -3,13 +3,13 @@ import torchimport osclass SD2ClipHModel(sd1_clip.SDClipModel):- def __init__(self, arch="ViT-H-14", device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, textmodel_path=None, dtype=None):+ def __init__(self, arch="ViT-H-14", device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, dtype=None):if layer == "penultimate":layer="hidden"- layer_idx=23+ layer_idx=-2textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sd2_clip_config.json")- super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, textmodel_path=textmodel_path, dtype=dtype, special_tokens={"start": 49406, "end": 49407, "pad": 0})+ super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype, special_tokens={"start": 49406, "end": 49407, "pad": 0})class SD2ClipHTokenizer(sd1_clip.SDTokenizer):def __init__(self, tokenizer_path=None, embedding_directory=None):diff --git a/comfy/sdxl_clip.py b/comfy/sdxl_clip.pyindex 673399e22..b35056bb9 100644--- a/comfy/sdxl_clip.py+++ b/comfy/sdxl_clip.py@@ -3,13 +3,13 @@ import torchimport osclass SDXLClipG(sd1_clip.SDClipModel):- def __init__(self, device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, textmodel_path=None, dtype=None):+ def __init__(self, device="cpu", max_length=77, freeze=True, layer="penultimate", layer_idx=None, dtype=None):if layer == "penultimate":layer="hidden"layer_idx=-2textmodel_json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_config_bigg.json")- super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, textmodel_path=textmodel_path, dtype=dtype,+ super().__init__(device=device, freeze=freeze, layer=layer, layer_idx=layer_idx, textmodel_json_config=textmodel_json_config, dtype=dtype,special_tokens={"start": 49406, "end": 49407, "pad": 0}, layer_norm_hidden_state=False)def load_sd(self, sd):@@ -37,7 +37,7 @@ class SDXLTokenizer:class SDXLClipModel(torch.nn.Module):def __init__(self, device="cpu", dtype=None):super().__init__()- self.clip_l = sd1_clip.SDClipModel(layer="hidden", layer_idx=11, device=device, dtype=dtype, layer_norm_hidden_state=False)+ self.clip_l = sd1_clip.SDClipModel(layer="hidden", layer_idx=-2, device=device, dtype=dtype, layer_norm_hidden_state=False)self.clip_g = SDXLClipG(device=device, dtype=dtype)def clip_layer(self, layer_idx):