Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Move lora code to comfy/lora.py

comfyanonymous15a7716fa6f0
diff --git a/comfy/lora.py b/comfy/lora.py
new file mode 100644
index 000000000..d685a455e
--- /dev/null
+++ b/comfy/lora.py
@@ -0,0 +1,186 @@
+import comfy.utils
+
+LORA_CLIP_MAP = {
+ "mlp.fc1": "mlp_fc1",
+ "mlp.fc2": "mlp_fc2",
+ "self_attn.k_proj": "self_attn_k_proj",
+ "self_attn.q_proj": "self_attn_q_proj",
+ "self_attn.v_proj": "self_attn_v_proj",
+ "self_attn.out_proj": "self_attn_out_proj",
+}
+
+
+def load_lora(lora, to_load):
+ patch_dict = {}
+ loaded_keys = set()
+ for x in to_load:
+ alpha_name = "{}.alpha".format(x)
+ alpha = None
+ if alpha_name in lora.keys():
+ alpha = lora[alpha_name].item()
+ loaded_keys.add(alpha_name)
+
+ regular_lora = "{}.lora_up.weight".format(x)
+ diffusers_lora = "{}_lora.up.weight".format(x)
+ transformers_lora = "{}.lora_linear_layer.up.weight".format(x)
+ A_name = None
+
+ if regular_lora in lora.keys():
+ A_name = regular_lora
+ B_name = "{}.lora_down.weight".format(x)
+ mid_name = "{}.lora_mid.weight".format(x)
+ elif diffusers_lora in lora.keys():
+ A_name = diffusers_lora
+ B_name = "{}_lora.down.weight".format(x)
+ mid_name = None
+ elif transformers_lora in lora.keys():
+ A_name = transformers_lora
+ B_name ="{}.lora_linear_layer.down.weight".format(x)
+ mid_name = None
+
+ if A_name is not None:
+ mid = None
+ if mid_name is not None and mid_name in lora.keys():
+ mid = lora[mid_name]
+ loaded_keys.add(mid_name)
+ patch_dict[to_load[x]] = (lora[A_name], lora[B_name], alpha, mid)
+ loaded_keys.add(A_name)
+ loaded_keys.add(B_name)
+
+
+ ######## loha
+ hada_w1_a_name = "{}.hada_w1_a".format(x)
+ hada_w1_b_name = "{}.hada_w1_b".format(x)
+ hada_w2_a_name = "{}.hada_w2_a".format(x)
+ hada_w2_b_name = "{}.hada_w2_b".format(x)
+ hada_t1_name = "{}.hada_t1".format(x)
+ hada_t2_name = "{}.hada_t2".format(x)
+ if hada_w1_a_name in lora.keys():
+ hada_t1 = None
+ hada_t2 = None
+ if hada_t1_name in lora.keys():
+ hada_t1 = lora[hada_t1_name]
+ hada_t2 = lora[hada_t2_name]
+ loaded_keys.add(hada_t1_name)
+ loaded_keys.add(hada_t2_name)
+
+ patch_dict[to_load[x]] = (lora[hada_w1_a_name], lora[hada_w1_b_name], alpha, lora[hada_w2_a_name], lora[hada_w2_b_name], hada_t1, hada_t2)
+ loaded_keys.add(hada_w1_a_name)
+ loaded_keys.add(hada_w1_b_name)
+ loaded_keys.add(hada_w2_a_name)
+ loaded_keys.add(hada_w2_b_name)
+
+
+ ######## lokr
+ lokr_w1_name = "{}.lokr_w1".format(x)
+ lokr_w2_name = "{}.lokr_w2".format(x)
+ lokr_w1_a_name = "{}.lokr_w1_a".format(x)
+ lokr_w1_b_name = "{}.lokr_w1_b".format(x)
+ lokr_t2_name = "{}.lokr_t2".format(x)
+ lokr_w2_a_name = "{}.lokr_w2_a".format(x)
+ lokr_w2_b_name = "{}.lokr_w2_b".format(x)
+
+ lokr_w1 = None
+ if lokr_w1_name in lora.keys():
+ lokr_w1 = lora[lokr_w1_name]
+ loaded_keys.add(lokr_w1_name)
+
+ lokr_w2 = None
+ if lokr_w2_name in lora.keys():
+ lokr_w2 = lora[lokr_w2_name]
+ loaded_keys.add(lokr_w2_name)
+
+ lokr_w1_a = None
+ if lokr_w1_a_name in lora.keys():
+ lokr_w1_a = lora[lokr_w1_a_name]
+ loaded_keys.add(lokr_w1_a_name)
+
+ lokr_w1_b = None
+ if lokr_w1_b_name in lora.keys():
+ lokr_w1_b = lora[lokr_w1_b_name]
+ loaded_keys.add(lokr_w1_b_name)
+
+ lokr_w2_a = None
+ if lokr_w2_a_name in lora.keys():
+ lokr_w2_a = lora[lokr_w2_a_name]
+ loaded_keys.add(lokr_w2_a_name)
+
+ lokr_w2_b = None
+ if lokr_w2_b_name in lora.keys():
+ lokr_w2_b = lora[lokr_w2_b_name]
+ loaded_keys.add(lokr_w2_b_name)
+
+ lokr_t2 = None
+ if lokr_t2_name in lora.keys():
+ lokr_t2 = lora[lokr_t2_name]
+ loaded_keys.add(lokr_t2_name)
+
+ if (lokr_w1 is not None) or (lokr_w2 is not None) or (lokr_w1_a is not None) or (lokr_w2_a is not None):
+ patch_dict[to_load[x]] = (lokr_w1, lokr_w2, alpha, lokr_w1_a, lokr_w1_b, lokr_w2_a, lokr_w2_b, lokr_t2)
+
+ for x in lora.keys():
+ if x not in loaded_keys:
+ print("lora key not loaded", x)
+ return patch_dict
+
+def model_lora_keys_clip(model, key_map={}):
+ sdk = model.state_dict().keys()
+
+ text_model_lora_key = "lora_te_text_model_encoder_layers_{}_{}"
+ clip_l_present = False
+ for b in range(32):
+ for c in LORA_CLIP_MAP:
+ k = "transformer.text_model.encoder.layers.{}.{}.weight".format(b, c)
+ if k in sdk:
+ lora_key = text_model_lora_key.format(b, LORA_CLIP_MAP[c])
+ key_map[lora_key] = k
+ lora_key = "lora_te1_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c])
+ key_map[lora_key] = k
+ lora_key = "text_encoder.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora
+ key_map[lora_key] = k
+
+ k = "clip_l.transformer.text_model.encoder.layers.{}.{}.weight".format(b, c)
+ if k in sdk:
+ lora_key = "lora_te1_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c]) #SDXL base
+ key_map[lora_key] = k
+ clip_l_present = True
+ lora_key = "text_encoder.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora
+ key_map[lora_key] = k
+
+ k = "clip_g.transformer.text_model.encoder.layers.{}.{}.weight".format(b, c)
+ if k in sdk:
+ if clip_l_present:
+ lora_key = "lora_te2_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c]) #SDXL base
+ key_map[lora_key] = k
+ lora_key = "text_encoder_2.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora
+ key_map[lora_key] = k
+ else:
+ lora_key = "lora_te_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c]) #TODO: test if this is correct for SDXL-Refiner
+ key_map[lora_key] = k
+ lora_key = "text_encoder.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora
+ key_map[lora_key] = k
+
+ return key_map
+
+def model_lora_keys_unet(model, key_map={}):
+ sdk = model.state_dict().keys()
+
+ for k in sdk:
+ if k.startswith("diffusion_model.") and k.endswith(".weight"):
+ key_lora = k[len("diffusion_model."):-len(".weight")].replace(".", "_")
+ key_map["lora_unet_{}".format(key_lora)] = k
+
+ diffusers_keys = comfy.utils.unet_to_diffusers(model.model_config.unet_config)
+ for k in diffusers_keys:
+ if k.endswith(".weight"):
+ unet_key = "diffusion_model.{}".format(diffusers_keys[k])
+ key_lora = k[:-len(".weight")].replace(".", "_")
+ key_map["lora_unet_{}".format(key_lora)] = unet_key
+
+ diffusers_lora_prefix = ["", "unet."]
+ for p in diffusers_lora_prefix:
+ diffusers_lora_key = "{}{}".format(p, k[:-len(".weight")].replace(".to_", ".processor.to_"))
+ if diffusers_lora_key.endswith(".to_out.0"):
+ diffusers_lora_key = diffusers_lora_key[:-2]
+ key_map[diffusers_lora_key] = unet_key
+ return key_map
diff --git a/comfy/sd.py b/comfy/sd.py
index 20d00952f..e42d4cdc8 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -22,6 +22,8 @@ from . import sd1_clip
from . import sd2_clip
from . import sdxl_clip
+import comfy.lora
+
def load_model_weights(model, sd):
m, u = model.load_state_dict(sd, strict=False)
m = set(m)
@@ -51,190 +53,7 @@ def load_clip_weights(model, sd):
sd = utils.transformers_convert(sd, "cond_stage_model.model.", "cond_stage_model.transformer.text_model.", 24)
return load_model_weights(model, sd)
-LORA_CLIP_MAP = {
- "mlp.fc1": "mlp_fc1",
- "mlp.fc2": "mlp_fc2",
- "self_attn.k_proj": "self_attn_k_proj",
- "self_attn.q_proj": "self_attn_q_proj",
- "self_attn.v_proj": "self_attn_v_proj",
- "self_attn.out_proj": "self_attn_out_proj",
-}
-
-
-def load_lora(lora, to_load):
- patch_dict = {}
- loaded_keys = set()
- for x in to_load:
- alpha_name = "{}.alpha".format(x)
- alpha = None
- if alpha_name in lora.keys():
- alpha = lora[alpha_name].item()
- loaded_keys.add(alpha_name)
-
- regular_lora = "{}.lora_up.weight".format(x)
- diffusers_lora = "{}_lora.up.weight".format(x)
- transformers_lora = "{}.lora_linear_layer.up.weight".format(x)
- A_name = None
-
- if regular_lora in lora.keys():
- A_name = regular_lora
- B_name = "{}.lora_down.weight".format(x)
- mid_name = "{}.lora_mid.weight".format(x)
- elif diffusers_lora in lora.keys():
- A_name = diffusers_lora
- B_name = "{}_lora.down.weight".format(x)
- mid_name = None
- elif transformers_lora in lora.keys():
- A_name = transformers_lora
- B_name ="{}.lora_linear_layer.down.weight".format(x)
- mid_name = None
-
- if A_name is not None:
- mid = None
- if mid_name is not None and mid_name in lora.keys():
- mid = lora[mid_name]
- loaded_keys.add(mid_name)
- patch_dict[to_load[x]] = (lora[A_name], lora[B_name], alpha, mid)
- loaded_keys.add(A_name)
- loaded_keys.add(B_name)
-
-
- ######## loha
- hada_w1_a_name = "{}.hada_w1_a".format(x)
- hada_w1_b_name = "{}.hada_w1_b".format(x)
- hada_w2_a_name = "{}.hada_w2_a".format(x)
- hada_w2_b_name = "{}.hada_w2_b".format(x)
- hada_t1_name = "{}.hada_t1".format(x)
- hada_t2_name = "{}.hada_t2".format(x)
- if hada_w1_a_name in lora.keys():
- hada_t1 = None
- hada_t2 = None
- if hada_t1_name in lora.keys():
- hada_t1 = lora[hada_t1_name]
- hada_t2 = lora[hada_t2_name]
- loaded_keys.add(hada_t1_name)
- loaded_keys.add(hada_t2_name)
-
- patch_dict[to_load[x]] = (lora[hada_w1_a_name], lora[hada_w1_b_name], alpha, lora[hada_w2_a_name], lora[hada_w2_b_name], hada_t1, hada_t2)
- loaded_keys.add(hada_w1_a_name)
- loaded_keys.add(hada_w1_b_name)
- loaded_keys.add(hada_w2_a_name)
- loaded_keys.add(hada_w2_b_name)
-
-
- ######## lokr
- lokr_w1_name = "{}.lokr_w1".format(x)
- lokr_w2_name = "{}.lokr_w2".format(x)
- lokr_w1_a_name = "{}.lokr_w1_a".format(x)
- lokr_w1_b_name = "{}.lokr_w1_b".format(x)
- lokr_t2_name = "{}.lokr_t2".format(x)
- lokr_w2_a_name = "{}.lokr_w2_a".format(x)
- lokr_w2_b_name = "{}.lokr_w2_b".format(x)
-
- lokr_w1 = None
- if lokr_w1_name in lora.keys():
- lokr_w1 = lora[lokr_w1_name]
- loaded_keys.add(lokr_w1_name)
-
- lokr_w2 = None
- if lokr_w2_name in lora.keys():
- lokr_w2 = lora[lokr_w2_name]
- loaded_keys.add(lokr_w2_name)
-
- lokr_w1_a = None
- if lokr_w1_a_name in lora.keys():
- lokr_w1_a = lora[lokr_w1_a_name]
- loaded_keys.add(lokr_w1_a_name)
-
- lokr_w1_b = None
- if lokr_w1_b_name in lora.keys():
- lokr_w1_b = lora[lokr_w1_b_name]
- loaded_keys.add(lokr_w1_b_name)
-
- lokr_w2_a = None
- if lokr_w2_a_name in lora.keys():
- lokr_w2_a = lora[lokr_w2_a_name]
- loaded_keys.add(lokr_w2_a_name)
-
- lokr_w2_b = None
- if lokr_w2_b_name in lora.keys():
- lokr_w2_b = lora[lokr_w2_b_name]
- loaded_keys.add(lokr_w2_b_name)
-
- lokr_t2 = None
- if lokr_t2_name in lora.keys():
- lokr_t2 = lora[lokr_t2_name]
- loaded_keys.add(lokr_t2_name)
-
- if (lokr_w1 is not None) or (lokr_w2 is not None) or (lokr_w1_a is not None) or (lokr_w2_a is not None):
- patch_dict[to_load[x]] = (lokr_w1, lokr_w2, alpha, lokr_w1_a, lokr_w1_b, lokr_w2_a, lokr_w2_b, lokr_t2)
-
- for x in lora.keys():
- if x not in loaded_keys:
- print("lora key not loaded", x)
- return patch_dict
-
-def model_lora_keys_clip(model, key_map={}):
- sdk = model.state_dict().keys()
-
- text_model_lora_key = "lora_te_text_model_encoder_layers_{}_{}"
- clip_l_present = False
- for b in range(32):
- for c in LORA_CLIP_MAP:
- k = "transformer.text_model.encoder.layers.{}.{}.weight".format(b, c)
- if k in sdk:
- lora_key = text_model_lora_key.format(b, LORA_CLIP_MAP[c])
- key_map[lora_key] = k
- lora_key = "lora_te1_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c])
- key_map[lora_key] = k
- lora_key = "text_encoder.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora
- key_map[lora_key] = k
-
- k = "clip_l.transformer.text_model.encoder.layers.{}.{}.weight".format(b, c)
- if k in sdk:
- lora_key = "lora_te1_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c]) #SDXL base
- key_map[lora_key] = k
- clip_l_present = True
- lora_key = "text_encoder.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora
- key_map[lora_key] = k
-
- k = "clip_g.transformer.text_model.encoder.layers.{}.{}.weight".format(b, c)
- if k in sdk:
- if clip_l_present:
- lora_key = "lora_te2_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c]) #SDXL base
- key_map[lora_key] = k
- lora_key = "text_encoder_2.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora
- key_map[lora_key] = k
- else:
- lora_key = "lora_te_text_model_encoder_layers_{}_{}".format(b, LORA_CLIP_MAP[c]) #TODO: test if this is correct for SDXL-Refiner
- key_map[lora_key] = k
- lora_key = "text_encoder.text_model.encoder.layers.{}.{}".format(b, c) #diffusers lora
- key_map[lora_key] = k
-
- return key_map
-def model_lora_keys_unet(model, key_map={}):
- sdk = model.state_dict().keys()
-
- for k in sdk:
- if k.startswith("diffusion_model.") and k.endswith(".weight"):
- key_lora = k[len("diffusion_model."):-len(".weight")].replace(".", "_")
- key_map["lora_unet_{}".format(key_lora)] = k
-
- diffusers_keys = utils.unet_to_diffusers(model.model_config.unet_config)
- for k in diffusers_keys:
- if k.endswith(".weight"):
- unet_key = "diffusion_model.{}".format(diffusers_keys[k])
- key_lora = k[:-len(".weight")].replace(".", "_")
- key_map["lora_unet_{}".format(key_lora)] = unet_key
-
- diffusers_lora_prefix = ["", "unet."]
- for p in diffusers_lora_prefix:
- diffusers_lora_key = "{}{}".format(p, k[:-len(".weight")].replace(".to_", ".processor.to_"))
- if diffusers_lora_key.endswith(".to_out.0"):
- diffusers_lora_key = diffusers_lora_key[:-2]
- key_map[diffusers_lora_key] = unet_key
- return key_map
def set_attr(obj, attr, value):
attrs = attr.split(".")
@@ -518,9 +337,9 @@ class ModelPatcher:
def load_lora_for_models(model, clip, lora, strength_model, strength_clip):
- key_map = model_lora_keys_unet(model.model)
- key_map = model_lora_keys_clip(clip.cond_stage_model, key_map)
- loaded = load_lora(lora, key_map)
+ key_map = comfy.lora.model_lora_keys_unet(model.model)
+ key_map = comfy.lora.model_lora_keys_clip(clip.cond_stage_model, key_map)
+ loaded = comfy.lora.load_lora(lora, key_map)
new_modelpatcher = model.clone()
k = new_modelpatcher.add_patches(loaded, strength_model)
new_clip = clip.clone()