Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

FLUX: Added full diffusers mapping for FLUX.1 schnell and dev. Adds full LoRA support from diffusers LoRAs. (#4302)

Jaret Burkett1765f1c60c86
diff --git a/comfy/utils.py b/comfy/utils.py
index e6736dbde..a1e9213f9 100644
--- a/comfy/utils.py
+++ b/comfy/utils.py
@@ -457,8 +457,23 @@ def flux_to_diffusers(mmdit_config, output_prefix=""):
key_map["{}add_k_proj.{}".format(k, end)] = (qkv, (0, hidden_size, hidden_size))
key_map["{}add_v_proj.{}".format(k, end)] = (qkv, (0, hidden_size * 2, hidden_size))
- block_map = {"attn.to_out.0.weight": "img_attn.proj.weight",
- "attn.to_out.0.bias": "img_attn.proj.bias",
+ block_map = {
+ "attn.to_out.0.weight": "img_attn.proj.weight",
+ "attn.to_out.0.bias": "img_attn.proj.bias",
+ "norm1.linear.weight": "img_mod.lin.weight",
+ "norm1.linear.bias": "img_mod.lin.bias",
+ "norm1_context.linear.weight": "txt_mod.lin.weight",
+ "norm1_context.linear.bias": "txt_mod.lin.bias",
+ "attn.to_add_out.weight": "txt_attn.proj.weight",
+ "attn.to_add_out.bias": "txt_attn.proj.bias",
+ "ff.net.0.proj.weight": "img_mlp.0.weight",
+ "ff.net.0.proj.bias": "img_mlp.0.bias",
+ "ff.net.2.weight": "img_mlp.2.weight",
+ "ff.net.2.bias": "img_mlp.2.bias",
+ "ff_context.net.0.proj.weight": "txt_mlp.0.weight",
+ "ff_context.net.0.proj.bias": "txt_mlp.0.bias",
+ "ff_context.net.2.weight": "txt_mlp.2.weight",
+ "ff_context.net.2.bias": "txt_mlp.2.bias",
}
for k in block_map:
@@ -474,15 +489,43 @@ def flux_to_diffusers(mmdit_config, output_prefix=""):
key_map["{}to_q.{}".format(k, end)] = (qkv, (0, 0, hidden_size))
key_map["{}to_k.{}".format(k, end)] = (qkv, (0, hidden_size, hidden_size))
key_map["{}to_v.{}".format(k, end)] = (qkv, (0, hidden_size * 2, hidden_size))
- key_map["{}proj_mlp.{}".format(k, end)] = (qkv, (0, hidden_size * 3, hidden_size))
+ key_map["{}.proj_mlp.{}".format(prefix_from, end)] = (qkv, (0, hidden_size * 3, hidden_size * 4))
- block_map = {#TODO
+ block_map = {
+ "norm.linear.weight": "modulation.lin.weight",
+ "norm.linear.bias": "modulation.lin.bias",
+ "proj_out.weight": "linear2.weight",
+ "proj_out.bias": "linear2.bias",
}
for k in block_map:
key_map["{}.{}".format(prefix_from, k)] = "{}.{}".format(prefix_to, block_map[k])
- MAP_BASIC = { #TODO
+ MAP_BASIC = {
+ ("final_layer.linear.bias", "proj_out.bias"),
+ ("final_layer.linear.weight", "proj_out.weight"),
+ ("img_in.bias", "x_embedder.bias"),
+ ("img_in.weight", "x_embedder.weight"),
+ ("time_in.in_layer.bias", "time_text_embed.timestep_embedder.linear_1.bias"),
+ ("time_in.in_layer.weight", "time_text_embed.timestep_embedder.linear_1.weight"),
+ ("time_in.out_layer.bias", "time_text_embed.timestep_embedder.linear_2.bias"),
+ ("time_in.out_layer.weight", "time_text_embed.timestep_embedder.linear_2.weight"),
+ ("txt_in.bias", "context_embedder.bias"),
+ ("txt_in.weight", "context_embedder.weight"),
+ ("vector_in.in_layer.bias", "time_text_embed.text_embedder.linear_1.bias"),
+ ("vector_in.in_layer.weight", "time_text_embed.text_embedder.linear_1.weight"),
+ ("vector_in.out_layer.bias", "time_text_embed.timestep_embedder.linear_2.bias"),
+ ("vector_in.out_layer.weight", "time_text_embed.text_embedder.linear_2.weight"),
+ ("guidance_in.in_layer.bias", "time_text_embed.guidance_embedder.linear_1.bias"),
+ ("guidance_in.in_layer.weight", "time_text_embed.guidance_embedder.linear_1.weight"),
+ ("guidance_in.out_layer.bias", "time_text_embed.guidance_embedder.linear_1.bias"),
+ ("guidance_in.out_layer.weight", "time_text_embed.guidance_embedder.linear_2.weight"),
+ ("final_layer.adaLN_modulation.1.bias", "norm_out.linear.bias", swap_scale_shift),
+ ("final_layer.adaLN_modulation.1.weight", "norm_out.linear.weight", swap_scale_shift),
+
+ # TODO: the values of these weights are different in Diffusers
+ ("guidance_in.out_layer.bias", "time_text_embed.guidance_embedder.linear_2.bias"),
+ ("vector_in.out_layer.bias", "time_text_embed.text_embedder.linear_2.bias"),
}
for k in MAP_BASIC: