The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Koala 700M and 1B support.
comfyanonymousb3e97fc71416
diff --git a/comfy/ldm/modules/diffusionmodules/openaimodel.py b/comfy/ldm/modules/diffusionmodules/openaimodel.pyindex 998afd977..c54770255 100644--- a/comfy/ldm/modules/diffusionmodules/openaimodel.py+++ b/comfy/ldm/modules/diffusionmodules/openaimodel.py@@ -708,27 +708,30 @@ class UNetModel(nn.Module):device=device,operations=operations)]- if transformer_depth_middle >= 0:- mid_block += [get_attention_layer( # always uses a self-attn- ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim,- disable_self_attn=disable_middle_self_attn, use_checkpoint=use_checkpoint- ),- get_resblock(- merge_factor=merge_factor,- merge_strategy=merge_strategy,- video_kernel_size=video_kernel_size,- ch=ch,- time_embed_dim=time_embed_dim,- dropout=dropout,- out_channels=None,- dims=dims,- use_checkpoint=use_checkpoint,- use_scale_shift_norm=use_scale_shift_norm,- dtype=self.dtype,- device=device,- operations=operations- )]- self.middle_block = TimestepEmbedSequential(*mid_block)++ self.middle_block = None+ if transformer_depth_middle >= -1:+ if transformer_depth_middle >= 0:+ mid_block += [get_attention_layer( # always uses a self-attn+ ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim,+ disable_self_attn=disable_middle_self_attn, use_checkpoint=use_checkpoint+ ),+ get_resblock(+ merge_factor=merge_factor,+ merge_strategy=merge_strategy,+ video_kernel_size=video_kernel_size,+ ch=ch,+ time_embed_dim=time_embed_dim,+ dropout=dropout,+ out_channels=None,+ dims=dims,+ use_checkpoint=use_checkpoint,+ use_scale_shift_norm=use_scale_shift_norm,+ dtype=self.dtype,+ device=device,+ operations=operations+ )]+ self.middle_block = TimestepEmbedSequential(*mid_block)self._feature_size += chself.output_blocks = nn.ModuleList([])@@ -858,7 +861,8 @@ class UNetModel(nn.Module):h = p(h, transformer_options)transformer_options["block"] = ("middle", 0)- h = forward_timestep_embed(self.middle_block, h, emb, context, transformer_options, time_context=time_context, num_video_frames=num_video_frames, image_only_indicator=image_only_indicator)+ if self.middle_block is not None:+ h = forward_timestep_embed(self.middle_block, h, emb, context, transformer_options, time_context=time_context, num_video_frames=num_video_frames, image_only_indicator=image_only_indicator)h = apply_control(h, control, 'middle')diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex 8fca6d8c8..07ee85708 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -151,8 +151,10 @@ def detect_unet_config(state_dict, key_prefix):channel_mult.append(last_channel_mult)if "{}middle_block.1.proj_in.weight".format(key_prefix) in state_dict_keys:transformer_depth_middle = count_blocks(state_dict_keys, '{}middle_block.1.transformer_blocks.'.format(key_prefix) + '{}')- else:+ elif "{}middle_block.0.in_layers.0.weight".format(key_prefix) in state_dict_keys:transformer_depth_middle = -1+ else:+ transformer_depth_middle = -2unet_config["in_channels"] = in_channelsunet_config["out_channels"] = out_channels@@ -242,6 +244,7 @@ def unet_config_from_diffusers_unet(state_dict, dtype=None):down_blocks = count_blocks(state_dict, "down_blocks.{}")for i in range(down_blocks):attn_blocks = count_blocks(state_dict, "down_blocks.{}.attentions.".format(i) + '{}')+ res_blocks = count_blocks(state_dict, "down_blocks.{}.resnets.".format(i) + '{}')for ab in range(attn_blocks):transformer_count = count_blocks(state_dict, "down_blocks.{}.attentions.{}.transformer_blocks.".format(i, ab) + '{}')transformer_depth.append(transformer_count)@@ -250,8 +253,8 @@ def unet_config_from_diffusers_unet(state_dict, dtype=None):attn_res *= 2if attn_blocks == 0:- transformer_depth.append(0)- transformer_depth.append(0)+ for i in range(res_blocks):+ transformer_depth.append(0)match["transformer_depth"] = transformer_depth@@ -329,7 +332,19 @@ def unet_config_from_diffusers_unet(state_dict, dtype=None):'channel_mult': [1, 2, 4], 'transformer_depth_middle': -1, 'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64,'use_temporal_attention': False, 'use_temporal_resblock': False}- supported_models = [SDXL, SDXL_refiner, SD21, SD15, SD21_uncliph, SD21_unclipl, SDXL_mid_cnet, SDXL_small_cnet, SDXL_diffusers_inpaint, SSD_1B, Segmind_Vega]+ KOALA_700M = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,+ 'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,+ 'num_res_blocks': [1, 1, 1], 'transformer_depth': [0, 2, 5], 'transformer_depth_output': [0, 0, 2, 2, 5, 5],+ 'channel_mult': [1, 2, 4], 'transformer_depth_middle': -2, 'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64,+ 'use_temporal_attention': False, 'use_temporal_resblock': False}++ KOALA_1B = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,+ 'num_classes': 'sequential', 'adm_in_channels': 2816, 'dtype': dtype, 'in_channels': 4, 'model_channels': 320,+ 'num_res_blocks': [1, 1, 1], 'transformer_depth': [0, 2, 6], 'transformer_depth_output': [0, 0, 2, 2, 6, 6],+ 'channel_mult': [1, 2, 4], 'transformer_depth_middle': 6, 'use_linear_in_transformer': True, 'context_dim': 2048, 'num_head_channels': 64,+ 'use_temporal_attention': False, 'use_temporal_resblock': False}++ supported_models = [SDXL, SDXL_refiner, SD21, SD15, SD21_uncliph, SD21_unclipl, SDXL_mid_cnet, SDXL_small_cnet, SDXL_diffusers_inpaint, SSD_1B, Segmind_Vega, KOALA_700M, KOALA_1B]for unet_config in supported_models:matches = Truediff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex 74908216c..375821032 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -234,6 +234,26 @@ class Segmind_Vega(SDXL):"use_temporal_attention": False,}+class KOALA_700M(SDXL):+ unet_config = {+ "model_channels": 320,+ "use_linear_in_transformer": True,+ "transformer_depth": [0, 2, 5],+ "context_dim": 2048,+ "adm_in_channels": 2816,+ "use_temporal_attention": False,+ }++class KOALA_1B(SDXL):+ unet_config = {+ "model_channels": 320,+ "use_linear_in_transformer": True,+ "transformer_depth": [0, 2, 6],+ "context_dim": 2048,+ "adm_in_channels": 2816,+ "use_temporal_attention": False,+ }+class SVD_img2vid(supported_models_base.BASE):unet_config = {"model_channels": 320,@@ -380,5 +400,5 @@ class Stable_Cascade_B(Stable_Cascade_C):return out-models = [Stable_Zero123, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXLRefiner, SDXL, SSD1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B]+models = [Stable_Zero123, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B]models += [SVD_img2vid]