The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support MOSS-image color-depth control (#17206)
Jukka Seppänene3bee7ffcb7d
diff --git a/comfy/lora.py b/comfy/lora.pyindex d5b5c0c35..a7224ce90 100644--- a/comfy/lora.py+++ b/comfy/lora.py@@ -473,6 +473,8 @@ def calculate_shape(patches, weight, key, original_weights=None):patch_type, patch_data = v[0], v[1]if patch_type == "diff" and len(patch_data) > 1 and patch_data[1]['pad_weight']:current_shape = patch_data[0].shape+ elif patch_type == "set":+ current_shape = patch_data[0].shapereturn current_shape@@ -527,7 +529,10 @@ def calculate_weight(patches, weight, key, intermediate_dtype=torch.float32, orielse:weight += function(strength * comfy.model_management.cast_to_device(diff, weight.device, weight.dtype))elif patch_type == "set":- weight.copy_(v[0])+ if v[0].shape != weight.shape:+ weight = comfy.model_management.cast_to_device(v[0], weight.device, weight.dtype, copy=True)+ else:+ weight.copy_(v[0])elif patch_type == "model_as_lora":target_weight: torch.Tensor = v[0]diff_weight = comfy.model_management.cast_to_device(target_weight, weight.device, intermediate_dtype) - \diff --git a/comfy/lora_convert.py b/comfy/lora_convert.pyindex 749e81df3..ec8cca8a0 100644--- a/comfy/lora_convert.py+++ b/comfy/lora_convert.py@@ -33,9 +33,23 @@ def convert_uso_lora(sd):return sd_out+def convert_moss_control_lora(sd): #MOSS-Image control adapters: the widened video_embedder replaces the model's+ sd_out = {}+ for k in sd:+ k_to = "diffusion_model.{}".format(k)+ if k == "video_embedder.weight":+ k_to = "diffusion_model.video_embedder.set_weight"+ elif k == "video_embedder.bias":+ k_to = "diffusion_model.video_embedder.bias.set_weight"+ sd_out[k_to] = sd[k]+ return sd_out++def convert_lora(sd):if "img_in.lora_A.weight" in sd and "single_blocks.0.norm.key_norm.scale" in sd:return convert_lora_bfl_control(sd)+ if "video_embedder.weight" in sd and "layers.0.proj.q.lora_A.weight" in sd:+ return convert_moss_control_lora(sd)if "lora_unet__blocks_0_cross_attn_k.lora_down.weight" in sd:return convert_lora_wan_fun(sd)if "single_blocks.37.processor.qkv_lora.up.weight" in sd and "double_blocks.18.processor.qkv_lora2.up.weight" in sd:diff --git a/comfy/model_base.py b/comfy/model_base.pyindex 12c61e53b..8e42a86b4 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -2808,6 +2808,18 @@ class MOSSImage(BaseModel):out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)return out+ def concat_cond(self, **kwargs):+ # control adapters widen video_embedder, zeros are no control+ noise = kwargs["noise"]+ channels = self.diffusion_model.video_embedder.weight.shape[1] - noise.shape[1]+ if channels <= 0:+ return None+ image = kwargs.get("concat_latent_image", None)+ if image is None:+ return noise.new_zeros((noise.shape[0], channels) + noise.shape[2:])+ image = utils.common_upscale(image.to(kwargs["device"]), noise.shape[-1], noise.shape[-2], "bilinear", "center")+ return utils.resize_to_batch_size(self.process_latent_in(image), noise.shape[0])+class HunyuanImage21(BaseModel):def __init__(self, model_config, model_type=ModelType.FLOW, device=None):super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.hunyuan_video.model.HunyuanVideo)diff --git a/comfy_extras/nodes_moss_image.py b/comfy_extras/nodes_moss_image.pyindex 6f8a4d16b..23732072f 100644--- a/comfy_extras/nodes_moss_image.py+++ b/comfy_extras/nodes_moss_image.py@@ -2,7 +2,11 @@ import jsonimport mathimport re+import torchfrom typing_extensions import override++import comfy.utils+import node_helpersfrom comfy_api.latest import ComfyExtension, io@@ -47,11 +51,51 @@ class MOSSImageLayoutPrompt(io.ComfyNode):return io.NodeOutput(json.dumps(layout, ensure_ascii=False, separators=(",", ":")))+class MOSSImageControl(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="MOSSImageControl",+ display_name="MOSS-Image Color/Depth Control",+ category="model/conditioning/moss_image",+ description="Adds the color region and depth maps for a MOSS-Image color/depth control adapter loaded with a LoRA loader.",+ inputs=[+ io.Conditioning.Input("positive"),+ io.Conditioning.Input("negative"),+ io.Vae.Input("vae"),+ io.Image.Input("color_map", optional=True, tooltip="One flat color per region, at the output size. The colors are only region IDs."),+ io.Image.Input("depth_map", optional=True, tooltip="Depth map at the output size, near is white."),+ io.Float.Input("strength", default=1.0, min=0.0, max=10.0, step=0.01, tooltip="Scales the maps' contribution to the image tokens. 0.0 is the same as no maps, 1.0 is the trained strength."),+ io.Float.Input("start_percent", default=0.0, min=0.0, max=1.0, step=0.001, tooltip="Sampling percent at which the control starts."),+ io.Float.Input("end_percent", default=1.0, min=0.0, max=1.0, step=0.001, tooltip="Sampling percent at which the control ends. Ending early keeps the layout and depth while the rest of the steps loosen the shapes."),+ ],+ outputs=[io.Conditioning.Output(display_name="positive"), io.Conditioning.Output(display_name="negative")],+ is_experimental=True,+ )++ @classmethod+ def execute(cls, positive, negative, vae, color_map=None, depth_map=None, strength=1.0, start_percent=0.0, end_percent=1.0) -> io.NodeOutput:+ latents = [vae.encode(m[:, :, :, :3]) if m is not None else None for m in (color_map, depth_map)]+ ref = next((latent for latent in latents if latent is not None), None)+ if ref is None:+ return io.NodeOutput(positive, negative)+ batch = max(latent.shape[0] for latent in latents if latent is not None)+ # [color | depth]; a missing map is a zero latent+ latents = [torch.zeros_like(ref) if latent is None else latent if latent.shape[-2:] == ref.shape[-2:]+ else comfy.utils.common_upscale(latent, ref.shape[-1], ref.shape[-2], "bilinear", "center") for latent in latents]+ # the embedder is linear and zeros are no control, so scaling the latent scales the control's contribution+ control = torch.cat([comfy.utils.resize_to_batch_size(latent, batch) for latent in latents], dim=1) * strength+ values = {"concat_latent_image": control}+ return io.NodeOutput(node_helpers.conditioning_set_values_with_timestep_range(positive, values, start_percent, end_percent),+ node_helpers.conditioning_set_values_with_timestep_range(negative, values, start_percent, end_percent))++class MOSSImageExtension(ComfyExtension):@overrideasync def get_node_list(self) -> list[type[io.ComfyNode]]:return [MOSSImageLayoutPrompt,+ MOSSImageControl,]