Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Support MOSS-image color-depth control (#17206)

Jukka Seppänene3bee7ffcb7d
diff --git a/comfy/lora.py b/comfy/lora.py
index d5b5c0c35..a7224ce90 100644
--- a/comfy/lora.py
+++ b/comfy/lora.py
@@ -473,6 +473,8 @@ def calculate_shape(patches, weight, key, original_weights=None):
patch_type, patch_data = v[0], v[1]
if patch_type == "diff" and len(patch_data) > 1 and patch_data[1]['pad_weight']:
current_shape = patch_data[0].shape
+ elif patch_type == "set":
+ current_shape = patch_data[0].shape
return current_shape
@@ -527,7 +529,10 @@ def calculate_weight(patches, weight, key, intermediate_dtype=torch.float32, ori
else:
weight += function(strength * comfy.model_management.cast_to_device(diff, weight.device, weight.dtype))
elif patch_type == "set":
- weight.copy_(v[0])
+ if v[0].shape != weight.shape:
+ weight = comfy.model_management.cast_to_device(v[0], weight.device, weight.dtype, copy=True)
+ else:
+ weight.copy_(v[0])
elif patch_type == "model_as_lora":
target_weight: torch.Tensor = v[0]
diff_weight = comfy.model_management.cast_to_device(target_weight, weight.device, intermediate_dtype) - \
diff --git a/comfy/lora_convert.py b/comfy/lora_convert.py
index 749e81df3..ec8cca8a0 100644
--- a/comfy/lora_convert.py
+++ b/comfy/lora_convert.py
@@ -33,9 +33,23 @@ def convert_uso_lora(sd):
return sd_out
+def convert_moss_control_lora(sd): #MOSS-Image control adapters: the widened video_embedder replaces the model's
+ sd_out = {}
+ for k in sd:
+ k_to = "diffusion_model.{}".format(k)
+ if k == "video_embedder.weight":
+ k_to = "diffusion_model.video_embedder.set_weight"
+ elif k == "video_embedder.bias":
+ k_to = "diffusion_model.video_embedder.bias.set_weight"
+ sd_out[k_to] = sd[k]
+ return sd_out
+
+
def convert_lora(sd):
if "img_in.lora_A.weight" in sd and "single_blocks.0.norm.key_norm.scale" in sd:
return convert_lora_bfl_control(sd)
+ if "video_embedder.weight" in sd and "layers.0.proj.q.lora_A.weight" in sd:
+ return convert_moss_control_lora(sd)
if "lora_unet__blocks_0_cross_attn_k.lora_down.weight" in sd:
return convert_lora_wan_fun(sd)
if "single_blocks.37.processor.qkv_lora.up.weight" in sd and "double_blocks.18.processor.qkv_lora2.up.weight" in sd:
diff --git a/comfy/model_base.py b/comfy/model_base.py
index 12c61e53b..8e42a86b4 100644
--- a/comfy/model_base.py
+++ b/comfy/model_base.py
@@ -2808,6 +2808,18 @@ class MOSSImage(BaseModel):
out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)
return out
+ def concat_cond(self, **kwargs):
+ # control adapters widen video_embedder, zeros are no control
+ noise = kwargs["noise"]
+ channels = self.diffusion_model.video_embedder.weight.shape[1] - noise.shape[1]
+ if channels <= 0:
+ return None
+ image = kwargs.get("concat_latent_image", None)
+ if image is None:
+ return noise.new_zeros((noise.shape[0], channels) + noise.shape[2:])
+ image = utils.common_upscale(image.to(kwargs["device"]), noise.shape[-1], noise.shape[-2], "bilinear", "center")
+ return utils.resize_to_batch_size(self.process_latent_in(image), noise.shape[0])
+
class HunyuanImage21(BaseModel):
def __init__(self, model_config, model_type=ModelType.FLOW, device=None):
super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.hunyuan_video.model.HunyuanVideo)
diff --git a/comfy_extras/nodes_moss_image.py b/comfy_extras/nodes_moss_image.py
index 6f8a4d16b..23732072f 100644
--- a/comfy_extras/nodes_moss_image.py
+++ b/comfy_extras/nodes_moss_image.py
@@ -2,7 +2,11 @@ import json
import math
import re
+import torch
from typing_extensions import override
+
+import comfy.utils
+import node_helpers
from comfy_api.latest import ComfyExtension, io
@@ -47,11 +51,51 @@ class MOSSImageLayoutPrompt(io.ComfyNode):
return io.NodeOutput(json.dumps(layout, ensure_ascii=False, separators=(",", ":")))
+class MOSSImageControl(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="MOSSImageControl",
+ display_name="MOSS-Image Color/Depth Control",
+ category="model/conditioning/moss_image",
+ description="Adds the color region and depth maps for a MOSS-Image color/depth control adapter loaded with a LoRA loader.",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Conditioning.Input("negative"),
+ io.Vae.Input("vae"),
+ io.Image.Input("color_map", optional=True, tooltip="One flat color per region, at the output size. The colors are only region IDs."),
+ io.Image.Input("depth_map", optional=True, tooltip="Depth map at the output size, near is white."),
+ io.Float.Input("strength", default=1.0, min=0.0, max=10.0, step=0.01, tooltip="Scales the maps' contribution to the image tokens. 0.0 is the same as no maps, 1.0 is the trained strength."),
+ io.Float.Input("start_percent", default=0.0, min=0.0, max=1.0, step=0.001, tooltip="Sampling percent at which the control starts."),
+ io.Float.Input("end_percent", default=1.0, min=0.0, max=1.0, step=0.001, tooltip="Sampling percent at which the control ends. Ending early keeps the layout and depth while the rest of the steps loosen the shapes."),
+ ],
+ outputs=[io.Conditioning.Output(display_name="positive"), io.Conditioning.Output(display_name="negative")],
+ is_experimental=True,
+ )
+
+ @classmethod
+ def execute(cls, positive, negative, vae, color_map=None, depth_map=None, strength=1.0, start_percent=0.0, end_percent=1.0) -> io.NodeOutput:
+ latents = [vae.encode(m[:, :, :, :3]) if m is not None else None for m in (color_map, depth_map)]
+ ref = next((latent for latent in latents if latent is not None), None)
+ if ref is None:
+ return io.NodeOutput(positive, negative)
+ batch = max(latent.shape[0] for latent in latents if latent is not None)
+ # [color | depth]; a missing map is a zero latent
+ latents = [torch.zeros_like(ref) if latent is None else latent if latent.shape[-2:] == ref.shape[-2:]
+ else comfy.utils.common_upscale(latent, ref.shape[-1], ref.shape[-2], "bilinear", "center") for latent in latents]
+ # the embedder is linear and zeros are no control, so scaling the latent scales the control's contribution
+ control = torch.cat([comfy.utils.resize_to_batch_size(latent, batch) for latent in latents], dim=1) * strength
+ values = {"concat_latent_image": control}
+ return io.NodeOutput(node_helpers.conditioning_set_values_with_timestep_range(positive, values, start_percent, end_percent),
+ node_helpers.conditioning_set_values_with_timestep_range(negative, values, start_percent, end_percent))
+
+
class MOSSImageExtension(ComfyExtension):
@override
async def get_node_list(self) -> list[type[io.ComfyNode]]:
return [
MOSSImageLayoutPrompt,
+ MOSSImageControl,
]