The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
feat: Marigold v2 support (CORE-431) (#16232)
Jukka Seppänen1d91a82dc623
diff --git a/comfy/model_sampling.py b/comfy/model_sampling.pyindex 7b00d6704..4a5565520 100644--- a/comfy/model_sampling.py+++ b/comfy/model_sampling.py@@ -118,6 +118,16 @@ class IMG_TO_IMG_FLOW(CONST):def inverse_noise_scaling(self, sigma, latent):return 1.0 - latent+class IMG_TO_IMG_VELOCITY(CONST):+ def calculate_denoised(self, sigma, model_output, model_input):+ return model_input - model_output++ def noise_scaling(self, sigma, noise, latent_image, max_denoise=False):+ return latent_image++ def inverse_noise_scaling(self, sigma, latent):+ return latent+class COSMOS_RFLOW:def calculate_input(self, sigma, noise):sigma = (sigma / (sigma + 1))diff --git a/comfy_extras/nodes_cond.py b/comfy_extras/nodes_cond.pyindex c8091b7a4..ac43a63b4 100644--- a/comfy_extras/nodes_cond.py+++ b/comfy_extras/nodes_cond.py@@ -1,5 +1,12 @@+import json+import os++import safetensors.torch+import torchfrom typing_extensions import override+import comfy.utils+import folder_pathsfrom comfy_api.latest import ComfyExtension, io@@ -57,12 +64,84 @@ class T5TokenizerOptions(io.ComfyNode):return io.NodeOutput(clip)+class ConditioningLoader(io.ComfyNode):+ @classmethod+ def define_schema(cls) -> io.Schema:+ return io.Schema(+ node_id="ConditioningLoader",+ display_name="Load Conditioning",+ category="model/loaders",+ description="Loads a conditioning saved with Save Conditioning, or any safetensors file with a 'conditioning' tensor, from the embeddings folder.",+ inputs=[+ io.Combo.Input("conditioning_name", options=folder_paths.get_filename_list("embeddings")),+ ],+ outputs=[io.Conditioning.Output()],+ )++ @classmethod+ def execute(cls, conditioning_name) -> io.NodeOutput:+ sd, metadata = comfy.utils.load_torch_file(folder_paths.get_full_path_or_raise("embeddings", conditioning_name), safe_load=True, return_metadata=True)+ cond = sd.pop("conditioning")+ options = json.loads((metadata or {}).get("conditioning_options", "{}"))+ lists = {}+ for k, v in sd.items():+ name, _, index = k.rpartition(".")+ if index.isdigit():+ lists.setdefault(name, {})[int(index)] = v+ else:+ options[k] = v+ options.update({k: [v[i] for i in sorted(v)] for k, v in lists.items()})+ return io.NodeOutput([[cond, options]])+++class SaveConditioning(io.ComfyNode):+ @classmethod+ def define_schema(cls) -> io.Schema:+ return io.Schema(+ node_id="SaveConditioning",+ search_aliases=["save conditioning", "export conditioning", "cache text encoder output"],+ display_name="Save Conditioning",+ category="model/conditioning",+ description="Saves a conditioning to the output folder as safetensors; move the file to models/embeddings to load it with Load Conditioning, e.g. to skip the text encoder.",+ is_output_node=True,+ inputs=[+ io.Conditioning.Input("conditioning"),+ io.String.Input("filename_prefix", default="conditioning/ComfyUI"),+ ],+ outputs=[io.Conditioning.Output(display_name="conditioning")],+ )++ @classmethod+ def execute(cls, conditioning, filename_prefix) -> io.NodeOutput:+ if len(conditioning) != 1:+ raise ValueError("Save Conditioning supports a single conditioning entry, save it before combining.")+ cond, options = conditioning[0]+ sd = {"conditioning": cond}+ values = {}+ for k, v in options.items():+ if isinstance(v, torch.Tensor):+ sd[k] = v+ elif isinstance(v, (list, tuple)) and all(isinstance(t, torch.Tensor) for t in v):+ sd.update({f"{k}.{i}": t for i, t in enumerate(v)})+ elif isinstance(v, (bool, int, float, str)):+ values[k] = v+ elif v is not None:+ raise ValueError(f"Conditioning option '{k}' ({type(v).__name__}) can't be saved.")+ full_output_folder, filename, counter, _, _ = folder_paths.get_save_image_path(filename_prefix, folder_paths.get_output_directory())+ safetensors.torch.save_file({k: v.detach().to("cpu", copy=True).contiguous() for k, v in sd.items()},+ os.path.join(full_output_folder, f"{filename}_{counter:05}_.safetensors"),+ metadata={"conditioning_options": json.dumps(values)})+ return io.NodeOutput(conditioning)++class CondExtension(ComfyExtension):@overrideasync def get_node_list(self) -> list[type[io.ComfyNode]]:return [CLIPTextEncodeControlnet,T5TokenizerOptions,+ ConditioningLoader,+ SaveConditioning,]diff --git a/comfy_extras/nodes_marigold.py b/comfy_extras/nodes_marigold.pynew file mode 100644index 000000000..539281b53--- /dev/null+++ b/comfy_extras/nodes_marigold.py@@ -0,0 +1,45 @@+from typing_extensions import override++import torch+import torch.nn.functional as F++from comfy_api.latest import ComfyExtension, io+++class MarigoldV2PostProcess(io.ComfyNode):+ @classmethod+ def define_schema(cls) -> io.Schema:+ return io.Schema(+ node_id="MarigoldV2PostProcess",+ display_name="Marigold V2 Post-Process",+ category="image/geometry estimation",+ description="Turns a decoded Marigold V2 prediction into an image: normalized depth with near as bright, unit surface normals, or sRGB albedo.",+ inputs=[+ io.Image.Input("image"),+ io.Combo.Input("prediction", options=["depth", "normals", "albedo"]),+ ],+ outputs=[io.Image.Output()],+ )++ @classmethod+ def execute(cls, image, prediction) -> io.NodeOutput:+ if prediction == "normals":+ return io.NodeOutput((F.normalize(image * 2.0 - 1.0, dim=-1) + 1.0) * 0.5)+ if prediction == "albedo":+ return io.NodeOutput(torch.where(image <= 0.0031308, image * 12.92, 1.055 * image.clamp(min=0.0031308) ** (1.0 / 2.4) - 0.055))+ d = image.mean(dim=-1, keepdim=True)+ lo = d.amin(dim=(1, 2, 3), keepdim=True)+ hi = d.amax(dim=(1, 2, 3), keepdim=True)+ return io.NodeOutput(((hi - d) / (hi - lo).clamp(min=1e-6)).repeat(1, 1, 1, 3))+++class MarigoldExtension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[io.ComfyNode]]:+ return [+ MarigoldV2PostProcess,+ ]+++async def comfy_entrypoint() -> MarigoldExtension:+ return MarigoldExtension()diff --git a/comfy_extras/nodes_model_advanced.py b/comfy_extras/nodes_model_advanced.pyindex 5e82ef791..97adc8cd8 100644--- a/comfy_extras/nodes_model_advanced.py+++ b/comfy_extras/nodes_model_advanced.py@@ -129,11 +129,13 @@ class ModelSamplingSD3:CATEGORY = "model/patch/stable diffusion"- def patch(self, model, shift, multiplier=1000):+ def patch(self, model, shift, multiplier=1000, sampling="flow"):m = model.clone()sampling_base = comfy.model_sampling.ModelSamplingDiscreteFlowsampling_type = comfy.model_sampling.CONST+ if sampling == "img_to_img_velocity":+ sampling_type = comfy.model_sampling.IMG_TO_IMG_VELOCITYclass ModelSamplingAdvanced(sampling_base, sampling_type):pass@@ -151,13 +153,15 @@ class ModelSamplingAuraFlow(ModelSamplingSD3):def INPUT_TYPES(s):return {"required": { "model": ("MODEL",),"shift": ("FLOAT", {"default": 1.73, "min": 0.0, "max": 100.0, "step":0.01}),+ },+ "optional": { "sampling": (["flow", "img_to_img_velocity"], {"default": "flow", "advanced": True}),}}FUNCTION = "patch_aura"CATEGORY = "model/patch"- def patch_aura(self, model, shift):- return self.patch(model, shift, multiplier=1.0)+ def patch_aura(self, model, shift, sampling="flow"):+ return self.patch(model, shift, multiplier=1.0, sampling=sampling)class ModelSamplingFlux:@classmethoddiff --git a/nodes.py b/nodes.pyindex 038b88350..b94b4d008 100644--- a/nodes.py+++ b/nodes.py@@ -2539,6 +2539,7 @@ async def init_builtin_extra_nodes():"nodes_seed.py","nodes_text.py","nodes_sam3d_body.py",+ "nodes_marigold.py",]import_failed = []