Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

feat: Marigold v2 support (CORE-431) (#16232)

Jukka Seppänen1d91a82dc623
diff --git a/comfy/model_sampling.py b/comfy/model_sampling.py
index 7b00d6704..4a5565520 100644
--- a/comfy/model_sampling.py
+++ b/comfy/model_sampling.py
@@ -118,6 +118,16 @@ class IMG_TO_IMG_FLOW(CONST):
def inverse_noise_scaling(self, sigma, latent):
return 1.0 - latent
+class IMG_TO_IMG_VELOCITY(CONST):
+ def calculate_denoised(self, sigma, model_output, model_input):
+ return model_input - model_output
+
+ def noise_scaling(self, sigma, noise, latent_image, max_denoise=False):
+ return latent_image
+
+ def inverse_noise_scaling(self, sigma, latent):
+ return latent
+
class COSMOS_RFLOW:
def calculate_input(self, sigma, noise):
sigma = (sigma / (sigma + 1))
diff --git a/comfy_extras/nodes_cond.py b/comfy_extras/nodes_cond.py
index c8091b7a4..ac43a63b4 100644
--- a/comfy_extras/nodes_cond.py
+++ b/comfy_extras/nodes_cond.py
@@ -1,5 +1,12 @@
+import json
+import os
+
+import safetensors.torch
+import torch
from typing_extensions import override
+import comfy.utils
+import folder_paths
from comfy_api.latest import ComfyExtension, io
@@ -57,12 +64,84 @@ class T5TokenizerOptions(io.ComfyNode):
return io.NodeOutput(clip)
+class ConditioningLoader(io.ComfyNode):
+ @classmethod
+ def define_schema(cls) -> io.Schema:
+ return io.Schema(
+ node_id="ConditioningLoader",
+ display_name="Load Conditioning",
+ category="model/loaders",
+ description="Loads a conditioning saved with Save Conditioning, or any safetensors file with a 'conditioning' tensor, from the embeddings folder.",
+ inputs=[
+ io.Combo.Input("conditioning_name", options=folder_paths.get_filename_list("embeddings")),
+ ],
+ outputs=[io.Conditioning.Output()],
+ )
+
+ @classmethod
+ def execute(cls, conditioning_name) -> io.NodeOutput:
+ sd, metadata = comfy.utils.load_torch_file(folder_paths.get_full_path_or_raise("embeddings", conditioning_name), safe_load=True, return_metadata=True)
+ cond = sd.pop("conditioning")
+ options = json.loads((metadata or {}).get("conditioning_options", "{}"))
+ lists = {}
+ for k, v in sd.items():
+ name, _, index = k.rpartition(".")
+ if index.isdigit():
+ lists.setdefault(name, {})[int(index)] = v
+ else:
+ options[k] = v
+ options.update({k: [v[i] for i in sorted(v)] for k, v in lists.items()})
+ return io.NodeOutput([[cond, options]])
+
+
+class SaveConditioning(io.ComfyNode):
+ @classmethod
+ def define_schema(cls) -> io.Schema:
+ return io.Schema(
+ node_id="SaveConditioning",
+ search_aliases=["save conditioning", "export conditioning", "cache text encoder output"],
+ display_name="Save Conditioning",
+ category="model/conditioning",
+ description="Saves a conditioning to the output folder as safetensors; move the file to models/embeddings to load it with Load Conditioning, e.g. to skip the text encoder.",
+ is_output_node=True,
+ inputs=[
+ io.Conditioning.Input("conditioning"),
+ io.String.Input("filename_prefix", default="conditioning/ComfyUI"),
+ ],
+ outputs=[io.Conditioning.Output(display_name="conditioning")],
+ )
+
+ @classmethod
+ def execute(cls, conditioning, filename_prefix) -> io.NodeOutput:
+ if len(conditioning) != 1:
+ raise ValueError("Save Conditioning supports a single conditioning entry, save it before combining.")
+ cond, options = conditioning[0]
+ sd = {"conditioning": cond}
+ values = {}
+ for k, v in options.items():
+ if isinstance(v, torch.Tensor):
+ sd[k] = v
+ elif isinstance(v, (list, tuple)) and all(isinstance(t, torch.Tensor) for t in v):
+ sd.update({f"{k}.{i}": t for i, t in enumerate(v)})
+ elif isinstance(v, (bool, int, float, str)):
+ values[k] = v
+ elif v is not None:
+ raise ValueError(f"Conditioning option '{k}' ({type(v).__name__}) can't be saved.")
+ full_output_folder, filename, counter, _, _ = folder_paths.get_save_image_path(filename_prefix, folder_paths.get_output_directory())
+ safetensors.torch.save_file({k: v.detach().to("cpu", copy=True).contiguous() for k, v in sd.items()},
+ os.path.join(full_output_folder, f"{filename}_{counter:05}_.safetensors"),
+ metadata={"conditioning_options": json.dumps(values)})
+ return io.NodeOutput(conditioning)
+
+
class CondExtension(ComfyExtension):
@override
async def get_node_list(self) -> list[type[io.ComfyNode]]:
return [
CLIPTextEncodeControlnet,
T5TokenizerOptions,
+ ConditioningLoader,
+ SaveConditioning,
]
diff --git a/comfy_extras/nodes_marigold.py b/comfy_extras/nodes_marigold.py
new file mode 100644
index 000000000..539281b53
--- /dev/null
+++ b/comfy_extras/nodes_marigold.py
@@ -0,0 +1,45 @@
+from typing_extensions import override
+
+import torch
+import torch.nn.functional as F
+
+from comfy_api.latest import ComfyExtension, io
+
+
+class MarigoldV2PostProcess(io.ComfyNode):
+ @classmethod
+ def define_schema(cls) -> io.Schema:
+ return io.Schema(
+ node_id="MarigoldV2PostProcess",
+ display_name="Marigold V2 Post-Process",
+ category="image/geometry estimation",
+ description="Turns a decoded Marigold V2 prediction into an image: normalized depth with near as bright, unit surface normals, or sRGB albedo.",
+ inputs=[
+ io.Image.Input("image"),
+ io.Combo.Input("prediction", options=["depth", "normals", "albedo"]),
+ ],
+ outputs=[io.Image.Output()],
+ )
+
+ @classmethod
+ def execute(cls, image, prediction) -> io.NodeOutput:
+ if prediction == "normals":
+ return io.NodeOutput((F.normalize(image * 2.0 - 1.0, dim=-1) + 1.0) * 0.5)
+ if prediction == "albedo":
+ return io.NodeOutput(torch.where(image <= 0.0031308, image * 12.92, 1.055 * image.clamp(min=0.0031308) ** (1.0 / 2.4) - 0.055))
+ d = image.mean(dim=-1, keepdim=True)
+ lo = d.amin(dim=(1, 2, 3), keepdim=True)
+ hi = d.amax(dim=(1, 2, 3), keepdim=True)
+ return io.NodeOutput(((hi - d) / (hi - lo).clamp(min=1e-6)).repeat(1, 1, 1, 3))
+
+
+class MarigoldExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ MarigoldV2PostProcess,
+ ]
+
+
+async def comfy_entrypoint() -> MarigoldExtension:
+ return MarigoldExtension()
diff --git a/comfy_extras/nodes_model_advanced.py b/comfy_extras/nodes_model_advanced.py
index 5e82ef791..97adc8cd8 100644
--- a/comfy_extras/nodes_model_advanced.py
+++ b/comfy_extras/nodes_model_advanced.py
@@ -129,11 +129,13 @@ class ModelSamplingSD3:
CATEGORY = "model/patch/stable diffusion"
- def patch(self, model, shift, multiplier=1000):
+ def patch(self, model, shift, multiplier=1000, sampling="flow"):
m = model.clone()
sampling_base = comfy.model_sampling.ModelSamplingDiscreteFlow
sampling_type = comfy.model_sampling.CONST
+ if sampling == "img_to_img_velocity":
+ sampling_type = comfy.model_sampling.IMG_TO_IMG_VELOCITY
class ModelSamplingAdvanced(sampling_base, sampling_type):
pass
@@ -151,13 +153,15 @@ class ModelSamplingAuraFlow(ModelSamplingSD3):
def INPUT_TYPES(s):
return {"required": { "model": ("MODEL",),
"shift": ("FLOAT", {"default": 1.73, "min": 0.0, "max": 100.0, "step":0.01}),
+ },
+ "optional": { "sampling": (["flow", "img_to_img_velocity"], {"default": "flow", "advanced": True}),
}}
FUNCTION = "patch_aura"
CATEGORY = "model/patch"
- def patch_aura(self, model, shift):
- return self.patch(model, shift, multiplier=1.0)
+ def patch_aura(self, model, shift, sampling="flow"):
+ return self.patch(model, shift, multiplier=1.0, sampling=sampling)
class ModelSamplingFlux:
@classmethod
diff --git a/nodes.py b/nodes.py
index 038b88350..b94b4d008 100644
--- a/nodes.py
+++ b/nodes.py
@@ -2539,6 +2539,7 @@ async def init_builtin_extra_nodes():
"nodes_seed.py",
"nodes_text.py",
"nodes_sam3d_body.py",
+ "nodes_marigold.py",
]
import_failed = []