The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Fix sampler issues for audio with minimax, support more samplers. (#15243)
Jukka Seppänenbdcb886a4705
diff --git a/comfy/ldm/minimax/model.py b/comfy/ldm/minimax/model.pyindex 494350d40..98bd841db 100644--- a/comfy/ldm/minimax/model.py+++ b/comfy/ldm/minimax/model.py@@ -9,8 +9,9 @@ The packed sequence is:Timestep domain: the model receives the *video* sigma from the sampler andderives per-token timesteps t = 1 - sigma internally; the audio stream runs onits own shifted schedule (sigma_shift video 12.0 / audio 3.0), mapped from the-video sigma in closed form. The audio velocity is returned scaled by the-schedule map's derivative d(sigma_a)/d(sigma_v).+video sigma in closed form. The sampler carries the audio latent scaled onto the+video schedule (ModelSamplingAV); forward() undoes that scale and converts the+velocity back, so _forward only ever sees the stream's own latent."""import math@@ -38,17 +39,6 @@ def time_shift_sigma(sigma, from_shift, to_shift):return to_shift * base / (1.0 + (to_shift - 1.0) * base)-def time_shift_slope(sigma, from_shift, to_shift):- """d(sigma_to)/d(sigma_from) at the same base-grid point.-- Scaling a stream's returned velocity by this slope makes the flat ODE that- any sampler integrates on the from-schedule equal to that stream's true ODE- on its own schedule.- """- base = sigma / (from_shift + sigma * (1.0 - from_shift))- return (to_shift * (1.0 + (from_shift - 1.0) * base) ** 2) / (from_shift * (1.0 + (to_shift - 1.0) * base) ** 2)--def patchify_video(latent, patch_size=(1, 2, 2)):# [B, C, T, H, W] -> [B*t*h*w, C*pt*ph*pw]b, c, t_full, h_full, w_full = latent.shape@@ -496,12 +486,30 @@ class MiniMaxH3Model(nn.Module):return torch.cat(rows, dim=0) if rows else Nonedef forward(self, x, timestep, context, transformer_options={}, minimax_payload=None, **kwargs):- return comfy.patcher_extension.WrapperExecutor.new_class_executor(+ # the sampler carries the audio as (sigma_v / sigma_a) * x_audio; undo it outside+ # the wrappers so they and the network see the stream's own latent and velocity+ scale = float((minimax_payload or {}).get("audio_scale", 1.0))+ audio_x = x[1]+ if scale != 1.0:+ shift_v = float(transformer_options.get("minimax_h3_sigma_shift_video", self.sigma_shift_video))+ shift_a = float(transformer_options.get("minimax_h3_sigma_shift_audio", self.sigma_shift_audio))+ sigma_v = (timestep.flatten()[0] / 1000.0).float().clamp(min=1e-6)+ sigma_a = time_shift_sigma(sigma_v, shift_v, shift_a)+ audio_x = audio_x * (sigma_a / sigma_v).to(audio_x.dtype)+ x = [x[0], audio_x]++ out = comfy.patcher_extension.WrapperExecutor.new_class_executor(self._forward,self,comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)).execute(x, timestep, context, transformer_options, minimax_payload=minimax_payload, **kwargs)+ if scale != 1.0:+ # d/d(sigma_v) of the carried variable+ out[1] = ((1.0 - scale) * audio_x+ + (1.0 + (scale - 1.0) * sigma_a).to(out[1].dtype) * out[1])+ return out+def _forward(self, x, timestep, context, transformer_options={}, minimax_payload=None, **kwargs):video_x, audio_x = x[0], x[1]orig_t, orig_h, orig_w = video_x.shape[2], video_x.shape[3], video_x.shape[4]@@ -639,8 +647,4 @@ class MiniMaxH3Model(nn.Module):video_out = video_out[:, :, :orig_t, :orig_h, :orig_w]audio_out = unpack_audio(a)- # The sampler integrates the flat ODE dX/dsigma_v = (X - denoised)/sigma_v.- # Scaling the audio velocity by d(sigma_a)/d(sigma_v) makes that ODE equal- # to the audio stream's true ODE on its own shifted schedule.- slope_a = time_shift_slope(sigma_v, shift_v, shift_a).to(audio_out.dtype)- return [-video_out.to(video_x.dtype), (-slope_a) * audio_out.to(audio_x.dtype)]+ return [-video_out.to(video_x.dtype), -audio_out.to(audio_x.dtype)]diff --git a/comfy/model_base.py b/comfy/model_base.pyindex 6f75d53b5..1b9247fe0 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -22,6 +22,7 @@ import torchimport loggingimport comfy.ldm.lightricks.av_modelimport comfy.ldm.minimax.model+import comfy.nested_tensorimport comfy.ldm.lightricks.symmetric_patchifierimport comfy.context_windowsfrom comfy.ldm.modules.diffusionmodules.openaimodel import UNetModel, Timestep@@ -100,6 +101,7 @@ class ModelType(Enum):FLOW_COSMOS = 10IMG_TO_IMG_FLOW = 11V_PREDICTION_DDPM = 12+ FLOW_AV = 13def model_sampling(model_config, model_type):@@ -136,6 +138,9 @@ def model_sampling(model_config, model_type):c = comfy.model_sampling.IMG_TO_IMG_FLOWelif model_type == ModelType.V_PREDICTION_DDPM:c = comfy.model_sampling.V_PREDICTION_DDPM+ elif model_type == ModelType.FLOW_AV:+ c = comfy.model_sampling.CONST+ s = comfy.model_sampling.ModelSamplingAVclass ModelSampling(s, c):pass@@ -180,6 +185,7 @@ class BaseModel(torch.nn.Module):self.model_type = model_typeself.model_sampling = model_sampling(model_config, model_type)+ self.latent_shapes = None # set by the sampler for models that pack several streams into one latentself.adm_channels = unet_config.get("adm_in_channels", None)if self.adm_channels is None:@@ -2065,9 +2071,33 @@ class Hunyuan3Dv2_1(BaseModel):return outclass MiniMaxH3(BaseModel):- def __init__(self, model_config, model_type=ModelType.FLOW, device=None):+ def __init__(self, model_config, model_type=ModelType.FLOW_AV, device=None):super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.minimax.model.MiniMaxH3Model)+ def audio_scale(self):+ """Scale the sampler carries the audio stream at, 1.0 when not sampling the packed latent."""+ if self.latent_shapes is None or len(self.latent_shapes) < 2:+ return 1.0+ return self.model_sampling.audio_scale++ def _scale_audio_slice(self, latent, scale):+ # the sampler carries the audio stream scaled onto the video schedule+ if scale == 1.0:+ return latent+ if latent.is_nested: # the x0 output hands back the unpacked view+ streams = latent.unbind()+ return comfy.nested_tensor.NestedTensor([streams[0], streams[1] * scale] + list(streams[2:]))+ n = math.prod(self.latent_shapes[0][1:])+ latent = latent.clone()+ latent[..., n:] *= scale+ return latent++ def process_latent_in(self, latent):+ return self._scale_audio_slice(super().process_latent_in(latent), self.audio_scale())++ def process_latent_out(self, latent):+ return super().process_latent_out(self._scale_audio_slice(latent, 1.0 / self.audio_scale()))+def extra_conds(self, **kwargs):out = super().extra_conds(**kwargs)cross_attn = kwargs.get("cross_attn", None)@@ -2102,6 +2132,8 @@ class MiniMaxH3(BaseModel):if kwargs.get("minimax_audio_cond_noise_aug", None) is not None:payload["audio_cond_noise_aug"] = kwargs["minimax_audio_cond_noise_aug"]payload["seed"] = kwargs.get("seed", 0)+ # same value process_latent_in/out used, so the model never undoes a scale that was not applied+ payload["audio_scale"] = self.audio_scale()if cross_attn is not None and latent_shapes is not None and len(latent_shapes) > 1:# packed layout built once per sampling run, h/w rounded up to the DiT's 2x2 patchvs = latent_shapes[0]diff --git a/comfy/model_sampling.py b/comfy/model_sampling.pyindex 5af336e76..7b00d6704 100644--- a/comfy/model_sampling.py+++ b/comfy/model_sampling.py@@ -325,6 +325,27 @@ class ModelSamplingDiscreteFlow(torch.nn.Module):return 0.0return time_snr_shift(self.shift, 1.0 - percent)+class ModelSamplingAV(ModelSamplingDiscreteFlow):+ """Flow sampling for packed audio-video latents whose audio stream has its own flow shift.++ Carrying the audio latent scaled onto the video schedule makes the pack an ordinary+ single-schedule flow latent whose audio target is scaled by audio_scale.+ """+ def __init__(self, model_config=None):+ super().__init__(model_config)+ sampling_settings = model_config.sampling_settings if model_config is not None else {}+ self.audio_shift = sampling_settings.get("audio_shift", None)++ def set_parameters(self, shift=1.0, audio_shift=None, timesteps=1000, multiplier=1000):+ self.audio_shift = audio_shift+ super().set_parameters(shift=shift, timesteps=timesteps, multiplier=multiplier)++ @property+ def audio_scale(self):+ if self.audio_shift is None:+ return 1.0+ return self.shift / self.audio_shift+class StableCascadeSampling(ModelSamplingDiscrete):def __init__(self, model_config=None):super().__init__()diff --git a/comfy/samplers.py b/comfy/samplers.pyindex a280f3bb6..1d6a4e104 100755--- a/comfy/samplers.py+++ b/comfy/samplers.py@@ -1218,6 +1218,8 @@ class CFGGuider:return sampling_function(self.inner_model, x, timestep, self.conds.get("negative", None), self.conds.get("positive", None), self.cfg, model_options=model_options, seed=seed)def inner_sample(self, noise, latent_image, device, sampler, sigmas, denoise_mask, callback, disable_pbar, seed, latent_shapes=None):+ self.inner_model.latent_shapes = latent_shapes+if latent_image is not None and torch.count_nonzero(latent_image) > 0: #Don't shift the empty latent image.latent_image = self.inner_model.process_latent_in(latent_image)diff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex 51b58ed1e..bc53594d7 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -963,6 +963,7 @@ class MiniMaxH3(supported_models_base.BASE):sampling_settings = {"shift": 12.0,+ "audio_shift": 3.0,}unet_extra_config = {}diff --git a/comfy_extras/nodes_lt.py b/comfy_extras/nodes_lt.pyindex b4806bd6a..8c85c92b1 100644--- a/comfy_extras/nodes_lt.py+++ b/comfy_extras/nodes_lt.py@@ -758,22 +758,60 @@ class LTXVConcatAVLatent(io.ComfyNode):],)+ @staticmethod+ def fit_audio(reference, audio, noise_mask):+ """Trim or zero-pad the audio stream to the length of the one it replaces.++ The padded tail is left unmasked so the model generates it, which is what a+ clip shorter than the video should do.+ """+ dims = [i for i in range(reference.ndim) if reference.shape[i] != audio.shape[i]]+ if len(dims) == 0:+ return audio, noise_mask+ if len(dims) > 1 or dims[0] < 2:+ raise ValueError("audio latent {} cannot be fitted to {}".format(tuple(audio.shape), tuple(reference.shape)))++ dim, length = dims[0], reference.shape[dims[0]]+ if noise_mask is not None: # masks carry their own shape until sampling resizes them+ noise_mask = comfy.utils.reshape_mask(noise_mask, audio.shape)++ if audio.shape[dim] > length:+ audio = audio.narrow(dim, 0, length)+ if noise_mask is not None:+ noise_mask = noise_mask.narrow(dim, 0, length)+ else:+ pad = torch.zeros_like(audio.narrow(dim, 0, 1)).repeat(+ [length - audio.shape[dim] if i == dim else 1 for i in range(audio.ndim)])+ audio = torch.cat([audio, pad], dim=dim)+ if noise_mask is not None:+ noise_mask = torch.cat([noise_mask, torch.ones_like(pad)], dim=dim)+ return audio, noise_mask+@classmethoddef execute(cls, video_latent, audio_latent) -> io.NodeOutput:output = {}output.update(video_latent)output.update(audio_latent)+ video_samples = video_latent["samples"]+ audio_samples = audio_latent["samples"]video_noise_mask = video_latent.get("noise_mask", None)audio_noise_mask = audio_latent.get("noise_mask", None)+ if video_samples.is_nested: # already an AV latent: keep its video and swap the audio stream+ streams = video_samples.unbind()+ video_samples = streams[0]+ if video_noise_mask is not None:+ video_noise_mask = video_noise_mask.unbind()[0]+ audio_samples, audio_noise_mask = cls.fit_audio(streams[1], audio_samples, audio_noise_mask)+if video_noise_mask is not None or audio_noise_mask is not None:if video_noise_mask is None:- video_noise_mask = torch.ones_like(video_latent["samples"])+ video_noise_mask = torch.ones_like(video_samples)if audio_noise_mask is None:- audio_noise_mask = torch.ones_like(audio_latent["samples"])+ audio_noise_mask = torch.ones_like(audio_samples)output["noise_mask"] = comfy.nested_tensor.NestedTensor((video_noise_mask, audio_noise_mask))- output["samples"] = comfy.nested_tensor.NestedTensor((video_latent["samples"], audio_latent["samples"]))+ output["samples"] = comfy.nested_tensor.NestedTensor((video_samples, audio_samples))return io.NodeOutput(output)diff --git a/comfy_extras/nodes_minimax_h3.py b/comfy_extras/nodes_minimax_h3.pyindex 22bc91cd4..0b1840e85 100644--- a/comfy_extras/nodes_minimax_h3.py+++ b/comfy_extras/nodes_minimax_h3.py@@ -283,9 +283,9 @@ class MiniMaxH3ReferenceToVideo(io.ComfyNode):class MiniMaxH3SigmaShift(io.ComfyNode):"""Set the video/audio flow shifts coherently.- The video shift drives the sampler's sigma schedule; both values are also- handed to the DiT, which inverts the video schedule to the shared base grid- and derives the audio schedule from it.+ The video shift drives the sampler's sigma schedule (ModelSamplingAV); both+ values are also handed to the DiT, which inverts the video schedule to the+ shared base grid and derives the audio schedule from it."""@classmethod@@ -293,7 +293,8 @@ class MiniMaxH3SigmaShift(io.ComfyNode):return io.Schema(node_id="MiniMaxH3SigmaShift",description="Set the video/audio flow shifts.",- display_name="MiniMax H3 Sigma Shift",+ display_name="ModelSamplingMiniMaxH3",+ search_aliases=["sigma shift", "minimax shift"],category="model/patch/minimax",inputs=[io.Model.Input("model"),@@ -307,12 +308,12 @@ class MiniMaxH3SigmaShift(io.ComfyNode):def execute(cls, model, shift_video, shift_audio) -> io.NodeOutput:m = model.clone()- class ModelSamplingAdvanced(comfy.model_sampling.ModelSamplingDiscreteFlow, comfy.model_sampling.CONST):+ class ModelSamplingAdvanced(comfy.model_sampling.ModelSamplingAV, comfy.model_sampling.CONST):passoriginal = m.get_model_object("model_sampling")model_sampling = ModelSamplingAdvanced(model.model.model_config)- model_sampling.set_parameters(shift=shift_video)+ model_sampling.set_parameters(shift=shift_video, audio_shift=shift_audio)if hasattr(original, "noise_scale"):model_sampling.set_noise_scale(original.noise_scale)m.add_object_patch("model_sampling", model_sampling)@@ -329,7 +330,7 @@ class MiniMaxH3Extension(ComfyExtension):EmptyMiniMaxH3LatentAV,MiniMaxH3ImageToVideo,MiniMaxH3ReferenceToVideo,- MiniMaxH3SigmaShift+ MiniMaxH3SigmaShift,]