The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Reduce MiniMax-H3 peak VRAM by releasing embedding temporaries before blocks
Tokha2332d6b73283af2
diff --git a/comfy/ldm/minimax/model.py b/comfy/ldm/minimax/model.pyindex c4b7a68aa..440d47ef3 100644--- a/comfy/ldm/minimax/model.py+++ b/comfy/ldm/minimax/model.py@@ -558,6 +558,49 @@ class MiniMaxH3Model(nn.Module):rows.append(r.to(device))return torch.cat(rows, dim=0) if rows else None+ def _embed_and_pack(self, video_x, audio_x, context, layout, payload, transformer_options):+ device = video_x.device+ dtype = context.dtype+ img_update = layout.img_update.to(device)+ audio_update = layout.audio_update.to(device)+ video_rows = patchify_video(video_x.to(torch.float32), self.patch_size)+ audio_rows = pack_audio(audio_x.to(torch.float32))+ cond_video_rows = self._cond_video_rows(payload, device)+ cond_audio_rows = self._cond_audio_rows(payload, device)++ all_video_rows = video_rows+ if cond_video_rows is not None:+ all_video_rows = torch.empty(img_update.shape[0], video_rows.shape[1], dtype=torch.float32, device=device)+ all_video_rows[~img_update] = cond_video_rows+ all_video_rows[img_update] = video_rows+ all_audio_rows = audio_rows+ if cond_audio_rows is not None:+ all_audio_rows = torch.empty(audio_update.shape[0], audio_rows.shape[1], dtype=torch.float32, device=device)+ all_audio_rows[~audio_update] = cond_audio_rows+ all_audio_rows[audio_update] = audio_rows++ video_embed = self.video_patch_proj(all_video_rows).to(dtype)+ audio_embed = self.audio_patch_proj(all_audio_rows).to(dtype)+ text_states = context[0]+ if text_states.shape[-1] != self.hidden_size:+ text_states = self.token_refiner(self.condition_proj(text_states),+ transformer_options=transformer_options)++ # segments are contiguous: assemble by slices, embed rows follow segment order+ h = torch.empty(layout.seq_len, self.hidden_size, dtype=dtype, device=device)+ voff = aoff = 0+ for a, b, kind in layout.segments:+ n = b - a+ if kind == "text":+ h[a:b] = text_states+ elif kind in ("cond", "ref_img", "video"):+ h[a:b] = video_embed[voff:voff + n]+ voff += n+ else: # ref_audio / audio+ h[a:b] = audio_embed[aoff:aoff + n]+ aoff += n+ return h+def forward(self, x, timestep, context, transformer_options={}, minimax_payload=None, denoise_mask=None, audio_denoise_mask=None, **kwargs):# the sampler carries the audio as (sigma_v / sigma_a) * x_audio; undo it outside# the wrappers so they and the network see the stream's own latent and velocity@@ -693,45 +736,7 @@ class MiniMaxH3Model(nn.Module):else:mod_segments.append((a, b, row_base + seg_tag[kind]))- # embed- img_update = layout.img_update.to(device)- audio_update = layout.audio_update.to(device)- video_rows = patchify_video(video_x.to(torch.float32), self.patch_size)- audio_rows = pack_audio(audio_x.to(torch.float32))- cond_video_rows = self._cond_video_rows(payload, device)- cond_audio_rows = self._cond_audio_rows(payload, device)-- all_video_rows = video_rows- if cond_video_rows is not None:- all_video_rows = torch.empty(img_update.shape[0], video_rows.shape[1], dtype=torch.float32, device=device)- all_video_rows[~img_update] = cond_video_rows- all_video_rows[img_update] = video_rows- all_audio_rows = audio_rows- if cond_audio_rows is not None:- all_audio_rows = torch.empty(audio_update.shape[0], audio_rows.shape[1], dtype=torch.float32, device=device)- all_audio_rows[~audio_update] = cond_audio_rows- all_audio_rows[audio_update] = audio_rows-- video_embed = self.video_patch_proj(all_video_rows).to(dtype)- audio_embed = self.audio_patch_proj(all_audio_rows).to(dtype)- text_states = context[0]- if text_states.shape[-1] != self.hidden_size:- text_states = self.token_refiner(self.condition_proj(text_states),- transformer_options=transformer_options)-- # segments are contiguous: assemble by slices, embed rows follow segment order- h = torch.empty(layout.seq_len, self.hidden_size, dtype=dtype, device=device)- voff = aoff = 0- for a, b, kind in layout.segments:- n = b - a- if kind == "text":- h[a:b] = text_states- elif kind in ("cond", "ref_img", "video"):- h[a:b] = video_embed[voff:voff + n]- voff += n- else: # ref_audio / audio- h[a:b] = audio_embed[aoff:aoff + n]- aoff += n+ h = self._embed_and_pack(video_x, audio_x, context, layout, payload, transformer_options)t_vals = torch.tensor(unique_t, dtype=torch.float32, device=device)if self.use_adaln_curves:diff --git a/tests-unit/comfy_test/test_minimax_h3_model.py b/tests-unit/comfy_test/test_minimax_h3_model.pyindex acf3d858d..4f07be944 100644--- a/tests-unit/comfy_test/test_minimax_h3_model.py+++ b/tests-unit/comfy_test/test_minimax_h3_model.py@@ -1,7 +1,10 @@+import weakref++import pytestimport torchfrom torch import nn-from comfy.ldm.minimax.model import MiniMaxH3Model, time_shift_sigma+from comfy.ldm.minimax.model import MiniMaxH3Model, PackedLayout, time_shift_sigmafrom comfy.model_sampling import CONST@@ -59,3 +62,54 @@ def test_forward_scales_audio_velocity_before_carry_conversion():expected = -3.0 * audio_src * carry + (1.0 + 3.0 * sigma_a) * audio_output * audio_masktorch.testing.assert_close(out[1], expected)++[email protected]("conditioning", ["none", "refs", "keyframes"])+def test_embed_and_pack_releases_intermediates(conditioning):+ model = MiniMaxH3Model(+ hidden_size=8, num_layers=0, token_refiner_num_layers=0,+ num_attention_heads=1, attention_head_dim=8, ffn_hidden_size=8,+ latents_dim=2, audio_latents_dim=3, text_dim=5,+ timestep_input_dim=8, time_embed_hidden_size=8, time_embed_dim=8,+ dtype=torch.float32, device="cpu", operations=nn,+ )+ video = torch.randn(1, 2, 1, 4, 6)+ audio = torch.randn(1, 3, 2, 2)+ context = torch.randn(1, 4, 8)+ payload = {}+ if conditioning != "none":+ cond_video = torch.randn_like(video)+ cond_audio = torch.randn(1, 3, 2, 1)+ payload.update(cond_video_latents=[cond_video], cond_audio_latents=[cond_audio])+ if conditioning == "refs":+ payload["refs"] = [{"kind": "image", "latent_h": 4, "latent_w": 6},+ {"kind": "audio", "ref_audio_t": 1}]+ else:+ payload["keyframes"] = [{"resolved_frame_index": 0, "latent": cond_video,+ "audio_latent": cond_audio}]+ layout = PackedLayout(4, 1, 4, 6, 2, refs=payload.get("refs"), keyframes=payload.get("keyframes"))+ outputs, refs = {}, []++ def remember(name):+ def hook(module, inputs, output):+ outputs[name] = output.clone()+ refs.extend([weakref.ref(inputs[0]), weakref.ref(output)])+ return hook++ handles = [model.video_patch_proj.register_forward_hook(remember("video")),+ model.audio_patch_proj.register_forward_hook(remember("audio"))]+ with torch.no_grad():+ packed = model._embed_and_pack(video, audio, context, layout, payload, {})+ for handle in handles:+ handle.remove()++ assert all(ref() is None for ref in refs)+ assert packed.shape == (layout.seq_len, 8)+ assert packed.dtype == torch.float32+ slices = {kind: torch.cat([packed[a:b] for a, b, name in layout.segments if name in kinds])+ for kind, kinds in {"video": {"video", "ref_img", "cond"},+ "audio": {"audio", "ref_audio", "cond_audio"},+ "text": {"text"}}.items()}+ assert torch.equal(slices["video"], outputs["video"])+ assert torch.equal(slices["audio"], outputs["audio"])+ assert torch.equal(slices["text"], context[0])