Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Reduce MiniMax-H3 peak VRAM by releasing embedding temporaries before blocks

Tokha2332d6b73283af2
diff --git a/comfy/ldm/minimax/model.py b/comfy/ldm/minimax/model.py
index c4b7a68aa..440d47ef3 100644
--- a/comfy/ldm/minimax/model.py
+++ b/comfy/ldm/minimax/model.py
@@ -558,6 +558,49 @@ class MiniMaxH3Model(nn.Module):
rows.append(r.to(device))
return torch.cat(rows, dim=0) if rows else None
+ def _embed_and_pack(self, video_x, audio_x, context, layout, payload, transformer_options):
+ device = video_x.device
+ dtype = context.dtype
+ img_update = layout.img_update.to(device)
+ audio_update = layout.audio_update.to(device)
+ video_rows = patchify_video(video_x.to(torch.float32), self.patch_size)
+ audio_rows = pack_audio(audio_x.to(torch.float32))
+ cond_video_rows = self._cond_video_rows(payload, device)
+ cond_audio_rows = self._cond_audio_rows(payload, device)
+
+ all_video_rows = video_rows
+ if cond_video_rows is not None:
+ all_video_rows = torch.empty(img_update.shape[0], video_rows.shape[1], dtype=torch.float32, device=device)
+ all_video_rows[~img_update] = cond_video_rows
+ all_video_rows[img_update] = video_rows
+ all_audio_rows = audio_rows
+ if cond_audio_rows is not None:
+ all_audio_rows = torch.empty(audio_update.shape[0], audio_rows.shape[1], dtype=torch.float32, device=device)
+ all_audio_rows[~audio_update] = cond_audio_rows
+ all_audio_rows[audio_update] = audio_rows
+
+ video_embed = self.video_patch_proj(all_video_rows).to(dtype)
+ audio_embed = self.audio_patch_proj(all_audio_rows).to(dtype)
+ text_states = context[0]
+ if text_states.shape[-1] != self.hidden_size:
+ text_states = self.token_refiner(self.condition_proj(text_states),
+ transformer_options=transformer_options)
+
+ # segments are contiguous: assemble by slices, embed rows follow segment order
+ h = torch.empty(layout.seq_len, self.hidden_size, dtype=dtype, device=device)
+ voff = aoff = 0
+ for a, b, kind in layout.segments:
+ n = b - a
+ if kind == "text":
+ h[a:b] = text_states
+ elif kind in ("cond", "ref_img", "video"):
+ h[a:b] = video_embed[voff:voff + n]
+ voff += n
+ else: # ref_audio / audio
+ h[a:b] = audio_embed[aoff:aoff + n]
+ aoff += n
+ return h
+
def forward(self, x, timestep, context, transformer_options={}, minimax_payload=None, denoise_mask=None, audio_denoise_mask=None, **kwargs):
# the sampler carries the audio as (sigma_v / sigma_a) * x_audio; undo it outside
# the wrappers so they and the network see the stream's own latent and velocity
@@ -693,45 +736,7 @@ class MiniMaxH3Model(nn.Module):
else:
mod_segments.append((a, b, row_base + seg_tag[kind]))
- # embed
- img_update = layout.img_update.to(device)
- audio_update = layout.audio_update.to(device)
- video_rows = patchify_video(video_x.to(torch.float32), self.patch_size)
- audio_rows = pack_audio(audio_x.to(torch.float32))
- cond_video_rows = self._cond_video_rows(payload, device)
- cond_audio_rows = self._cond_audio_rows(payload, device)
-
- all_video_rows = video_rows
- if cond_video_rows is not None:
- all_video_rows = torch.empty(img_update.shape[0], video_rows.shape[1], dtype=torch.float32, device=device)
- all_video_rows[~img_update] = cond_video_rows
- all_video_rows[img_update] = video_rows
- all_audio_rows = audio_rows
- if cond_audio_rows is not None:
- all_audio_rows = torch.empty(audio_update.shape[0], audio_rows.shape[1], dtype=torch.float32, device=device)
- all_audio_rows[~audio_update] = cond_audio_rows
- all_audio_rows[audio_update] = audio_rows
-
- video_embed = self.video_patch_proj(all_video_rows).to(dtype)
- audio_embed = self.audio_patch_proj(all_audio_rows).to(dtype)
- text_states = context[0]
- if text_states.shape[-1] != self.hidden_size:
- text_states = self.token_refiner(self.condition_proj(text_states),
- transformer_options=transformer_options)
-
- # segments are contiguous: assemble by slices, embed rows follow segment order
- h = torch.empty(layout.seq_len, self.hidden_size, dtype=dtype, device=device)
- voff = aoff = 0
- for a, b, kind in layout.segments:
- n = b - a
- if kind == "text":
- h[a:b] = text_states
- elif kind in ("cond", "ref_img", "video"):
- h[a:b] = video_embed[voff:voff + n]
- voff += n
- else: # ref_audio / audio
- h[a:b] = audio_embed[aoff:aoff + n]
- aoff += n
+ h = self._embed_and_pack(video_x, audio_x, context, layout, payload, transformer_options)
t_vals = torch.tensor(unique_t, dtype=torch.float32, device=device)
if self.use_adaln_curves:
diff --git a/tests-unit/comfy_test/test_minimax_h3_model.py b/tests-unit/comfy_test/test_minimax_h3_model.py
index acf3d858d..4f07be944 100644
--- a/tests-unit/comfy_test/test_minimax_h3_model.py
+++ b/tests-unit/comfy_test/test_minimax_h3_model.py
@@ -1,7 +1,10 @@
+import weakref
+
+import pytest
import torch
from torch import nn
-from comfy.ldm.minimax.model import MiniMaxH3Model, time_shift_sigma
+from comfy.ldm.minimax.model import MiniMaxH3Model, PackedLayout, time_shift_sigma
from comfy.model_sampling import CONST
@@ -59,3 +62,54 @@ def test_forward_scales_audio_velocity_before_carry_conversion():
expected = -3.0 * audio_src * carry + (1.0 + 3.0 * sigma_a) * audio_output * audio_mask
torch.testing.assert_close(out[1], expected)
+
+
[email protected]("conditioning", ["none", "refs", "keyframes"])
+def test_embed_and_pack_releases_intermediates(conditioning):
+ model = MiniMaxH3Model(
+ hidden_size=8, num_layers=0, token_refiner_num_layers=0,
+ num_attention_heads=1, attention_head_dim=8, ffn_hidden_size=8,
+ latents_dim=2, audio_latents_dim=3, text_dim=5,
+ timestep_input_dim=8, time_embed_hidden_size=8, time_embed_dim=8,
+ dtype=torch.float32, device="cpu", operations=nn,
+ )
+ video = torch.randn(1, 2, 1, 4, 6)
+ audio = torch.randn(1, 3, 2, 2)
+ context = torch.randn(1, 4, 8)
+ payload = {}
+ if conditioning != "none":
+ cond_video = torch.randn_like(video)
+ cond_audio = torch.randn(1, 3, 2, 1)
+ payload.update(cond_video_latents=[cond_video], cond_audio_latents=[cond_audio])
+ if conditioning == "refs":
+ payload["refs"] = [{"kind": "image", "latent_h": 4, "latent_w": 6},
+ {"kind": "audio", "ref_audio_t": 1}]
+ else:
+ payload["keyframes"] = [{"resolved_frame_index": 0, "latent": cond_video,
+ "audio_latent": cond_audio}]
+ layout = PackedLayout(4, 1, 4, 6, 2, refs=payload.get("refs"), keyframes=payload.get("keyframes"))
+ outputs, refs = {}, []
+
+ def remember(name):
+ def hook(module, inputs, output):
+ outputs[name] = output.clone()
+ refs.extend([weakref.ref(inputs[0]), weakref.ref(output)])
+ return hook
+
+ handles = [model.video_patch_proj.register_forward_hook(remember("video")),
+ model.audio_patch_proj.register_forward_hook(remember("audio"))]
+ with torch.no_grad():
+ packed = model._embed_and_pack(video, audio, context, layout, payload, {})
+ for handle in handles:
+ handle.remove()
+
+ assert all(ref() is None for ref in refs)
+ assert packed.shape == (layout.seq_len, 8)
+ assert packed.dtype == torch.float32
+ slices = {kind: torch.cat([packed[a:b] for a, b, name in layout.segments if name in kinds])
+ for kind, kinds in {"video": {"video", "ref_img", "cond"},
+ "audio": {"audio", "ref_audio", "cond_audio"},
+ "text": {"text"}}.items()}
+ assert torch.equal(slices["video"], outputs["video"])
+ assert torch.equal(slices["audio"], outputs["audio"])
+ assert torch.equal(slices["text"], context[0])