Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

feat: Support PH3Image (CORE-697) (#17506)

Jukka Seppänen576b642e77b4
diff --git a/comfy/ldm/minimax/vae.py b/comfy/ldm/minimax/vae.py
index 93ae0d9bb..9faf45919 100644
--- a/comfy/ldm/minimax/vae.py
+++ b/comfy/ldm/minimax/vae.py
@@ -17,6 +17,10 @@ ops = comfy.ops.disable_weight_init
IMAGENET_MEAN = (0.485, 0.456, 0.406)
IMAGENET_STD = (0.229, 0.224, 0.225)
+# p-h3-image: rope time coordinate of the decoded frame, and the seed of the posterior sample its encode conditions on
+PH3_IMAGE_FRAME_POSITION = 2.0 * (0.5 / 7) - 1.0
+PH3_IMAGE_ENCODE_SEED = 42
+
LATENTS_MEAN = [
0.858090341091156, -0.9606591463088989, 1.0661640167236328, -0.5090325474739075,
-0.2727581858634949, -1.3675414323806763, -0.2553254961967468, -0.26907554268836975,
@@ -394,6 +398,9 @@ class ViT3DDecoder(nn.Module):
self.norm_out = ops.LayerNorm(dim, elementwise_affine=True, eps=eps)
self.proj_out = ops.Linear(dim, out_channels * patch_size_t * patch_size * patch_size)
+ def _token_ids(self, T, H, W, device, dtype):
+ return create_token_ids((T, H, W), device, dtype)
+
def forward(self, x):
B, C, latent_T, latent_H, latent_W = x.shape
@@ -404,7 +411,7 @@ class ViT3DDecoder(nn.Module):
h = torch.cat([h, comfy.ops.cast_to_input(self.register_tokens, h).expand(B, -1, -1), torch.zeros_like(h[:, 0:1, :])], dim=1)
- img_ids = create_token_ids((latent_T, latent_H, latent_W), x.device, x.dtype).expand(B, -1, -1)
+ img_ids = self._token_ids(latent_T, latent_H, latent_W, x.device, x.dtype).expand(B, -1, -1)
suffix_ids = torch.zeros((B, num_suffix, 3), device=x.device, dtype=img_ids.dtype)
img_ids = torch.cat([img_ids, suffix_ids], dim=1)
@@ -419,6 +426,18 @@ class ViT3DDecoder(nn.Module):
self.out_channels, self.patch_size_t, self.patch_size)
+class PH3ImageDecoder(ViT3DDecoder):
+ """p-h3-image decoder: one frame per latent, at a fixed rope time coordinate."""
+
+ def __init__(self, **kwargs):
+ super().__init__(patch_size_t=1, **kwargs)
+ del self.mask_token
+
+ def _token_ids(self, T, H, W, device, dtype):
+ ids = create_token_ids((H, W), device, dtype)
+ return torch.cat([torch.full_like(ids[..., :1], PH3_IMAGE_FRAME_POSITION), ids], dim=-1)
+
+
class NonCausalViT3DDecoder(nn.Module):
"""Sol-H3-VAE decoder: the ViT decoder pruned to 24 layers and distilled to decode a whole clip at once, so
there is no spatial tiling and no temporal overlap recompute. Attention is local: 16x16-token spatial windows
@@ -990,3 +1009,25 @@ class MiniMaxH3VideoVAE(nn.Module):
output_buffer.copy_(dec)
return output_buffer
return self.decode_temporal(z, output_buffer)
+
+
+class PH3ImageVAE(MiniMaxH3VideoVAE):
+ """p-h3-image: the H3 encoder with a single-frame decoder."""
+
+ def __init__(self, operations=ops, num_layers=36):
+ super().__init__(still_frame=0, decoder=False)
+ self.post_quant_conv = ops.Conv3d(24, 24, 1)
+ self.decoder = PH3ImageDecoder(patch_size=self.vae_ratio, num_layers=num_layers, operations=operations)
+
+ def encode(self, x, device=None):
+ # x: [B, 3, 1, H, W] in [-1, 1]
+ moments = self._adaptive_encode(self._normalize_pixels(x.to(device=device)))
+ mean, logvar = moments.float().chunk(2, dim=1)
+ noise = torch.randn(mean.shape, generator=torch.Generator().manual_seed(PH3_IMAGE_ENCODE_SEED)).to(mean.device)
+ latents = torch.addcmul(mean, torch.exp(0.5 * logvar.clamp(-30.0, 20.0)), noise)
+ return (latents - self.latents_mean.view(1, -1, 1, 1, 1).to(latents)) / self.latents_std.view(1, -1, 1, 1, 1).to(latents)
+
+ def decode(self, z, output_buffer=None):
+ if z.shape[2] != 1:
+ raise ValueError(f"The p-h3-image VAE decodes single-frame image latents only, got {z.shape[2]} latent frames")
+ return super().decode(z, output_buffer)
diff --git a/comfy/sd.py b/comfy/sd.py
index 5069b463d..ca61c01f7 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -1051,7 +1051,12 @@ class VAE:
still_frame = int(metadata.get("h3_t1_output_slice", 3))
# LightVAE Lite encoders prune resnet blocks down to their shortcut
minimax_kept = {(int(k.split(".")[2]), int(k.split(".")[4])) for k in sd if k.startswith("encoder.down.") and k.endswith(".conv1.weight")}
- self.first_stage_model = comfy.ldm.minimax.vae.MiniMaxH3VideoVAE(operations=minimax_ops, num_layers=minimax_layers, still_frame=still_frame, encoder_kept_blocks=minimax_kept, encoder="encoder.conv_in.weight" in sd, decoder=minimax_layers > 0, noncausal_decoder=noncausal)
+ if "decoder.proj_out.weight" in sd and sd["decoder.proj_out.weight"].shape[0] == 768: # p-h3-image: single-frame decoder
+ still_frame = 0
+ self.not_video = True
+ self.first_stage_model = comfy.ldm.minimax.vae.PH3ImageVAE(operations=minimax_ops, num_layers=minimax_layers)
+ else:
+ self.first_stage_model = comfy.ldm.minimax.vae.MiniMaxH3VideoVAE(operations=minimax_ops, num_layers=minimax_layers, still_frame=still_frame, encoder_kept_blocks=minimax_kept, encoder="encoder.conv_in.weight" in sd, decoder=minimax_layers > 0, noncausal_decoder=noncausal)
self.latent_channels = 24
self.latent_dim = 3
# frames 17k+5 <-> latents 5k+2, 16x spatial
@@ -2062,6 +2067,10 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip
clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})
clip_target.clip = comfy.text_encoders.fibo2.te(**llama_detect(clip_data))
clip_target.tokenizer = comfy.text_encoders.fibo2.Fibo2Tokenizer
+ elif clip_type == CLIPType.MINIMAX and "text_connector.proj.weight" in clip_data[0]: # p-h3-image: Qwen3-VL-4B, 5-layer tap -> text connector -> 5120.
+ clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})
+ clip_target.clip = comfy.text_encoders.minimax.te(**llama_detect(clip_data), te_model=comfy.text_encoders.minimax.PH3ImageTEModel)
+ clip_target.tokenizer = comfy.text_encoders.minimax.PH3ImageTokenizer
elif clip_type == CLIPType.MAGE and te_model == TEModel.QWEN3VL_4B: # Mage-Flow: full Qwen3-VL-4B, last hidden state, Qwen-Image-style templates.
clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})
clip_target.clip = comfy.text_encoders.mage_flow.te(**llama_detect(clip_data))
diff --git a/comfy/text_encoders/minimax.py b/comfy/text_encoders/minimax.py
index b248e70c7..3c9e8f7e4 100644
--- a/comfy/text_encoders/minimax.py
+++ b/comfy/text_encoders/minimax.py
@@ -23,6 +23,7 @@ import math
import torch
import torch.nn.functional as F
+import comfy.ops
import comfy.sd1_clip
from .qwen3vl import Qwen3VL, Qwen3VLSDTokenizer
@@ -34,6 +35,9 @@ MINIMAX_EXTRA_TOKENS = {"<d>": 151669, "</d>": 151670, "<|cutoff|>": 151671,
"<|caption_start|>": 151674, "<|caption_end|>": 151675}
QWEN_IMAGE_MEAN = [0.5, 0.5, 0.5]
QWEN_IMAGE_STD = [0.5, 0.5, 0.5]
+# p-h3-image: Qwen3-VL-4B hidden states read by the text connector, the last one normed
+PH3_IMAGE_HIDDEN_STATES = [13, 19, 25, 31, 36]
+PH3_IMAGE_TEXT_DIM = 5120
def process_video_block(frames, patch_size=16, temporal_patch_size=2, merge_size=2,
@@ -103,11 +107,45 @@ class MiniMaxQwen3VL(Qwen3VL):
dtype=dtype, embeds_info=embeds_info, **kwargs)
+class TextConnector(torch.nn.Module):
+ def __init__(self, in_channels, out_channels, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.input_mean = torch.nn.Parameter(torch.empty(in_channels, dtype=dtype, device=device))
+ self.input_std = torch.nn.Parameter(torch.empty(in_channels, dtype=dtype, device=device))
+ self.proj = operations.Linear(in_channels, out_channels, bias=False, dtype=dtype, device=device)
+ self.output_mean = torch.nn.Parameter(torch.empty(out_channels, dtype=dtype, device=device))
+ self.output_std = torch.nn.Parameter(torch.empty(out_channels, dtype=dtype, device=device))
+ self.sink_embedding = torch.nn.Parameter(torch.empty(out_channels, dtype=dtype, device=device))
+
+ def forward(self, hidden_states):
+ # [B, layers, T, C] -> [B, T, layers * C] standardized, projected and rescaled; the first token is replaced by the sink
+ x = hidden_states.movedim(1, 2).flatten(2)
+ x = (x - comfy.ops.cast_to_input(self.input_mean, x)) / comfy.ops.cast_to_input(self.input_std, x)
+ x = self.proj(x)
+ x = torch.addcmul(comfy.ops.cast_to_input(self.output_mean, x), x, comfy.ops.cast_to_input(self.output_std, x))
+ sink = comfy.ops.cast_to_input(self.sink_embedding, x).expand(x.shape[0], 1, -1)
+ return torch.cat([sink, x[:, 1:]], dim=1)
+
+
+class PH3ImageQwen3VL(MiniMaxQwen3VL):
+ model_type = "qwen3vl_4b"
+
+ def __init__(self, config_dict, dtype, device, operations):
+ super().__init__(config_dict, dtype, device, operations)
+ # float32 and never quantized: the mixed-precision ops of a quantized file would load it at the compute dtype
+ self.text_connector = TextConnector(len(PH3_IMAGE_HIDDEN_STATES) * self.model.config.hidden_size, PH3_IMAGE_TEXT_DIM,
+ dtype=torch.float32, device=device, operations=comfy.ops.manual_cast)
+
+ def forward(self, *args, **kwargs):
+ x, hidden_states = super().forward(*args, **kwargs)
+ return x, self.text_connector(hidden_states)
+
+
class MiniMaxH3ClipModel(comfy.sd1_clip.SDClipModel):
- def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, model_options={}):
- super().__init__(device=device, layer="last", layer_idx=None, textmodel_json_config={},
+ def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, model_class=MiniMaxQwen3VL, model_options={}):
+ super().__init__(device=device, layer=layer, layer_idx=None, textmodel_json_config={},
dtype=dtype, special_tokens={"pad": 151643}, layer_norm_hidden_state=False,
- model_class=MiniMaxQwen3VL, enable_attention_masks=False,
+ model_class=model_class, enable_attention_masks=False,
return_attention_masks=False, model_options=model_options)
def encode_token_weights(self, token_weight_pairs):
@@ -120,12 +158,23 @@ class MiniMaxH3ClipModel(comfy.sd1_clip.SDClipModel):
return out
+class PH3ImageClipModel(MiniMaxH3ClipModel):
+ def __init__(self, device="cpu", dtype=None, model_options={}):
+ super().__init__(device=device, layer=PH3_IMAGE_HIDDEN_STATES, dtype=dtype, model_class=PH3ImageQwen3VL, model_options=model_options)
+
+
class MiniMaxH3TEModel(comfy.sd1_clip.SD1ClipModel):
def __init__(self, device="cpu", dtype=None, model_options={}):
super().__init__(device=device, dtype=dtype, name="qwen3vl_32b",
clip_model=MiniMaxH3ClipModel, model_options=model_options)
+class PH3ImageTEModel(comfy.sd1_clip.SD1ClipModel):
+ def __init__(self, device="cpu", dtype=None, model_options={}):
+ super().__init__(device=device, dtype=dtype, name="qwen3vl_4b",
+ clip_model=PH3ImageClipModel, model_options=model_options)
+
+
class MiniMaxQwenSDTokenizer(Qwen3VLSDTokenizer):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -152,7 +201,7 @@ class MiniMaxH3Tokenizer(comfy.sd1_clip.SD1Tokenizer):
def add_text(s):
if not s:
return
- token_batches = self.qwen3vl_32b.tokenize_with_weights(
+ token_batches = getattr(self, self.clip).tokenize_with_weights(
s,
return_word_ids=False,
disable_weights=True,
@@ -199,14 +248,17 @@ class MiniMaxH3Tokenizer(comfy.sd1_clip.SD1Tokenizer):
entries.append((151643, 1.0))
if return_word_ids:
entries = [t + (0,) for t in entries]
- return {"qwen3vl_32b": [entries]}
+ return {self.clip: [entries]}
+
- def untokenize(self, token_weight_pair):
- return self.qwen3vl_32b.untokenize(token_weight_pair)
+class PH3ImageTokenizer(MiniMaxH3Tokenizer):
+ def __init__(self, embedding_directory=None, tokenizer_data={}):
+ tokenizer = lambda *a, **kw: Qwen3VLSDTokenizer(*a, **kw, embedding_size=2560, embedding_key="qwen3vl_4b")
+ comfy.sd1_clip.SD1Tokenizer.__init__(self, embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, name="qwen3vl_4b", tokenizer=tokenizer)
-def te(dtype_llama=None, llama_quantization_metadata=None, **kwargs):
- class MiniMaxH3TEModel_(MiniMaxH3TEModel):
+def te(dtype_llama=None, llama_quantization_metadata=None, te_model=MiniMaxH3TEModel, **kwargs):
+ class MiniMaxH3TEModel_(te_model):
def __init__(self, device="cpu", dtype=None, model_options={}):
if dtype_llama is not None:
dtype = dtype_llama
diff --git a/comfy_extras/nodes_minimax_h3.py b/comfy_extras/nodes_minimax_h3.py
index 6989453f8..b0abbc717 100644
--- a/comfy_extras/nodes_minimax_h3.py
+++ b/comfy_extras/nodes_minimax_h3.py
@@ -10,6 +10,7 @@ audio stream's shifted schedule internally).
"""
import math
+import re
import torch
import torch.nn.functional as F
@@ -34,6 +35,12 @@ REF_IMAGE_SHORT_EDGE = 2048
FPS = 24
AUDIO_LATENT_FPS = 40
+PH3_IMAGE_SIZE = 2912
+PH3_IMAGE_VISION_PIXELS = 1000000
+PH3_IMAGE_EDIT = "Edit <Picture 1>. {}"
+# an edit brief from the upsampler, or an instruction that already names the picture to edit, is encoded as it is
+PH3_IMAGE_EDIT_VERBATIM = re.compile(r"\s*(Edit <Picture \d+>|subject_definitions:)")
+
def align_frame_count(n):
while n % 17 != 5:
@@ -82,13 +89,17 @@ def _encode_ref_audio(audio_vae, audio):
return z, z.shape[-1]
-def _empty_av_latent(width, height, length, batch_size=1):
- frame_count, latent_t, audio_t = temporal_shape(length)
+def _av_latent(width, height, latent_t, audio_t, batch_size=1):
video = torch.zeros([batch_size, 24, latent_t, height // 16, width // 16],
device=comfy.model_management.intermediate_device())
audio = torch.zeros([batch_size, 32, 2, audio_t],
device=comfy.model_management.intermediate_device())
- return {"samples": comfy.nested_tensor.NestedTensor((video, audio))}, frame_count
+ return {"samples": comfy.nested_tensor.NestedTensor((video, audio))}
+
+
+def _empty_av_latent(width, height, length, batch_size=1):
+ frame_count, latent_t, audio_t = temporal_shape(length)
+ return _av_latent(width, height, latent_t, audio_t, batch_size), frame_count
class EmptyMiniMaxH3LatentAV(io.ComfyNode):
@@ -654,6 +665,72 @@ class MiniMaxH3LatentUpscaleWithModel(io.ComfyNode):
return io.NodeOutput(out)
+def _size_within_area(width, height, pixels):
+ """Aspect-preserving scale (down only) to `pixels`, rounded to the canvas grid."""
+ scale = min(1.0, math.sqrt(pixels / (width * height)))
+ return tuple(max(CANVAS_MULTIPLE, round(side * scale / CANVAS_MULTIPLE) * CANVAS_MULTIPLE) for side in (width, height))
+
+
+def _resize_to(image, size):
+ # lanczos round-trips through 8 bit: an image already at its size stays untouched
+ if (image.shape[2], image.shape[1]) == size:
+ return image[..., :3]
+ return _resize(image, size[0], size[1], "disabled")
+
+
+class PH3ImageConditioning(io.ComfyNode):
+ """p-h3-image text-to-image and editing: prompt + images -> conditioning + image latent."""
+
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="PH3ImageConditioning",
+ display_name="p-h3-image Conditioning",
+ category="model/conditioning/minimax",
+ description="Text-to-image, or with images an edit of image_1 with the others as references. The prompt refers to them as <Picture 1>, <Picture 2>, ... The model needs ModelSamplingMiniMaxH3 at 1.0/1.0.",
+ inputs=[
+ io.Clip.Input("clip"),
+ io.Vae.Input("vae", optional=True, tooltip="The p-h3-image VAE, needed when images are connected."),
+ io.String.Input("prompt", multiline=True, dynamic_prompts=True,
+ tooltip="With images a short instruction gets \"Edit <Picture 1>. \" in front; an instruction that starts with \"Edit <Picture\" and an edit brief from the prompt upsampler are used as they are."),
+ io.Int.Input("width", default=0, min=0, max=nodes.MAX_RESOLUTION, step=CANVAS_MULTIPLE,
+ tooltip="0 uses the width of image_1 after scaling, or 2912 without images."),
+ io.Int.Input("height", default=0, min=0, max=nodes.MAX_RESOLUTION, step=CANVAS_MULTIPLE,
+ tooltip="0 uses the height of image_1 after scaling, or 2912 without images."),
+ io.Autogrow.Input("images", optional=True, template=io.Autogrow.TemplateNames(
+ io.Image.Input("image"), names=["image_{}".format(i) for i in range(1, 10)], min=0)),
+ ],
+ outputs=[io.Conditioning.Output(display_name="positive"), io.Latent.Output()],
+ )
+
+ @classmethod
+ def execute(cls, clip, prompt, width, height, vae=None, images=None) -> io.NodeOutput:
+ # image_1 is the image to edit, so the slots keep their order
+ names = sorted(images or {}, key=lambda name: int(name.rsplit("_", 1)[-1]))
+ images = [images[name][:1] for name in names if images[name] is not None]
+ if images and vae is None:
+ raise ValueError("p-h3-image images need the vae input")
+
+ area = (width * height) or PH3_IMAGE_SIZE ** 2
+ vision_images = []
+ refs = []
+ for image in images:
+ size = (image.shape[2], image.shape[1])
+ vision_images.append(_resize_to(image, _size_within_area(*size, PH3_IMAGE_VISION_PIXELS)))
+ z = vae.encode(_resize_to(image, _size_within_area(*size, area)))
+ refs.append({"kind": "image", "latent_h": z.shape[-2], "latent_w": z.shape[-1], "latent": z})
+
+ if refs:
+ prompt = prompt.strip() if PH3_IMAGE_EDIT_VERBATIM.match(prompt) else PH3_IMAGE_EDIT.format(prompt)
+ cond = clip.encode_from_tokens_scheduled(clip.tokenize(prompt, images=vision_images))
+ if refs:
+ cond = node_helpers.conditioning_set_values(cond, {"minimax_refs": refs})
+ latent_width = width or (refs[0]["latent_w"] * 16 if refs else PH3_IMAGE_SIZE)
+ latent_height = height or (refs[0]["latent_h"] * 16 if refs else PH3_IMAGE_SIZE)
+ # a still image, with the two audio latents it spans
+ return io.NodeOutput(cond, _av_latent(latent_width, latent_height, 1, 2))
+
+
class MiniMaxH3Extension(ComfyExtension):
async def get_node_list(self):
return [
@@ -664,6 +741,7 @@ class MiniMaxH3Extension(ComfyExtension):
MiniMaxH3SigmaShift,
MiniMaxH3FunControlNetApply,
MiniMaxH3LatentUpscaleWithModel,
+ PH3ImageConditioning,
]