The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
feat: Support PH3Image (CORE-697) (#17506)
Jukka Seppänen576b642e77b4
diff --git a/comfy/ldm/minimax/vae.py b/comfy/ldm/minimax/vae.pyindex 93ae0d9bb..9faf45919 100644--- a/comfy/ldm/minimax/vae.py+++ b/comfy/ldm/minimax/vae.py@@ -17,6 +17,10 @@ ops = comfy.ops.disable_weight_initIMAGENET_MEAN = (0.485, 0.456, 0.406)IMAGENET_STD = (0.229, 0.224, 0.225)+# p-h3-image: rope time coordinate of the decoded frame, and the seed of the posterior sample its encode conditions on+PH3_IMAGE_FRAME_POSITION = 2.0 * (0.5 / 7) - 1.0+PH3_IMAGE_ENCODE_SEED = 42+LATENTS_MEAN = [0.858090341091156, -0.9606591463088989, 1.0661640167236328, -0.5090325474739075,-0.2727581858634949, -1.3675414323806763, -0.2553254961967468, -0.26907554268836975,@@ -394,6 +398,9 @@ class ViT3DDecoder(nn.Module):self.norm_out = ops.LayerNorm(dim, elementwise_affine=True, eps=eps)self.proj_out = ops.Linear(dim, out_channels * patch_size_t * patch_size * patch_size)+ def _token_ids(self, T, H, W, device, dtype):+ return create_token_ids((T, H, W), device, dtype)+def forward(self, x):B, C, latent_T, latent_H, latent_W = x.shape@@ -404,7 +411,7 @@ class ViT3DDecoder(nn.Module):h = torch.cat([h, comfy.ops.cast_to_input(self.register_tokens, h).expand(B, -1, -1), torch.zeros_like(h[:, 0:1, :])], dim=1)- img_ids = create_token_ids((latent_T, latent_H, latent_W), x.device, x.dtype).expand(B, -1, -1)+ img_ids = self._token_ids(latent_T, latent_H, latent_W, x.device, x.dtype).expand(B, -1, -1)suffix_ids = torch.zeros((B, num_suffix, 3), device=x.device, dtype=img_ids.dtype)img_ids = torch.cat([img_ids, suffix_ids], dim=1)@@ -419,6 +426,18 @@ class ViT3DDecoder(nn.Module):self.out_channels, self.patch_size_t, self.patch_size)+class PH3ImageDecoder(ViT3DDecoder):+ """p-h3-image decoder: one frame per latent, at a fixed rope time coordinate."""++ def __init__(self, **kwargs):+ super().__init__(patch_size_t=1, **kwargs)+ del self.mask_token++ def _token_ids(self, T, H, W, device, dtype):+ ids = create_token_ids((H, W), device, dtype)+ return torch.cat([torch.full_like(ids[..., :1], PH3_IMAGE_FRAME_POSITION), ids], dim=-1)++class NonCausalViT3DDecoder(nn.Module):"""Sol-H3-VAE decoder: the ViT decoder pruned to 24 layers and distilled to decode a whole clip at once, sothere is no spatial tiling and no temporal overlap recompute. Attention is local: 16x16-token spatial windows@@ -990,3 +1009,25 @@ class MiniMaxH3VideoVAE(nn.Module):output_buffer.copy_(dec)return output_bufferreturn self.decode_temporal(z, output_buffer)+++class PH3ImageVAE(MiniMaxH3VideoVAE):+ """p-h3-image: the H3 encoder with a single-frame decoder."""++ def __init__(self, operations=ops, num_layers=36):+ super().__init__(still_frame=0, decoder=False)+ self.post_quant_conv = ops.Conv3d(24, 24, 1)+ self.decoder = PH3ImageDecoder(patch_size=self.vae_ratio, num_layers=num_layers, operations=operations)++ def encode(self, x, device=None):+ # x: [B, 3, 1, H, W] in [-1, 1]+ moments = self._adaptive_encode(self._normalize_pixels(x.to(device=device)))+ mean, logvar = moments.float().chunk(2, dim=1)+ noise = torch.randn(mean.shape, generator=torch.Generator().manual_seed(PH3_IMAGE_ENCODE_SEED)).to(mean.device)+ latents = torch.addcmul(mean, torch.exp(0.5 * logvar.clamp(-30.0, 20.0)), noise)+ return (latents - self.latents_mean.view(1, -1, 1, 1, 1).to(latents)) / self.latents_std.view(1, -1, 1, 1, 1).to(latents)++ def decode(self, z, output_buffer=None):+ if z.shape[2] != 1:+ raise ValueError(f"The p-h3-image VAE decodes single-frame image latents only, got {z.shape[2]} latent frames")+ return super().decode(z, output_buffer)diff --git a/comfy/sd.py b/comfy/sd.pyindex 5069b463d..ca61c01f7 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -1051,7 +1051,12 @@ class VAE:still_frame = int(metadata.get("h3_t1_output_slice", 3))# LightVAE Lite encoders prune resnet blocks down to their shortcutminimax_kept = {(int(k.split(".")[2]), int(k.split(".")[4])) for k in sd if k.startswith("encoder.down.") and k.endswith(".conv1.weight")}- self.first_stage_model = comfy.ldm.minimax.vae.MiniMaxH3VideoVAE(operations=minimax_ops, num_layers=minimax_layers, still_frame=still_frame, encoder_kept_blocks=minimax_kept, encoder="encoder.conv_in.weight" in sd, decoder=minimax_layers > 0, noncausal_decoder=noncausal)+ if "decoder.proj_out.weight" in sd and sd["decoder.proj_out.weight"].shape[0] == 768: # p-h3-image: single-frame decoder+ still_frame = 0+ self.not_video = True+ self.first_stage_model = comfy.ldm.minimax.vae.PH3ImageVAE(operations=minimax_ops, num_layers=minimax_layers)+ else:+ self.first_stage_model = comfy.ldm.minimax.vae.MiniMaxH3VideoVAE(operations=minimax_ops, num_layers=minimax_layers, still_frame=still_frame, encoder_kept_blocks=minimax_kept, encoder="encoder.conv_in.weight" in sd, decoder=minimax_layers > 0, noncausal_decoder=noncausal)self.latent_channels = 24self.latent_dim = 3# frames 17k+5 <-> latents 5k+2, 16x spatial@@ -2062,6 +2067,10 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clipclip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})clip_target.clip = comfy.text_encoders.fibo2.te(**llama_detect(clip_data))clip_target.tokenizer = comfy.text_encoders.fibo2.Fibo2Tokenizer+ elif clip_type == CLIPType.MINIMAX and "text_connector.proj.weight" in clip_data[0]: # p-h3-image: Qwen3-VL-4B, 5-layer tap -> text connector -> 5120.+ clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})+ clip_target.clip = comfy.text_encoders.minimax.te(**llama_detect(clip_data), te_model=comfy.text_encoders.minimax.PH3ImageTEModel)+ clip_target.tokenizer = comfy.text_encoders.minimax.PH3ImageTokenizerelif clip_type == CLIPType.MAGE and te_model == TEModel.QWEN3VL_4B: # Mage-Flow: full Qwen3-VL-4B, last hidden state, Qwen-Image-style templates.clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})clip_target.clip = comfy.text_encoders.mage_flow.te(**llama_detect(clip_data))diff --git a/comfy/text_encoders/minimax.py b/comfy/text_encoders/minimax.pyindex b248e70c7..3c9e8f7e4 100644--- a/comfy/text_encoders/minimax.py+++ b/comfy/text_encoders/minimax.py@@ -23,6 +23,7 @@ import mathimport torchimport torch.nn.functional as F+import comfy.opsimport comfy.sd1_clipfrom .qwen3vl import Qwen3VL, Qwen3VLSDTokenizer@@ -34,6 +35,9 @@ MINIMAX_EXTRA_TOKENS = {"<d>": 151669, "</d>": 151670, "<|cutoff|>": 151671,"<|caption_start|>": 151674, "<|caption_end|>": 151675}QWEN_IMAGE_MEAN = [0.5, 0.5, 0.5]QWEN_IMAGE_STD = [0.5, 0.5, 0.5]+# p-h3-image: Qwen3-VL-4B hidden states read by the text connector, the last one normed+PH3_IMAGE_HIDDEN_STATES = [13, 19, 25, 31, 36]+PH3_IMAGE_TEXT_DIM = 5120def process_video_block(frames, patch_size=16, temporal_patch_size=2, merge_size=2,@@ -103,11 +107,45 @@ class MiniMaxQwen3VL(Qwen3VL):dtype=dtype, embeds_info=embeds_info, **kwargs)+class TextConnector(torch.nn.Module):+ def __init__(self, in_channels, out_channels, dtype=None, device=None, operations=None):+ super().__init__()+ self.input_mean = torch.nn.Parameter(torch.empty(in_channels, dtype=dtype, device=device))+ self.input_std = torch.nn.Parameter(torch.empty(in_channels, dtype=dtype, device=device))+ self.proj = operations.Linear(in_channels, out_channels, bias=False, dtype=dtype, device=device)+ self.output_mean = torch.nn.Parameter(torch.empty(out_channels, dtype=dtype, device=device))+ self.output_std = torch.nn.Parameter(torch.empty(out_channels, dtype=dtype, device=device))+ self.sink_embedding = torch.nn.Parameter(torch.empty(out_channels, dtype=dtype, device=device))++ def forward(self, hidden_states):+ # [B, layers, T, C] -> [B, T, layers * C] standardized, projected and rescaled; the first token is replaced by the sink+ x = hidden_states.movedim(1, 2).flatten(2)+ x = (x - comfy.ops.cast_to_input(self.input_mean, x)) / comfy.ops.cast_to_input(self.input_std, x)+ x = self.proj(x)+ x = torch.addcmul(comfy.ops.cast_to_input(self.output_mean, x), x, comfy.ops.cast_to_input(self.output_std, x))+ sink = comfy.ops.cast_to_input(self.sink_embedding, x).expand(x.shape[0], 1, -1)+ return torch.cat([sink, x[:, 1:]], dim=1)+++class PH3ImageQwen3VL(MiniMaxQwen3VL):+ model_type = "qwen3vl_4b"++ def __init__(self, config_dict, dtype, device, operations):+ super().__init__(config_dict, dtype, device, operations)+ # float32 and never quantized: the mixed-precision ops of a quantized file would load it at the compute dtype+ self.text_connector = TextConnector(len(PH3_IMAGE_HIDDEN_STATES) * self.model.config.hidden_size, PH3_IMAGE_TEXT_DIM,+ dtype=torch.float32, device=device, operations=comfy.ops.manual_cast)++ def forward(self, *args, **kwargs):+ x, hidden_states = super().forward(*args, **kwargs)+ return x, self.text_connector(hidden_states)++class MiniMaxH3ClipModel(comfy.sd1_clip.SDClipModel):- def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, model_options={}):- super().__init__(device=device, layer="last", layer_idx=None, textmodel_json_config={},+ def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, model_class=MiniMaxQwen3VL, model_options={}):+ super().__init__(device=device, layer=layer, layer_idx=None, textmodel_json_config={},dtype=dtype, special_tokens={"pad": 151643}, layer_norm_hidden_state=False,- model_class=MiniMaxQwen3VL, enable_attention_masks=False,+ model_class=model_class, enable_attention_masks=False,return_attention_masks=False, model_options=model_options)def encode_token_weights(self, token_weight_pairs):@@ -120,12 +158,23 @@ class MiniMaxH3ClipModel(comfy.sd1_clip.SDClipModel):return out+class PH3ImageClipModel(MiniMaxH3ClipModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ super().__init__(device=device, layer=PH3_IMAGE_HIDDEN_STATES, dtype=dtype, model_class=PH3ImageQwen3VL, model_options=model_options)++class MiniMaxH3TEModel(comfy.sd1_clip.SD1ClipModel):def __init__(self, device="cpu", dtype=None, model_options={}):super().__init__(device=device, dtype=dtype, name="qwen3vl_32b",clip_model=MiniMaxH3ClipModel, model_options=model_options)+class PH3ImageTEModel(comfy.sd1_clip.SD1ClipModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ super().__init__(device=device, dtype=dtype, name="qwen3vl_4b",+ clip_model=PH3ImageClipModel, model_options=model_options)++class MiniMaxQwenSDTokenizer(Qwen3VLSDTokenizer):def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)@@ -152,7 +201,7 @@ class MiniMaxH3Tokenizer(comfy.sd1_clip.SD1Tokenizer):def add_text(s):if not s:return- token_batches = self.qwen3vl_32b.tokenize_with_weights(+ token_batches = getattr(self, self.clip).tokenize_with_weights(s,return_word_ids=False,disable_weights=True,@@ -199,14 +248,17 @@ class MiniMaxH3Tokenizer(comfy.sd1_clip.SD1Tokenizer):entries.append((151643, 1.0))if return_word_ids:entries = [t + (0,) for t in entries]- return {"qwen3vl_32b": [entries]}+ return {self.clip: [entries]}+- def untokenize(self, token_weight_pair):- return self.qwen3vl_32b.untokenize(token_weight_pair)+class PH3ImageTokenizer(MiniMaxH3Tokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ tokenizer = lambda *a, **kw: Qwen3VLSDTokenizer(*a, **kw, embedding_size=2560, embedding_key="qwen3vl_4b")+ comfy.sd1_clip.SD1Tokenizer.__init__(self, embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, name="qwen3vl_4b", tokenizer=tokenizer)-def te(dtype_llama=None, llama_quantization_metadata=None, **kwargs):- class MiniMaxH3TEModel_(MiniMaxH3TEModel):+def te(dtype_llama=None, llama_quantization_metadata=None, te_model=MiniMaxH3TEModel, **kwargs):+ class MiniMaxH3TEModel_(te_model):def __init__(self, device="cpu", dtype=None, model_options={}):if dtype_llama is not None:dtype = dtype_llamadiff --git a/comfy_extras/nodes_minimax_h3.py b/comfy_extras/nodes_minimax_h3.pyindex 6989453f8..b0abbc717 100644--- a/comfy_extras/nodes_minimax_h3.py+++ b/comfy_extras/nodes_minimax_h3.py@@ -10,6 +10,7 @@ audio stream's shifted schedule internally)."""import math+import reimport torchimport torch.nn.functional as F@@ -34,6 +35,12 @@ REF_IMAGE_SHORT_EDGE = 2048FPS = 24AUDIO_LATENT_FPS = 40+PH3_IMAGE_SIZE = 2912+PH3_IMAGE_VISION_PIXELS = 1000000+PH3_IMAGE_EDIT = "Edit <Picture 1>. {}"+# an edit brief from the upsampler, or an instruction that already names the picture to edit, is encoded as it is+PH3_IMAGE_EDIT_VERBATIM = re.compile(r"\s*(Edit <Picture \d+>|subject_definitions:)")+def align_frame_count(n):while n % 17 != 5:@@ -82,13 +89,17 @@ def _encode_ref_audio(audio_vae, audio):return z, z.shape[-1]-def _empty_av_latent(width, height, length, batch_size=1):- frame_count, latent_t, audio_t = temporal_shape(length)+def _av_latent(width, height, latent_t, audio_t, batch_size=1):video = torch.zeros([batch_size, 24, latent_t, height // 16, width // 16],device=comfy.model_management.intermediate_device())audio = torch.zeros([batch_size, 32, 2, audio_t],device=comfy.model_management.intermediate_device())- return {"samples": comfy.nested_tensor.NestedTensor((video, audio))}, frame_count+ return {"samples": comfy.nested_tensor.NestedTensor((video, audio))}+++def _empty_av_latent(width, height, length, batch_size=1):+ frame_count, latent_t, audio_t = temporal_shape(length)+ return _av_latent(width, height, latent_t, audio_t, batch_size), frame_countclass EmptyMiniMaxH3LatentAV(io.ComfyNode):@@ -654,6 +665,72 @@ class MiniMaxH3LatentUpscaleWithModel(io.ComfyNode):return io.NodeOutput(out)+def _size_within_area(width, height, pixels):+ """Aspect-preserving scale (down only) to `pixels`, rounded to the canvas grid."""+ scale = min(1.0, math.sqrt(pixels / (width * height)))+ return tuple(max(CANVAS_MULTIPLE, round(side * scale / CANVAS_MULTIPLE) * CANVAS_MULTIPLE) for side in (width, height))+++def _resize_to(image, size):+ # lanczos round-trips through 8 bit: an image already at its size stays untouched+ if (image.shape[2], image.shape[1]) == size:+ return image[..., :3]+ return _resize(image, size[0], size[1], "disabled")+++class PH3ImageConditioning(io.ComfyNode):+ """p-h3-image text-to-image and editing: prompt + images -> conditioning + image latent."""++ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="PH3ImageConditioning",+ display_name="p-h3-image Conditioning",+ category="model/conditioning/minimax",+ description="Text-to-image, or with images an edit of image_1 with the others as references. The prompt refers to them as <Picture 1>, <Picture 2>, ... The model needs ModelSamplingMiniMaxH3 at 1.0/1.0.",+ inputs=[+ io.Clip.Input("clip"),+ io.Vae.Input("vae", optional=True, tooltip="The p-h3-image VAE, needed when images are connected."),+ io.String.Input("prompt", multiline=True, dynamic_prompts=True,+ tooltip="With images a short instruction gets \"Edit <Picture 1>. \" in front; an instruction that starts with \"Edit <Picture\" and an edit brief from the prompt upsampler are used as they are."),+ io.Int.Input("width", default=0, min=0, max=nodes.MAX_RESOLUTION, step=CANVAS_MULTIPLE,+ tooltip="0 uses the width of image_1 after scaling, or 2912 without images."),+ io.Int.Input("height", default=0, min=0, max=nodes.MAX_RESOLUTION, step=CANVAS_MULTIPLE,+ tooltip="0 uses the height of image_1 after scaling, or 2912 without images."),+ io.Autogrow.Input("images", optional=True, template=io.Autogrow.TemplateNames(+ io.Image.Input("image"), names=["image_{}".format(i) for i in range(1, 10)], min=0)),+ ],+ outputs=[io.Conditioning.Output(display_name="positive"), io.Latent.Output()],+ )++ @classmethod+ def execute(cls, clip, prompt, width, height, vae=None, images=None) -> io.NodeOutput:+ # image_1 is the image to edit, so the slots keep their order+ names = sorted(images or {}, key=lambda name: int(name.rsplit("_", 1)[-1]))+ images = [images[name][:1] for name in names if images[name] is not None]+ if images and vae is None:+ raise ValueError("p-h3-image images need the vae input")++ area = (width * height) or PH3_IMAGE_SIZE ** 2+ vision_images = []+ refs = []+ for image in images:+ size = (image.shape[2], image.shape[1])+ vision_images.append(_resize_to(image, _size_within_area(*size, PH3_IMAGE_VISION_PIXELS)))+ z = vae.encode(_resize_to(image, _size_within_area(*size, area)))+ refs.append({"kind": "image", "latent_h": z.shape[-2], "latent_w": z.shape[-1], "latent": z})++ if refs:+ prompt = prompt.strip() if PH3_IMAGE_EDIT_VERBATIM.match(prompt) else PH3_IMAGE_EDIT.format(prompt)+ cond = clip.encode_from_tokens_scheduled(clip.tokenize(prompt, images=vision_images))+ if refs:+ cond = node_helpers.conditioning_set_values(cond, {"minimax_refs": refs})+ latent_width = width or (refs[0]["latent_w"] * 16 if refs else PH3_IMAGE_SIZE)+ latent_height = height or (refs[0]["latent_h"] * 16 if refs else PH3_IMAGE_SIZE)+ # a still image, with the two audio latents it spans+ return io.NodeOutput(cond, _av_latent(latent_width, latent_height, 1, 2))++class MiniMaxH3Extension(ComfyExtension):async def get_node_list(self):return [@@ -664,6 +741,7 @@ class MiniMaxH3Extension(ComfyExtension):MiniMaxH3SigmaShift,MiniMaxH3FunControlNetApply,MiniMaxH3LatentUpscaleWithModel,+ PH3ImageConditioning,]