The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support Ideogram4 (#14259)
Jukka Seppänen24f9a020ce0c
diff --git a/comfy/ldm/ideogram4/model.py b/comfy/ldm/ideogram4/model.pynew file mode 100644index 000000000..3b02a243a--- /dev/null+++ b/comfy/ldm/ideogram4/model.py@@ -0,0 +1,297 @@+"""+The Ideogram 4 transformer is a NextDiT/Lumina2-family single-stream model+consumes Qwen3-VL hidden-state features (concatenated from 13 layers -> 53248 dims)+packs ``[text tokens, image tokens]`` into one sequence with block-diagonal segment attention and 3D interleaved MRoPE.+"""++from __future__ import annotations++import math++import torch+import torch.nn as nn+import torch.nn.functional as F++import comfy.patcher_extension+from comfy.ldm.lumina.model import FeedForward+from comfy.ldm.modules.attention import optimized_attention_masked+from comfy.text_encoders.llama import apply_rope, precompute_freqs_cis++# Per-token role indicators+SEQUENCE_PADDING_INDICATOR = -1+OUTPUT_IMAGE_INDICATOR = 2+LLM_TOKEN_INDICATOR = 3+# Image grid coordinates are offset so they never collide with text positions+IMAGE_POSITION_OFFSET = 65536+++class Ideogram4Attention(nn.Module):+ def __init__(self, hidden_size, num_heads, eps=1e-5, dtype=None, device=None, operations=None):+ super().__init__()+ self.num_heads = num_heads+ self.head_dim = hidden_size // num_heads+ self.hidden_size = hidden_size++ self.qkv = operations.Linear(hidden_size, hidden_size * 3, bias=False, dtype=dtype, device=device)+ self.norm_q = operations.RMSNorm(self.head_dim, eps=eps, elementwise_affine=True, dtype=dtype, device=device)+ self.norm_k = operations.RMSNorm(self.head_dim, eps=eps, elementwise_affine=True, dtype=dtype, device=device)+ self.o = operations.Linear(hidden_size, hidden_size, bias=False, dtype=dtype, device=device)++ def forward(self, x, attn_mask, freqs_cis, transformer_options={}):+ batch_size, seq_len, _ = x.shape+ qkv = self.qkv(x).view(batch_size, seq_len, 3, self.num_heads, self.head_dim)+ q, k, v = qkv.unbind(dim=2)++ q = self.norm_q(q)+ k = self.norm_k(k)++ # (B, heads, L, head_dim)+ q = q.transpose(1, 2)+ k = k.transpose(1, 2)+ v = v.transpose(1, 2)++ q, k = apply_rope(q, k, freqs_cis)++ out = optimized_attention_masked(q, k, v, self.num_heads, attn_mask, skip_reshape=True, transformer_options=transformer_options)+ return self.o(out)+++class Ideogram4TransformerBlock(nn.Module):+ def __init__(self, hidden_size, intermediate_size, num_heads, norm_eps, adaln_dim, dtype=None, device=None, operations=None):+ super().__init__()+ self.attention = Ideogram4Attention(hidden_size, num_heads, eps=1e-5, dtype=dtype, device=device, operations=operations)+ self.feed_forward = FeedForward(+ dim=hidden_size, hidden_dim=intermediate_size, multiple_of=1, ffn_dim_multiplier=None,+ operation_settings={"operations": operations, "dtype": dtype, "device": device},+ )++ self.attention_norm1 = operations.RMSNorm(hidden_size, eps=norm_eps, elementwise_affine=True, dtype=dtype, device=device)+ self.ffn_norm1 = operations.RMSNorm(hidden_size, eps=norm_eps, elementwise_affine=True, dtype=dtype, device=device)+ self.attention_norm2 = operations.RMSNorm(hidden_size, eps=norm_eps, elementwise_affine=True, dtype=dtype, device=device)+ self.ffn_norm2 = operations.RMSNorm(hidden_size, eps=norm_eps, elementwise_affine=True, dtype=dtype, device=device)++ self.adaln_modulation = operations.Linear(adaln_dim, 4 * hidden_size, bias=True, dtype=dtype, device=device)++ def forward(self, x, attn_mask, freqs_cis, adaln_input, transformer_options={}):+ mod = self.adaln_modulation(adaln_input)+ scale_msa, gate_msa, scale_mlp, gate_mlp = mod.chunk(4, dim=-1)+ gate_msa = torch.tanh(gate_msa)+ gate_mlp = torch.tanh(gate_mlp)+ scale_msa = 1.0 + scale_msa+ scale_mlp = 1.0 + scale_mlp++ attn_out = self.attention(self.attention_norm1(x) * scale_msa, attn_mask, freqs_cis, transformer_options=transformer_options)+ x = x + gate_msa * self.attention_norm2(attn_out)+ x = x + gate_mlp * self.ffn_norm2(self.feed_forward(self.ffn_norm1(x) * scale_mlp))+ return x+++def _sinusoidal_embedding(t, dim, scale=1e4):+ t = t.to(torch.float32)+ half = dim // 2+ freq = math.log(scale) / (half - 1)+ freq = torch.exp(torch.arange(half, dtype=torch.float32, device=t.device) * -freq)+ emb = t.unsqueeze(-1) * freq+ emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=-1)+ if dim % 2 == 1:+ emb = F.pad(emb, (0, 1))+ return emb+++class Ideogram4EmbedScalar(nn.Module):+ def __init__(self, dim, input_range=(0.0, 1.0), dtype=None, device=None, operations=None):+ super().__init__()+ self.dim = dim+ self.range_min, self.range_max = input_range+ self.mlp_in = operations.Linear(dim, dim, bias=True, dtype=dtype, device=device)+ self.mlp_out = operations.Linear(dim, dim, bias=True, dtype=dtype, device=device)++ def forward(self, x):+ x = x.to(torch.float32)+ scaled = 1e4 * (x - self.range_min) / (self.range_max - self.range_min)+ emb = _sinusoidal_embedding(scaled, self.dim)+ emb = emb.to(self.mlp_in.weight.dtype)+ emb = F.silu(self.mlp_in(emb))+ return self.mlp_out(emb)+++class Ideogram4FinalLayer(nn.Module):+ def __init__(self, hidden_size, out_channels, adaln_dim, dtype=None, device=None, operations=None):+ super().__init__()+ self.norm_final = operations.LayerNorm(hidden_size, eps=1e-6, elementwise_affine=False, dtype=dtype, device=device)+ self.linear = operations.Linear(hidden_size, out_channels, bias=True, dtype=dtype, device=device)+ self.adaln_modulation = operations.Linear(adaln_dim, hidden_size, bias=True, dtype=dtype, device=device)++ def forward(self, x, c):+ scale = 1.0 + self.adaln_modulation(F.silu(c))+ return self.linear(self.norm_final(x) * scale)+++class Ideogram4Transformer(nn.Module):+ """A single Ideogram 4 backbone operating on a packed token sequence."""++ def __init__(self, emb_dim, num_layers, num_heads, intermediate_size, adaln_dim,+ in_channels, llm_features_dim, rope_theta, mrope_section, norm_eps,+ dtype=None, device=None, operations=None):+ super().__init__()+ self.head_dim = emb_dim // num_heads+ self.rope_theta = rope_theta+ self.mrope_section = tuple(mrope_section)++ self.input_proj = operations.Linear(in_channels, emb_dim, bias=True, dtype=dtype, device=device)+ self.llm_cond_norm = operations.RMSNorm(llm_features_dim, eps=1e-6, elementwise_affine=True, dtype=dtype, device=device)+ self.llm_cond_proj = operations.Linear(llm_features_dim, emb_dim, bias=True, dtype=dtype, device=device)+ self.t_embedding = Ideogram4EmbedScalar(emb_dim, input_range=(0.0, 1.0), dtype=dtype, device=device, operations=operations)+ self.adaln_proj = operations.Linear(emb_dim, adaln_dim, bias=True, dtype=dtype, device=device)++ self.embed_image_indicator = operations.Embedding(2, emb_dim, dtype=dtype, device=device)++ self.layers = nn.ModuleList([+ Ideogram4TransformerBlock(emb_dim, intermediate_size, num_heads, norm_eps, adaln_dim,+ dtype=dtype, device=device, operations=operations)+ for _ in range(num_layers)+ ])++ self.final_layer = Ideogram4FinalLayer(emb_dim, in_channels, adaln_dim, dtype=dtype, device=device, operations=operations)++ def _backbone(self, llm_features, x, t, position_ids, attn_mask, indicator, transformer_options={}):+ indicator = indicator.to(torch.long)+ output_image_mask = (indicator == OUTPUT_IMAGE_INDICATOR).to(x.dtype).unsqueeze(-1)++ x = x * output_image_mask+ h = self.input_proj(x) * output_image_mask++ t_cond = self.t_embedding(t)+ if t.dim() == 1:+ t_cond = t_cond.unsqueeze(1)+ adaln_input = F.silu(self.adaln_proj(t_cond))++ # h is zero on the text rows (content lives only on image rows), add writes the text features in place+ if llm_features is not None:+ L_text = llm_features.shape[1]+ text_mask = (indicator[:, :L_text] == LLM_TOKEN_INDICATOR).to(x.dtype).unsqueeze(-1)+ llm = self.llm_cond_norm(llm_features * text_mask)+ llm = self.llm_cond_proj(llm) * text_mask+ h[:, :L_text] = h[:, :L_text] + llm++ h = h + self.embed_image_indicator((indicator == OUTPUT_IMAGE_INDICATOR).to(torch.long))++ # Qwen3-VL interleaved MRoPE; position_ids (B, L, 3) -> (3, L) (same across batch).+ freqs_cis = precompute_freqs_cis(+ self.head_dim, position_ids[0].transpose(0, 1), self.rope_theta,+ rope_dims=self.mrope_section, interleaved_mrope=True, device=position_ids.device,+ )++ if attn_mask is not None and attn_mask.dtype == torch.bool:+ attn_mask = torch.zeros_like(attn_mask, dtype=h.dtype).masked_fill_(~attn_mask, -torch.finfo(h.dtype).max)++ for layer in self.layers:+ h = layer(h, attn_mask, freqs_cis, adaln_input, transformer_options=transformer_options)++ return self.final_layer(h, adaln_input)+++class Ideogram4Transformer2DModel(Ideogram4Transformer):+ """Ideogram 4 single-stream DiT.++ Runs a packed ``[text, image]`` sequence when text context is supplied, or an image-only sequence when ``context is None``.+ """++ def __init__(self, image_model=None, in_channels=128, num_layers=34, num_attention_heads=18, attention_head_dim=256, intermediate_size=12288,+ adaln_dim=512, llm_features_dim=53248, rope_theta=5000000, mrope_section=(24, 20, 20), norm_eps=1e-5,+ dtype=None, device=None, operations=None, **kwargs):+ emb_dim = num_attention_heads * attention_head_dim+ super().__init__(+ emb_dim=emb_dim, num_layers=num_layers, num_heads=num_attention_heads,+ intermediate_size=intermediate_size, adaln_dim=adaln_dim, in_channels=in_channels,+ llm_features_dim=llm_features_dim, rope_theta=rope_theta, mrope_section=mrope_section,+ norm_eps=norm_eps, dtype=dtype, device=device, operations=operations)+ self.dtype = dtype+ self.in_channels = in_channels+ self.out_channels = in_channels+ # 128-dim token = patch (2x2) * ae_channels (32).+ self.patch_size = 2+ self.ae_channels = in_channels // (self.patch_size * self.patch_size)++ def _img_to_tokens(self, x):+ B, C, gh, gw = x.shape+ x = x.view(B, self.ae_channels, self.patch_size, self.patch_size, gh, gw)+ x = x.permute(0, 4, 5, 2, 3, 1) # (B, gh, gw, pi, pj, c)+ return x.reshape(B, gh * gw, C)++ def _tokens_to_img(self, tokens, gh, gw):+ B = tokens.shape[0]+ C = tokens.shape[-1]+ x = tokens.reshape(B, gh, gw, self.patch_size, self.patch_size, self.ae_channels)+ x = x.permute(0, 5, 3, 4, 1, 2) # (B, c, pi, pj, gh, gw)+ return x.reshape(B, C, gh, gw)++ def _image_position_ids(self, gh, gw, device):+ h_idx = torch.arange(gh, device=device).view(-1, 1).expand(gh, gw).reshape(-1)+ w_idx = torch.arange(gw, device=device).view(1, -1).expand(gh, gw).reshape(-1)+ t_idx = torch.zeros_like(h_idx)+ return torch.stack([t_idx, h_idx, w_idx], dim=1) + IMAGE_POSITION_OFFSET # (L_img, 3)++ def _run_conditional(self, x_chunk, context_chunk, attn_mask_chunk, t_chunk, gh, gw, transformer_options):+ B = x_chunk.shape[0]+ device = x_chunk.device+ img_tokens = self._img_to_tokens(x_chunk).to(self.dtype)+ L_img = img_tokens.shape[1]+ L_text = context_chunk.shape[1]+ L = L_text + L_img+ latent_dim = img_tokens.shape[-1]++ x_full = torch.zeros(B, L, latent_dim, dtype=img_tokens.dtype, device=device)+ x_full[:, L_text:] = img_tokens++ text_pos = torch.arange(L_text, device=device).view(-1, 1).expand(L_text, 3)+ img_pos = self._image_position_ids(gh, gw, device)+ position_ids = torch.cat([text_pos, img_pos], dim=0).unsqueeze(0).expand(B, L, 3)++ indicator = torch.empty(B, L, dtype=torch.long, device=device)+ indicator[:, :L_text] = LLM_TOKEN_INDICATOR+ indicator[:, L_text:] = OUTPUT_IMAGE_INDICATOR++ attn_mask = None+ if attn_mask_chunk is not None:+ segment_ids = torch.ones(B, L, dtype=torch.long, device=device)+ pad = (attn_mask_chunk == 0)+ segment_ids[:, :L_text][pad] = SEQUENCE_PADDING_INDICATOR+ indicator[:, :L_text][pad] = 0+ # Block-diagonal mask from segment ids: (B, 1, L, L), True = attend.+ attn_mask = (segment_ids.unsqueeze(2) == segment_ids.unsqueeze(1)).unsqueeze(1)++ out = self._backbone(context_chunk, x_full, t_chunk, position_ids, attn_mask, indicator,+ transformer_options=transformer_options)+ return self._tokens_to_img(out[:, L_text:], gh, gw)++ def _run_image_only(self, x_chunk, t_chunk, gh, gw, transformer_options):+ B = x_chunk.shape[0]+ device = x_chunk.device+ img_tokens = self._img_to_tokens(x_chunk).to(self.dtype)+ L_img = img_tokens.shape[1]++ position_ids = self._image_position_ids(gh, gw, device).unsqueeze(0).expand(B, L_img, 3)+ indicator = torch.full((B, L_img), OUTPUT_IMAGE_INDICATOR, dtype=torch.long, device=device)++ # Image-only sequence is a single segment -> no mask, full attention, no LLM context.+ out = self._backbone(None, img_tokens, t_chunk, position_ids, None, indicator, transformer_options=transformer_options)+ return self._tokens_to_img(out, gh, gw)++ def forward(self, x, timesteps, context=None, attention_mask=None, transformer_options={}, **kwargs):+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(+ self._forward,+ self,+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options),+ ).execute(x, timesteps, context, attention_mask, transformer_options, **kwargs)++ def _forward(self, x, timesteps, context=None, attention_mask=None, transformer_options={}, **kwargs):+ bs, c, gh, gw = x.shape++ timesteps = 1.0 - timesteps++ # unconditional pass+ if context is None:+ return -self._run_image_only(x, timesteps, gh, gw, transformer_options)++ return -self._run_conditional(x, context, attention_mask, timesteps, gh, gw, transformer_options)diff --git a/comfy/model_base.py b/comfy/model_base.pyindex 3e2d4e930..042804771 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -55,6 +55,7 @@ import comfy.ldm.pixeldit.pidimport comfy.ldm.ace.modelimport comfy.ldm.omnigen.omnigen2import comfy.ldm.qwen_image.model+import comfy.ldm.ideogram4.modelimport comfy.ldm.kandinsky5.modelimport comfy.ldm.anima.modelimport comfy.ldm.ace.ace_step15@@ -2018,6 +2019,21 @@ class QwenImage(BaseModel):out['ref_latents'] = list([1, 16, sum(map(lambda a: math.prod(a.size()), ref_latents)) // 16])return out+class Ideogram4(BaseModel):+ def __init__(self, model_config, model_type=ModelType.FLOW, device=None):+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.ideogram4.model.Ideogram4Transformer2DModel)++ def extra_conds(self, **kwargs):+ out = super().extra_conds(**kwargs)+ attention_mask = kwargs.get("attention_mask", None)+ if attention_mask is not None:+ if torch.numel(attention_mask) != attention_mask.sum():+ out['attention_mask'] = comfy.conds.CONDRegular(attention_mask)+ cross_attn = kwargs.get("cross_attn", None)+ if cross_attn is not None:+ out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)+ return out+class HunyuanImage21(BaseModel):def __init__(self, model_config, model_type=ModelType.FLOW, device=None):super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.hunyuan_video.model.HunyuanVideo)diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex 24e742a7f..74c838d13 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -815,6 +815,13 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):dit_config["default_ref_method"] = "negative_index"return dit_config+ if '{}embed_image_indicator.weight'.format(key_prefix) in state_dict_keys: # Ideogram 4+ dit_config = {}+ dit_config["image_model"] = "ideogram4"+ dit_config["in_channels"] = state_dict['{}input_proj.weight'.format(key_prefix)].shape[1]+ dit_config["num_layers"] = count_blocks(state_dict_keys, '{}layers.'.format(key_prefix) + '{}.')+ return dit_config+if '{}visual_transformer_blocks.0.cross_attention.key_norm.weight'.format(key_prefix) in state_dict_keys: # Kandinsky 5dit_config = {}model_dim = state_dict['{}visual_embeddings.in_layer.bias'.format(key_prefix)].shape[0]diff --git a/comfy/sd.py b/comfy/sd.pyindex 9a2d31930..a66ba1bfb 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -58,6 +58,7 @@ import comfy.text_encoders.omnigen2import comfy.text_encoders.qwen_imageimport comfy.text_encoders.hunyuan_imageimport comfy.text_encoders.z_image+import comfy.text_encoders.ideogram4import comfy.text_encoders.ovisimport comfy.text_encoders.kandinsky5import comfy.text_encoders.jina_clip_2@@ -1298,6 +1299,7 @@ class CLIPType(Enum):COGVIDEOX = 27LENS = 28PIXELDIT = 29+ IDEOGRAM4 = 30@@ -1596,8 +1598,12 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clipclip_target.clip = comfy.text_encoders.ovis.te(**llama_detect(clip_data))clip_target.tokenizer = comfy.text_encoders.ovis.OvisTokenizerelif te_model == TEModel.QWEN3_8B:- clip_target.clip = comfy.text_encoders.flux.klein_te(**llama_detect(clip_data), model_type="qwen3_8b")- clip_target.tokenizer = comfy.text_encoders.flux.KleinTokenizer8B+ if clip_type == CLIPType.IDEOGRAM4:+ clip_target.clip = comfy.text_encoders.ideogram4.te(**llama_detect(clip_data))+ clip_target.tokenizer = comfy.text_encoders.ideogram4.Ideogram4Tokenizer+ else:+ clip_target.clip = comfy.text_encoders.flux.klein_te(**llama_detect(clip_data), model_type="qwen3_8b")+ clip_target.tokenizer = comfy.text_encoders.flux.KleinTokenizer8Belif te_model == TEModel.JINA_CLIP_2:clip_target.clip = comfy.text_encoders.jina_clip_2.JinaClip2TextModelWrapperclip_target.tokenizer = comfy.text_encoders.jina_clip_2.JinaClip2TokenizerWrapperdiff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex 0872b0e27..478489ed8 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -24,6 +24,7 @@ import comfy.text_encoders.qwen_imageimport comfy.text_encoders.hunyuan_imageimport comfy.text_encoders.kandinsky5import comfy.text_encoders.z_image+import comfy.text_encoders.ideogram4import comfy.text_encoders.animaimport comfy.text_encoders.ace15import comfy.text_encoders.longcat_image@@ -1746,6 +1747,44 @@ class Omnigen2(supported_models_base.BASE):hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen25_3b.transformer.".format(pref))return supported_models_base.ClipTarget(comfy.text_encoders.omnigen2.Omnigen2Tokenizer, comfy.text_encoders.omnigen2.te(**hunyuan_detect))+class Ideogram4(supported_models_base.BASE):+ unet_config = {+ "image_model": "ideogram4",+ }++ sampling_settings = {+ "multiplier": 1.0,+ "shift": 1.0,+ }++ memory_usage_factor = 1.8 # TODO++ unet_extra_config = {+ "num_attention_heads": 18,+ "attention_head_dim": 256,+ "intermediate_size": 12288,+ "adaln_dim": 512,+ "llm_features_dim": 53248,+ "rope_theta": 5000000,+ "mrope_section": [24, 20, 20],+ "norm_eps": 1e-5,+ }+ latent_format = latent_formats.Flux2++ supported_inference_dtypes = [torch.bfloat16, torch.float32]++ vae_key_prefix = ["vae."]+ text_encoder_key_prefix = ["text_encoders."]++ def get_model(self, state_dict, prefix="", device=None):+ out = model_base.Ideogram4(self, device=device)+ return out++ def clip_target(self, state_dict={}):+ pref = self.text_encoder_key_prefix[0]+ hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_8b.transformer.".format(pref))+ return supported_models_base.ClipTarget(comfy.text_encoders.ideogram4.Ideogram4Tokenizer, comfy.text_encoders.ideogram4.te(**hunyuan_detect))+class QwenImage(supported_models_base.BASE):unet_config = {"image_model": "qwen_image",@@ -2233,6 +2272,7 @@ models = [ACEStep15,Omnigen2,QwenImage,+ Ideogram4,Flux2,Lens,Kandinsky5Image,diff --git a/comfy/text_encoders/ideogram4.py b/comfy/text_encoders/ideogram4.pynew file mode 100644index 000000000..55e655d67--- /dev/null+++ b/comfy/text_encoders/ideogram4.py@@ -0,0 +1,77 @@+"""Ideogram 4 text encoder: Qwen3-VL-8B language model, 13-layer tap.++Ideogram 4 conditions on the concatenation of hidden states from 13 layers of+Qwen3-VL (layers 0,3,...,33,35), giving a 4096*13 = 53248-dim feature per token.+"""++import os++from transformers import Qwen2Tokenizer++import comfy.text_encoders.llama+from comfy import sd1_clip++# Reference taps outputs of layers (0,3,...,35); comfy captures layer inputs, offset by +1.+IDEOGRAM4_TAP_LAYERS = [1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31, 34, 36]+++class Qwen3VLTokenizer(sd1_clip.SDTokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "qwen25_tokenizer")+ super().__init__(tokenizer_path, pad_with_end=False, embedding_directory=embedding_directory,+ embedding_size=4096, embedding_key='qwen3vl_8b', tokenizer_class=Qwen2Tokenizer,+ has_start_token=False, has_end_token=False, pad_to_max_length=False,+ max_length=99999999, min_length=1, pad_token=151643, tokenizer_data=tokenizer_data)+++class Ideogram4Tokenizer(sd1_clip.SD1Tokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data,+ name="qwen3vl_8b", tokenizer=Qwen3VLTokenizer)++ self.llama_template = "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"++ def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, **kwargs):+ if llama_template is None:+ llama_text = self.llama_template.format(text)+ else:+ llama_text = llama_template.format(text)+ return super().tokenize_with_weights(llama_text, return_word_ids=return_word_ids, disable_weights=True, **kwargs)+++# Qwen3-VL-8B = 5e6 (vs plain Qwen3-8B's 1e6)+# final_norm/lm_head off -> Ideogram only reads raw tapped hidden states+QWEN3VL_8B_CONFIG = {"rope_theta": 5000000.0, "final_norm": False, "lm_head": False}+++class Qwen3VL8BModel(sd1_clip.SDClipModel):+ def __init__(self, device="cpu", layer="hidden", layer_idx=None, dtype=None, attention_mask=True, model_options={}):+ super().__init__(device=device, layer=IDEOGRAM4_TAP_LAYERS, layer_idx=None,+ textmodel_json_config=dict(QWEN3VL_8B_CONFIG),+ dtype=dtype, special_tokens={"pad": 151643}, layer_norm_hidden_state=False,+ model_class=comfy.text_encoders.llama.Qwen3_8B,+ enable_attention_masks=attention_mask, return_attention_masks=attention_mask,+ model_options=model_options)+++class Ideogram4TEModel(sd1_clip.SD1ClipModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ super().__init__(device=device, dtype=dtype, name="qwen3vl_8b", clip_model=Qwen3VL8BModel, model_options=model_options)++ def encode_token_weights(self, token_weight_pairs):+ out, pooled, extra = super().encode_token_weights(token_weight_pairs)+ b, n, seq, h = out.shape # (B, n_taps=13, seq, 4096) stacked in ascending layer order.+ out = out.permute(0, 2, 3, 1).reshape(b, seq, h * n) # (B, seq, 4096*13). permute -> (B, seq, H, taps).+ return out, pooled, extra+++def te(dtype_llama=None, llama_quantization_metadata=None):+ class Ideogram4TEModel_(Ideogram4TEModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ if dtype_llama is not None:+ dtype = dtype_llama+ if llama_quantization_metadata is not None:+ model_options = model_options.copy()+ model_options["quantization_metadata"] = llama_quantization_metadata+ super().__init__(device=device, dtype=dtype, model_options=model_options)+ return Ideogram4TEModel_diff --git a/comfy_extras/nodes_custom_sampler.py b/comfy_extras/nodes_custom_sampler.pyindex c3346bf09..b790d7aac 100644--- a/comfy_extras/nodes_custom_sampler.py+++ b/comfy_extras/nodes_custom_sampler.py@@ -1,5 +1,7 @@import mathimport comfy.samplers+import comfy.sampler_helpers+import comfy.patcher_extensionimport comfy.samplefrom comfy.k_diffusion import sampling as k_diffusion_samplingfrom comfy.k_diffusion import sa_solver@@ -894,6 +896,83 @@ class DualCFGGuider(io.ComfyNode):get_guider = execute+class Guider_DualModel(comfy.samplers.CFGGuider):+ # Runs the positive (cond) pass on the main model and the negative (uncond) pass on a separate model+ def __init__(self, model_patcher, uncond_model_patcher):+ super().__init__(model_patcher)+ self.uncond_model_patcher = uncond_model_patcher+ self.uncond_inner = None++ def outer_sample(self, noise, latent_image, sampler, sigmas, denoise_mask=None, callback=None, disable_pbar=False, seed=None, latent_shapes=None):+ self.uncond_inner = None+ self.uncond_loaded = []+ self._uncond_neg = None+ # skip at cfg 1.0+ if not math.isclose(self.cfg, 1.0):+ uc = {"negative": list(map(lambda a: a.copy(), self.conds["negative"]))}+ self.uncond_inner, uc, self.uncond_loaded = comfy.sampler_helpers.prepare_sampling(+ self.uncond_model_patcher, noise.shape, uc, self.uncond_model_patcher.model_options)+ self._uncond_neg = uc["negative"]+ self.uncond_model_patcher.pre_run()+ try:+ return super().outer_sample(noise, latent_image, sampler, sigmas, denoise_mask, callback, disable_pbar, seed, latent_shapes=latent_shapes)+ finally:+ if self.uncond_inner is not None:+ self.uncond_model_patcher.cleanup()+ comfy.sampler_helpers.cleanup_models({"negative": self._uncond_neg}, self.uncond_loaded)+ self.uncond_inner = None++ def inner_sample(self, noise, latent_image, device, sampler, sigmas, denoise_mask, callback, disable_pbar, seed, latent_shapes=None):+ if self.uncond_inner is not None:+ li = latent_image+ if li is not None and torch.count_nonzero(li) > 0:+ li = self.uncond_inner.process_latent_in(li)+ self._uncond_conds = comfy.samplers.process_conds(+ self.uncond_inner, noise, {"negative": self._uncond_neg}, device, li, denoise_mask, seed, latent_shapes=latent_shapes)["negative"]+ return super().inner_sample(noise, latent_image, device, sampler, sigmas, denoise_mask, callback, disable_pbar, seed, latent_shapes=latent_shapes)++ def predict_noise(self, x, timestep, model_options={}, seed=None):+ positive = self.conds.get("positive", None)+ if self.uncond_inner is None: # cfg == 1 or no negative -> single model, cond only+ return comfy.samplers.calc_cond_batch(self.inner_model, [positive], x, timestep, model_options)[0]+ cond = comfy.samplers.calc_cond_batch(self.inner_model, [positive], x, timestep, model_options)[0]++ uncond_model_options = model_options+ if "multigpu_clones" in model_options: # TODO: support multigpu instead of just running uncond on a single GPU+ uncond_model_options = {k: v for k, v in model_options.items() if k != "multigpu_clones"}+ uncond = comfy.samplers.calc_cond_batch(self.uncond_inner, [self._uncond_conds], x, timestep, uncond_model_options)[0]+ return comfy.samplers.cfg_function(self.inner_model, cond, uncond, self.cfg, x, timestep,+ model_options=model_options, cond=positive, uncond=self._uncond_conds)++class DualModelGuider(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="DualModelGuider",+ display_name="Dual Model CFG Guider",+ category="model/sampling/guiders",+ inputs=[+ io.Model.Input("model", tooltip="Model used for the positive (conditional) pass."),+ io.Model.Input("model_negative", optional=True, tooltip="Model used for the negative (unconditional) pass. Use the same model for ordinary CFG."),+ io.Conditioning.Input("positive"),+ io.Float.Input("cfg", default=4.0, min=0.0, max=100.0, step=0.1, round=0.01),+ io.Conditioning.Input("negative", optional=True, tooltip="Negative conditioning run on the negative model. Leave unconnected for a text-free (image-only) unconditional pass."),+ ],+ outputs=[io.Guider.Output()],+ )++ @classmethod+ def execute(cls, model, positive, cfg, model_negative=None, negative=None) -> io.NodeOutput:+ if negative is None:+ negative = [[None, {}]] # null cond -> no cross_attn -> model runs image-only++ guider = Guider_DualModel(model, model_negative) if model_negative is not None else comfy.samplers.CFGGuider(model)+ guider.set_conds(positive, negative)+ guider.set_cfg(cfg)+ return io.NodeOutput(guider)++ get_guider = execute+class DisableNoise(io.ComfyNode):@classmethoddef define_schema(cls):@@ -1054,11 +1133,53 @@ class ManualSigmas(io.ComfyNode):sigmas = torch.FloatTensor(sigmas)return io.NodeOutput(sigmas)+class CFGOverride(io.ComfyNode):+ @classmethod+ def define_schema(cls) -> io.Schema:+ return io.Schema(+ node_id="CFGOverride",+ display_name="CFG Override",+ description="Override cfg to a fixed value over a [start, end] percent slice of the steps. "+ "With multiple overrides, the one nearest the sampler wins on overlap.",+ category="sampling/custom_sampling",+ inputs=[+ io.Model.Input("model"),+ io.Float.Input("cfg", default=1.0, min=0.0, max=100.0, step=0.1, round=0.01),+ io.Float.Input("start_percent", default=0.0, min=0.0, max=1.0, step=0.001),+ io.Float.Input("end_percent", default=1.0, min=0.0, max=1.0, step=0.001),+ ],+ outputs=[io.Model.Output()],+ )++ @classmethod+ def execute(cls, model, cfg, start_percent, end_percent) -> io.NodeOutput:+ ms = model.get_model_object("model_sampling")+ sigma_hi = ms.percent_to_sigma(start_percent) # percent->sigma decreasing, so hi >= lo+ sigma_lo = ms.percent_to_sigma(end_percent)++ def predict_noise_wrapper(executor, *args, **kwargs):+ sigma = float(args[1].flatten()[0]) # args = (x, timestep, model_options, seed)+ if not (sigma_lo <= sigma <= sigma_hi):+ return executor(*args, **kwargs)+ guider = executor.class_obj # guider.cfg feeds cond_scale+ saved = guider.cfg+ guider.cfg = cfg+ try:+ return executor(*args, **kwargs)+ finally:+ guider.cfg = saved # restore for other steps/overrides++ m = model.clone()+ m.add_wrapper(comfy.patcher_extension.WrappersMP.PREDICT_NOISE, predict_noise_wrapper)+ return io.NodeOutput(m)++class CustomSamplersExtension(ComfyExtension):@overrideasync def get_node_list(self) -> list[type[io.ComfyNode]]:return [SamplerCustom,+ CFGOverride,BasicScheduler,KarrasScheduler,ExponentialScheduler,@@ -1087,6 +1208,7 @@ class CustomSamplersExtension(ComfyExtension):SamplingPercentToSigma,CFGGuider,DualCFGGuider,+ DualModelGuider,BasicGuider,RandomNoise,DisableNoise,diff --git a/comfy_extras/nodes_ideogram4.py b/comfy_extras/nodes_ideogram4.pynew file mode 100644index 000000000..d5827db4f--- /dev/null+++ b/comfy_extras/nodes_ideogram4.py@@ -0,0 +1,64 @@+"""Ideogram 4 sampling helper+"""++import math++import torch+from typing_extensions import override+from comfy_api.latest import ComfyExtension, io++_LOGSNR_MIN = -15.0+_LOGSNR_MAX = 18.0+++def _logit_normal_schedule(u, mean, std):+ # Reference time (0=noise..1=clean) via the probit/ndtri quantile.+ u = torch.as_tensor(u, dtype=torch.float64)+ t = 1.0 - torch.special.expit(mean + std * torch.special.ndtri(u))+ t_min = 1.0 / (1.0 + math.exp(0.5 * _LOGSNR_MAX))+ t_max = 1.0 / (1.0 + math.exp(0.5 * _LOGSNR_MIN))+ return t.clamp(t_min, t_max)+++def ideogram4_sigmas(num_steps, width, height, mu, std):+ """Descending sigmas (len num_steps+1) for the reference schedule.++ mu + the resolution term form the logSNR shift; std is the spread.+ """+ mean = mu + 0.5 * math.log((width * height) / (512 * 512))+ u = torch.linspace(0.0, 1.0, num_steps + 1, dtype=torch.float64)+ sigmas = (1.0 - _logit_normal_schedule(u, mean, std)).flip(0)+ sigmas[-1] = 0.0 # clamp leaves ~6e-4; force full denoise+ return sigmas.to(torch.float32)+++class Ideogram4Scheduler(io.ComfyNode):+ @classmethod+ def define_schema(cls) -> io.Schema:+ return io.Schema(+ node_id="Ideogram4Scheduler",+ display_name="Ideogram 4 Scheduler",+ category="sampling/custom_sampling/schedulers",+ inputs=[+ io.Int.Input("steps", default=20, min=1, max=200),+ io.Int.Input("width", default=1024, min=256, max=8192, step=16),+ io.Int.Input("height", default=1024, min=256, max=8192, step=16),+ io.Float.Input("mu", default=0.0, min=-10.0, max=10.0, step=0.05),+ io.Float.Input("std", default=1.75, min=0.1, max=5.0, step=0.05),+ ],+ outputs=[io.Sigmas.Output()],+ )++ @classmethod+ def execute(cls, steps, width, height, mu, std) -> io.NodeOutput:+ return io.NodeOutput(ideogram4_sigmas(steps, width, height, mu, std))+++class Ideogram4Extension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[io.ComfyNode]]:+ return [Ideogram4Scheduler]+++async def comfy_entrypoint() -> Ideogram4Extension:+ return Ideogram4Extension()diff --git a/nodes.py b/nodes.pyindex 331425b87..2f5a478b5 100644--- a/nodes.py+++ b/nodes.py@@ -969,7 +969,7 @@ class CLIPLoader:@classmethoddef INPUT_TYPES(s):return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ),- "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit"], ),+ "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4"], ),},"optional": {"device": (["default", "cpu"], {"advanced": True}),@@ -2362,6 +2362,7 @@ async def init_builtin_extra_nodes():"nodes_model_downscale.py","nodes_images.py","nodes_video_model.py",+ "nodes_ideogram4.py","nodes_train.py","nodes_dataset.py","nodes_sag.py",