The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support Flux3-image (#17064)
Jukka Seppänen20b46065ef7f
diff --git a/comfy/ldm/flux3/model.py b/comfy/ldm/flux3/model.pynew file mode 100644index 000000000..6abad685f--- /dev/null+++ b/comfy/ldm/flux3/model.py@@ -0,0 +1,216 @@+import torch+from torch import nn++import comfy.model_management+import comfy.model_prefetch+import comfy.ops+import comfy.patcher_extension+import comfy.quant_ops+import comfy.utils+from comfy.ldm.flux.layers import EmbedND, LastLayer, MLPEmbedder, Modulation, modulated_norm, timestep_embedding+from comfy.ldm.flux.math import apply_rope1+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention+++class QKNorm(nn.Module):+ def __init__(self, dim, dtype=None, device=None, operations=None):+ super().__init__()+ self.query_norm = operations.RMSNorm(dim, eps=1e-6, dtype=dtype, device=device)+ self.key_norm = operations.RMSNorm(dim, eps=1e-6, dtype=dtype, device=device)++ def forward(self, q, k, pe):+ if comfy.model_management.in_training:+ return apply_rope1(self.query_norm(q), pe), apply_rope1(self.key_norm(k), pe)+ q_scale, _, q_stream = comfy.ops.cast_bias_weight(self.query_norm, q, offloadable=True)+ k_scale, _, k_stream = comfy.ops.cast_bias_weight(self.key_norm, k, offloadable=True)+ q, k = comfy.quant_ops.ck.rms_rope(q, k, pe, q_scale, k_scale, self.query_norm.eps)+ comfy.ops.uncast_bias_weight(self.query_norm, q_scale, None, q_stream)+ comfy.ops.uncast_bias_weight(self.key_norm, k_scale, None, k_stream)+ return q, k+++class Block(nn.Module):+ # parallel attention + SwiGLU MLP under one adaLN and one gate, used for the per-mode and the joint blocks+ def __init__(self, hidden_size, num_heads, mlp_ratio, fused_qkv=False, dtype=None, device=None, operations=None):+ super().__init__()+ self.comfy_attention = ComfyAttention()+ self.num_heads = num_heads+ self.fused_qkv = fused_qkv+ mlp_hidden_dim = int(hidden_size * mlp_ratio)+ if fused_qkv:+ # one GEMM and one activation quantization instead of three+ self.qkv = operations.Linear(hidden_size, 3 * hidden_size, bias=False, dtype=dtype, device=device)+ else:+ self.q_proj = operations.Linear(hidden_size, hidden_size, bias=False, dtype=dtype, device=device)+ self.k_proj = operations.Linear(hidden_size, hidden_size, bias=False, dtype=dtype, device=device)+ self.v_proj = operations.Linear(hidden_size, hidden_size, bias=False, dtype=dtype, device=device)+ self.mlp_in = operations.Linear(hidden_size, 2 * mlp_hidden_dim, bias=False, dtype=dtype, device=device)+ self.attn_out = operations.Linear(hidden_size, hidden_size, bias=False, dtype=dtype, device=device)+ self.mlp_out = operations.Linear(mlp_hidden_dim, hidden_size, bias=False, dtype=dtype, device=device)+ self.norm = QKNorm(hidden_size // num_heads, dtype=dtype, device=device, operations=operations)+ self.pre_norm = operations.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device)++ def forward(self, x, spans, pe, transformer_options={}):+ # spans: (start, end, ModulationOut) row ranges; the first modulates every row, the others redo their rows+ B, N, _ = x.shape+ mod = spans[0][2]+ x_mod = modulated_norm(x, self.pre_norm, mod.scale, mod.shift)+ for start, end, mod in spans[1:]:+ x_mod[:, start:end] = modulated_norm(x[:, start:end], self.pre_norm, mod.scale, mod.shift)++ if self.fused_qkv:+ q, k, v = self.qkv(x_mod).chunk(3, dim=-1)+ else:+ q, k, v = self.q_proj(x_mod), self.k_proj(x_mod), self.v_proj(x_mod)+ mlp = self.mlp_in(x_mod)+ del x_mod+ q, k = self.norm(q.view(B, N, self.num_heads, -1), k.view(B, N, self.num_heads, -1), pe)+ q, k, v = AttentionTensorContainer(q.flatten(2)), AttentionTensorContainer(k.flatten(2)), AttentionTensorContainer(v)+ out = self.attn_out(optimized_attention(q, k, v, self.num_heads, transformer_options=transformer_options, preferred_attention=self.comfy_attention))+ # attention + MLP sum folded into the mlp_out GEMM epilogue+ out = comfy.ops.linear_input_act(self.mlp_out, mlp, "swiglu", residual=out, residual_scale=out.new_ones(out.shape[-1]))+ del mlp++ for start, end, mod in spans:+ x[:, start:end].addcmul_(out[:, start:end], mod.gate)+ return x+++class Flux3(nn.Module):+ def __init__(self, hidden_size=3072, num_heads=24, depth=5, depth_single_blocks=28, fused_qkv=False, flow_map=False, image_model=None, dtype=None, device=None, operations=None):+ super().__init__()+ self.dtype = dtype+ self.in_channels = 128+ self.image_rope_scale = 0.5+ self.flow_map = flow_map+ content_modes = ("image", "image_cond")++ self.pe_embedder = EmbedND(dim=hidden_size // num_heads, theta=10000, axes_dim=[32, 32, 32, 32])+ self.emb_in = nn.ModuleDict({m: operations.Linear(self.in_channels, hidden_size, bias=False, dtype=dtype, device=device) for m in content_modes})+ self.txt_in = operations.Linear(20480, hidden_size, bias=False, dtype=dtype, device=device)+ self.time_in = MLPEmbedder(256, hidden_size, bias=False, dtype=dtype, device=device, operations=operations)+ if flow_map:+ # step-distilled flow map: embeds the time each step jumps to+ self.ref_time_in = MLPEmbedder(256, hidden_size, bias=False, dtype=dtype, device=device, operations=operations)++ modulation = lambda: Modulation(hidden_size, double=False, bias=False, dtype=dtype, device=device, operations=operations)+ self.early_stream_modulations = nn.ModuleDict({m: modulation() for m in content_modes + ("txt",)})+ self.single_stream_modulations = nn.ModuleDict({m: modulation() for m in content_modes + ("txt",)})++ block = lambda: Block(hidden_size, num_heads, 3.0, fused_qkv=fused_qkv, dtype=dtype, device=device, operations=operations)+ self.content_mode_blocks = nn.ModuleDict({m: nn.ModuleList([block() for _ in range(depth)]) for m in content_modes})+ self.txt_mode_blocks = nn.ModuleList([block() for _ in range(depth)])+ self.single_blocks = nn.ModuleList([block() for _ in range(depth_single_blocks)])+ self.final_layer = nn.ModuleDict({"image": LastLayer(hidden_size, 1, self.in_channels, bias=False, dtype=dtype, device=device, operations=operations)})++ def image_ids(self, h, w, index, device):+ # (t, row, col, l); references are told apart on the l axis+ ids = torch.zeros(h, w, 4, device=device, dtype=torch.float32)+ ids[..., 1] = torch.arange(h, device=device, dtype=torch.float32)[:, None] * self.image_rope_scale+ ids[..., 2] = torch.arange(w, device=device, dtype=torch.float32)[None, :] * self.image_rope_scale+ ids[..., 3] = index+ return ids.flatten(0, 1)++ def ref_ids(self, ref_shapes, device):+ return torch.cat([self.image_ids(h, w, -1 - i, device) for i, (h, w) in enumerate(ref_shapes)])++ def text_ids(self, length, device):+ ids = torch.zeros(length, 4, device=device, dtype=torch.float32)+ ids[:, 3] = torch.arange(length, device=device, dtype=torch.float32)+ return ids++ def rope(self, ids):+ return self.pe_embedder(ids.unsqueeze(0)).transpose(1, 2).contiguous()++ def preprocess_conds(self, context, ref_latents=None, vec_target=None, transformer_options={}):+ # text and references only attend to their own mode at t = 0 in the early blocks, so these depend on the conditioning alone+ device = context.device+ if ref_latents:+ # a batch of references stays one per sample+ batch = max([context.shape[0]] + [ref.shape[0] for ref in ref_latents])+ context = comfy.utils.repeat_to_batch_size(context, batch)+ ref_latents = [comfy.utils.repeat_to_batch_size(ref, batch) for ref in ref_latents]+ vec = self.time_in(timestep_embedding(context.new_zeros(1), 256))+ if vec_target is not None:+ vec = vec + vec_target+ txt = self.txt_in(context)+ streams = [(txt, "txt", self.text_ids(txt.shape[1], device), self.txt_mode_blocks)]+ refs = None+ if ref_latents:+ refs = self.emb_in["image_cond"](torch.cat([ref.flatten(2).transpose(1, 2) for ref in ref_latents], dim=1))+ streams.append((refs, "image_cond", self.ref_ids([ref.shape[-2:] for ref in ref_latents], device), self.content_mode_blocks["image_cond"]))+ prefetch_queue = comfy.model_prefetch.make_prefetch_queue([block for *_, blocks in streams for block in blocks], device, transformer_options)+ for tokens, mode, ids, blocks in streams:+ mod = self.early_stream_modulations[mode](vec)[0]+ pe = self.rope(ids)+ for block in blocks:+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, block, tokens.dtype)+ block(tokens, [(0, tokens.shape[1], mod)], pe)+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, None)+ return txt, refs++ def forward(self, x, timestep, context=None, ref_latents=None, transformer_options={}, **kwargs):+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(+ self._forward,+ self,+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)+ ).execute(x, timestep, context, ref_latents, transformer_options, **kwargs)++ def _forward(self, x, timestep, context=None, ref_latents=None, transformer_options={}, txt_states=None, ref_states=None, ref_shapes=None, **kwargs):+ B, _, H, W = x.shape+ dtype, device = x.dtype, x.device+ vec_target = None+ if self.flow_map:+ # every mode's modulation also gets the time this step jumps to: the next sigma of the schedule+ sample_sigmas = transformer_options["sample_sigmas"]+ i = int((sample_sigmas - timestep[0]).abs().argmin())+ target = sample_sigmas[min(i + 1, sample_sigmas.shape[0] - 1)]+ vec_target = self.ref_time_in(timestep_embedding(target.reshape(1).to(device=device, dtype=dtype), 256))++ # txt_states/ref_states are the early-block outputs of preprocess_conds, normally run once from extra_conds+ if txt_states is None:+ txt_states, ref_states = self.preprocess_conds(context, ref_latents, vec_target, transformer_options)+ ref_shapes = [ref.shape[-2:] for ref in ref_latents] if ref_latents else None+ transformer_options = transformer_options.copy()++ # the reference embeds t in the compute dtype; image_cond and text sit at t = 0+ vec = self.time_in(timestep_embedding(torch.cat([timestep, timestep.new_zeros(1)]).to(dtype), 256))+ if vec_target is not None:+ vec = vec + vec_target+ vec_img, vec_zero = vec[:-1], vec[-1:]+ early = self.early_stream_modulations["image"](vec_img)[0]++ # per sample the joint sequence is [image, image_cond, text]+ img = self.emb_in["image"](x.flatten(2).transpose(1, 2))+ parts = [(img, "image", self.image_ids(H, W, 0, device))]+ if ref_states is not None:+ parts.append((ref_states, "image_cond", self.ref_ids(ref_shapes, device)))+ parts.append((txt_states, "txt", self.text_ids(txt_states.shape[1], device)))+ pe = self.rope(torch.cat([ids for _, _, ids in parts]))+ spans, start = [], 0+ for tokens, mode, _ in parts:+ spans.append((start, start + tokens.shape[1], self.single_stream_modulations[mode](vec_img if mode == "image" else vec_zero)[0]))+ start += tokens.shape[1]+ # joint sequence allocated outside the memory compiler region, filled after the image's early blocks+ x = torch.empty(B, start, img.shape[-1], device=device, dtype=img.dtype)++ queue = list(self.content_mode_blocks["image"]) + list(self.single_blocks)+ prefetch_queue = comfy.model_prefetch.make_prefetch_queue(queue, device, transformer_options)+ comfy.model_prefetch.malloc_graph_begin(device)+ for block in self.content_mode_blocks["image"]:+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, block, dtype, malloc_scope="block")+ img = block(img, [(0, H * W, early)], pe[:, :H * W], transformer_options)++ torch.cat([img] + [tokens for tokens, _, _ in parts[1:]], dim=1, out=x)+ del img, parts+ transformer_options["total_blocks"] = len(self.single_blocks)+ transformer_options["block_type"] = "single"+ for i, block in enumerate(self.single_blocks):+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, block, dtype, malloc_scope="block")+ transformer_options["block_index"] = i+ x = block(x, spans, pe, transformer_options)+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, None, malloc_scope="block")+ comfy.model_prefetch.malloc_graph_end()++ out = self.final_layer["image"](x[:, :H * W], vec_img)+ return out.transpose(1, 2).reshape(B, self.in_channels, H, W)diff --git a/comfy/lora.py b/comfy/lora.pyindex 36f286cc3..061550067 100644--- a/comfy/lora.py+++ b/comfy/lora.py@@ -343,6 +343,15 @@ def model_lora_keys_unet(model, key_map={}):key_map["transformer.{}".format(key_lora)] = tokey_map["lycoris_{}".format(key_lora.replace(".", "_"))] = to #SimpleTuner lycoris format+ if isinstance(model, comfy.model_base.Flux3):+ for k in sdk:+ if k.endswith(".qkv.weight"): # fused q/k/v files; LoRAs address the separate projections+ third = sd[k].shape[0] // 3+ for i, proj in enumerate(("q_proj", "k_proj", "v_proj")):+ key_lora = "{}.{}".format(k[:-len(".qkv.weight")], proj)+ key_map[key_lora] = (k, (0, i * third, third))+ key_map["lora_unet_{}".format(key_lora[len("diffusion_model."):].replace(".", "_"))] = (k, (0, i * third, third))+if isinstance(model, comfy.model_base.Krea2):diffusers_keys = comfy.utils.krea2_to_diffusers(model.model_config.unet_config, output_prefix="diffusion_model.")for k in diffusers_keys:diff --git a/comfy/model_base.py b/comfy/model_base.pyindex 4f731e7cd..f7514770f 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -40,6 +40,7 @@ import comfy.ldm.hydit.modelsimport comfy.ldm.audio.ditimport comfy.ldm.audio.embeddersimport comfy.ldm.flux.model+import comfy.ldm.flux3.modelimport comfy.ldm.lens.modelimport comfy.ldm.lightricks.modelimport comfy.ldm.hunyuan_video.model@@ -1098,6 +1099,39 @@ class Flux2(Flux):return out+class Flux3(BaseModel):+ def __init__(self, model_config, model_type=ModelType.FLUX, device=None):+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.flux3.model.Flux3)+ self.memory_usage_factor_conds = ("ref_latents",)++ def extra_conds(self, **kwargs):+ out = super().extra_conds(**kwargs)+ cross_attn = kwargs.get("cross_attn", None)+ ref_latents = kwargs.get("reference_latents", None)+ if ref_latents is not None:+ ref_latents = [self.process_latent_in(lat) for lat in ref_latents]+ # the DiT attends to every text token, so no padding: cond and uncond of different lengths run separately+ if cross_attn is not None and torch.is_inference_mode_enabled() and kwargs.get("hooks") is None and not self.diffusion_model.flow_map:+ # the t = 0 early blocks of text and references depend on the conditioning alone: run them once, not every step+ # (a flow map also modulates them with each step's target time)+ device, dtype = kwargs["device"], self.get_dtype_inference()+ refs = [lat.to(device=device, dtype=dtype) for lat in ref_latents] if ref_latents else None+ txt_states, ref_states = self.diffusion_model.preprocess_conds(cross_attn.to(device=device, dtype=dtype), refs)+ del out['c_crossattn']+ out['txt_states'] = comfy.conds.CONDRegular(txt_states)+ if ref_states is not None:+ out['ref_states'] = comfy.conds.CONDRegular(ref_states)+ out['ref_shapes'] = comfy.conds.CONDConstant([tuple(lat.shape[-2:]) for lat in ref_latents])+ return out+ if cross_attn is not None:+ out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)+ if ref_latents is not None:+ out['ref_latents'] = comfy.conds.CONDList(ref_latents)+ return out++ extra_conds_shapes = Flux.extra_conds_shapes++class Lens(BaseModel):def __init__(self, model_config, model_type=ModelType.FLUX, device=None):super().__init__(diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex 29618dea2..78e71d86b 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -260,6 +260,17 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):dit_config["meanflow_sum"] = Falsereturn dit_config+ if '{}content_mode_blocks.image.0.attn_out.weight'.format(key_prefix) in state_dict_keys and '{}single_stream_modulations.txt.lin.weight'.format(key_prefix) in state_dict_keys: # FLUX.3+ dit_config = {}+ dit_config["image_model"] = "flux3"+ dit_config["hidden_size"] = state_dict['{}content_mode_blocks.image.0.attn_out.weight'.format(key_prefix)].shape[0]+ dit_config["num_heads"] = dit_config["hidden_size"] // 128+ dit_config["fused_qkv"] = '{}single_blocks.0.qkv.weight'.format(key_prefix) in state_dict_keys+ dit_config["flow_map"] = '{}ref_time_in.in_layer.weight'.format(key_prefix) in state_dict_keys+ dit_config["depth"] = count_blocks(state_dict_keys, '{}txt_mode_blocks.'.format(key_prefix) + '{}.')+ dit_config["depth_single_blocks"] = count_blocks(state_dict_keys, '{}single_blocks.'.format(key_prefix) + '{}.')+ return dit_config+if any_suffix_in(state_dict_keys, key_prefix, 'double_blocks.0.img_attn.norm.key_norm.', ["weight", "scale"]) and ('{}img_in.weight'.format(key_prefix) in state_dict_keys or any_suffix_in(state_dict_keys, key_prefix, 'distilled_guidance_layer.norms.0.', ["weight", "scale"])): #Flux, Chroma or Chroma Radiance (has no img_in.weight)dit_config = {}if '{}double_stream_modulation_img.lin.weight'.format(key_prefix) in state_dict_keys:diff --git a/comfy/sd.py b/comfy/sd.pyindex ac62bf795..02f8f3bb1 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -1608,6 +1608,7 @@ class CLIPType(Enum):MAGE = 34MINIMAX = 35YUE2 = 36+ FLUX3 = 37@@ -1977,6 +1978,10 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clipclip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})clip_target.clip = comfy.text_encoders.krea2.te(**llama_detect(clip_data))clip_target.tokenizer = comfy.text_encoders.krea2.Krea2Tokenizer+ elif clip_type == CLIPType.FLUX3 and te_model == TEModel.QWEN3VL_4B: # FLUX.3: Qwen3-VL-4B text only, 8-layer tap -> 20480.+ clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})+ clip_target.clip = comfy.text_encoders.flux.flux3_te(**llama_detect(clip_data))+ clip_target.tokenizer = comfy.text_encoders.flux.Flux3Tokenizerelif clip_type == CLIPType.MAGE and te_model == TEModel.QWEN3VL_4B: # Mage-Flow: full Qwen3-VL-4B, last hidden state, Qwen-Image-style templates.clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})clip_target.clip = comfy.text_encoders.mage_flow.te(**llama_detect(clip_data))diff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex 1be399a09..939cbea9b 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -842,6 +842,26 @@ class Flux2(Flux):return None+class Flux3(Flux2):+ unet_config = {+ "image_model": "flux3",+ }++ sampling_settings = {+ "shift": 1.93586, # mu = ln(6.93), the reference timestep shift+ }++ supported_inference_dtypes = [torch.bfloat16, torch.float32]++ def get_model(self, state_dict, prefix="", device=None):+ return model_base.Flux3(self, device=device)++ def clip_target(self, state_dict={}):+ pref = self.text_encoder_key_prefix[0]+ detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_4b.transformer.".format(pref))+ return supported_models_base.ClipTarget(comfy.text_encoders.flux.Flux3Tokenizer, comfy.text_encoders.flux.flux3_te(**detect))++class Lens(supported_models_base.BASE):"""Microsoft Lens (3.8B dual-stream MMDiT, GPT-OSS-20B text features, Flux2 VAE)."""@@ -2665,6 +2685,7 @@ models = [Ideogram4,Krea2,Flux2,+ Flux3,Lens,Kandinsky5Image,Kandinsky5,diff --git a/comfy/text_encoders/flux.py b/comfy/text_encoders/flux.pyindex 300ac5c11..26a596dba 100644--- a/comfy/text_encoders/flux.py+++ b/comfy/text_encoders/flux.py@@ -2,6 +2,7 @@ from comfy import sd1_clipimport comfy.text_encoders.t5import comfy.text_encoders.sd3_clipimport comfy.text_encoders.llama+import comfy.text_encoders.qwen3vlimport comfy.model_managementfrom transformers import T5TokenizerFast, Qwen2Tokenizerfrom .bpe_tokenizer import from_tekken_json@@ -198,3 +199,47 @@ def klein_te(dtype_llama=None, llama_quantization_metadata=None, model_type="qwedtype = dtype_llamasuper().__init__(device=device, dtype=dtype, name=model_type, model_options=model_options, clip_model=model)return Flux2TEModel_++class Flux3Tokenizer(comfy.text_encoders.qwen3vl.Qwen3VLTokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type="qwen3vl_4b")++ def tokenize_with_weights(self, text, return_word_ids=False, thinking=True, **kwargs):+ # instruct template without a think block, right padded to a multiple of 80 tokens+ tokens = super().tokenize_with_weights(text, return_word_ids=return_word_ids, thinking=thinking, **kwargs)+ pad = (151643, 1.0, 0) if return_word_ids else (151643, 1.0)+ for batch in tokens[self.clip_name]:+ batch.extend([pad] * (-len(batch) % 80))+ return tokens++class Flux3Qwen3VL(comfy.text_encoders.qwen3vl.Qwen3VL):+ model_type = "qwen3vl_4b"++ def forward(self, input_ids, attention_mask=None, **kwargs):+ if attention_mask is not None:+ # transformers 4.57, which the reference pins, puts the padding at rope position 1; the DiT attends to it+ kwargs["position_ids"] = (attention_mask.cumsum(-1) - 1).masked_fill(attention_mask == 0, 1)+ return super().forward(input_ids, attention_mask=attention_mask, **kwargs)++class Flux3Qwen3VLModel(sd1_clip.SDClipModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ super().__init__(device=device, layer=[4, 8, 12, 16, 20, 24, 28, 32], layer_idx=None, textmodel_json_config={}, dtype=dtype, special_tokens={"pad": 151643}, layer_norm_hidden_state=False, model_class=Flux3Qwen3VL, enable_attention_masks=True, model_options=model_options)++class Flux3TEModel(sd1_clip.SD1ClipModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ super().__init__(device=device, dtype=dtype, name="qwen3vl_4b", clip_model=Flux3Qwen3VLModel, model_options=model_options)++ def encode_token_weights(self, token_weight_pairs):+ out, pooled = super().encode_token_weights(token_weight_pairs)+ return out.movedim(1, 2).flatten(2), pooled++def flux3_te(dtype_llama=None, llama_quantization_metadata=None):+ class Flux3TEModel_(Flux3TEModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ if llama_quantization_metadata is not None:+ model_options = model_options.copy()+ model_options["quantization_metadata"] = llama_quantization_metadata+ if dtype_llama is not None:+ dtype = dtype_llama+ super().__init__(device=device, dtype=dtype, model_options=model_options)+ return Flux3TEModel_diff --git a/comfy_extras/nodes_flux.py b/comfy_extras/nodes_flux.pyindex e9986c9e7..53f1712b6 100644--- a/comfy_extras/nodes_flux.py+++ b/comfy_extras/nodes_flux.py@@ -1,3 +1,4 @@+import jsonimport node_helpersimport comfy.utilsfrom typing_extensions import override@@ -300,6 +301,35 @@ class FluxKVCache(io.ComfyNode):return io.NodeOutput(m)+class Flux3LayoutPrompt(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="Flux3LayoutPrompt",+ display_name="FLUX.3 Layout Prompt",+ category="model/conditioning/flux",+ description="Appends the boxes from Create Bounding Boxes to a FLUX.3 prompt as its layout table.",+ inputs=[+ io.String.Input("prompt", multiline=True, dynamic_prompts=True,+ tooltip="The prompt text. Cite box N as <obj_N> to tie words to it."),+ io.Array.Input("element", tooltip="Prompt elements from the node Create Bounding Boxes. Colour palettes are not used, describe colours in the description."),+ ],+ outputs=[io.String.Output(display_name="prompt")],+ is_experimental=True,+ )++ @classmethod+ def execute(cls, prompt, element) -> io.NodeOutput:+ rows = []+ for i, el in enumerate(element if isinstance(element, list) else []):+ desc = el.get("desc", "")+ if el.get("type") == "text" and el.get("text"):+ desc = f'{desc} "{el["text"]}"'.strip()+ rows.append({"id": f"obj_{i + 1}", "bbox": el["bbox"], "desc": desc})+ if not rows:+ return io.NodeOutput(prompt)+ return io.NodeOutput(f"{prompt.rstrip()} {json.dumps(rows, ensure_ascii=False)}")+class FluxExtension(ComfyExtension):@overrideasync def get_node_list(self) -> list[type[io.ComfyNode]]:@@ -312,6 +342,7 @@ class FluxExtension(ComfyExtension):EmptyFlux2LatentImage,Flux2Scheduler,FluxKVCache,+ Flux3LayoutPrompt,]diff --git a/nodes.py b/nodes.pyindex c0dd5563f..e0ef36232 100644--- a/nodes.py+++ b/nodes.py@@ -1018,7 +1018,7 @@ class CLIPLoader:@classmethoddef INPUT_TYPES(s):return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ),- "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2", "joyimage", "mage", "minimax", "yue2"], ),+ "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2", "joyimage", "mage", "minimax", "yue2", "flux3"], ),},"optional": {"device": (["default", "cpu"], {"advanced": True}),@@ -1028,7 +1028,7 @@ class CLIPLoader:CATEGORY = "model/loaders"- DESCRIPTION = "Recipes:\nsd: clip-l\nstable cascade: clip-g\nsd3: t5 xxl / clip-g / clip-l\nstable audio: t5 base\nmochi: t5 xxl\ncogvideox: t5 xxl (226-token padding)\ncosmos: old t5 xxl\nlumina2: gemma 2 2B\nwan: umt5 xxl\nhidream: llama-3.1 (Recommend) or t5\nomnigen2: qwen vl 2.5 3B\njoyimage: qwen3-vl 8B\nlens: gpt-oss-20b\npixeldit: gemma 2 2B elm\nminimax: MiniMax H3 Qwen3-VL or Music3 Qwen/RVQ"+ DESCRIPTION = "Recipes:\nsd: clip-l\nstable cascade: clip-g\nsd3: t5 xxl / clip-g / clip-l\nstable audio: t5 base\nmochi: t5 xxl\ncogvideox: t5 xxl (226-token padding)\ncosmos: old t5 xxl\nlumina2: gemma 2 2B\nwan: umt5 xxl\nhidream: llama-3.1 (Recommend) or t5\nomnigen2: qwen vl 2.5 3B\njoyimage: qwen3-vl 8B\nlens: gpt-oss-20b\npixeldit: gemma 2 2B elm\nminimax: MiniMax H3 Qwen3-VL or Music3 Qwen/RVQ\nflux3: qwen3-vl 4B"def load_clip(self, clip_name, type="stable_diffusion", device="default"):clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION)