Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Support Flux3-image (#17064)

Jukka Seppänen20b46065ef7f
diff --git a/comfy/ldm/flux3/model.py b/comfy/ldm/flux3/model.py
new file mode 100644
index 000000000..6abad685f
--- /dev/null
+++ b/comfy/ldm/flux3/model.py
@@ -0,0 +1,216 @@
+import torch
+from torch import nn
+
+import comfy.model_management
+import comfy.model_prefetch
+import comfy.ops
+import comfy.patcher_extension
+import comfy.quant_ops
+import comfy.utils
+from comfy.ldm.flux.layers import EmbedND, LastLayer, MLPEmbedder, Modulation, modulated_norm, timestep_embedding
+from comfy.ldm.flux.math import apply_rope1
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
+
+
+class QKNorm(nn.Module):
+ def __init__(self, dim, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.query_norm = operations.RMSNorm(dim, eps=1e-6, dtype=dtype, device=device)
+ self.key_norm = operations.RMSNorm(dim, eps=1e-6, dtype=dtype, device=device)
+
+ def forward(self, q, k, pe):
+ if comfy.model_management.in_training:
+ return apply_rope1(self.query_norm(q), pe), apply_rope1(self.key_norm(k), pe)
+ q_scale, _, q_stream = comfy.ops.cast_bias_weight(self.query_norm, q, offloadable=True)
+ k_scale, _, k_stream = comfy.ops.cast_bias_weight(self.key_norm, k, offloadable=True)
+ q, k = comfy.quant_ops.ck.rms_rope(q, k, pe, q_scale, k_scale, self.query_norm.eps)
+ comfy.ops.uncast_bias_weight(self.query_norm, q_scale, None, q_stream)
+ comfy.ops.uncast_bias_weight(self.key_norm, k_scale, None, k_stream)
+ return q, k
+
+
+class Block(nn.Module):
+ # parallel attention + SwiGLU MLP under one adaLN and one gate, used for the per-mode and the joint blocks
+ def __init__(self, hidden_size, num_heads, mlp_ratio, fused_qkv=False, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.comfy_attention = ComfyAttention()
+ self.num_heads = num_heads
+ self.fused_qkv = fused_qkv
+ mlp_hidden_dim = int(hidden_size * mlp_ratio)
+ if fused_qkv:
+ # one GEMM and one activation quantization instead of three
+ self.qkv = operations.Linear(hidden_size, 3 * hidden_size, bias=False, dtype=dtype, device=device)
+ else:
+ self.q_proj = operations.Linear(hidden_size, hidden_size, bias=False, dtype=dtype, device=device)
+ self.k_proj = operations.Linear(hidden_size, hidden_size, bias=False, dtype=dtype, device=device)
+ self.v_proj = operations.Linear(hidden_size, hidden_size, bias=False, dtype=dtype, device=device)
+ self.mlp_in = operations.Linear(hidden_size, 2 * mlp_hidden_dim, bias=False, dtype=dtype, device=device)
+ self.attn_out = operations.Linear(hidden_size, hidden_size, bias=False, dtype=dtype, device=device)
+ self.mlp_out = operations.Linear(mlp_hidden_dim, hidden_size, bias=False, dtype=dtype, device=device)
+ self.norm = QKNorm(hidden_size // num_heads, dtype=dtype, device=device, operations=operations)
+ self.pre_norm = operations.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device)
+
+ def forward(self, x, spans, pe, transformer_options={}):
+ # spans: (start, end, ModulationOut) row ranges; the first modulates every row, the others redo their rows
+ B, N, _ = x.shape
+ mod = spans[0][2]
+ x_mod = modulated_norm(x, self.pre_norm, mod.scale, mod.shift)
+ for start, end, mod in spans[1:]:
+ x_mod[:, start:end] = modulated_norm(x[:, start:end], self.pre_norm, mod.scale, mod.shift)
+
+ if self.fused_qkv:
+ q, k, v = self.qkv(x_mod).chunk(3, dim=-1)
+ else:
+ q, k, v = self.q_proj(x_mod), self.k_proj(x_mod), self.v_proj(x_mod)
+ mlp = self.mlp_in(x_mod)
+ del x_mod
+ q, k = self.norm(q.view(B, N, self.num_heads, -1), k.view(B, N, self.num_heads, -1), pe)
+ q, k, v = AttentionTensorContainer(q.flatten(2)), AttentionTensorContainer(k.flatten(2)), AttentionTensorContainer(v)
+ out = self.attn_out(optimized_attention(q, k, v, self.num_heads, transformer_options=transformer_options, preferred_attention=self.comfy_attention))
+ # attention + MLP sum folded into the mlp_out GEMM epilogue
+ out = comfy.ops.linear_input_act(self.mlp_out, mlp, "swiglu", residual=out, residual_scale=out.new_ones(out.shape[-1]))
+ del mlp
+
+ for start, end, mod in spans:
+ x[:, start:end].addcmul_(out[:, start:end], mod.gate)
+ return x
+
+
+class Flux3(nn.Module):
+ def __init__(self, hidden_size=3072, num_heads=24, depth=5, depth_single_blocks=28, fused_qkv=False, flow_map=False, image_model=None, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.dtype = dtype
+ self.in_channels = 128
+ self.image_rope_scale = 0.5
+ self.flow_map = flow_map
+ content_modes = ("image", "image_cond")
+
+ self.pe_embedder = EmbedND(dim=hidden_size // num_heads, theta=10000, axes_dim=[32, 32, 32, 32])
+ self.emb_in = nn.ModuleDict({m: operations.Linear(self.in_channels, hidden_size, bias=False, dtype=dtype, device=device) for m in content_modes})
+ self.txt_in = operations.Linear(20480, hidden_size, bias=False, dtype=dtype, device=device)
+ self.time_in = MLPEmbedder(256, hidden_size, bias=False, dtype=dtype, device=device, operations=operations)
+ if flow_map:
+ # step-distilled flow map: embeds the time each step jumps to
+ self.ref_time_in = MLPEmbedder(256, hidden_size, bias=False, dtype=dtype, device=device, operations=operations)
+
+ modulation = lambda: Modulation(hidden_size, double=False, bias=False, dtype=dtype, device=device, operations=operations)
+ self.early_stream_modulations = nn.ModuleDict({m: modulation() for m in content_modes + ("txt",)})
+ self.single_stream_modulations = nn.ModuleDict({m: modulation() for m in content_modes + ("txt",)})
+
+ block = lambda: Block(hidden_size, num_heads, 3.0, fused_qkv=fused_qkv, dtype=dtype, device=device, operations=operations)
+ self.content_mode_blocks = nn.ModuleDict({m: nn.ModuleList([block() for _ in range(depth)]) for m in content_modes})
+ self.txt_mode_blocks = nn.ModuleList([block() for _ in range(depth)])
+ self.single_blocks = nn.ModuleList([block() for _ in range(depth_single_blocks)])
+ self.final_layer = nn.ModuleDict({"image": LastLayer(hidden_size, 1, self.in_channels, bias=False, dtype=dtype, device=device, operations=operations)})
+
+ def image_ids(self, h, w, index, device):
+ # (t, row, col, l); references are told apart on the l axis
+ ids = torch.zeros(h, w, 4, device=device, dtype=torch.float32)
+ ids[..., 1] = torch.arange(h, device=device, dtype=torch.float32)[:, None] * self.image_rope_scale
+ ids[..., 2] = torch.arange(w, device=device, dtype=torch.float32)[None, :] * self.image_rope_scale
+ ids[..., 3] = index
+ return ids.flatten(0, 1)
+
+ def ref_ids(self, ref_shapes, device):
+ return torch.cat([self.image_ids(h, w, -1 - i, device) for i, (h, w) in enumerate(ref_shapes)])
+
+ def text_ids(self, length, device):
+ ids = torch.zeros(length, 4, device=device, dtype=torch.float32)
+ ids[:, 3] = torch.arange(length, device=device, dtype=torch.float32)
+ return ids
+
+ def rope(self, ids):
+ return self.pe_embedder(ids.unsqueeze(0)).transpose(1, 2).contiguous()
+
+ def preprocess_conds(self, context, ref_latents=None, vec_target=None, transformer_options={}):
+ # text and references only attend to their own mode at t = 0 in the early blocks, so these depend on the conditioning alone
+ device = context.device
+ if ref_latents:
+ # a batch of references stays one per sample
+ batch = max([context.shape[0]] + [ref.shape[0] for ref in ref_latents])
+ context = comfy.utils.repeat_to_batch_size(context, batch)
+ ref_latents = [comfy.utils.repeat_to_batch_size(ref, batch) for ref in ref_latents]
+ vec = self.time_in(timestep_embedding(context.new_zeros(1), 256))
+ if vec_target is not None:
+ vec = vec + vec_target
+ txt = self.txt_in(context)
+ streams = [(txt, "txt", self.text_ids(txt.shape[1], device), self.txt_mode_blocks)]
+ refs = None
+ if ref_latents:
+ refs = self.emb_in["image_cond"](torch.cat([ref.flatten(2).transpose(1, 2) for ref in ref_latents], dim=1))
+ streams.append((refs, "image_cond", self.ref_ids([ref.shape[-2:] for ref in ref_latents], device), self.content_mode_blocks["image_cond"]))
+ prefetch_queue = comfy.model_prefetch.make_prefetch_queue([block for *_, blocks in streams for block in blocks], device, transformer_options)
+ for tokens, mode, ids, blocks in streams:
+ mod = self.early_stream_modulations[mode](vec)[0]
+ pe = self.rope(ids)
+ for block in blocks:
+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, block, tokens.dtype)
+ block(tokens, [(0, tokens.shape[1], mod)], pe)
+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, None)
+ return txt, refs
+
+ def forward(self, x, timestep, context=None, ref_latents=None, transformer_options={}, **kwargs):
+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+ self._forward,
+ self,
+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)
+ ).execute(x, timestep, context, ref_latents, transformer_options, **kwargs)
+
+ def _forward(self, x, timestep, context=None, ref_latents=None, transformer_options={}, txt_states=None, ref_states=None, ref_shapes=None, **kwargs):
+ B, _, H, W = x.shape
+ dtype, device = x.dtype, x.device
+ vec_target = None
+ if self.flow_map:
+ # every mode's modulation also gets the time this step jumps to: the next sigma of the schedule
+ sample_sigmas = transformer_options["sample_sigmas"]
+ i = int((sample_sigmas - timestep[0]).abs().argmin())
+ target = sample_sigmas[min(i + 1, sample_sigmas.shape[0] - 1)]
+ vec_target = self.ref_time_in(timestep_embedding(target.reshape(1).to(device=device, dtype=dtype), 256))
+
+ # txt_states/ref_states are the early-block outputs of preprocess_conds, normally run once from extra_conds
+ if txt_states is None:
+ txt_states, ref_states = self.preprocess_conds(context, ref_latents, vec_target, transformer_options)
+ ref_shapes = [ref.shape[-2:] for ref in ref_latents] if ref_latents else None
+ transformer_options = transformer_options.copy()
+
+ # the reference embeds t in the compute dtype; image_cond and text sit at t = 0
+ vec = self.time_in(timestep_embedding(torch.cat([timestep, timestep.new_zeros(1)]).to(dtype), 256))
+ if vec_target is not None:
+ vec = vec + vec_target
+ vec_img, vec_zero = vec[:-1], vec[-1:]
+ early = self.early_stream_modulations["image"](vec_img)[0]
+
+ # per sample the joint sequence is [image, image_cond, text]
+ img = self.emb_in["image"](x.flatten(2).transpose(1, 2))
+ parts = [(img, "image", self.image_ids(H, W, 0, device))]
+ if ref_states is not None:
+ parts.append((ref_states, "image_cond", self.ref_ids(ref_shapes, device)))
+ parts.append((txt_states, "txt", self.text_ids(txt_states.shape[1], device)))
+ pe = self.rope(torch.cat([ids for _, _, ids in parts]))
+ spans, start = [], 0
+ for tokens, mode, _ in parts:
+ spans.append((start, start + tokens.shape[1], self.single_stream_modulations[mode](vec_img if mode == "image" else vec_zero)[0]))
+ start += tokens.shape[1]
+ # joint sequence allocated outside the memory compiler region, filled after the image's early blocks
+ x = torch.empty(B, start, img.shape[-1], device=device, dtype=img.dtype)
+
+ queue = list(self.content_mode_blocks["image"]) + list(self.single_blocks)
+ prefetch_queue = comfy.model_prefetch.make_prefetch_queue(queue, device, transformer_options)
+ comfy.model_prefetch.malloc_graph_begin(device)
+ for block in self.content_mode_blocks["image"]:
+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, block, dtype, malloc_scope="block")
+ img = block(img, [(0, H * W, early)], pe[:, :H * W], transformer_options)
+
+ torch.cat([img] + [tokens for tokens, _, _ in parts[1:]], dim=1, out=x)
+ del img, parts
+ transformer_options["total_blocks"] = len(self.single_blocks)
+ transformer_options["block_type"] = "single"
+ for i, block in enumerate(self.single_blocks):
+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, block, dtype, malloc_scope="block")
+ transformer_options["block_index"] = i
+ x = block(x, spans, pe, transformer_options)
+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, None, malloc_scope="block")
+ comfy.model_prefetch.malloc_graph_end()
+
+ out = self.final_layer["image"](x[:, :H * W], vec_img)
+ return out.transpose(1, 2).reshape(B, self.in_channels, H, W)
diff --git a/comfy/lora.py b/comfy/lora.py
index 36f286cc3..061550067 100644
--- a/comfy/lora.py
+++ b/comfy/lora.py
@@ -343,6 +343,15 @@ def model_lora_keys_unet(model, key_map={}):
key_map["transformer.{}".format(key_lora)] = to
key_map["lycoris_{}".format(key_lora.replace(".", "_"))] = to #SimpleTuner lycoris format
+ if isinstance(model, comfy.model_base.Flux3):
+ for k in sdk:
+ if k.endswith(".qkv.weight"): # fused q/k/v files; LoRAs address the separate projections
+ third = sd[k].shape[0] // 3
+ for i, proj in enumerate(("q_proj", "k_proj", "v_proj")):
+ key_lora = "{}.{}".format(k[:-len(".qkv.weight")], proj)
+ key_map[key_lora] = (k, (0, i * third, third))
+ key_map["lora_unet_{}".format(key_lora[len("diffusion_model."):].replace(".", "_"))] = (k, (0, i * third, third))
+
if isinstance(model, comfy.model_base.Krea2):
diffusers_keys = comfy.utils.krea2_to_diffusers(model.model_config.unet_config, output_prefix="diffusion_model.")
for k in diffusers_keys:
diff --git a/comfy/model_base.py b/comfy/model_base.py
index 4f731e7cd..f7514770f 100644
--- a/comfy/model_base.py
+++ b/comfy/model_base.py
@@ -40,6 +40,7 @@ import comfy.ldm.hydit.models
import comfy.ldm.audio.dit
import comfy.ldm.audio.embedders
import comfy.ldm.flux.model
+import comfy.ldm.flux3.model
import comfy.ldm.lens.model
import comfy.ldm.lightricks.model
import comfy.ldm.hunyuan_video.model
@@ -1098,6 +1099,39 @@ class Flux2(Flux):
return out
+class Flux3(BaseModel):
+ def __init__(self, model_config, model_type=ModelType.FLUX, device=None):
+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.flux3.model.Flux3)
+ self.memory_usage_factor_conds = ("ref_latents",)
+
+ def extra_conds(self, **kwargs):
+ out = super().extra_conds(**kwargs)
+ cross_attn = kwargs.get("cross_attn", None)
+ ref_latents = kwargs.get("reference_latents", None)
+ if ref_latents is not None:
+ ref_latents = [self.process_latent_in(lat) for lat in ref_latents]
+ # the DiT attends to every text token, so no padding: cond and uncond of different lengths run separately
+ if cross_attn is not None and torch.is_inference_mode_enabled() and kwargs.get("hooks") is None and not self.diffusion_model.flow_map:
+ # the t = 0 early blocks of text and references depend on the conditioning alone: run them once, not every step
+ # (a flow map also modulates them with each step's target time)
+ device, dtype = kwargs["device"], self.get_dtype_inference()
+ refs = [lat.to(device=device, dtype=dtype) for lat in ref_latents] if ref_latents else None
+ txt_states, ref_states = self.diffusion_model.preprocess_conds(cross_attn.to(device=device, dtype=dtype), refs)
+ del out['c_crossattn']
+ out['txt_states'] = comfy.conds.CONDRegular(txt_states)
+ if ref_states is not None:
+ out['ref_states'] = comfy.conds.CONDRegular(ref_states)
+ out['ref_shapes'] = comfy.conds.CONDConstant([tuple(lat.shape[-2:]) for lat in ref_latents])
+ return out
+ if cross_attn is not None:
+ out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)
+ if ref_latents is not None:
+ out['ref_latents'] = comfy.conds.CONDList(ref_latents)
+ return out
+
+ extra_conds_shapes = Flux.extra_conds_shapes
+
+
class Lens(BaseModel):
def __init__(self, model_config, model_type=ModelType.FLUX, device=None):
super().__init__(
diff --git a/comfy/model_detection.py b/comfy/model_detection.py
index 29618dea2..78e71d86b 100644
--- a/comfy/model_detection.py
+++ b/comfy/model_detection.py
@@ -260,6 +260,17 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
dit_config["meanflow_sum"] = False
return dit_config
+ if '{}content_mode_blocks.image.0.attn_out.weight'.format(key_prefix) in state_dict_keys and '{}single_stream_modulations.txt.lin.weight'.format(key_prefix) in state_dict_keys: # FLUX.3
+ dit_config = {}
+ dit_config["image_model"] = "flux3"
+ dit_config["hidden_size"] = state_dict['{}content_mode_blocks.image.0.attn_out.weight'.format(key_prefix)].shape[0]
+ dit_config["num_heads"] = dit_config["hidden_size"] // 128
+ dit_config["fused_qkv"] = '{}single_blocks.0.qkv.weight'.format(key_prefix) in state_dict_keys
+ dit_config["flow_map"] = '{}ref_time_in.in_layer.weight'.format(key_prefix) in state_dict_keys
+ dit_config["depth"] = count_blocks(state_dict_keys, '{}txt_mode_blocks.'.format(key_prefix) + '{}.')
+ dit_config["depth_single_blocks"] = count_blocks(state_dict_keys, '{}single_blocks.'.format(key_prefix) + '{}.')
+ return dit_config
+
if any_suffix_in(state_dict_keys, key_prefix, 'double_blocks.0.img_attn.norm.key_norm.', ["weight", "scale"]) and ('{}img_in.weight'.format(key_prefix) in state_dict_keys or any_suffix_in(state_dict_keys, key_prefix, 'distilled_guidance_layer.norms.0.', ["weight", "scale"])): #Flux, Chroma or Chroma Radiance (has no img_in.weight)
dit_config = {}
if '{}double_stream_modulation_img.lin.weight'.format(key_prefix) in state_dict_keys:
diff --git a/comfy/sd.py b/comfy/sd.py
index ac62bf795..02f8f3bb1 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -1608,6 +1608,7 @@ class CLIPType(Enum):
MAGE = 34
MINIMAX = 35
YUE2 = 36
+ FLUX3 = 37
@@ -1977,6 +1978,10 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip
clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})
clip_target.clip = comfy.text_encoders.krea2.te(**llama_detect(clip_data))
clip_target.tokenizer = comfy.text_encoders.krea2.Krea2Tokenizer
+ elif clip_type == CLIPType.FLUX3 and te_model == TEModel.QWEN3VL_4B: # FLUX.3: Qwen3-VL-4B text only, 8-layer tap -> 20480.
+ clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})
+ clip_target.clip = comfy.text_encoders.flux.flux3_te(**llama_detect(clip_data))
+ clip_target.tokenizer = comfy.text_encoders.flux.Flux3Tokenizer
elif clip_type == CLIPType.MAGE and te_model == TEModel.QWEN3VL_4B: # Mage-Flow: full Qwen3-VL-4B, last hidden state, Qwen-Image-style templates.
clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})
clip_target.clip = comfy.text_encoders.mage_flow.te(**llama_detect(clip_data))
diff --git a/comfy/supported_models.py b/comfy/supported_models.py
index 1be399a09..939cbea9b 100644
--- a/comfy/supported_models.py
+++ b/comfy/supported_models.py
@@ -842,6 +842,26 @@ class Flux2(Flux):
return None
+class Flux3(Flux2):
+ unet_config = {
+ "image_model": "flux3",
+ }
+
+ sampling_settings = {
+ "shift": 1.93586, # mu = ln(6.93), the reference timestep shift
+ }
+
+ supported_inference_dtypes = [torch.bfloat16, torch.float32]
+
+ def get_model(self, state_dict, prefix="", device=None):
+ return model_base.Flux3(self, device=device)
+
+ def clip_target(self, state_dict={}):
+ pref = self.text_encoder_key_prefix[0]
+ detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_4b.transformer.".format(pref))
+ return supported_models_base.ClipTarget(comfy.text_encoders.flux.Flux3Tokenizer, comfy.text_encoders.flux.flux3_te(**detect))
+
+
class Lens(supported_models_base.BASE):
"""Microsoft Lens (3.8B dual-stream MMDiT, GPT-OSS-20B text features, Flux2 VAE)."""
@@ -2665,6 +2685,7 @@ models = [
Ideogram4,
Krea2,
Flux2,
+ Flux3,
Lens,
Kandinsky5Image,
Kandinsky5,
diff --git a/comfy/text_encoders/flux.py b/comfy/text_encoders/flux.py
index 300ac5c11..26a596dba 100644
--- a/comfy/text_encoders/flux.py
+++ b/comfy/text_encoders/flux.py
@@ -2,6 +2,7 @@ from comfy import sd1_clip
import comfy.text_encoders.t5
import comfy.text_encoders.sd3_clip
import comfy.text_encoders.llama
+import comfy.text_encoders.qwen3vl
import comfy.model_management
from transformers import T5TokenizerFast, Qwen2Tokenizer
from .bpe_tokenizer import from_tekken_json
@@ -198,3 +199,47 @@ def klein_te(dtype_llama=None, llama_quantization_metadata=None, model_type="qwe
dtype = dtype_llama
super().__init__(device=device, dtype=dtype, name=model_type, model_options=model_options, clip_model=model)
return Flux2TEModel_
+
+class Flux3Tokenizer(comfy.text_encoders.qwen3vl.Qwen3VLTokenizer):
+ def __init__(self, embedding_directory=None, tokenizer_data={}):
+ super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type="qwen3vl_4b")
+
+ def tokenize_with_weights(self, text, return_word_ids=False, thinking=True, **kwargs):
+ # instruct template without a think block, right padded to a multiple of 80 tokens
+ tokens = super().tokenize_with_weights(text, return_word_ids=return_word_ids, thinking=thinking, **kwargs)
+ pad = (151643, 1.0, 0) if return_word_ids else (151643, 1.0)
+ for batch in tokens[self.clip_name]:
+ batch.extend([pad] * (-len(batch) % 80))
+ return tokens
+
+class Flux3Qwen3VL(comfy.text_encoders.qwen3vl.Qwen3VL):
+ model_type = "qwen3vl_4b"
+
+ def forward(self, input_ids, attention_mask=None, **kwargs):
+ if attention_mask is not None:
+ # transformers 4.57, which the reference pins, puts the padding at rope position 1; the DiT attends to it
+ kwargs["position_ids"] = (attention_mask.cumsum(-1) - 1).masked_fill(attention_mask == 0, 1)
+ return super().forward(input_ids, attention_mask=attention_mask, **kwargs)
+
+class Flux3Qwen3VLModel(sd1_clip.SDClipModel):
+ def __init__(self, device="cpu", dtype=None, model_options={}):
+ super().__init__(device=device, layer=[4, 8, 12, 16, 20, 24, 28, 32], layer_idx=None, textmodel_json_config={}, dtype=dtype, special_tokens={"pad": 151643}, layer_norm_hidden_state=False, model_class=Flux3Qwen3VL, enable_attention_masks=True, model_options=model_options)
+
+class Flux3TEModel(sd1_clip.SD1ClipModel):
+ def __init__(self, device="cpu", dtype=None, model_options={}):
+ super().__init__(device=device, dtype=dtype, name="qwen3vl_4b", clip_model=Flux3Qwen3VLModel, model_options=model_options)
+
+ def encode_token_weights(self, token_weight_pairs):
+ out, pooled = super().encode_token_weights(token_weight_pairs)
+ return out.movedim(1, 2).flatten(2), pooled
+
+def flux3_te(dtype_llama=None, llama_quantization_metadata=None):
+ class Flux3TEModel_(Flux3TEModel):
+ def __init__(self, device="cpu", dtype=None, model_options={}):
+ if llama_quantization_metadata is not None:
+ model_options = model_options.copy()
+ model_options["quantization_metadata"] = llama_quantization_metadata
+ if dtype_llama is not None:
+ dtype = dtype_llama
+ super().__init__(device=device, dtype=dtype, model_options=model_options)
+ return Flux3TEModel_
diff --git a/comfy_extras/nodes_flux.py b/comfy_extras/nodes_flux.py
index e9986c9e7..53f1712b6 100644
--- a/comfy_extras/nodes_flux.py
+++ b/comfy_extras/nodes_flux.py
@@ -1,3 +1,4 @@
+import json
import node_helpers
import comfy.utils
from typing_extensions import override
@@ -300,6 +301,35 @@ class FluxKVCache(io.ComfyNode):
return io.NodeOutput(m)
+class Flux3LayoutPrompt(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="Flux3LayoutPrompt",
+ display_name="FLUX.3 Layout Prompt",
+ category="model/conditioning/flux",
+ description="Appends the boxes from Create Bounding Boxes to a FLUX.3 prompt as its layout table.",
+ inputs=[
+ io.String.Input("prompt", multiline=True, dynamic_prompts=True,
+ tooltip="The prompt text. Cite box N as <obj_N> to tie words to it."),
+ io.Array.Input("element", tooltip="Prompt elements from the node Create Bounding Boxes. Colour palettes are not used, describe colours in the description."),
+ ],
+ outputs=[io.String.Output(display_name="prompt")],
+ is_experimental=True,
+ )
+
+ @classmethod
+ def execute(cls, prompt, element) -> io.NodeOutput:
+ rows = []
+ for i, el in enumerate(element if isinstance(element, list) else []):
+ desc = el.get("desc", "")
+ if el.get("type") == "text" and el.get("text"):
+ desc = f'{desc} "{el["text"]}"'.strip()
+ rows.append({"id": f"obj_{i + 1}", "bbox": el["bbox"], "desc": desc})
+ if not rows:
+ return io.NodeOutput(prompt)
+ return io.NodeOutput(f"{prompt.rstrip()} {json.dumps(rows, ensure_ascii=False)}")
+
class FluxExtension(ComfyExtension):
@override
async def get_node_list(self) -> list[type[io.ComfyNode]]:
@@ -312,6 +342,7 @@ class FluxExtension(ComfyExtension):
EmptyFlux2LatentImage,
Flux2Scheduler,
FluxKVCache,
+ Flux3LayoutPrompt,
]
diff --git a/nodes.py b/nodes.py
index c0dd5563f..e0ef36232 100644
--- a/nodes.py
+++ b/nodes.py
@@ -1018,7 +1018,7 @@ class CLIPLoader:
@classmethod
def INPUT_TYPES(s):
return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ),
- "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2", "joyimage", "mage", "minimax", "yue2"], ),
+ "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2", "joyimage", "mage", "minimax", "yue2", "flux3"], ),
},
"optional": {
"device": (["default", "cpu"], {"advanced": True}),
@@ -1028,7 +1028,7 @@ class CLIPLoader:
CATEGORY = "model/loaders"
- DESCRIPTION = "Recipes:\nsd: clip-l\nstable cascade: clip-g\nsd3: t5 xxl / clip-g / clip-l\nstable audio: t5 base\nmochi: t5 xxl\ncogvideox: t5 xxl (226-token padding)\ncosmos: old t5 xxl\nlumina2: gemma 2 2B\nwan: umt5 xxl\nhidream: llama-3.1 (Recommend) or t5\nomnigen2: qwen vl 2.5 3B\njoyimage: qwen3-vl 8B\nlens: gpt-oss-20b\npixeldit: gemma 2 2B elm\nminimax: MiniMax H3 Qwen3-VL or Music3 Qwen/RVQ"
+ DESCRIPTION = "Recipes:\nsd: clip-l\nstable cascade: clip-g\nsd3: t5 xxl / clip-g / clip-l\nstable audio: t5 base\nmochi: t5 xxl\ncogvideox: t5 xxl (226-token padding)\ncosmos: old t5 xxl\nlumina2: gemma 2 2B\nwan: umt5 xxl\nhidream: llama-3.1 (Recommend) or t5\nomnigen2: qwen vl 2.5 3B\njoyimage: qwen3-vl 8B\nlens: gpt-oss-20b\npixeldit: gemma 2 2B elm\nminimax: MiniMax H3 Qwen3-VL or Music3 Qwen/RVQ\nflux3: qwen3-vl 4B"
def load_clip(self, clip_name, type="stable_diffusion", device="default"):
clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION)