Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

feat: Add Fibo 2.0 support (#17240)

Jukka Seppänen87ed5a066d9c
diff --git a/comfy/ldm/fibo2/model.py b/comfy/ldm/fibo2/model.py
new file mode 100644
index 000000000..1a318834b
--- /dev/null
+++ b/comfy/ldm/fibo2/model.py
@@ -0,0 +1,219 @@
+import math
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+import comfy.model_management
+import comfy.model_prefetch
+import comfy.ops
+import comfy.patcher_extension
+import comfy.utils
+from comfy.ldm.flux.layers import EmbedND
+from comfy.ldm.lumina.model import FinalLayer, JointAttention, apply_gate, clamp_fp16, modulate, pos_ids_x
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
+from comfy.ldm.modules.diffusionmodules.mmdit import TimestepEmbedder
+
+
+def residual(x, y, gate=None):
+ # in place inside the memory compiler region; out of place keeps x for backward when training
+ if comfy.model_management.in_training:
+ return x + y if gate is None else torch.addcmul(x, y, gate)
+ return x.add_(y) if gate is None else x.addcmul_(y, gate)
+
+
+def gist_count(text_len, max_gist):
+ # 256 gist tokens up to 256 text tokens, rising linearly to max_gist at 2048, in steps of 64
+ ratio = min(1.0, max(0.0, (text_len - 256) / (2048 - 256)))
+ return max(256, min(max_gist, math.ceil((256 + ratio * (max_gist - 256)) / 64) * 64))
+
+
+class Attention(nn.Module):
+ # no RoPE: the cross-attention to the text, and the Perceiver's self-attention
+ def __init__(self, dim, n_heads, kv_dim, norm_eps, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.comfy_attention = ComfyAttention()
+ self.n_heads = n_heads
+ self.to_q = operations.Linear(dim, dim, bias=False, dtype=dtype, device=device)
+ self.to_kv = operations.Linear(kv_dim, 2 * dim, bias=False, dtype=dtype, device=device)
+ self.to_out = operations.Linear(dim, dim, bias=False, dtype=dtype, device=device)
+ self.norm_q = operations.RMSNorm(dim // n_heads, eps=norm_eps, dtype=dtype, device=device)
+ self.norm_k = operations.RMSNorm(dim // n_heads, eps=norm_eps, dtype=dtype, device=device)
+
+ def forward(self, x, context, transformer_options={}):
+ q = self.norm_q(self.to_q(x).unflatten(-1, (self.n_heads, -1)))
+ k, v = self.to_kv(context).unflatten(-1, (2, self.n_heads, -1)).unbind(2)
+ k = self.norm_k(k)
+ q, k, v = (AttentionTensorContainer(t.movedim(1, 2)) for t in (q, k, v))
+ return self.to_out(optimized_attention(q, k, v, self.n_heads, skip_reshape=True, transformer_options=transformer_options, preferred_attention=self.comfy_attention))
+
+
+class FeedForward(nn.Module):
+ # SwiGLU; fused files carry w1 and w3 as one w13, the hidden width zero padded to a multiple of 256 so int8 weights quantize swiglu(x) inside w2's GEMM
+ def __init__(self, dim, hidden_dim, fused, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.fused = fused
+ if fused:
+ self.w13 = operations.Linear(dim, 2 * hidden_dim, bias=False, dtype=dtype, device=device)
+ else:
+ self.w1 = operations.Linear(dim, hidden_dim, bias=False, dtype=dtype, device=device)
+ self.w3 = operations.Linear(dim, hidden_dim, bias=False, dtype=dtype, device=device)
+ self.w2 = operations.Linear(hidden_dim, dim, bias=False, dtype=dtype, device=device)
+
+ def forward(self, x):
+ if self.fused:
+ return comfy.ops.linear_input_act(self.w2, self.w13(x), "swiglu")
+ return self.w2(F.silu(self.w1(x)) * self.w3(x))
+
+
+class Block(nn.Module):
+ # Z-Image's block with in place residuals; an injection block also cross-attends to its own text bundle
+ def __init__(self, dim, n_heads, ffn_dim, fused_ffn, norm_eps, cap_feat_dim=None, operation_settings={}):
+ super().__init__()
+ operations, device, dtype = operation_settings["operations"], operation_settings["device"], operation_settings["dtype"]
+ self.attention = JointAttention(dim, n_heads, None, True, operation_settings=operation_settings)
+ self.attention.q_norm.eps = self.attention.k_norm.eps = norm_eps
+ self.feed_forward = FeedForward(dim, ffn_dim, fused_ffn, dtype=dtype, device=device, operations=operations)
+ self.attention_norm1 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)
+ self.ffn_norm1 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)
+ self.attention_norm2 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)
+ self.ffn_norm2 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)
+ self.adaLN_modulation = nn.Sequential(operations.Linear(256, 4 * dim, bias=True, dtype=dtype, device=device))
+ self.cross_attention = None
+ if cap_feat_dim is not None:
+ self.cross_attn_norm1 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)
+ self.cross_attention = Attention(dim, n_heads, cap_feat_dim, norm_eps, dtype=dtype, device=device, operations=operations)
+ self.cross_attn_norm2 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)
+ self.cross_attn_gate = nn.Parameter(torch.empty(1, dtype=dtype, device=device))
+
+ def forward(self, x, freqs_cis, adaln_input, timestep_zero_index=None, context=None, transformer_options={}):
+ scale_msa, gate_msa, scale_mlp, gate_mlp = self.adaLN_modulation(adaln_input).chunk(4, dim=1)
+ y = self.attention(modulate(self.attention_norm1(x), scale_msa, timestep_zero_index=timestep_zero_index), None, freqs_cis, transformer_options=transformer_options)
+ x = residual(x, apply_gate(gate_msa.unsqueeze(1).tanh(), self.attention_norm2(clamp_fp16(y)), timestep_zero_index=timestep_zero_index))
+ if self.cross_attention is not None:
+ y = self.cross_attention(self.cross_attn_norm1(x), context, transformer_options)
+ x = residual(x, self.cross_attn_norm2(y), comfy.ops.cast_to_input(self.cross_attn_gate, y).tanh())
+ y = self.feed_forward(modulate(self.ffn_norm1(x), scale_mlp, timestep_zero_index=timestep_zero_index))
+ return residual(x, apply_gate(gate_mlp.unsqueeze(1).tanh(), self.ffn_norm2(clamp_fp16(y)), timestep_zero_index=timestep_zero_index))
+
+
+class PerceiverLayer(nn.Module):
+ def __init__(self, dim, n_heads, ffn_dim, fused_ffn, cap_feat_dim, norm_eps, operation_settings={}):
+ super().__init__()
+ operations, device, dtype = operation_settings["operations"], operation_settings["device"], operation_settings["dtype"]
+ self.cross_attn_norm = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)
+ self.cross_attention = Attention(dim, n_heads, cap_feat_dim, norm_eps, dtype=dtype, device=device, operations=operations)
+ self.self_attn_norm = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)
+ self.self_attention = Attention(dim, n_heads, dim, norm_eps, dtype=dtype, device=device, operations=operations)
+ self.ffn_norm = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)
+ self.ffn = FeedForward(dim, ffn_dim, fused_ffn, dtype=dtype, device=device, operations=operations)
+
+ def forward(self, x, context, transformer_options={}):
+ x = x + self.cross_attention(self.cross_attn_norm(x), context, transformer_options)
+ y = self.self_attn_norm(x)
+ x = x + self.self_attention(y, y, transformer_options)
+ return x + self.ffn(self.ffn_norm(x))
+
+
+class Perceiver(nn.Module):
+ # squeezes the text encoder's hidden states into gist tokens
+ def __init__(self, dim, n_heads, ffn_dim, fused_ffn, cap_feat_dim, num_queries, num_layers, norm_eps, operation_settings={}):
+ super().__init__()
+ operations, device, dtype = operation_settings["operations"], operation_settings["device"], operation_settings["dtype"]
+ self.latent_queries = nn.Parameter(torch.empty(num_queries, dim, dtype=dtype, device=device))
+ self.layers = nn.ModuleList([PerceiverLayer(dim, n_heads, ffn_dim, fused_ffn, cap_feat_dim, norm_eps, operation_settings=operation_settings) for _ in range(num_layers)])
+ self.norm_out = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)
+
+ def forward(self, context, num_gist, transformer_options={}):
+ x = comfy.ops.cast_to_input(self.latent_queries[:num_gist], context).unsqueeze(0).expand(context.shape[0], -1, -1)
+ prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.layers), context.device, transformer_options)
+ for layer in self.layers:
+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, context.device, layer, context.dtype)
+ x = layer(x, context, transformer_options)
+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, context.device, None)
+ return self.norm_out(x)
+
+
+class Fibo2(nn.Module):
+ def __init__(self, in_channels=128, dim=3584, n_heads=28, n_layers=44, n_refiner_layers=2, injection_layers=(6, 14, 22, 30, 38), cap_feat_dim=7680,
+ perceiver_layers=2, max_gist_tokens=512, ffn_dim=9728, fused_ffn=True, norm_eps=1e-5, image_model=None, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.dtype = dtype
+ operation_settings = {"operations": operations, "device": device, "dtype": dtype}
+ self.max_gist_tokens = max_gist_tokens
+ self.injection_layers = injection_layers
+
+ self.x_embedder = operations.Linear(in_channels, dim, bias=True, dtype=dtype, device=device)
+ self.t_embedder = TimestepEmbedder(1024, output_size=256, **operation_settings)
+ self.noise_refiner = nn.ModuleList([Block(dim, n_heads, ffn_dim, fused_ffn, norm_eps, operation_settings=operation_settings) for _ in range(n_refiner_layers)])
+ self.perceiver = Perceiver(dim, n_heads, ffn_dim, fused_ffn, cap_feat_dim, max_gist_tokens, perceiver_layers, norm_eps, operation_settings=operation_settings)
+ self.layers = nn.ModuleList([Block(dim, n_heads, ffn_dim, fused_ffn, norm_eps, cap_feat_dim if i in injection_layers else None, operation_settings=operation_settings) for i in range(n_layers)])
+ self.final_layer = FinalLayer(dim, 1, in_channels, z_image_modulation=True, operation_settings=operation_settings)
+ self.rope_embedder = EmbedND(dim=dim // n_heads, theta=256.0, axes_dim=[32, 48, 48])
+
+ def forward(self, x, timestep, context, transformer_options={}, **kwargs):
+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(
+ self._forward,
+ self,
+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)
+ ).execute(x, timestep, context, transformer_options, **kwargs)
+
+ def _forward(self, x, timestep, context, transformer_options={}, ref_latents=[], **kwargs):
+ B, C, H, W = x.shape
+ dtype, device = x.dtype, x.device
+ transformer_options = transformer_options.copy()
+
+ # the image and the reference tokens are noisy and clean: the second half of the adaln batch modulates the image
+ adaln_mixed = self.t_embedder(torch.cat((torch.zeros_like(timestep), timestep * 1000)), dtype)
+ adaln_noisy = adaln_mixed[B:]
+
+ # gist tokens on rope planes 1..num_gist, the image on plane max_gist_tokens + 1 whatever the gist count, reference k on the k-th plane after it
+ num_gist = gist_count(context.shape[1], self.max_gist_tokens)
+ plane = self.max_gist_tokens + 1
+ tokens = [x.flatten(2).transpose(1, 2)]
+ ids = [torch.zeros(1, num_gist, 3, dtype=torch.float32, device=device), pos_ids_x(plane, H, W, 1, device, transformer_options=transformer_options)]
+ ids[0][0, :, 0] = torch.arange(1, num_gist + 1, dtype=torch.float32, device=device)
+ for k, ref in enumerate(ref_latents):
+ tokens.append(comfy.utils.repeat_to_batch_size(ref, B).flatten(2).transpose(1, 2))
+ ids.append(pos_ids_x(plane + 1 + k, ref.shape[-2], ref.shape[-1], 1, device, transformer_options=transformer_options))
+ img = self.x_embedder(torch.cat(tokens, dim=1))
+ freqs_cis = self.rope_embedder(torch.cat(ids, dim=1)).movedim(1, 2)
+ del tokens
+
+ n_img = H * W
+ refiner_adaln, refiner_index = (adaln_mixed, [(0, n_img)]) if len(ref_latents) > 0 else (adaln_noisy, None)
+ prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.noise_refiner), device, transformer_options)
+ for block in self.noise_refiner:
+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, block, dtype)
+ img = block(img, freqs_cis[:, num_gist:], refiner_adaln, refiner_index, transformer_options=transformer_options)
+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, None)
+
+ # the Perceiver reads the first text bundle, each injection block its own
+ bundles = context.chunk(1 + len(self.injection_layers), dim=-1)
+ gist = self.perceiver(bundles[0], num_gist, transformer_options)
+ text = dict(zip(self.injection_layers, bundles[1:]))
+
+ # allocated outside the memory compiler region, the blocks update it in place
+ x = torch.cat((gist, img), dim=1)
+ del gist, img
+ image_index = [(num_gist, num_gist + n_img)]
+
+ blocks_replace = transformer_options.get("patches_replace", {}).get("dit", {})
+ transformer_options["total_blocks"] = len(self.layers)
+ transformer_options["block_type"] = "single"
+ prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.layers), device, transformer_options)
+ comfy.model_prefetch.malloc_graph_begin(device)
+ for i, block in enumerate(self.layers):
+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, block, dtype, malloc_scope="block")
+ transformer_options["block_index"] = i
+ if ("single_block", i) in blocks_replace:
+ def block_wrap(args):
+ return {"img": block(args["img"], args["pe"], args["vec"], image_index, text.get(i), args["transformer_options"])}
+ x = blocks_replace[("single_block", i)]({"img": x, "vec": adaln_mixed, "pe": freqs_cis, "transformer_options": transformer_options}, {"original_block": block_wrap})["img"]
+ else:
+ x = block(x, freqs_cis, adaln_mixed, image_index, text.get(i), transformer_options)
+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, None, malloc_scope="block")
+ comfy.model_prefetch.malloc_graph_end()
+
+ out = self.final_layer(x[:, num_gist:num_gist + n_img], adaln_noisy)
+ return out.transpose(1, 2).reshape(B, C, H, W)
diff --git a/comfy/model_base.py b/comfy/model_base.py
index 8e42a86b4..8ee80152e 100644
--- a/comfy/model_base.py
+++ b/comfy/model_base.py
@@ -84,6 +84,7 @@ import comfy.ldm.sensenova.model
from comfy.ldm.sensenova.sampling import SenseNovaModelSampling, time_snr_shift
import comfy.ldm.depth_anything_3.model
import comfy.ldm.moss_image.model
+import comfy.ldm.fibo2.model
import comfy.model_management
import comfy.patcher_extension
@@ -2820,6 +2821,24 @@ class MOSSImage(BaseModel):
image = utils.common_upscale(image.to(kwargs["device"]), noise.shape[-1], noise.shape[-2], "bilinear", "center")
return utils.resize_to_batch_size(self.process_latent_in(image), noise.shape[0])
+class Fibo2(BaseModel):
+ def __init__(self, model_config, model_type=ModelType.FLUX, device=None):
+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.fibo2.model.Fibo2)
+ self.memory_usage_factor_conds = ("ref_latents",)
+
+ def extra_conds(self, **kwargs):
+ out = super().extra_conds(**kwargs)
+ cross_attn = kwargs.get("cross_attn", None)
+ # the gist token count follows the true text length, so no repeat padding: cond and uncond of different lengths run separately
+ if cross_attn is not None:
+ out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)
+ ref_latents = kwargs.get("reference_latents", None)
+ if ref_latents is not None:
+ out['ref_latents'] = comfy.conds.CONDList([self.process_latent_in(lat) for lat in ref_latents])
+ return out
+
+ extra_conds_shapes = Flux.extra_conds_shapes
+
class HunyuanImage21(BaseModel):
def __init__(self, model_config, model_type=ModelType.FLOW, device=None):
super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.hunyuan_video.model.HunyuanVideo)
diff --git a/comfy/model_detection.py b/comfy/model_detection.py
index 79c0e143e..04d76db2a 100644
--- a/comfy/model_detection.py
+++ b/comfy/model_detection.py
@@ -1022,6 +1022,25 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
dit_config["num_layers"] = count_blocks(state_dict_keys, '{}layers.'.format(key_prefix) + '{}.')
return dit_config
+ if ('{}perceiver.latent_queries'.format(key_prefix) in state_dict_keys and '{}layers.0.attention.qkv.weight'.format(key_prefix) in state_dict_keys and '{}final_layer.linear.weight'.format(key_prefix) in state_dict_keys
+ and ('{}layers.0.feed_forward.w13.weight'.format(key_prefix) in state_dict_keys or '{}layers.0.feed_forward.w1.weight'.format(key_prefix) in state_dict_keys)): # Fibo 2.0 (repacked)
+ dit_config = {}
+ dit_config["image_model"] = "fibo2"
+ dit_config["max_gist_tokens"], dit_config["dim"] = state_dict['{}perceiver.latent_queries'.format(key_prefix)].shape
+ dit_config["n_heads"] = dit_config["dim"] // 128
+ dit_config["in_channels"] = state_dict['{}final_layer.linear.weight'.format(key_prefix)].shape[0]
+ dit_config["n_layers"] = count_blocks(state_dict_keys, '{}layers.'.format(key_prefix) + '{}.')
+ dit_config["n_refiner_layers"] = count_blocks(state_dict_keys, '{}noise_refiner.'.format(key_prefix) + '{}.')
+ dit_config["perceiver_layers"] = count_blocks(state_dict_keys, '{}perceiver.layers.'.format(key_prefix) + '{}.')
+ dit_config["injection_layers"] = [i for i in range(dit_config["n_layers"]) if '{}layers.{}.cross_attn_gate'.format(key_prefix, i) in state_dict_keys]
+ w13 = '{}layers.0.feed_forward.w13.weight'.format(key_prefix)
+ if w13 in state_dict_keys: # w1 and w3 fused, hidden width padded for the fused int8 swiglu
+ dit_config["ffn_dim"] = state_dict[w13].shape[0] // 2
+ else:
+ dit_config["ffn_dim"] = state_dict['{}layers.0.feed_forward.w1.weight'.format(key_prefix)].shape[0]
+ dit_config["fused_ffn"] = False
+ return dit_config
+
if '{}shared_block_modulation.weight'.format(key_prefix) in state_dict_keys and '{}layers.0.proj.ffn_up_gate.weight'.format(key_prefix) in state_dict_keys: # MOSS-Image
dit_config = {}
dit_config["image_model"] = "moss_image"
diff --git a/comfy/sd.py b/comfy/sd.py
index e0b9d276b..11b4f6b73 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -93,6 +93,7 @@ import comfy.text_encoders.sa3
import comfy.text_encoders.gpt_oss
import comfy.text_encoders.joyimage
import comfy.text_encoders.moss_image
+import comfy.text_encoders.fibo2
import comfy.model_patcher
import comfy.lora
@@ -1611,6 +1612,7 @@ class CLIPType(Enum):
YUE2 = 36
FLUX3 = 37
MOSS_IMAGE = 38
+ FIBO2 = 39
@@ -1988,6 +1990,10 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip
clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})
clip_target.clip = comfy.text_encoders.flux.flux3_te(**llama_detect(clip_data))
clip_target.tokenizer = comfy.text_encoders.flux.Flux3Tokenizer
+ elif clip_type == CLIPType.FIBO2 and te_model == TEModel.QWEN3VL_4B: # Fibo 2.0: full Qwen3-VL-4B, 15-layer tap -> six 7680 bundles.
+ clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})
+ clip_target.clip = comfy.text_encoders.fibo2.te(**llama_detect(clip_data))
+ clip_target.tokenizer = comfy.text_encoders.fibo2.Fibo2Tokenizer
elif clip_type == CLIPType.MAGE and te_model == TEModel.QWEN3VL_4B: # Mage-Flow: full Qwen3-VL-4B, last hidden state, Qwen-Image-style templates.
clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})
clip_target.clip = comfy.text_encoders.mage_flow.te(**llama_detect(clip_data))
diff --git a/comfy/supported_models.py b/comfy/supported_models.py
index c136c95f1..3d2015bbd 100644
--- a/comfy/supported_models.py
+++ b/comfy/supported_models.py
@@ -43,6 +43,7 @@ import comfy.text_encoders.hidream_o1
import comfy.text_encoders.sensenova
import comfy.text_encoders.pixeldit
import comfy.text_encoders.moss_image
+import comfy.text_encoders.fibo2
from . import supported_models_base
from . import latent_formats
@@ -2067,6 +2068,34 @@ class MOSSImage(supported_models_base.BASE):
hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen35_9b.transformer.".format(pref))
return supported_models_base.ClipTarget(comfy.text_encoders.moss_image.MOSSImageTokenizer, comfy.text_encoders.moss_image.te(**hunyuan_detect))
+class Fibo2(supported_models_base.BASE):
+ unet_config = {
+ "image_model": "fibo2",
+ }
+
+ sampling_settings = {
+ "shift": 1.15,
+ }
+
+ memory_usage_factor = 17.0
+
+ unet_extra_config = {}
+ latent_format = latent_formats.Flux2
+
+ supported_inference_dtypes = [torch.bfloat16, torch.float32]
+
+ vae_key_prefix = ["vae."]
+ text_encoder_key_prefix = ["text_encoders."]
+
+ def get_model(self, state_dict, prefix="", device=None):
+ out = model_base.Fibo2(self, device=device)
+ return out
+
+ def clip_target(self, state_dict={}):
+ pref = self.text_encoder_key_prefix[0]
+ hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_4b.transformer.".format(pref))
+ return supported_models_base.ClipTarget(comfy.text_encoders.fibo2.Fibo2Tokenizer, comfy.text_encoders.fibo2.te(**hunyuan_detect))
+
class MageFlow(supported_models_base.BASE):
unet_config = {
"image_model": "mage_flow",
@@ -2719,6 +2748,7 @@ models = [
Ideogram4,
Krea2,
MOSSImage,
+ Fibo2,
Flux2,
Flux3,
Lens,
diff --git a/comfy/text_encoders/fibo2.py b/comfy/text_encoders/fibo2.py
new file mode 100644
index 000000000..d16df37c1
--- /dev/null
+++ b/comfy/text_encoders/fibo2.py
@@ -0,0 +1,188 @@
+"""Fibo 2.0 text encoder: Qwen3-VL-4B, 15 hidden layers at unit RMS in six 7680-dim bundles (Perceiver, then each injection block)."""
+
+import json
+import math
+
+import numpy as np
+import torch
+from PIL import Image, ImageDraw, ImageFont
+
+import comfy.rmsnorm
+import comfy.text_encoders.qwen3vl
+from comfy import sd1_clip
+
+VISION_MARKER = "<|vision_start|><|image_pad|><|vision_end|>"
+TEMPLATE = "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"
+
+# the null caption training used for classifier-free guidance, a whole chat turn so the tokenizer takes it as it is, uncleaned
+NULL_CAPTION = TEMPLATE.format('{"short_description":"","objects":[],"background_setting":"","lighting":{},"aesthetics":{},"photographic_characteristics":{},"style_medium":"","context":"","artistic_style":""}')
+
+BUNDLE_LAYERS = ((9, 20, 31), (5, 16, 27), (7, 18, 29), (9, 20, 31), (11, 22, 33), (13, 24, 35))
+TAP_LAYERS = sorted({layer for bundle in BUNDLE_LAYERS for layer in bundle})
+BUNDLE_INDEX = [TAP_LAYERS.index(layer) for bundle in BUNDLE_LAYERS for layer in bundle]
+
+SCORES = ("aesthetic_score", "preference_score")
+
+
+def dumps(value):
+ return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
+
+
+def empty(value):
+ if isinstance(value, float):
+ return math.isnan(value)
+ return value is None or (isinstance(value, (str, dict, list)) and not value)
+
+
+def clean(value):
+ if isinstance(value, dict):
+ value = {k: clean(v) for k, v in value.items()}
+ return {k: v for k, v in value.items() if not empty(v)}
+ if isinstance(value, list):
+ return [v for v in map(clean, value) if not empty(v)]
+ return value
+
+
+def parse_prompt(prompt):
+ # the caption as training saw it, and the edit instruction; a prompt that isn't a JSON object comes back as it is
+ try:
+ caption = json.loads(prompt)
+ except json.JSONDecodeError:
+ return prompt, None
+ if not isinstance(caption, dict):
+ return prompt, None
+ instruction = caption.pop("edit_instruction", None)
+ if "structured_caption" in caption:
+ caption = caption.pop("structured_caption")
+ if not isinstance(caption, dict):
+ return caption or None, instruction
+
+ keys = set(caption)
+ caption = clean({key: {k: v for k, v in value.items() if k not in SCORES} if isinstance(value, dict) else value
+ for key, value in caption.items() if key not in SCORES})
+ # exactly a short_description and a text_render list keeps both, the list even when empty: the image has no text
+ if keys == {"short_description", "text_render"}:
+ caption = {"short_description": caption.get("short_description") or "", "text_render": caption.get("text_render") or []}
+ return caption or None, instruction
+
+
+def has_glyph(font, notdef, character):
+ # a font without the glyph draws its notdef box, the one it draws for U+FFFE, which no font has
+ try:
+ mask = font.getmask(character)
+ except (OSError, ValueError): # as in training, a font that can't say is taken to have the glyph
+ return True
+ return not (mask.size == notdef.size and bytes(mask) == bytes(notdef))
+
+
+def draw_text_image(text, font):
+ # Pillow's built-in font, black on white, whitespace collapsed, split every 256 characters (training's non-ASCII fallback fonts aren't shipped)
+ line = " ".join(text.split())
+ lines = [line[i:i + 256] for i in range(0, len(line), 256)] or [""]
+ notdef = font.getmask(chr(0xFFFE))
+ if all(ord(c) < 128 or has_glyph(font, notdef, c) for c in set(line)):
+ # each line placed by its bounding box, at least as tall as "Ag"
+ min_box = font.getbbox("Ag")
+ boxes = [font.getbbox(line or " ") for line in lines]
+ heights = [max(1, min_box[3] - min_box[1], box[3] - box[1]) for box in boxes]
+ width = max([1] + [box[2] - box[0] for box in boxes])
+ origins, anchor = [(2 - box[0], -box[1]) for box in boxes], None
+ else:
+ # a character the font lacks (drawn as its notdef box) puts every line on the baseline, as tall as the font
+ ascent, descent = font.getmetrics()
+ lines = [line or " " for line in lines]
+ heights = [max(1, ascent + descent)] * len(lines)
+ width = max(max(1, round(font.getlength(line))) for line in lines)
+ origins, anchor = [(2.0, ascent)] * len(lines), "ls"
+
+ image = Image.new("RGB", (width + 4, sum(heights) + 4 * len(lines)), "white")
+ draw = ImageDraw.Draw(image)
+ y = 2
+ for line, (x, dy), height in zip(lines, origins, heights):
+ draw.text((x, y + dy), line, font=font, fill=(0, 0, 0), anchor=anchor)
+ y += height + 4
+ return torch.from_numpy(np.array(image)).float().div(255.0).unsqueeze(0)
+
+
+def resize_8bit(image, height, width):
+ # antialiased bicubic rounded to 8 bit, as PIL and Qwen3-VL's processor resize
+ image = torch.nn.functional.interpolate(image.movedim(-1, 1), size=(height, width), mode="bicubic", antialias=True)
+ return image.mul(255.0).round().clamp(0, 255).div(255.0).movedim(1, -1)
+
+
+def thumbnail(image, size=768):
+ # Qwen3-VL sees each image to edit at most 768 pixels a side, as in training
+ scale = size / max(image.shape[1], image.shape[2])
+ if scale >= 1:
+ return image
+ return resize_8bit(image, max(1, round(image.shape[1] * scale)), max(1, round(image.shape[2] * scale)))
+
+
+def to_qwen_grid(image):
+ # the processor's 32 pixel grid, so ComfyUI's own bilinear resize is a no-op
+ h, w = image.shape[1], image.shape[2]
+ height, width = round(h / 32) * 32, round(w / 32) * 32
+ if height * width > 1605632:
+ beta = math.sqrt(h * w / 1605632)
+ height, width = max(32, math.floor(h / beta / 32) * 32), max(32, math.floor(w / beta / 32) * 32)
+ elif height * width < 3136:
+ beta = math.sqrt(3136 / (h * w))
+ height, width = math.ceil(h * beta / 32) * 32, math.ceil(w * beta / 32) * 32
+ return resize_8bit(image, height, width)
+
+
+class Fibo2Tokenizer(comfy.text_encoders.qwen3vl.Qwen3VLTokenizer):
+ def __init__(self, embedding_directory=None, tokenizer_data={}):
+ super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type="qwen3vl_4b")
+ self.llama_template_images = TEMPLATE # the images sit inside the JSON, not before it
+ self.font = ImageFont.load_default(20)
+
+ def tokenize_with_weights(self, text, return_word_ids=False, images=[], text_images=True, **kwargs):
+ # an empty prompt is the null caption, without the images: the negative of a generation or an edit
+ if text.strip() == "":
+ return super().tokenize_with_weights(NULL_CAPTION, return_word_ids, **kwargs)
+ # text_images=False gives the caption a generation starts from: each text entry's marker left empty, no images
+ caption, instruction = parse_prompt(text)
+ if len(images) > 0 and instruction is None:
+ caption, instruction = None, text
+ entries = [e for e in caption.get("text_render") or [] if isinstance(e, dict) and isinstance(e.get("text"), str) and e["text"]] if isinstance(caption, dict) else []
+ for entry in entries:
+ entry["image_tokens"] = VISION_MARKER if text_images else ""
+
+ if len(images) > 0:
+ edit = {"image": VISION_MARKER} if len(images) == 1 else {f"image_{k}": VISION_MARKER for k in range(1, len(images) + 1)}
+ if caption:
+ edit["structured_caption"] = caption
+ edit["edit_instruction"] = instruction
+ text = dumps(edit)
+ elif not isinstance(caption, str):
+ text = dumps(caption or {})
+
+ images = [thumbnail(image[..., :3]) for image in images]
+ if text_images:
+ images += [draw_text_image(entry["text"], self.font) for entry in entries]
+ return super().tokenize_with_weights(text, return_word_ids, images=[to_qwen_grid(image) for image in images], **kwargs)
+
+
+class Fibo2TEModel(sd1_clip.SD1ClipModel):
+ def __init__(self, device="cpu", dtype=None, model_options={}):
+ clip_model = lambda **kw: comfy.text_encoders.qwen3vl.Qwen3VLClipModel(**kw, layer=TAP_LAYERS, layer_idx=None, model_type="qwen3vl_4b") # noqa: E731
+ super().__init__(device=device, dtype=dtype, name="qwen3vl_4b", clip_model=clip_model, model_options=model_options)
+
+ def encode_token_weights(self, token_weight_pairs):
+ out, pooled, extra = super().encode_token_weights(token_weight_pairs) # (B, 15, seq, 2560) float32
+ # bf16 like the reference's text encoder output, which also halves every cond
+ out = comfy.rmsnorm.rms_norm(out).to(torch.bfloat16)
+ return out[:, BUNDLE_INDEX].permute(0, 2, 1, 3).flatten(2), pooled, extra
+
+
+def te(dtype_llama=None, llama_quantization_metadata=None):
+ class Fibo2TEModel_(Fibo2TEModel):
+ def __init__(self, device="cpu", dtype=None, model_options={}):
+ if llama_quantization_metadata is not None:
+ model_options = model_options.copy()
+ model_options["quantization_metadata"] = llama_quantization_metadata
+ if dtype_llama is not None:
+ dtype = dtype_llama
+ super().__init__(device=device, dtype=dtype, model_options=model_options)
+ return Fibo2TEModel_
diff --git a/comfy_extras/nodes_fibo2.py b/comfy_extras/nodes_fibo2.py
new file mode 100644
index 000000000..3e49a0f58
--- /dev/null
+++ b/comfy_extras/nodes_fibo2.py
@@ -0,0 +1,100 @@
+import math
+
+import torch
+from typing_extensions import override
+
+import comfy.model_management
+import comfy.utils
+import node_helpers
+import nodes
+from comfy_api.latest import ComfyExtension, io
+
+# sigma 0.9 at the default shift 1.15: a generation shows Qwen3-VL the images of its caption's text from there on
+TEXT_IMAGES_START = math.exp(1.15) / (9 + math.exp(1.15))
+
+
+def has_images(tokens):
+ return any(isinstance(t[0], dict) for batches in tokens.values() for batch in batches for t in batch)
+
+
+def trained_size(aspect_ratio):
+ # Bria's (width, height) for an aspect ratio: of the multiples of 16 within 10% of 1 MP, the least cropping, then the closest to 1 MP
+ area = 1024 * 1024
+ sizes = [(w, h) for w in range(64, 2 * area // 64, 16) for h in range(max(64, 16 * math.ceil(0.9 * area / w / 16)), 16 * math.floor(1.1 * area / w / 16) + 1, 16)]
+ return min(sizes, key=lambda s: (round(1 - min(s[0] / s[1] / aspect_ratio, aspect_ratio * s[1] / s[0]), 9), abs(s[0] * s[1] - area), -s[0]))
+
+
+class TextEncodeFibo2(io.ComfyNode):
+
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="TextEncodeFibo2",
+ category="model/conditioning/fibo",
+ description="Encode a Fibo 2.0 prompt, a structured JSON caption or plain text. With images, the prompt is the edit instruction, or a JSON caption of the result with an edit_instruction key, and the images become reference latents of both outputs when a vae is connected. Sample the latent output, the result's size the images are cropped for.",
+ inputs=[
+ io.Clip.Input("clip"),
+ io.String.Input("prompt", multiline=True),
+ io.String.Input("negative_prompt", multiline=True, advanced=True, tooltip="Empty is the null caption Fibo 2.0 was trained with for classifier-free guidance."),
+ io.Vae.Input("vae", optional=True),
+ io.Autogrow.Input(
+ "images",
+ template=io.Autogrow.TemplateNames(
+ io.Image.Input("image"),
+ names=[f"image_{i}" for i in range(1, 17)],
+ min=0,
+ ),
+ tooltip="Images to edit, referred to as <image_1>, <image_2> and so on in the instruction. A single image is scaled and center cropped to the result's size, since the edit lines up with it pixel for pixel; several images each to their own trained size.",
+ ),
+ io.Int.Input("width", default=0, min=0, max=nodes.MAX_RESOLUTION, step=16, tooltip="Width of the result. 0 = the size Fibo 2.0 was trained at for the first image's aspect ratio, about 1 megapixel, or 1024 without images."),
+ io.Int.Input("height", default=0, min=0, max=nodes.MAX_RESOLUTION, step=16, tooltip="Height of the result. 0 = the size Fibo 2.0 was trained at for the first image's aspect ratio, about 1 megapixel, or 1024 without images."),
+ io.Int.Input("batch_size", default=1, min=1, max=4096),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive"),
+ io.Conditioning.Output(display_name="negative"),
+ io.Latent.Output(display_name="latent"),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, clip, prompt, negative_prompt="", vae=None, images: io.Autogrow.Type = None, width=0, height=0, batch_size=1) -> io.NodeOutput:
+ images = images or {}
+ images = [images[name] for name in sorted(images, key=lambda n: int(n.rsplit("_", 1)[-1])) if images[name] is not None]
+ images = [image[i:i + 1] for image in images for i in range(image.shape[0])]
+
+ # as Bria's pipeline: a single image is cropped to the result's size, several images each to their own trained size
+ default = trained_size(images[0].shape[2] / images[0].shape[1]) if len(images) > 0 else (1024, 1024)
+ width, height = max(16, (width or default[0]) // 16 * 16), max(16, (height or default[1]) // 16 * 16)
+ sizes = [(width, height)] if len(images) == 1 else [trained_size(image.shape[2] / image.shape[1]) for image in images]
+ images = [image if (image.shape[2], image.shape[1]) == size else comfy.utils.common_upscale(image.movedim(-1, 1), *size, "lanczos", "center").movedim(1, -1)
+ for image, size in zip(images, sizes)]
+
+ tokens = clip.tokenize(prompt, images=images)
+ positive = clip.encode_from_tokens_scheduled(tokens)
+ if len(images) == 0 and has_images(tokens):
+ base = clip.encode_from_tokens_scheduled(clip.tokenize(prompt, text_images=False))
+ # ranges include both ends: the base stops just above sigma 0.9, which already has the text images
+ positive = (node_helpers.conditioning_set_values(base, {"end_percent": TEXT_IMAGES_START - 1e-6})
+ + node_helpers.conditioning_set_values(positive, {"start_percent": TEXT_IMAGES_START}))
+ # the negative is text only; an edit's reference latents condition both
+ negative = clip.encode_from_tokens_scheduled(clip.tokenize(negative_prompt))
+
+ if vae is not None and len(images) > 0:
+ ref_latents = [vae.encode(image) for image in images]
+ positive = node_helpers.conditioning_set_values(positive, {"reference_latents": ref_latents}, append=True)
+ negative = node_helpers.conditioning_set_values(negative, {"reference_latents": ref_latents}, append=True)
+ latent = torch.zeros([batch_size, 128, height // 16, width // 16], device=comfy.model_management.intermediate_device())
+ return io.NodeOutput(positive, negative, {"samples": latent})
+
+
+class Fibo2Extension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ TextEncodeFibo2,
+ ]
+
+
+async def comfy_entrypoint() -> Fibo2Extension:
+ return Fibo2Extension()
diff --git a/nodes.py b/nodes.py
index fc30bed26..e279ae2fe 100644
--- a/nodes.py
+++ b/nodes.py
@@ -1018,7 +1018,7 @@ class CLIPLoader:
@classmethod
def INPUT_TYPES(s):
return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ),
- "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2", "joyimage", "mage", "minimax", "yue2", "flux3", "moss_image"], ),
+ "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2", "joyimage", "mage", "minimax", "yue2", "flux3", "moss_image", "fibo2"], ),
},
"optional": {
"device": (["default", "cpu"], {"advanced": True}),
@@ -1028,7 +1028,7 @@ class CLIPLoader:
CATEGORY = "model/loaders"
- DESCRIPTION = "Recipes:\nsd: clip-l\nstable cascade: clip-g\nsd3: t5 xxl / clip-g / clip-l\nstable audio: t5 base\nmochi: t5 xxl\ncogvideox: t5 xxl (226-token padding)\ncosmos: old t5 xxl\nlumina2: gemma 2 2B\nwan: umt5 xxl\nhidream: llama-3.1 (Recommend) or t5\nomnigen2: qwen vl 2.5 3B\njoyimage: qwen3-vl 8B\nlens: gpt-oss-20b\npixeldit: gemma 2 2B elm\nminimax: MiniMax H3 Qwen3-VL or Music3 Qwen/RVQ\nflux3: qwen3-vl 4B"
+ DESCRIPTION = "Recipes:\nsd: clip-l\nstable cascade: clip-g\nsd3: t5 xxl / clip-g / clip-l\nstable audio: t5 base\nmochi: t5 xxl\ncogvideox: t5 xxl (226-token padding)\ncosmos: old t5 xxl\nlumina2: gemma 2 2B\nwan: umt5 xxl\nhidream: llama-3.1 (Recommend) or t5\nomnigen2: qwen vl 2.5 3B\njoyimage: qwen3-vl 8B\nlens: gpt-oss-20b\npixeldit: gemma 2 2B elm\nminimax: MiniMax H3 Qwen3-VL or Music3 Qwen/RVQ\nflux3: qwen3-vl 4B\nfibo2: qwen3-vl 4B"
def load_clip(self, clip_name, type="stable_diffusion", device="default"):
clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION)
@@ -2508,6 +2508,7 @@ async def init_builtin_extra_nodes():
"nodes_joyimage.py",
"nodes_boogu.py",
"nodes_moss_image.py",
+ "nodes_fibo2.py",
"nodes_chroma_radiance.py",
"nodes_pid.py",
"nodes_model_patch.py",