The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
feat: Add Fibo 2.0 support (#17240)
Jukka Seppänen87ed5a066d9c
diff --git a/comfy/ldm/fibo2/model.py b/comfy/ldm/fibo2/model.pynew file mode 100644index 000000000..1a318834b--- /dev/null+++ b/comfy/ldm/fibo2/model.py@@ -0,0 +1,219 @@+import math++import torch+import torch.nn.functional as F+from torch import nn++import comfy.model_management+import comfy.model_prefetch+import comfy.ops+import comfy.patcher_extension+import comfy.utils+from comfy.ldm.flux.layers import EmbedND+from comfy.ldm.lumina.model import FinalLayer, JointAttention, apply_gate, clamp_fp16, modulate, pos_ids_x+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention+from comfy.ldm.modules.diffusionmodules.mmdit import TimestepEmbedder+++def residual(x, y, gate=None):+ # in place inside the memory compiler region; out of place keeps x for backward when training+ if comfy.model_management.in_training:+ return x + y if gate is None else torch.addcmul(x, y, gate)+ return x.add_(y) if gate is None else x.addcmul_(y, gate)+++def gist_count(text_len, max_gist):+ # 256 gist tokens up to 256 text tokens, rising linearly to max_gist at 2048, in steps of 64+ ratio = min(1.0, max(0.0, (text_len - 256) / (2048 - 256)))+ return max(256, min(max_gist, math.ceil((256 + ratio * (max_gist - 256)) / 64) * 64))+++class Attention(nn.Module):+ # no RoPE: the cross-attention to the text, and the Perceiver's self-attention+ def __init__(self, dim, n_heads, kv_dim, norm_eps, dtype=None, device=None, operations=None):+ super().__init__()+ self.comfy_attention = ComfyAttention()+ self.n_heads = n_heads+ self.to_q = operations.Linear(dim, dim, bias=False, dtype=dtype, device=device)+ self.to_kv = operations.Linear(kv_dim, 2 * dim, bias=False, dtype=dtype, device=device)+ self.to_out = operations.Linear(dim, dim, bias=False, dtype=dtype, device=device)+ self.norm_q = operations.RMSNorm(dim // n_heads, eps=norm_eps, dtype=dtype, device=device)+ self.norm_k = operations.RMSNorm(dim // n_heads, eps=norm_eps, dtype=dtype, device=device)++ def forward(self, x, context, transformer_options={}):+ q = self.norm_q(self.to_q(x).unflatten(-1, (self.n_heads, -1)))+ k, v = self.to_kv(context).unflatten(-1, (2, self.n_heads, -1)).unbind(2)+ k = self.norm_k(k)+ q, k, v = (AttentionTensorContainer(t.movedim(1, 2)) for t in (q, k, v))+ return self.to_out(optimized_attention(q, k, v, self.n_heads, skip_reshape=True, transformer_options=transformer_options, preferred_attention=self.comfy_attention))+++class FeedForward(nn.Module):+ # SwiGLU; fused files carry w1 and w3 as one w13, the hidden width zero padded to a multiple of 256 so int8 weights quantize swiglu(x) inside w2's GEMM+ def __init__(self, dim, hidden_dim, fused, dtype=None, device=None, operations=None):+ super().__init__()+ self.fused = fused+ if fused:+ self.w13 = operations.Linear(dim, 2 * hidden_dim, bias=False, dtype=dtype, device=device)+ else:+ self.w1 = operations.Linear(dim, hidden_dim, bias=False, dtype=dtype, device=device)+ self.w3 = operations.Linear(dim, hidden_dim, bias=False, dtype=dtype, device=device)+ self.w2 = operations.Linear(hidden_dim, dim, bias=False, dtype=dtype, device=device)++ def forward(self, x):+ if self.fused:+ return comfy.ops.linear_input_act(self.w2, self.w13(x), "swiglu")+ return self.w2(F.silu(self.w1(x)) * self.w3(x))+++class Block(nn.Module):+ # Z-Image's block with in place residuals; an injection block also cross-attends to its own text bundle+ def __init__(self, dim, n_heads, ffn_dim, fused_ffn, norm_eps, cap_feat_dim=None, operation_settings={}):+ super().__init__()+ operations, device, dtype = operation_settings["operations"], operation_settings["device"], operation_settings["dtype"]+ self.attention = JointAttention(dim, n_heads, None, True, operation_settings=operation_settings)+ self.attention.q_norm.eps = self.attention.k_norm.eps = norm_eps+ self.feed_forward = FeedForward(dim, ffn_dim, fused_ffn, dtype=dtype, device=device, operations=operations)+ self.attention_norm1 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)+ self.ffn_norm1 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)+ self.attention_norm2 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)+ self.ffn_norm2 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)+ self.adaLN_modulation = nn.Sequential(operations.Linear(256, 4 * dim, bias=True, dtype=dtype, device=device))+ self.cross_attention = None+ if cap_feat_dim is not None:+ self.cross_attn_norm1 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)+ self.cross_attention = Attention(dim, n_heads, cap_feat_dim, norm_eps, dtype=dtype, device=device, operations=operations)+ self.cross_attn_norm2 = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)+ self.cross_attn_gate = nn.Parameter(torch.empty(1, dtype=dtype, device=device))++ def forward(self, x, freqs_cis, adaln_input, timestep_zero_index=None, context=None, transformer_options={}):+ scale_msa, gate_msa, scale_mlp, gate_mlp = self.adaLN_modulation(adaln_input).chunk(4, dim=1)+ y = self.attention(modulate(self.attention_norm1(x), scale_msa, timestep_zero_index=timestep_zero_index), None, freqs_cis, transformer_options=transformer_options)+ x = residual(x, apply_gate(gate_msa.unsqueeze(1).tanh(), self.attention_norm2(clamp_fp16(y)), timestep_zero_index=timestep_zero_index))+ if self.cross_attention is not None:+ y = self.cross_attention(self.cross_attn_norm1(x), context, transformer_options)+ x = residual(x, self.cross_attn_norm2(y), comfy.ops.cast_to_input(self.cross_attn_gate, y).tanh())+ y = self.feed_forward(modulate(self.ffn_norm1(x), scale_mlp, timestep_zero_index=timestep_zero_index))+ return residual(x, apply_gate(gate_mlp.unsqueeze(1).tanh(), self.ffn_norm2(clamp_fp16(y)), timestep_zero_index=timestep_zero_index))+++class PerceiverLayer(nn.Module):+ def __init__(self, dim, n_heads, ffn_dim, fused_ffn, cap_feat_dim, norm_eps, operation_settings={}):+ super().__init__()+ operations, device, dtype = operation_settings["operations"], operation_settings["device"], operation_settings["dtype"]+ self.cross_attn_norm = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)+ self.cross_attention = Attention(dim, n_heads, cap_feat_dim, norm_eps, dtype=dtype, device=device, operations=operations)+ self.self_attn_norm = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)+ self.self_attention = Attention(dim, n_heads, dim, norm_eps, dtype=dtype, device=device, operations=operations)+ self.ffn_norm = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)+ self.ffn = FeedForward(dim, ffn_dim, fused_ffn, dtype=dtype, device=device, operations=operations)++ def forward(self, x, context, transformer_options={}):+ x = x + self.cross_attention(self.cross_attn_norm(x), context, transformer_options)+ y = self.self_attn_norm(x)+ x = x + self.self_attention(y, y, transformer_options)+ return x + self.ffn(self.ffn_norm(x))+++class Perceiver(nn.Module):+ # squeezes the text encoder's hidden states into gist tokens+ def __init__(self, dim, n_heads, ffn_dim, fused_ffn, cap_feat_dim, num_queries, num_layers, norm_eps, operation_settings={}):+ super().__init__()+ operations, device, dtype = operation_settings["operations"], operation_settings["device"], operation_settings["dtype"]+ self.latent_queries = nn.Parameter(torch.empty(num_queries, dim, dtype=dtype, device=device))+ self.layers = nn.ModuleList([PerceiverLayer(dim, n_heads, ffn_dim, fused_ffn, cap_feat_dim, norm_eps, operation_settings=operation_settings) for _ in range(num_layers)])+ self.norm_out = operations.RMSNorm(dim, eps=norm_eps, dtype=dtype, device=device)++ def forward(self, context, num_gist, transformer_options={}):+ x = comfy.ops.cast_to_input(self.latent_queries[:num_gist], context).unsqueeze(0).expand(context.shape[0], -1, -1)+ prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.layers), context.device, transformer_options)+ for layer in self.layers:+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, context.device, layer, context.dtype)+ x = layer(x, context, transformer_options)+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, context.device, None)+ return self.norm_out(x)+++class Fibo2(nn.Module):+ def __init__(self, in_channels=128, dim=3584, n_heads=28, n_layers=44, n_refiner_layers=2, injection_layers=(6, 14, 22, 30, 38), cap_feat_dim=7680,+ perceiver_layers=2, max_gist_tokens=512, ffn_dim=9728, fused_ffn=True, norm_eps=1e-5, image_model=None, dtype=None, device=None, operations=None):+ super().__init__()+ self.dtype = dtype+ operation_settings = {"operations": operations, "device": device, "dtype": dtype}+ self.max_gist_tokens = max_gist_tokens+ self.injection_layers = injection_layers++ self.x_embedder = operations.Linear(in_channels, dim, bias=True, dtype=dtype, device=device)+ self.t_embedder = TimestepEmbedder(1024, output_size=256, **operation_settings)+ self.noise_refiner = nn.ModuleList([Block(dim, n_heads, ffn_dim, fused_ffn, norm_eps, operation_settings=operation_settings) for _ in range(n_refiner_layers)])+ self.perceiver = Perceiver(dim, n_heads, ffn_dim, fused_ffn, cap_feat_dim, max_gist_tokens, perceiver_layers, norm_eps, operation_settings=operation_settings)+ self.layers = nn.ModuleList([Block(dim, n_heads, ffn_dim, fused_ffn, norm_eps, cap_feat_dim if i in injection_layers else None, operation_settings=operation_settings) for i in range(n_layers)])+ self.final_layer = FinalLayer(dim, 1, in_channels, z_image_modulation=True, operation_settings=operation_settings)+ self.rope_embedder = EmbedND(dim=dim // n_heads, theta=256.0, axes_dim=[32, 48, 48])++ def forward(self, x, timestep, context, transformer_options={}, **kwargs):+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(+ self._forward,+ self,+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)+ ).execute(x, timestep, context, transformer_options, **kwargs)++ def _forward(self, x, timestep, context, transformer_options={}, ref_latents=[], **kwargs):+ B, C, H, W = x.shape+ dtype, device = x.dtype, x.device+ transformer_options = transformer_options.copy()++ # the image and the reference tokens are noisy and clean: the second half of the adaln batch modulates the image+ adaln_mixed = self.t_embedder(torch.cat((torch.zeros_like(timestep), timestep * 1000)), dtype)+ adaln_noisy = adaln_mixed[B:]++ # gist tokens on rope planes 1..num_gist, the image on plane max_gist_tokens + 1 whatever the gist count, reference k on the k-th plane after it+ num_gist = gist_count(context.shape[1], self.max_gist_tokens)+ plane = self.max_gist_tokens + 1+ tokens = [x.flatten(2).transpose(1, 2)]+ ids = [torch.zeros(1, num_gist, 3, dtype=torch.float32, device=device), pos_ids_x(plane, H, W, 1, device, transformer_options=transformer_options)]+ ids[0][0, :, 0] = torch.arange(1, num_gist + 1, dtype=torch.float32, device=device)+ for k, ref in enumerate(ref_latents):+ tokens.append(comfy.utils.repeat_to_batch_size(ref, B).flatten(2).transpose(1, 2))+ ids.append(pos_ids_x(plane + 1 + k, ref.shape[-2], ref.shape[-1], 1, device, transformer_options=transformer_options))+ img = self.x_embedder(torch.cat(tokens, dim=1))+ freqs_cis = self.rope_embedder(torch.cat(ids, dim=1)).movedim(1, 2)+ del tokens++ n_img = H * W+ refiner_adaln, refiner_index = (adaln_mixed, [(0, n_img)]) if len(ref_latents) > 0 else (adaln_noisy, None)+ prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.noise_refiner), device, transformer_options)+ for block in self.noise_refiner:+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, block, dtype)+ img = block(img, freqs_cis[:, num_gist:], refiner_adaln, refiner_index, transformer_options=transformer_options)+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, None)++ # the Perceiver reads the first text bundle, each injection block its own+ bundles = context.chunk(1 + len(self.injection_layers), dim=-1)+ gist = self.perceiver(bundles[0], num_gist, transformer_options)+ text = dict(zip(self.injection_layers, bundles[1:]))++ # allocated outside the memory compiler region, the blocks update it in place+ x = torch.cat((gist, img), dim=1)+ del gist, img+ image_index = [(num_gist, num_gist + n_img)]++ blocks_replace = transformer_options.get("patches_replace", {}).get("dit", {})+ transformer_options["total_blocks"] = len(self.layers)+ transformer_options["block_type"] = "single"+ prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.layers), device, transformer_options)+ comfy.model_prefetch.malloc_graph_begin(device)+ for i, block in enumerate(self.layers):+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, block, dtype, malloc_scope="block")+ transformer_options["block_index"] = i+ if ("single_block", i) in blocks_replace:+ def block_wrap(args):+ return {"img": block(args["img"], args["pe"], args["vec"], image_index, text.get(i), args["transformer_options"])}+ x = blocks_replace[("single_block", i)]({"img": x, "vec": adaln_mixed, "pe": freqs_cis, "transformer_options": transformer_options}, {"original_block": block_wrap})["img"]+ else:+ x = block(x, freqs_cis, adaln_mixed, image_index, text.get(i), transformer_options)+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, None, malloc_scope="block")+ comfy.model_prefetch.malloc_graph_end()++ out = self.final_layer(x[:, num_gist:num_gist + n_img], adaln_noisy)+ return out.transpose(1, 2).reshape(B, C, H, W)diff --git a/comfy/model_base.py b/comfy/model_base.pyindex 8e42a86b4..8ee80152e 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -84,6 +84,7 @@ import comfy.ldm.sensenova.modelfrom comfy.ldm.sensenova.sampling import SenseNovaModelSampling, time_snr_shiftimport comfy.ldm.depth_anything_3.modelimport comfy.ldm.moss_image.model+import comfy.ldm.fibo2.modelimport comfy.model_managementimport comfy.patcher_extension@@ -2820,6 +2821,24 @@ class MOSSImage(BaseModel):image = utils.common_upscale(image.to(kwargs["device"]), noise.shape[-1], noise.shape[-2], "bilinear", "center")return utils.resize_to_batch_size(self.process_latent_in(image), noise.shape[0])+class Fibo2(BaseModel):+ def __init__(self, model_config, model_type=ModelType.FLUX, device=None):+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.fibo2.model.Fibo2)+ self.memory_usage_factor_conds = ("ref_latents",)++ def extra_conds(self, **kwargs):+ out = super().extra_conds(**kwargs)+ cross_attn = kwargs.get("cross_attn", None)+ # the gist token count follows the true text length, so no repeat padding: cond and uncond of different lengths run separately+ if cross_attn is not None:+ out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)+ ref_latents = kwargs.get("reference_latents", None)+ if ref_latents is not None:+ out['ref_latents'] = comfy.conds.CONDList([self.process_latent_in(lat) for lat in ref_latents])+ return out++ extra_conds_shapes = Flux.extra_conds_shapes+class HunyuanImage21(BaseModel):def __init__(self, model_config, model_type=ModelType.FLOW, device=None):super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.hunyuan_video.model.HunyuanVideo)diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex 79c0e143e..04d76db2a 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -1022,6 +1022,25 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):dit_config["num_layers"] = count_blocks(state_dict_keys, '{}layers.'.format(key_prefix) + '{}.')return dit_config+ if ('{}perceiver.latent_queries'.format(key_prefix) in state_dict_keys and '{}layers.0.attention.qkv.weight'.format(key_prefix) in state_dict_keys and '{}final_layer.linear.weight'.format(key_prefix) in state_dict_keys+ and ('{}layers.0.feed_forward.w13.weight'.format(key_prefix) in state_dict_keys or '{}layers.0.feed_forward.w1.weight'.format(key_prefix) in state_dict_keys)): # Fibo 2.0 (repacked)+ dit_config = {}+ dit_config["image_model"] = "fibo2"+ dit_config["max_gist_tokens"], dit_config["dim"] = state_dict['{}perceiver.latent_queries'.format(key_prefix)].shape+ dit_config["n_heads"] = dit_config["dim"] // 128+ dit_config["in_channels"] = state_dict['{}final_layer.linear.weight'.format(key_prefix)].shape[0]+ dit_config["n_layers"] = count_blocks(state_dict_keys, '{}layers.'.format(key_prefix) + '{}.')+ dit_config["n_refiner_layers"] = count_blocks(state_dict_keys, '{}noise_refiner.'.format(key_prefix) + '{}.')+ dit_config["perceiver_layers"] = count_blocks(state_dict_keys, '{}perceiver.layers.'.format(key_prefix) + '{}.')+ dit_config["injection_layers"] = [i for i in range(dit_config["n_layers"]) if '{}layers.{}.cross_attn_gate'.format(key_prefix, i) in state_dict_keys]+ w13 = '{}layers.0.feed_forward.w13.weight'.format(key_prefix)+ if w13 in state_dict_keys: # w1 and w3 fused, hidden width padded for the fused int8 swiglu+ dit_config["ffn_dim"] = state_dict[w13].shape[0] // 2+ else:+ dit_config["ffn_dim"] = state_dict['{}layers.0.feed_forward.w1.weight'.format(key_prefix)].shape[0]+ dit_config["fused_ffn"] = False+ return dit_config+if '{}shared_block_modulation.weight'.format(key_prefix) in state_dict_keys and '{}layers.0.proj.ffn_up_gate.weight'.format(key_prefix) in state_dict_keys: # MOSS-Imagedit_config = {}dit_config["image_model"] = "moss_image"diff --git a/comfy/sd.py b/comfy/sd.pyindex e0b9d276b..11b4f6b73 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -93,6 +93,7 @@ import comfy.text_encoders.sa3import comfy.text_encoders.gpt_ossimport comfy.text_encoders.joyimageimport comfy.text_encoders.moss_image+import comfy.text_encoders.fibo2import comfy.model_patcherimport comfy.lora@@ -1611,6 +1612,7 @@ class CLIPType(Enum):YUE2 = 36FLUX3 = 37MOSS_IMAGE = 38+ FIBO2 = 39@@ -1988,6 +1990,10 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clipclip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})clip_target.clip = comfy.text_encoders.flux.flux3_te(**llama_detect(clip_data))clip_target.tokenizer = comfy.text_encoders.flux.Flux3Tokenizer+ elif clip_type == CLIPType.FIBO2 and te_model == TEModel.QWEN3VL_4B: # Fibo 2.0: full Qwen3-VL-4B, 15-layer tap -> six 7680 bundles.+ clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})+ clip_target.clip = comfy.text_encoders.fibo2.te(**llama_detect(clip_data))+ clip_target.tokenizer = comfy.text_encoders.fibo2.Fibo2Tokenizerelif clip_type == CLIPType.MAGE and te_model == TEModel.QWEN3VL_4B: # Mage-Flow: full Qwen3-VL-4B, last hidden state, Qwen-Image-style templates.clip_data[0] = comfy.utils.state_dict_prefix_replace(clip_data[0], {"model.language_model.": "model.", "model.visual.": "visual.", "lm_head.": "model.lm_head."})clip_target.clip = comfy.text_encoders.mage_flow.te(**llama_detect(clip_data))diff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex c136c95f1..3d2015bbd 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -43,6 +43,7 @@ import comfy.text_encoders.hidream_o1import comfy.text_encoders.sensenovaimport comfy.text_encoders.pixelditimport comfy.text_encoders.moss_image+import comfy.text_encoders.fibo2from . import supported_models_basefrom . import latent_formats@@ -2067,6 +2068,34 @@ class MOSSImage(supported_models_base.BASE):hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen35_9b.transformer.".format(pref))return supported_models_base.ClipTarget(comfy.text_encoders.moss_image.MOSSImageTokenizer, comfy.text_encoders.moss_image.te(**hunyuan_detect))+class Fibo2(supported_models_base.BASE):+ unet_config = {+ "image_model": "fibo2",+ }++ sampling_settings = {+ "shift": 1.15,+ }++ memory_usage_factor = 17.0++ unet_extra_config = {}+ latent_format = latent_formats.Flux2++ supported_inference_dtypes = [torch.bfloat16, torch.float32]++ vae_key_prefix = ["vae."]+ text_encoder_key_prefix = ["text_encoders."]++ def get_model(self, state_dict, prefix="", device=None):+ out = model_base.Fibo2(self, device=device)+ return out++ def clip_target(self, state_dict={}):+ pref = self.text_encoder_key_prefix[0]+ hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_4b.transformer.".format(pref))+ return supported_models_base.ClipTarget(comfy.text_encoders.fibo2.Fibo2Tokenizer, comfy.text_encoders.fibo2.te(**hunyuan_detect))+class MageFlow(supported_models_base.BASE):unet_config = {"image_model": "mage_flow",@@ -2719,6 +2748,7 @@ models = [Ideogram4,Krea2,MOSSImage,+ Fibo2,Flux2,Flux3,Lens,diff --git a/comfy/text_encoders/fibo2.py b/comfy/text_encoders/fibo2.pynew file mode 100644index 000000000..d16df37c1--- /dev/null+++ b/comfy/text_encoders/fibo2.py@@ -0,0 +1,188 @@+"""Fibo 2.0 text encoder: Qwen3-VL-4B, 15 hidden layers at unit RMS in six 7680-dim bundles (Perceiver, then each injection block)."""++import json+import math++import numpy as np+import torch+from PIL import Image, ImageDraw, ImageFont++import comfy.rmsnorm+import comfy.text_encoders.qwen3vl+from comfy import sd1_clip++VISION_MARKER = "<|vision_start|><|image_pad|><|vision_end|>"+TEMPLATE = "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"++# the null caption training used for classifier-free guidance, a whole chat turn so the tokenizer takes it as it is, uncleaned+NULL_CAPTION = TEMPLATE.format('{"short_description":"","objects":[],"background_setting":"","lighting":{},"aesthetics":{},"photographic_characteristics":{},"style_medium":"","context":"","artistic_style":""}')++BUNDLE_LAYERS = ((9, 20, 31), (5, 16, 27), (7, 18, 29), (9, 20, 31), (11, 22, 33), (13, 24, 35))+TAP_LAYERS = sorted({layer for bundle in BUNDLE_LAYERS for layer in bundle})+BUNDLE_INDEX = [TAP_LAYERS.index(layer) for bundle in BUNDLE_LAYERS for layer in bundle]++SCORES = ("aesthetic_score", "preference_score")+++def dumps(value):+ return json.dumps(value, separators=(",", ":"), ensure_ascii=False)+++def empty(value):+ if isinstance(value, float):+ return math.isnan(value)+ return value is None or (isinstance(value, (str, dict, list)) and not value)+++def clean(value):+ if isinstance(value, dict):+ value = {k: clean(v) for k, v in value.items()}+ return {k: v for k, v in value.items() if not empty(v)}+ if isinstance(value, list):+ return [v for v in map(clean, value) if not empty(v)]+ return value+++def parse_prompt(prompt):+ # the caption as training saw it, and the edit instruction; a prompt that isn't a JSON object comes back as it is+ try:+ caption = json.loads(prompt)+ except json.JSONDecodeError:+ return prompt, None+ if not isinstance(caption, dict):+ return prompt, None+ instruction = caption.pop("edit_instruction", None)+ if "structured_caption" in caption:+ caption = caption.pop("structured_caption")+ if not isinstance(caption, dict):+ return caption or None, instruction++ keys = set(caption)+ caption = clean({key: {k: v for k, v in value.items() if k not in SCORES} if isinstance(value, dict) else value+ for key, value in caption.items() if key not in SCORES})+ # exactly a short_description and a text_render list keeps both, the list even when empty: the image has no text+ if keys == {"short_description", "text_render"}:+ caption = {"short_description": caption.get("short_description") or "", "text_render": caption.get("text_render") or []}+ return caption or None, instruction+++def has_glyph(font, notdef, character):+ # a font without the glyph draws its notdef box, the one it draws for U+FFFE, which no font has+ try:+ mask = font.getmask(character)+ except (OSError, ValueError): # as in training, a font that can't say is taken to have the glyph+ return True+ return not (mask.size == notdef.size and bytes(mask) == bytes(notdef))+++def draw_text_image(text, font):+ # Pillow's built-in font, black on white, whitespace collapsed, split every 256 characters (training's non-ASCII fallback fonts aren't shipped)+ line = " ".join(text.split())+ lines = [line[i:i + 256] for i in range(0, len(line), 256)] or [""]+ notdef = font.getmask(chr(0xFFFE))+ if all(ord(c) < 128 or has_glyph(font, notdef, c) for c in set(line)):+ # each line placed by its bounding box, at least as tall as "Ag"+ min_box = font.getbbox("Ag")+ boxes = [font.getbbox(line or " ") for line in lines]+ heights = [max(1, min_box[3] - min_box[1], box[3] - box[1]) for box in boxes]+ width = max([1] + [box[2] - box[0] for box in boxes])+ origins, anchor = [(2 - box[0], -box[1]) for box in boxes], None+ else:+ # a character the font lacks (drawn as its notdef box) puts every line on the baseline, as tall as the font+ ascent, descent = font.getmetrics()+ lines = [line or " " for line in lines]+ heights = [max(1, ascent + descent)] * len(lines)+ width = max(max(1, round(font.getlength(line))) for line in lines)+ origins, anchor = [(2.0, ascent)] * len(lines), "ls"++ image = Image.new("RGB", (width + 4, sum(heights) + 4 * len(lines)), "white")+ draw = ImageDraw.Draw(image)+ y = 2+ for line, (x, dy), height in zip(lines, origins, heights):+ draw.text((x, y + dy), line, font=font, fill=(0, 0, 0), anchor=anchor)+ y += height + 4+ return torch.from_numpy(np.array(image)).float().div(255.0).unsqueeze(0)+++def resize_8bit(image, height, width):+ # antialiased bicubic rounded to 8 bit, as PIL and Qwen3-VL's processor resize+ image = torch.nn.functional.interpolate(image.movedim(-1, 1), size=(height, width), mode="bicubic", antialias=True)+ return image.mul(255.0).round().clamp(0, 255).div(255.0).movedim(1, -1)+++def thumbnail(image, size=768):+ # Qwen3-VL sees each image to edit at most 768 pixels a side, as in training+ scale = size / max(image.shape[1], image.shape[2])+ if scale >= 1:+ return image+ return resize_8bit(image, max(1, round(image.shape[1] * scale)), max(1, round(image.shape[2] * scale)))+++def to_qwen_grid(image):+ # the processor's 32 pixel grid, so ComfyUI's own bilinear resize is a no-op+ h, w = image.shape[1], image.shape[2]+ height, width = round(h / 32) * 32, round(w / 32) * 32+ if height * width > 1605632:+ beta = math.sqrt(h * w / 1605632)+ height, width = max(32, math.floor(h / beta / 32) * 32), max(32, math.floor(w / beta / 32) * 32)+ elif height * width < 3136:+ beta = math.sqrt(3136 / (h * w))+ height, width = math.ceil(h * beta / 32) * 32, math.ceil(w * beta / 32) * 32+ return resize_8bit(image, height, width)+++class Fibo2Tokenizer(comfy.text_encoders.qwen3vl.Qwen3VLTokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, model_type="qwen3vl_4b")+ self.llama_template_images = TEMPLATE # the images sit inside the JSON, not before it+ self.font = ImageFont.load_default(20)++ def tokenize_with_weights(self, text, return_word_ids=False, images=[], text_images=True, **kwargs):+ # an empty prompt is the null caption, without the images: the negative of a generation or an edit+ if text.strip() == "":+ return super().tokenize_with_weights(NULL_CAPTION, return_word_ids, **kwargs)+ # text_images=False gives the caption a generation starts from: each text entry's marker left empty, no images+ caption, instruction = parse_prompt(text)+ if len(images) > 0 and instruction is None:+ caption, instruction = None, text+ entries = [e for e in caption.get("text_render") or [] if isinstance(e, dict) and isinstance(e.get("text"), str) and e["text"]] if isinstance(caption, dict) else []+ for entry in entries:+ entry["image_tokens"] = VISION_MARKER if text_images else ""++ if len(images) > 0:+ edit = {"image": VISION_MARKER} if len(images) == 1 else {f"image_{k}": VISION_MARKER for k in range(1, len(images) + 1)}+ if caption:+ edit["structured_caption"] = caption+ edit["edit_instruction"] = instruction+ text = dumps(edit)+ elif not isinstance(caption, str):+ text = dumps(caption or {})++ images = [thumbnail(image[..., :3]) for image in images]+ if text_images:+ images += [draw_text_image(entry["text"], self.font) for entry in entries]+ return super().tokenize_with_weights(text, return_word_ids, images=[to_qwen_grid(image) for image in images], **kwargs)+++class Fibo2TEModel(sd1_clip.SD1ClipModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ clip_model = lambda **kw: comfy.text_encoders.qwen3vl.Qwen3VLClipModel(**kw, layer=TAP_LAYERS, layer_idx=None, model_type="qwen3vl_4b") # noqa: E731+ super().__init__(device=device, dtype=dtype, name="qwen3vl_4b", clip_model=clip_model, model_options=model_options)++ def encode_token_weights(self, token_weight_pairs):+ out, pooled, extra = super().encode_token_weights(token_weight_pairs) # (B, 15, seq, 2560) float32+ # bf16 like the reference's text encoder output, which also halves every cond+ out = comfy.rmsnorm.rms_norm(out).to(torch.bfloat16)+ return out[:, BUNDLE_INDEX].permute(0, 2, 1, 3).flatten(2), pooled, extra+++def te(dtype_llama=None, llama_quantization_metadata=None):+ class Fibo2TEModel_(Fibo2TEModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ if llama_quantization_metadata is not None:+ model_options = model_options.copy()+ model_options["quantization_metadata"] = llama_quantization_metadata+ if dtype_llama is not None:+ dtype = dtype_llama+ super().__init__(device=device, dtype=dtype, model_options=model_options)+ return Fibo2TEModel_diff --git a/comfy_extras/nodes_fibo2.py b/comfy_extras/nodes_fibo2.pynew file mode 100644index 000000000..3e49a0f58--- /dev/null+++ b/comfy_extras/nodes_fibo2.py@@ -0,0 +1,100 @@+import math++import torch+from typing_extensions import override++import comfy.model_management+import comfy.utils+import node_helpers+import nodes+from comfy_api.latest import ComfyExtension, io++# sigma 0.9 at the default shift 1.15: a generation shows Qwen3-VL the images of its caption's text from there on+TEXT_IMAGES_START = math.exp(1.15) / (9 + math.exp(1.15))+++def has_images(tokens):+ return any(isinstance(t[0], dict) for batches in tokens.values() for batch in batches for t in batch)+++def trained_size(aspect_ratio):+ # Bria's (width, height) for an aspect ratio: of the multiples of 16 within 10% of 1 MP, the least cropping, then the closest to 1 MP+ area = 1024 * 1024+ sizes = [(w, h) for w in range(64, 2 * area // 64, 16) for h in range(max(64, 16 * math.ceil(0.9 * area / w / 16)), 16 * math.floor(1.1 * area / w / 16) + 1, 16)]+ return min(sizes, key=lambda s: (round(1 - min(s[0] / s[1] / aspect_ratio, aspect_ratio * s[1] / s[0]), 9), abs(s[0] * s[1] - area), -s[0]))+++class TextEncodeFibo2(io.ComfyNode):++ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="TextEncodeFibo2",+ category="model/conditioning/fibo",+ description="Encode a Fibo 2.0 prompt, a structured JSON caption or plain text. With images, the prompt is the edit instruction, or a JSON caption of the result with an edit_instruction key, and the images become reference latents of both outputs when a vae is connected. Sample the latent output, the result's size the images are cropped for.",+ inputs=[+ io.Clip.Input("clip"),+ io.String.Input("prompt", multiline=True),+ io.String.Input("negative_prompt", multiline=True, advanced=True, tooltip="Empty is the null caption Fibo 2.0 was trained with for classifier-free guidance."),+ io.Vae.Input("vae", optional=True),+ io.Autogrow.Input(+ "images",+ template=io.Autogrow.TemplateNames(+ io.Image.Input("image"),+ names=[f"image_{i}" for i in range(1, 17)],+ min=0,+ ),+ tooltip="Images to edit, referred to as <image_1>, <image_2> and so on in the instruction. A single image is scaled and center cropped to the result's size, since the edit lines up with it pixel for pixel; several images each to their own trained size.",+ ),+ io.Int.Input("width", default=0, min=0, max=nodes.MAX_RESOLUTION, step=16, tooltip="Width of the result. 0 = the size Fibo 2.0 was trained at for the first image's aspect ratio, about 1 megapixel, or 1024 without images."),+ io.Int.Input("height", default=0, min=0, max=nodes.MAX_RESOLUTION, step=16, tooltip="Height of the result. 0 = the size Fibo 2.0 was trained at for the first image's aspect ratio, about 1 megapixel, or 1024 without images."),+ io.Int.Input("batch_size", default=1, min=1, max=4096),+ ],+ outputs=[+ io.Conditioning.Output(display_name="positive"),+ io.Conditioning.Output(display_name="negative"),+ io.Latent.Output(display_name="latent"),+ ],+ )++ @classmethod+ def execute(cls, clip, prompt, negative_prompt="", vae=None, images: io.Autogrow.Type = None, width=0, height=0, batch_size=1) -> io.NodeOutput:+ images = images or {}+ images = [images[name] for name in sorted(images, key=lambda n: int(n.rsplit("_", 1)[-1])) if images[name] is not None]+ images = [image[i:i + 1] for image in images for i in range(image.shape[0])]++ # as Bria's pipeline: a single image is cropped to the result's size, several images each to their own trained size+ default = trained_size(images[0].shape[2] / images[0].shape[1]) if len(images) > 0 else (1024, 1024)+ width, height = max(16, (width or default[0]) // 16 * 16), max(16, (height or default[1]) // 16 * 16)+ sizes = [(width, height)] if len(images) == 1 else [trained_size(image.shape[2] / image.shape[1]) for image in images]+ images = [image if (image.shape[2], image.shape[1]) == size else comfy.utils.common_upscale(image.movedim(-1, 1), *size, "lanczos", "center").movedim(1, -1)+ for image, size in zip(images, sizes)]++ tokens = clip.tokenize(prompt, images=images)+ positive = clip.encode_from_tokens_scheduled(tokens)+ if len(images) == 0 and has_images(tokens):+ base = clip.encode_from_tokens_scheduled(clip.tokenize(prompt, text_images=False))+ # ranges include both ends: the base stops just above sigma 0.9, which already has the text images+ positive = (node_helpers.conditioning_set_values(base, {"end_percent": TEXT_IMAGES_START - 1e-6})+ + node_helpers.conditioning_set_values(positive, {"start_percent": TEXT_IMAGES_START}))+ # the negative is text only; an edit's reference latents condition both+ negative = clip.encode_from_tokens_scheduled(clip.tokenize(negative_prompt))++ if vae is not None and len(images) > 0:+ ref_latents = [vae.encode(image) for image in images]+ positive = node_helpers.conditioning_set_values(positive, {"reference_latents": ref_latents}, append=True)+ negative = node_helpers.conditioning_set_values(negative, {"reference_latents": ref_latents}, append=True)+ latent = torch.zeros([batch_size, 128, height // 16, width // 16], device=comfy.model_management.intermediate_device())+ return io.NodeOutput(positive, negative, {"samples": latent})+++class Fibo2Extension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[io.ComfyNode]]:+ return [+ TextEncodeFibo2,+ ]+++async def comfy_entrypoint() -> Fibo2Extension:+ return Fibo2Extension()diff --git a/nodes.py b/nodes.pyindex fc30bed26..e279ae2fe 100644--- a/nodes.py+++ b/nodes.py@@ -1018,7 +1018,7 @@ class CLIPLoader:@classmethoddef INPUT_TYPES(s):return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ),- "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2", "joyimage", "mage", "minimax", "yue2", "flux3", "moss_image"], ),+ "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2", "joyimage", "mage", "minimax", "yue2", "flux3", "moss_image", "fibo2"], ),},"optional": {"device": (["default", "cpu"], {"advanced": True}),@@ -1028,7 +1028,7 @@ class CLIPLoader:CATEGORY = "model/loaders"- DESCRIPTION = "Recipes:\nsd: clip-l\nstable cascade: clip-g\nsd3: t5 xxl / clip-g / clip-l\nstable audio: t5 base\nmochi: t5 xxl\ncogvideox: t5 xxl (226-token padding)\ncosmos: old t5 xxl\nlumina2: gemma 2 2B\nwan: umt5 xxl\nhidream: llama-3.1 (Recommend) or t5\nomnigen2: qwen vl 2.5 3B\njoyimage: qwen3-vl 8B\nlens: gpt-oss-20b\npixeldit: gemma 2 2B elm\nminimax: MiniMax H3 Qwen3-VL or Music3 Qwen/RVQ\nflux3: qwen3-vl 4B"+ DESCRIPTION = "Recipes:\nsd: clip-l\nstable cascade: clip-g\nsd3: t5 xxl / clip-g / clip-l\nstable audio: t5 base\nmochi: t5 xxl\ncogvideox: t5 xxl (226-token padding)\ncosmos: old t5 xxl\nlumina2: gemma 2 2B\nwan: umt5 xxl\nhidream: llama-3.1 (Recommend) or t5\nomnigen2: qwen vl 2.5 3B\njoyimage: qwen3-vl 8B\nlens: gpt-oss-20b\npixeldit: gemma 2 2B elm\nminimax: MiniMax H3 Qwen3-VL or Music3 Qwen/RVQ\nflux3: qwen3-vl 4B\nfibo2: qwen3-vl 4B"def load_clip(self, clip_name, type="stable_diffusion", device="default"):clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION)@@ -2508,6 +2508,7 @@ async def init_builtin_extra_nodes():"nodes_joyimage.py","nodes_boogu.py","nodes_moss_image.py",+ "nodes_fibo2.py","nodes_chroma_radiance.py","nodes_pid.py","nodes_model_patch.py",