The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Hidream: Allow loading hidream text encoders in CLIPLoader and DualCLIPLoader (#7676)
power88f43e1d7f4153
diff --git a/comfy/sd.py b/comfy/sd.pyindex d97873ba2..8aba5d655 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -703,6 +703,7 @@ class CLIPType(Enum):COSMOS = 11LUMINA2 = 12WAN = 13+ HIDREAM = 14def load_clip(ckpt_paths, embedding_directory=None, clip_type=CLIPType.STABLE_DIFFUSION, model_options={}):@@ -791,6 +792,9 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clipelif clip_type == CLIPType.SD3:clip_target.clip = comfy.text_encoders.sd3_clip.sd3_clip(clip_l=False, clip_g=True, t5=False)clip_target.tokenizer = comfy.text_encoders.sd3_clip.SD3Tokenizer+ elif clip_type == CLIPType.HIDREAM:+ clip_target.clip = comfy.text_encoders.hidream.hidream_clip(clip_l=False, clip_g=True, t5=False, llama=False, dtype_t5=None, dtype_llama=None, t5xxl_scaled_fp8=None, llama_scaled_fp8=None)+ clip_target.tokenizer = comfy.text_encoders.hidream.HiDreamTokenizerelse:clip_target.clip = sdxl_clip.SDXLRefinerClipModelclip_target.tokenizer = sdxl_clip.SDXLTokenizer@@ -811,6 +815,10 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clipclip_target.clip = comfy.text_encoders.wan.te(**t5xxl_detect(clip_data))clip_target.tokenizer = comfy.text_encoders.wan.WanT5Tokenizertokenizer_data["spiece_model"] = clip_data[0].get("spiece_model", None)+ elif clip_type == CLIPType.HIDREAM:+ clip_target.clip = comfy.text_encoders.hidream.hidream_clip(**t5xxl_detect(clip_data),+ clip_l=False, clip_g=False, t5=True, llama=False, dtype_llama=None, llama_scaled_fp8=None)+ clip_target.tokenizer = comfy.text_encoders.hidream.HiDreamTokenizerelse: #CLIPType.MOCHIclip_target.clip = comfy.text_encoders.genmo.mochi_te(**t5xxl_detect(clip_data))clip_target.tokenizer = comfy.text_encoders.genmo.MochiT5Tokenizer@@ -827,10 +835,18 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clipclip_target.clip = comfy.text_encoders.lumina2.te(**llama_detect(clip_data))clip_target.tokenizer = comfy.text_encoders.lumina2.LuminaTokenizertokenizer_data["spiece_model"] = clip_data[0].get("spiece_model", None)+ elif te_model == TEModel.LLAMA3_8:+ clip_target.clip = comfy.text_encoders.hidream.hidream_clip(**llama_detect(clip_data),+ clip_l=False, clip_g=False, t5=False, llama=True, dtype_t5=None, t5xxl_scaled_fp8=None)+ clip_target.tokenizer = comfy.text_encoders.hidream.HiDreamTokenizerelse:+ # clip_lif clip_type == CLIPType.SD3:clip_target.clip = comfy.text_encoders.sd3_clip.sd3_clip(clip_l=True, clip_g=False, t5=False)clip_target.tokenizer = comfy.text_encoders.sd3_clip.SD3Tokenizer+ elif clip_type == CLIPType.HIDREAM:+ clip_target.clip = comfy.text_encoders.hidream.hidream_clip(clip_l=True, clip_g=False, t5=False, llama=False, dtype_t5=None, dtype_llama=None, t5xxl_scaled_fp8=None, llama_scaled_fp8=None)+ clip_target.tokenizer = comfy.text_encoders.hidream.HiDreamTokenizerelse:clip_target.clip = sd1_clip.SD1ClipModelclip_target.tokenizer = sd1_clip.SD1Tokenizer@@ -848,6 +864,24 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clipelif clip_type == CLIPType.HUNYUAN_VIDEO:clip_target.clip = comfy.text_encoders.hunyuan_video.hunyuan_video_clip(**llama_detect(clip_data))clip_target.tokenizer = comfy.text_encoders.hunyuan_video.HunyuanVideoTokenizer+ elif clip_type == CLIPType.HIDREAM:+ # Detect+ hidream_dualclip_classes = []+ for hidream_te in clip_data:+ te_model = detect_te_model(hidream_te)+ hidream_dualclip_classes.append(te_model)++ clip_l = TEModel.CLIP_L in hidream_dualclip_classes+ clip_g = TEModel.CLIP_G in hidream_dualclip_classes+ t5 = TEModel.T5_XXL in hidream_dualclip_classes+ llama = TEModel.LLAMA3_8 in hidream_dualclip_classes++ # Initialize t5xxl_detect and llama_detect kwargs if needed+ t5_kwargs = t5xxl_detect(clip_data) if t5 else {}+ llama_kwargs = llama_detect(clip_data) if llama else {}++ clip_target.clip = comfy.text_encoders.hidream.hidream_clip(clip_l=clip_l, clip_g=clip_g, t5=t5, llama=llama, **t5_kwargs, **llama_kwargs)+ clip_target.tokenizer = comfy.text_encoders.hidream.HiDreamTokenizerelse:clip_target.clip = sdxl_clip.SDXLClipModelclip_target.tokenizer = sdxl_clip.SDXLTokenizerdiff --git a/comfy/text_encoders/hidream.py b/comfy/text_encoders/hidream.pyindex 6c34c5572..ca54eaa78 100644--- a/comfy/text_encoders/hidream.py+++ b/comfy/text_encoders/hidream.py@@ -109,11 +109,15 @@ class HiDreamTEModel(torch.nn.Module):if self.t5xxl is not None:t5_output = self.t5xxl.encode_token_weights(token_weight_pairs_t5)t5_out, t5_pooled = t5_output[:2]+ else:+ t5_out = Noneif self.llama is not None:ll_output = self.llama.encode_token_weights(token_weight_pairs_llama)ll_out, ll_pooled = ll_output[:2]ll_out = ll_out[:, 1:]+ else:+ ll_out = Noneif t5_out is None:t5_out = torch.zeros((1, 1, 4096), device=comfy.model_management.intermediate_device())diff --git a/nodes.py b/nodes.pyindex d4082d19d..b1ab62aad 100644--- a/nodes.py+++ b/nodes.py@@ -917,7 +917,7 @@ class CLIPLoader:@classmethoddef INPUT_TYPES(s):return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ),- "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan"], ),+ "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream"], ),},"optional": {"device": (["default", "cpu"], {"advanced": True}),@@ -927,7 +927,7 @@ class CLIPLoader:CATEGORY = "advanced/loaders"- DESCRIPTION = "[Recipes]\n\nstable_diffusion: clip-l\nstable_cascade: clip-g\nsd3: t5 xxl/ clip-g / clip-l\nstable_audio: t5 base\nmochi: t5 xxl\ncosmos: old t5 xxl\nlumina2: gemma 2 2B\nwan: umt5 xxl"+ DESCRIPTION = "[Recipes]\n\nstable_diffusion: clip-l\nstable_cascade: clip-g\nsd3: t5 xxl/ clip-g / clip-l\nstable_audio: t5 base\nmochi: t5 xxl\ncosmos: old t5 xxl\nlumina2: gemma 2 2B\nwan: umt5 xxl\n hidream: llama-3.1 (Recommend) or t5"def load_clip(self, clip_name, type="stable_diffusion", device="default"):clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION)@@ -945,7 +945,7 @@ class DualCLIPLoader:def INPUT_TYPES(s):return {"required": { "clip_name1": (folder_paths.get_filename_list("text_encoders"), ),"clip_name2": (folder_paths.get_filename_list("text_encoders"), ),- "type": (["sdxl", "sd3", "flux", "hunyuan_video"], ),+ "type": (["sdxl", "sd3", "flux", "hunyuan_video", "hidream"], ),},"optional": {"device": (["default", "cpu"], {"advanced": True}),@@ -955,7 +955,7 @@ class DualCLIPLoader:CATEGORY = "advanced/loaders"- DESCRIPTION = "[Recipes]\n\nsdxl: clip-l, clip-g\nsd3: clip-l, clip-g / clip-l, t5 / clip-g, t5\nflux: clip-l, t5"+ DESCRIPTION = "[Recipes]\n\nsdxl: clip-l, clip-g\nsd3: clip-l, clip-g / clip-l, t5 / clip-g, t5\nflux: clip-l, t5\nhidream: at least one of t5 or llama, recommended t5 and llama"def load_clip(self, clip_name1, clip_name2, type, device="default"):clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION)