The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add TextGenerateV2 with Autogrow image inputs (#17492)
Jukka Seppänen6b904b5fafd0
diff --git a/app/node_replace_manager.py b/app/node_replace_manager.pyindex 8d87aa6d4..1e4f7c757 100644--- a/app/node_replace_manager.py+++ b/app/node_replace_manager.py@@ -94,7 +94,7 @@ class NodeReplaceManager:for input_map in replacement.input_mapping:if "set_value" in input_map:new_node_struct["inputs"][input_map["new_id"]] = input_map["set_value"]- elif "old_id" in input_map:+ elif "old_id" in input_map and input_map["old_id"] in node_struct["inputs"]:new_node_struct["inputs"][input_map["new_id"]] = node_struct["inputs"][input_map["old_id"]]# finalize input replacementprompt[node_number] = new_node_structdiff --git a/comfy/text_encoders/gemma4.py b/comfy/text_encoders/gemma4.pyindex bb350802f..c2969ad1b 100644--- a/comfy/text_encoders/gemma4.py+++ b/comfy/text_encoders/gemma4.py@@ -1470,21 +1470,32 @@ class Gemma4_Tokenizer():up_slopes = slopes[:, 2:] / filter_diff[1:]return np.maximum(np.zeros(1), np.minimum(down_slopes, up_slopes))- def tokenize_with_weights(self, text, return_word_ids=False, image=None, audio=None, video=None, llama_template=None, skip_template=True, thinking=False, system_prompt="", **kwargs):+ def tokenize_with_weights(self, text, return_word_ids=False, image=None, audio=None, video=None, llama_template=None, skip_template=True, thinking=False, system_prompt="", images=[], audios=[], videos=[], **kwargs):# Process audio+ if audio is not None and len(audios) == 0:+ audios = [audio]audio_features = []- if audio is not None:+ audio_token_counts = []+ for audio in audios:waveform = audio["waveform"].squeeze(0) if hasattr(audio, "__getitem__") else audiosample_rate = audio.get("sample_rate", 16000) if hasattr(audio, "get") else 16000feat, feat_mask = self._extract_audio_features(waveform, sample_rate)- audio_features = [(feat.unsqueeze(0), feat_mask.unsqueeze(0))] # ([1, T, D], [1, T])+ audio_features.append((feat.unsqueeze(0), feat_mask.unsqueeze(0))) # ([1, T, D], [1, T])+ # Compute audio token count (always at 16kHz)+ num_samples = int(waveform.shape[-1] * 16000 / sample_rate) if sample_rate != 16000 else waveform.shape[-1]+ audio_token_counts.append(self._audio_token_count(num_samples))# Process image/video frames- is_video = video is not None- source = video if is_video else image- images = []- if source is not None:+ if video is not None and len(videos) == 0:+ videos = [video]+ if image is not None and len(images) == 0:+ images = [image]+ sources = [(source, False) for source in images] + [(source, True) for source in videos]+ image_frames = []+ video_frames = []+ video_frame_counts = []+ for source, is_video in sources:samples = source.movedim(-1, 1) # [B, C, H, W]num_frames = samples.shape[0]@@ -1497,6 +1508,7 @@ class Gemma4_Tokenizer():indices = [0]samples = samples[indices]num_frames = len(indices)+ video_frame_counts.append(num_frames)h, w = samples.shape[2], samples.shape[3]patch_size = 16@@ -1505,13 +1517,14 @@ class Gemma4_Tokenizer():max_patches = max_soft_tokens * pooling_k * pooling_ktarget_h, target_w = _get_aspect_ratio_preserving_size(h, w, patch_size, max_patches, pooling_k)+ frames = video_frames if is_video else image_framesfor i in range(num_frames):# rescaling to match reference codes = (samples[i].clamp(0, 1) * 255).to(torch.uint8) # [C, H, W] uint8if target_h != h or target_w != w:s = TVF.resize(s, [target_h, target_w], interpolation=TVF.InterpolationMode.BICUBIC, antialias=True)s = s.float() * (1.0 / 255.0)- images.append({"pixels": s.unsqueeze(0).movedim(1, -1)[:, :, :, :3], "max_soft_tokens": max_soft_tokens})+ frames.append({"pixels": s.unsqueeze(0).movedim(1, -1)[:, :, :, :3], "max_soft_tokens": max_soft_tokens})if text.startswith('<|turn>'):skip_template = True@@ -1526,25 +1539,21 @@ class Gemma4_Tokenizer():think = "<|think|>\n" if thinking else ""system = f"<|turn>system\n{think}{system_prompt}<turn|>\n" if thinking or system_prompt else ""media = ""- if len(images) > 0:- if is_video:- media += "\n\n"- for i in range(len(images)):- ts = f"{int(i // 60):02d}:{int(i % 60):02d}"- sep = "" if i == 0 else " "- media += f"{sep}{ts} <|image><|video|><image|>"- media += "\n\n"- else:- media += "\n\n"- for i in range(len(images)):- if i > 0:- media += "\n\n\n\n"- media += "<|image><|image|><image|>"- media += "\n\n"- if len(audio_features) > 0:- # Compute audio token count (always at 16kHz)- num_samples = int(waveform.shape[-1] * 16000 / sample_rate) if sample_rate != 16000 else waveform.shape[-1]- n_audio_tokens = self._audio_token_count(num_samples)+ if len(image_frames) > 0:+ media += "\n\n"+ for i in range(len(image_frames)):+ if i > 0:+ media += "\n\n\n\n"+ media += "<|image><|image|><image|>"+ media += "\n\n"+ for num_frames in video_frame_counts:+ media += "\n\n"+ for i in range(num_frames):+ ts = f"{int(i // 60):02d}:{int(i % 60):02d}"+ sep = "" if i == 0 else " "+ media += f"{sep}{ts} <|image><|video|><image|>"+ media += "\n\n"+ for n_audio_tokens in audio_token_counts:media += "<|audio>" + "<|audio|>" * n_audio_tokens + "<audio|>"# 12B/31B prime a closed thought block for non-thinking mode, E2B/E4B must not: it cues them into reasoning inline.model_open = "<|channel>thought\n<channel|>" if self.prime_empty_thought and not thinking else ""@@ -1566,9 +1575,8 @@ class Gemma4_Tokenizer():else:i += 1- if len(images) > 0:- img_token_id = 258884 if is_video else 258880- img_embeds = [{"type": "image", "data": img["pixels"], "max_soft_tokens": img["max_soft_tokens"]} for img in images]+ for img_token_id, frames in ((258880, image_frames), (258884, video_frames)):+ img_embeds = [{"type": "image", "data": img["pixels"], "max_soft_tokens": img["max_soft_tokens"]} for img in frames]for r in text_tokens:_replace_placeholders(r, img_token_id, img_embeds)diff --git a/comfy/text_encoders/lt.py b/comfy/text_encoders/lt.pyindex d362fe073..5f28bcbc0 100644--- a/comfy/text_encoders/lt.py+++ b/comfy/text_encoders/lt.py@@ -27,13 +27,15 @@ class Gemma3_Tokenizer():def state_dict(self):return {"spiece_model": self.tokenizer.serialize_model()}- def tokenize_with_weights(self, text, return_word_ids=False, image=None, llama_template=None, skip_template=True, system_prompt="", **kwargs):+ def tokenize_with_weights(self, text, return_word_ids=False, image=None, llama_template=None, skip_template=True, system_prompt="", images=[], **kwargs):self.llama_template = "<start_of_turn>system\nYou are a helpful assistant.<end_of_turn>\n<start_of_turn>user\n{}<end_of_turn>\n<start_of_turn>model\n"self.llama_template_images = "<start_of_turn>system\nYou are a helpful assistant.<end_of_turn>\n<start_of_turn>user\n\n<image_soft_token>{}<end_of_turn>\n\n<start_of_turn>model\n"- if image is None:- images = []- else:+ if image is not None and len(images) == 0:+ images = [image]++ resized = []+ for image in images:samples = image.movedim(-1, 1)total = int(896 * 896)@@ -42,7 +44,8 @@ class Gemma3_Tokenizer():height = round(samples.shape[2] * scale_by)s = comfy.utils.common_upscale(samples, width, height, "area", "disabled").movedim(1, -1)- images = [s[:, :, :, :3]]+ resized.append(s[:, :, :, :3])+ images = resizedif text.startswith('<start_of_turn>'):skip_template = True@@ -52,7 +55,7 @@ class Gemma3_Tokenizer():else:if llama_template is None:if len(images) > 0:- llama_text = self.llama_template_images.format(text)+ llama_text = self.llama_template_images.replace("<image_soft_token>", "\n\n\n\n".join(["<image_soft_token>"] * len(images)), 1).format(text)else:llama_text = self.llama_template.format(text)else:diff --git a/comfy_extras/nodes_replacements.py b/comfy_extras/nodes_replacements.pyindex 7684e854c..23f37103c 100644--- a/comfy_extras/nodes_replacements.py+++ b/comfy_extras/nodes_replacements.py@@ -13,6 +13,7 @@ async def register_replacements():await register_replacements_preview3d()await register_replacements_svdimg2vid()await register_replacements_conditioningavg()+ await register_replacements_textgenerate()async def register_replacements_longeredge():# No dynamic inputs here@@ -92,6 +93,23 @@ async def register_replacements_conditioningavg():old_node_id="ConditioningAverage ",))+async def register_replacements_textgenerate():+ # TextGenerateV2 uses Autogrow, widget order is the sampling_mode "on" layout+ sampling = [f"sampling_mode.{name}" for name in ["temperature", "top_k", "top_p", "min_p", "repetition_penalty", "seed", "presence_penalty"]]+ widgets = ["prompt", "max_length", "sampling_mode", *sampling, "thinking", "use_default_template", "mtp"]+ await api.node_replacement.register(io.NodeReplace(+ new_node_id="TextGenerateV2",+ old_node_id="TextGenerate",+ old_widget_ids=widgets,+ input_mapping=[+ {"new_id": "images.image1", "old_id": "image"},+ {"new_id": "videos.video1", "old_id": "video"},+ {"new_id": "audios.audio1", "old_id": "audio"},+ {"new_id": "sampling_mode.presence_penalty", "set_value": 0.0},+ *[{"new_id": name, "old_id": name} for name in ["clip", "system_prompt", *widgets]],+ ],+ ))+class NodeReplacementsExtension(ComfyExtension):async def on_load(self) -> None:await register_replacements()diff --git a/comfy_extras/nodes_textgen.py b/comfy_extras/nodes_textgen.pyindex 0e7d3a9af..832346a09 100644--- a/comfy_extras/nodes_textgen.py+++ b/comfy_extras/nodes_textgen.py@@ -1,7 +1,9 @@from comfy_api.latest import ComfyExtension, iofrom typing_extensions import override-class TextGenerate(io.ComfyNode):+VIDEO_TOOLTIP = "Video frames as image batch. Assumed to be 24 FPS; subsampled to 1 FPS internally."++class TextGenerateV2(io.ComfyNode):@classmethoddef define_schema(cls):# Define dynamic combo options for sampling mode@@ -14,8 +16,8 @@ class TextGenerate(io.ComfyNode):io.Float.Input("top_p", default=0.95, min=0.0, max=1.0, step=0.01),io.Float.Input("min_p", default=0.05, min=0.0, max=1.0, step=0.01),io.Float.Input("repetition_penalty", default=1.05, min=0.0, max=5.0, step=0.01),+ io.Float.Input("presence_penalty", default=0.0, min=0.0, max=5.0, step=0.01),io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),- io.Float.Input("presence_penalty", optional=True, default=0.0, min=0.0, max=5.0, step=0.01),]),io.DynamicCombo.Option(@@ -25,16 +27,16 @@ class TextGenerate(io.ComfyNode):]return io.Schema(- node_id="TextGenerate",+ node_id="TextGenerateV2",display_name="Generate Text",category="text",search_aliases=["LLM", "gemma"],inputs=[io.Clip.Input("clip"),io.String.Input("prompt", multiline=True, dynamic_prompts=True, default=""),- io.Image.Input("image", optional=True),- io.Image.Input("video", optional=True, tooltip="Video frames as image batch. Assumed to be 24 FPS; subsampled to 1 FPS internally."),- io.Audio.Input("audio", optional=True),+ io.Autogrow.Input("images", optional=True, template=io.Autogrow.TemplateNames(io.Image.Input("image"), names=[f"image{i}" for i in range(1, 51)], min=0)),+ io.Autogrow.Input("videos", optional=True, template=io.Autogrow.TemplateNames(io.Image.Input("video", tooltip=VIDEO_TOOLTIP), names=[f"video{i}" for i in range(1, 101)], min=0)),+ io.Autogrow.Input("audios", optional=True, template=io.Autogrow.TemplateNames(io.Audio.Input("audio"), names=[f"audio{i}" for i in range(1, 101)], min=0)),io.Int.Input("max_length", default=512, min=1, max=32768),io.DynamicCombo.Input("sampling_mode", options=sampling_options, display_name="Sampling Mode"),io.Boolean.Input("thinking", optional=True, default=False, tooltip="Operate in thinking mode if the model supports it."),@@ -49,11 +51,15 @@ class TextGenerate(io.ComfyNode):)@classmethod- def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto", system_prompt="") -> io.NodeOutput:+ def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto", system_prompt="", images: io.Autogrow.Type = None, videos: io.Autogrow.Type = None, audios: io.Autogrow.Type = None) -> io.NodeOutput:mtp = {"off": False, "auto": True}.get(mtp, mtp)- tokens = clip.tokenize(prompt, image=image, skip_template=not use_default_template, min_length=1, thinking=thinking, video=video, audio=audio, system_prompt=system_prompt if use_default_template else "")+ images = [batch[i:i + 1] for batch in (images or {}).values() if batch is not None for i in range(batch.shape[0])]+ videos = [v for v in (videos or {}).values() if v is not None]+ audios = [a for a in (audios or {}).values() if a is not None]++ tokens = clip.tokenize(prompt, image=image, images=images, skip_template=not use_default_template, min_length=1, thinking=thinking, video=video, videos=videos, audio=audio, audios=audios, system_prompt=system_prompt if use_default_template else "")# Get sampling parameters from dynamic combodo_sample = sampling_mode.get("sampling_mode") == "on"@@ -93,6 +99,28 @@ class TextGenerate(io.ComfyNode):return io.NodeOutput(text.strip(), reasoning.replace("<think>", "", 1).strip())+class TextGenerate(TextGenerateV2):+ @classmethod+ def define_schema(cls):+ parent_schema = super().define_schema()+ old_inputs = {+ "images": io.Image.Input("image", optional=True),+ "videos": io.Image.Input("video", optional=True, tooltip=VIDEO_TOOLTIP),+ "audios": io.Audio.Input("audio", optional=True),+ }+ # optional widgets follow required ones, which keeps seed before presence_penalty in saved workflows+ sampling_on = next(i for i in parent_schema.inputs if i.id == "sampling_mode").options[0]+ next(i for i in sampling_on.inputs if i.id == "presence_penalty").optional = True+ return io.Schema(+ node_id="TextGenerate",+ display_name="Generate Text (DEPRECATED)",+ category=parent_schema.category,+ is_deprecated=True,+ inputs=[old_inputs.get(i.id, i) for i in parent_schema.inputs],+ outputs=parent_schema.outputs,+ )++LTX2_T2V_SYSTEM_PROMPT = """You are a Creative Assistant. Given a user's raw input prompt describing a scene or concept, expand it into a detailed video generation prompt with specific visuals and integrated audio to guide a text-to-video model.#### Guidelines- Strictly follow all aspects of the user's raw input: include every element requested (style, visuals, motions, actions, camera movement, audio).@@ -278,6 +306,7 @@ class TextgenExtension(ComfyExtension):@overrideasync def get_node_list(self) -> list[type[io.ComfyNode]]:return [+ TextGenerateV2,TextGenerate,TextGenerateLTX2Prompt,]