Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Add TextGenerateV2 with Autogrow image inputs (#17492)

Jukka Seppänen6b904b5fafd0
diff --git a/app/node_replace_manager.py b/app/node_replace_manager.py
index 8d87aa6d4..1e4f7c757 100644
--- a/app/node_replace_manager.py
+++ b/app/node_replace_manager.py
@@ -94,7 +94,7 @@ class NodeReplaceManager:
for input_map in replacement.input_mapping:
if "set_value" in input_map:
new_node_struct["inputs"][input_map["new_id"]] = input_map["set_value"]
- elif "old_id" in input_map:
+ elif "old_id" in input_map and input_map["old_id"] in node_struct["inputs"]:
new_node_struct["inputs"][input_map["new_id"]] = node_struct["inputs"][input_map["old_id"]]
# finalize input replacement
prompt[node_number] = new_node_struct
diff --git a/comfy/text_encoders/gemma4.py b/comfy/text_encoders/gemma4.py
index bb350802f..c2969ad1b 100644
--- a/comfy/text_encoders/gemma4.py
+++ b/comfy/text_encoders/gemma4.py
@@ -1470,21 +1470,32 @@ class Gemma4_Tokenizer():
up_slopes = slopes[:, 2:] / filter_diff[1:]
return np.maximum(np.zeros(1), np.minimum(down_slopes, up_slopes))
- def tokenize_with_weights(self, text, return_word_ids=False, image=None, audio=None, video=None, llama_template=None, skip_template=True, thinking=False, system_prompt="", **kwargs):
+ def tokenize_with_weights(self, text, return_word_ids=False, image=None, audio=None, video=None, llama_template=None, skip_template=True, thinking=False, system_prompt="", images=[], audios=[], videos=[], **kwargs):
# Process audio
+ if audio is not None and len(audios) == 0:
+ audios = [audio]
audio_features = []
- if audio is not None:
+ audio_token_counts = []
+ for audio in audios:
waveform = audio["waveform"].squeeze(0) if hasattr(audio, "__getitem__") else audio
sample_rate = audio.get("sample_rate", 16000) if hasattr(audio, "get") else 16000
feat, feat_mask = self._extract_audio_features(waveform, sample_rate)
- audio_features = [(feat.unsqueeze(0), feat_mask.unsqueeze(0))] # ([1, T, D], [1, T])
+ audio_features.append((feat.unsqueeze(0), feat_mask.unsqueeze(0))) # ([1, T, D], [1, T])
+ # Compute audio token count (always at 16kHz)
+ num_samples = int(waveform.shape[-1] * 16000 / sample_rate) if sample_rate != 16000 else waveform.shape[-1]
+ audio_token_counts.append(self._audio_token_count(num_samples))
# Process image/video frames
- is_video = video is not None
- source = video if is_video else image
- images = []
- if source is not None:
+ if video is not None and len(videos) == 0:
+ videos = [video]
+ if image is not None and len(images) == 0:
+ images = [image]
+ sources = [(source, False) for source in images] + [(source, True) for source in videos]
+ image_frames = []
+ video_frames = []
+ video_frame_counts = []
+ for source, is_video in sources:
samples = source.movedim(-1, 1) # [B, C, H, W]
num_frames = samples.shape[0]
@@ -1497,6 +1508,7 @@ class Gemma4_Tokenizer():
indices = [0]
samples = samples[indices]
num_frames = len(indices)
+ video_frame_counts.append(num_frames)
h, w = samples.shape[2], samples.shape[3]
patch_size = 16
@@ -1505,13 +1517,14 @@ class Gemma4_Tokenizer():
max_patches = max_soft_tokens * pooling_k * pooling_k
target_h, target_w = _get_aspect_ratio_preserving_size(h, w, patch_size, max_patches, pooling_k)
+ frames = video_frames if is_video else image_frames
for i in range(num_frames):
# rescaling to match reference code
s = (samples[i].clamp(0, 1) * 255).to(torch.uint8) # [C, H, W] uint8
if target_h != h or target_w != w:
s = TVF.resize(s, [target_h, target_w], interpolation=TVF.InterpolationMode.BICUBIC, antialias=True)
s = s.float() * (1.0 / 255.0)
- images.append({"pixels": s.unsqueeze(0).movedim(1, -1)[:, :, :, :3], "max_soft_tokens": max_soft_tokens})
+ frames.append({"pixels": s.unsqueeze(0).movedim(1, -1)[:, :, :, :3], "max_soft_tokens": max_soft_tokens})
if text.startswith('<|turn>'):
skip_template = True
@@ -1526,25 +1539,21 @@ class Gemma4_Tokenizer():
think = "<|think|>\n" if thinking else ""
system = f"<|turn>system\n{think}{system_prompt}<turn|>\n" if thinking or system_prompt else ""
media = ""
- if len(images) > 0:
- if is_video:
- media += "\n\n"
- for i in range(len(images)):
- ts = f"{int(i // 60):02d}:{int(i % 60):02d}"
- sep = "" if i == 0 else " "
- media += f"{sep}{ts} <|image><|video|><image|>"
- media += "\n\n"
- else:
- media += "\n\n"
- for i in range(len(images)):
- if i > 0:
- media += "\n\n\n\n"
- media += "<|image><|image|><image|>"
- media += "\n\n"
- if len(audio_features) > 0:
- # Compute audio token count (always at 16kHz)
- num_samples = int(waveform.shape[-1] * 16000 / sample_rate) if sample_rate != 16000 else waveform.shape[-1]
- n_audio_tokens = self._audio_token_count(num_samples)
+ if len(image_frames) > 0:
+ media += "\n\n"
+ for i in range(len(image_frames)):
+ if i > 0:
+ media += "\n\n\n\n"
+ media += "<|image><|image|><image|>"
+ media += "\n\n"
+ for num_frames in video_frame_counts:
+ media += "\n\n"
+ for i in range(num_frames):
+ ts = f"{int(i // 60):02d}:{int(i % 60):02d}"
+ sep = "" if i == 0 else " "
+ media += f"{sep}{ts} <|image><|video|><image|>"
+ media += "\n\n"
+ for n_audio_tokens in audio_token_counts:
media += "<|audio>" + "<|audio|>" * n_audio_tokens + "<audio|>"
# 12B/31B prime a closed thought block for non-thinking mode, E2B/E4B must not: it cues them into reasoning inline.
model_open = "<|channel>thought\n<channel|>" if self.prime_empty_thought and not thinking else ""
@@ -1566,9 +1575,8 @@ class Gemma4_Tokenizer():
else:
i += 1
- if len(images) > 0:
- img_token_id = 258884 if is_video else 258880
- img_embeds = [{"type": "image", "data": img["pixels"], "max_soft_tokens": img["max_soft_tokens"]} for img in images]
+ for img_token_id, frames in ((258880, image_frames), (258884, video_frames)):
+ img_embeds = [{"type": "image", "data": img["pixels"], "max_soft_tokens": img["max_soft_tokens"]} for img in frames]
for r in text_tokens:
_replace_placeholders(r, img_token_id, img_embeds)
diff --git a/comfy/text_encoders/lt.py b/comfy/text_encoders/lt.py
index d362fe073..5f28bcbc0 100644
--- a/comfy/text_encoders/lt.py
+++ b/comfy/text_encoders/lt.py
@@ -27,13 +27,15 @@ class Gemma3_Tokenizer():
def state_dict(self):
return {"spiece_model": self.tokenizer.serialize_model()}
- def tokenize_with_weights(self, text, return_word_ids=False, image=None, llama_template=None, skip_template=True, system_prompt="", **kwargs):
+ def tokenize_with_weights(self, text, return_word_ids=False, image=None, llama_template=None, skip_template=True, system_prompt="", images=[], **kwargs):
self.llama_template = "<start_of_turn>system\nYou are a helpful assistant.<end_of_turn>\n<start_of_turn>user\n{}<end_of_turn>\n<start_of_turn>model\n"
self.llama_template_images = "<start_of_turn>system\nYou are a helpful assistant.<end_of_turn>\n<start_of_turn>user\n\n<image_soft_token>{}<end_of_turn>\n\n<start_of_turn>model\n"
- if image is None:
- images = []
- else:
+ if image is not None and len(images) == 0:
+ images = [image]
+
+ resized = []
+ for image in images:
samples = image.movedim(-1, 1)
total = int(896 * 896)
@@ -42,7 +44,8 @@ class Gemma3_Tokenizer():
height = round(samples.shape[2] * scale_by)
s = comfy.utils.common_upscale(samples, width, height, "area", "disabled").movedim(1, -1)
- images = [s[:, :, :, :3]]
+ resized.append(s[:, :, :, :3])
+ images = resized
if text.startswith('<start_of_turn>'):
skip_template = True
@@ -52,7 +55,7 @@ class Gemma3_Tokenizer():
else:
if llama_template is None:
if len(images) > 0:
- llama_text = self.llama_template_images.format(text)
+ llama_text = self.llama_template_images.replace("<image_soft_token>", "\n\n\n\n".join(["<image_soft_token>"] * len(images)), 1).format(text)
else:
llama_text = self.llama_template.format(text)
else:
diff --git a/comfy_extras/nodes_replacements.py b/comfy_extras/nodes_replacements.py
index 7684e854c..23f37103c 100644
--- a/comfy_extras/nodes_replacements.py
+++ b/comfy_extras/nodes_replacements.py
@@ -13,6 +13,7 @@ async def register_replacements():
await register_replacements_preview3d()
await register_replacements_svdimg2vid()
await register_replacements_conditioningavg()
+ await register_replacements_textgenerate()
async def register_replacements_longeredge():
# No dynamic inputs here
@@ -92,6 +93,23 @@ async def register_replacements_conditioningavg():
old_node_id="ConditioningAverage ",
))
+async def register_replacements_textgenerate():
+ # TextGenerateV2 uses Autogrow, widget order is the sampling_mode "on" layout
+ sampling = [f"sampling_mode.{name}" for name in ["temperature", "top_k", "top_p", "min_p", "repetition_penalty", "seed", "presence_penalty"]]
+ widgets = ["prompt", "max_length", "sampling_mode", *sampling, "thinking", "use_default_template", "mtp"]
+ await api.node_replacement.register(io.NodeReplace(
+ new_node_id="TextGenerateV2",
+ old_node_id="TextGenerate",
+ old_widget_ids=widgets,
+ input_mapping=[
+ {"new_id": "images.image1", "old_id": "image"},
+ {"new_id": "videos.video1", "old_id": "video"},
+ {"new_id": "audios.audio1", "old_id": "audio"},
+ {"new_id": "sampling_mode.presence_penalty", "set_value": 0.0},
+ *[{"new_id": name, "old_id": name} for name in ["clip", "system_prompt", *widgets]],
+ ],
+ ))
+
class NodeReplacementsExtension(ComfyExtension):
async def on_load(self) -> None:
await register_replacements()
diff --git a/comfy_extras/nodes_textgen.py b/comfy_extras/nodes_textgen.py
index 0e7d3a9af..832346a09 100644
--- a/comfy_extras/nodes_textgen.py
+++ b/comfy_extras/nodes_textgen.py
@@ -1,7 +1,9 @@
from comfy_api.latest import ComfyExtension, io
from typing_extensions import override
-class TextGenerate(io.ComfyNode):
+VIDEO_TOOLTIP = "Video frames as image batch. Assumed to be 24 FPS; subsampled to 1 FPS internally."
+
+class TextGenerateV2(io.ComfyNode):
@classmethod
def define_schema(cls):
# Define dynamic combo options for sampling mode
@@ -14,8 +16,8 @@ class TextGenerate(io.ComfyNode):
io.Float.Input("top_p", default=0.95, min=0.0, max=1.0, step=0.01),
io.Float.Input("min_p", default=0.05, min=0.0, max=1.0, step=0.01),
io.Float.Input("repetition_penalty", default=1.05, min=0.0, max=5.0, step=0.01),
+ io.Float.Input("presence_penalty", default=0.0, min=0.0, max=5.0, step=0.01),
io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
- io.Float.Input("presence_penalty", optional=True, default=0.0, min=0.0, max=5.0, step=0.01),
]
),
io.DynamicCombo.Option(
@@ -25,16 +27,16 @@ class TextGenerate(io.ComfyNode):
]
return io.Schema(
- node_id="TextGenerate",
+ node_id="TextGenerateV2",
display_name="Generate Text",
category="text",
search_aliases=["LLM", "gemma"],
inputs=[
io.Clip.Input("clip"),
io.String.Input("prompt", multiline=True, dynamic_prompts=True, default=""),
- io.Image.Input("image", optional=True),
- io.Image.Input("video", optional=True, tooltip="Video frames as image batch. Assumed to be 24 FPS; subsampled to 1 FPS internally."),
- io.Audio.Input("audio", optional=True),
+ io.Autogrow.Input("images", optional=True, template=io.Autogrow.TemplateNames(io.Image.Input("image"), names=[f"image{i}" for i in range(1, 51)], min=0)),
+ io.Autogrow.Input("videos", optional=True, template=io.Autogrow.TemplateNames(io.Image.Input("video", tooltip=VIDEO_TOOLTIP), names=[f"video{i}" for i in range(1, 101)], min=0)),
+ io.Autogrow.Input("audios", optional=True, template=io.Autogrow.TemplateNames(io.Audio.Input("audio"), names=[f"audio{i}" for i in range(1, 101)], min=0)),
io.Int.Input("max_length", default=512, min=1, max=32768),
io.DynamicCombo.Input("sampling_mode", options=sampling_options, display_name="Sampling Mode"),
io.Boolean.Input("thinking", optional=True, default=False, tooltip="Operate in thinking mode if the model supports it."),
@@ -49,11 +51,15 @@ class TextGenerate(io.ComfyNode):
)
@classmethod
- def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto", system_prompt="") -> io.NodeOutput:
+ def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None, mtp="auto", system_prompt="", images: io.Autogrow.Type = None, videos: io.Autogrow.Type = None, audios: io.Autogrow.Type = None) -> io.NodeOutput:
mtp = {"off": False, "auto": True}.get(mtp, mtp)
- tokens = clip.tokenize(prompt, image=image, skip_template=not use_default_template, min_length=1, thinking=thinking, video=video, audio=audio, system_prompt=system_prompt if use_default_template else "")
+ images = [batch[i:i + 1] for batch in (images or {}).values() if batch is not None for i in range(batch.shape[0])]
+ videos = [v for v in (videos or {}).values() if v is not None]
+ audios = [a for a in (audios or {}).values() if a is not None]
+
+ tokens = clip.tokenize(prompt, image=image, images=images, skip_template=not use_default_template, min_length=1, thinking=thinking, video=video, videos=videos, audio=audio, audios=audios, system_prompt=system_prompt if use_default_template else "")
# Get sampling parameters from dynamic combo
do_sample = sampling_mode.get("sampling_mode") == "on"
@@ -93,6 +99,28 @@ class TextGenerate(io.ComfyNode):
return io.NodeOutput(text.strip(), reasoning.replace("<think>", "", 1).strip())
+class TextGenerate(TextGenerateV2):
+ @classmethod
+ def define_schema(cls):
+ parent_schema = super().define_schema()
+ old_inputs = {
+ "images": io.Image.Input("image", optional=True),
+ "videos": io.Image.Input("video", optional=True, tooltip=VIDEO_TOOLTIP),
+ "audios": io.Audio.Input("audio", optional=True),
+ }
+ # optional widgets follow required ones, which keeps seed before presence_penalty in saved workflows
+ sampling_on = next(i for i in parent_schema.inputs if i.id == "sampling_mode").options[0]
+ next(i for i in sampling_on.inputs if i.id == "presence_penalty").optional = True
+ return io.Schema(
+ node_id="TextGenerate",
+ display_name="Generate Text (DEPRECATED)",
+ category=parent_schema.category,
+ is_deprecated=True,
+ inputs=[old_inputs.get(i.id, i) for i in parent_schema.inputs],
+ outputs=parent_schema.outputs,
+ )
+
+
LTX2_T2V_SYSTEM_PROMPT = """You are a Creative Assistant. Given a user's raw input prompt describing a scene or concept, expand it into a detailed video generation prompt with specific visuals and integrated audio to guide a text-to-video model.
#### Guidelines
- Strictly follow all aspects of the user's raw input: include every element requested (style, visuals, motions, actions, camera movement, audio).
@@ -278,6 +306,7 @@ class TextgenExtension(ComfyExtension):
@override
async def get_node_list(self) -> list[type[io.ComfyNode]]:
return [
+ TextGenerateV2,
TextGenerate,
TextGenerateLTX2Prompt,
]