Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Add video input support to Qwen 3.x (#17582)

Jukka Seppänenc85957ad7f0b
diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.py
index 196f5327b..72a21a196 100644
--- a/comfy/text_encoders/qwen35.py
+++ b/comfy/text_encoders/qwen35.py
@@ -892,6 +892,10 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):
# Qwen3.5 normalizes to [-1, 1] (mean/std 0.5), same as Qwen3-VL.
image, grid = comfy.text_encoders.qwen_vl.process_qwen2vl_images(embed["data"], patch_size=16, image_mean=[0.5, 0.5, 0.5], image_std=[0.5, 0.5, 0.5])
return self.visual(image.to(device, dtype=torch.float32), grid), grid
+ if embed["type"] == "video":
+ height, width = embed["size"]
+ video, grid = comfy.text_encoders.qwen_vl.process_qwen2vl_video(embed["data"], height, width, patch_size=16, image_mean=[0.5, 0.5, 0.5], image_std=[0.5, 0.5, 0.5])
+ return self.visual(video.to(device, dtype=torch.float32), grid), grid
return None, None
def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, embeds_info=[], past_key_values=None):
@@ -1211,10 +1215,36 @@ class Qwen35ImageTokenizer(sd1_clip.SD1Tokenizer):
self.llama_template = "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"
self.llama_template_images = "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n"
- def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], prevent_empty_text=False, thinking=False, system_prompt="", **kwargs):
+ def video_embeds(self, frames, fps):
+ # Qwen3-VL sampling: 2 fps, 4 to 768 frames, consecutive pairs form one temporal patch
+ # introduced by its midpoint timestamp. Returns (prompt text, one embed per pair).
+ total = frames.shape[0]
+ num_frames = min(max(int(total / fps * 2), 4), 768, total)
+ indices = [round(i * (total - 1) / max(num_frames - 1, 1)) for i in range(num_frames)]
+ if len(indices) % 2 == 1:
+ indices.append(indices[-1])
+ height, width = comfy.text_encoders.qwen_vl.qwen2vl_video_smart_resize(num_frames, frames.shape[1], frames.shape[2])
+ text = ""
+ embeds = []
+ for i in range(0, len(indices), 2):
+ seconds = (indices[i] + indices[i + 1]) / (2 * fps)
+ text += f"<{seconds:.1f} seconds><|vision_start|><|video_pad|><|vision_end|>"
+ embeds.append({"type": "video", "data": frames[indices[i:i + 2]], "size": (height, width), "original_type": "video"})
+ return text, embeds
+
+ def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], videos=[], prevent_empty_text=False, thinking=False, system_prompt="", **kwargs):
image = kwargs.get("image", None)
if image is not None and len(images) == 0:
images = [image[i:i + 1] for i in range(image.shape[0])]
+ video = kwargs.get("video", None)
+ if video is not None and len(videos) == 0:
+ videos = [video]
+ video_text = ""
+ video_embeds = []
+ for v in videos:
+ t, e = self.video_embeds(v, kwargs.get("fps", 24))
+ video_text += t
+ video_embeds += e
skip_template = False
if text.startswith('<|im_start|>'):
@@ -1227,13 +1257,12 @@ class Qwen35ImageTokenizer(sd1_clip.SD1Tokenizer):
else:
if llama_template is not None:
template = llama_template
- elif len(images) == 0:
+ elif len(images) == 0 and len(video_embeds) == 0:
template = self.llama_template
else:
template = self.llama_template_images
- if len(images) > 1:
- vision_block = "<|vision_start|><|image_pad|><|vision_end|>"
- template = template.replace(vision_block, vision_block * len(images), 1)
+ vision_block = "<|vision_start|><|image_pad|><|vision_end|>"
+ template = template.replace(vision_block, vision_block * len(images) + video_text, 1)
llama_text = template.format(text)
if system_prompt:
llama_text = "<|im_start|>system\n" + system_prompt + "<|im_end|>\n" + llama_text
@@ -1243,6 +1272,7 @@ class Qwen35ImageTokenizer(sd1_clip.SD1Tokenizer):
tokens = super().tokenize_with_weights(llama_text, return_word_ids=return_word_ids, disable_weights=True, **kwargs)
key_name = next(iter(tokens))
embed_count = 0
+ video_count = 0
qwen_tokens = tokens[key_name]
for r in qwen_tokens:
for i in range(len(r)):
@@ -1250,6 +1280,10 @@ class Qwen35ImageTokenizer(sd1_clip.SD1Tokenizer):
if len(images) > embed_count:
r[i] = ({"type": "image", "data": images[embed_count], "original_type": "image"},) + r[i][1:]
embed_count += 1
+ elif isinstance(r[i][0], (int, float)) and r[i][0] == 248057: # <|video_pad|>
+ if len(video_embeds) > video_count:
+ r[i] = (video_embeds[video_count],) + r[i][1:]
+ video_count += 1
return tokens
diff --git a/comfy/text_encoders/qwen_vl.py b/comfy/text_encoders/qwen_vl.py
index 818b149e6..e68f993d6 100644
--- a/comfy/text_encoders/qwen_vl.py
+++ b/comfy/text_encoders/qwen_vl.py
@@ -24,12 +24,9 @@ def process_qwen2vl_images(
images = images[..., :3]
batch_size, height, width, channels = images.shape
- device = images.device
- # dtype = images.dtype
images = images.permute(0, 3, 1, 2)
- grid_thw_list = []
img = images[0]
factor = patch_size * merge_size
@@ -56,17 +53,40 @@ def process_qwen2vl_images(
for c in range(3):
normalized[c] = (img_resized[c] - image_mean[c]) / image_std[c]
- grid_h = h_bar // patch_size
- grid_w = w_bar // patch_size
- grid_thw = torch.tensor([1, grid_h, grid_w], device=device, dtype=torch.long)
+ return qwen2vl_patchify(normalized.unsqueeze(0).repeat(temporal_patch_size, 1, 1, 1), patch_size, temporal_patch_size, merge_size)
- pixel_values = normalized
- grid_thw_list.append(grid_thw)
- image_grid_thw = torch.stack(grid_thw_list)
- grid_t = 1
- channel = pixel_values.shape[0]
- pixel_values = pixel_values.unsqueeze(0).repeat(2, 1, 1, 1)
+def qwen2vl_video_smart_resize(num_frames, height, width, factor=32, temporal_factor=2, min_pixels=128 * 128, max_pixels=32 * 32 * 768):
+ # Qwen3-VL video budget: total t*h*w pixels, t rounded up to the temporal patch.
+ h_bar = round(height / factor) * factor
+ w_bar = round(width / factor) * factor
+ t_bar = math.ceil(num_frames / temporal_factor) * temporal_factor
+ if t_bar * h_bar * w_bar > max_pixels:
+ beta = math.sqrt((num_frames * height * width) / max_pixels)
+ h_bar = max(factor, math.floor(height / beta / factor) * factor)
+ w_bar = max(factor, math.floor(width / beta / factor) * factor)
+ elif t_bar * h_bar * w_bar < min_pixels:
+ beta = math.sqrt(min_pixels / (num_frames * height * width))
+ h_bar = math.ceil(height * beta / factor) * factor
+ w_bar = math.ceil(width * beta / factor) * factor
+ return h_bar, w_bar
+
+
+def process_qwen2vl_video(frames, height, width, patch_size=14, temporal_patch_size=2, merge_size=2, image_mean=None, image_std=None, interpolation="bilinear"):
+ # frames: [T, H, W, C] in [0, 1], T a multiple of temporal_patch_size, already sized by qwen2vl_video_smart_resize.
+ frames = frames[..., :3].permute(0, 3, 1, 2)
+ frames = F.interpolate(frames, size=(height, width), mode=interpolation, align_corners=False)
+ mean = torch.tensor(image_mean, device=frames.device, dtype=frames.dtype).view(1, 3, 1, 1)
+ std = torch.tensor(image_std, device=frames.device, dtype=frames.dtype).view(1, 3, 1, 1)
+ return qwen2vl_patchify((frames - mean) / std, patch_size, temporal_patch_size, merge_size)
+
+
+def qwen2vl_patchify(pixel_values, patch_size, temporal_patch_size, merge_size):
+ # pixel_values: [T, C, H, W] -> (flattened patches, grid_thw [1, 3])
+ grid_t = pixel_values.shape[0] // temporal_patch_size
+ channel = pixel_values.shape[1]
+ grid_h = pixel_values.shape[2] // patch_size
+ grid_w = pixel_values.shape[3] // patch_size
patches = pixel_values.reshape(
grid_t,
@@ -86,16 +106,17 @@ def process_qwen2vl_images(
channel * temporal_patch_size * patch_size * patch_size
)
- return flatten_patches, image_grid_thw
+ return flatten_patches, torch.tensor([[grid_t, grid_h, grid_w]], device=pixel_values.device, dtype=torch.long)
def qwen2vl_mrope_position_ids(embeds_info, seq_len, device):
# (3, seq_len) T/H/W MRoPE position ids: text runs sequentially, each image span gets its grid positions.
+ # Video frame pairs arrive as separate t=1 spans split by their timestamp text, so they index like images.
# Returns None when there are no image embeds. `extra` is the image grid_thw, or a dict carrying it under "grid".
position_ids = None
offset = 0
for e in embeds_info:
- if e.get("type") == "image":
+ if e.get("type") in ("image", "video"):
extra = e.get("extra", None)
grid = extra["grid"] if isinstance(extra, dict) else extra
start = e.get("index")
diff --git a/comfy_extras/nodes_textgen.py b/comfy_extras/nodes_textgen.py
index 832346a09..8d1368c29 100644
--- a/comfy_extras/nodes_textgen.py
+++ b/comfy_extras/nodes_textgen.py
@@ -1,7 +1,7 @@
from comfy_api.latest import ComfyExtension, io
from typing_extensions import override
-VIDEO_TOOLTIP = "Video frames as image batch. Assumed to be 24 FPS; subsampled to 1 FPS internally."
+VIDEO_TOOLTIP = "Video frames as image batch. Assumed to be 24 FPS; subsampled to the model's frame rate internally."
class TextGenerateV2(io.ComfyNode):
@classmethod