The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add video input support to Qwen 3.x (#17582)
Jukka Seppänenc85957ad7f0b
diff --git a/comfy/text_encoders/qwen35.py b/comfy/text_encoders/qwen35.pyindex 196f5327b..72a21a196 100644--- a/comfy/text_encoders/qwen35.py+++ b/comfy/text_encoders/qwen35.py@@ -892,6 +892,10 @@ class Qwen35(BaseLlama, BaseGenerate, torch.nn.Module):# Qwen3.5 normalizes to [-1, 1] (mean/std 0.5), same as Qwen3-VL.image, grid = comfy.text_encoders.qwen_vl.process_qwen2vl_images(embed["data"], patch_size=16, image_mean=[0.5, 0.5, 0.5], image_std=[0.5, 0.5, 0.5])return self.visual(image.to(device, dtype=torch.float32), grid), grid+ if embed["type"] == "video":+ height, width = embed["size"]+ video, grid = comfy.text_encoders.qwen_vl.process_qwen2vl_video(embed["data"], height, width, patch_size=16, image_mean=[0.5, 0.5, 0.5], image_std=[0.5, 0.5, 0.5])+ return self.visual(video.to(device, dtype=torch.float32), grid), gridreturn None, Nonedef forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, embeds_info=[], past_key_values=None):@@ -1211,10 +1215,36 @@ class Qwen35ImageTokenizer(sd1_clip.SD1Tokenizer):self.llama_template = "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"self.llama_template_images = "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n"- def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], prevent_empty_text=False, thinking=False, system_prompt="", **kwargs):+ def video_embeds(self, frames, fps):+ # Qwen3-VL sampling: 2 fps, 4 to 768 frames, consecutive pairs form one temporal patch+ # introduced by its midpoint timestamp. Returns (prompt text, one embed per pair).+ total = frames.shape[0]+ num_frames = min(max(int(total / fps * 2), 4), 768, total)+ indices = [round(i * (total - 1) / max(num_frames - 1, 1)) for i in range(num_frames)]+ if len(indices) % 2 == 1:+ indices.append(indices[-1])+ height, width = comfy.text_encoders.qwen_vl.qwen2vl_video_smart_resize(num_frames, frames.shape[1], frames.shape[2])+ text = ""+ embeds = []+ for i in range(0, len(indices), 2):+ seconds = (indices[i] + indices[i + 1]) / (2 * fps)+ text += f"<{seconds:.1f} seconds><|vision_start|><|video_pad|><|vision_end|>"+ embeds.append({"type": "video", "data": frames[indices[i:i + 2]], "size": (height, width), "original_type": "video"})+ return text, embeds++ def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], videos=[], prevent_empty_text=False, thinking=False, system_prompt="", **kwargs):image = kwargs.get("image", None)if image is not None and len(images) == 0:images = [image[i:i + 1] for i in range(image.shape[0])]+ video = kwargs.get("video", None)+ if video is not None and len(videos) == 0:+ videos = [video]+ video_text = ""+ video_embeds = []+ for v in videos:+ t, e = self.video_embeds(v, kwargs.get("fps", 24))+ video_text += t+ video_embeds += eskip_template = Falseif text.startswith('<|im_start|>'):@@ -1227,13 +1257,12 @@ class Qwen35ImageTokenizer(sd1_clip.SD1Tokenizer):else:if llama_template is not None:template = llama_template- elif len(images) == 0:+ elif len(images) == 0 and len(video_embeds) == 0:template = self.llama_templateelse:template = self.llama_template_images- if len(images) > 1:- vision_block = "<|vision_start|><|image_pad|><|vision_end|>"- template = template.replace(vision_block, vision_block * len(images), 1)+ vision_block = "<|vision_start|><|image_pad|><|vision_end|>"+ template = template.replace(vision_block, vision_block * len(images) + video_text, 1)llama_text = template.format(text)if system_prompt:llama_text = "<|im_start|>system\n" + system_prompt + "<|im_end|>\n" + llama_text@@ -1243,6 +1272,7 @@ class Qwen35ImageTokenizer(sd1_clip.SD1Tokenizer):tokens = super().tokenize_with_weights(llama_text, return_word_ids=return_word_ids, disable_weights=True, **kwargs)key_name = next(iter(tokens))embed_count = 0+ video_count = 0qwen_tokens = tokens[key_name]for r in qwen_tokens:for i in range(len(r)):@@ -1250,6 +1280,10 @@ class Qwen35ImageTokenizer(sd1_clip.SD1Tokenizer):if len(images) > embed_count:r[i] = ({"type": "image", "data": images[embed_count], "original_type": "image"},) + r[i][1:]embed_count += 1+ elif isinstance(r[i][0], (int, float)) and r[i][0] == 248057: # <|video_pad|>+ if len(video_embeds) > video_count:+ r[i] = (video_embeds[video_count],) + r[i][1:]+ video_count += 1return tokensdiff --git a/comfy/text_encoders/qwen_vl.py b/comfy/text_encoders/qwen_vl.pyindex 818b149e6..e68f993d6 100644--- a/comfy/text_encoders/qwen_vl.py+++ b/comfy/text_encoders/qwen_vl.py@@ -24,12 +24,9 @@ def process_qwen2vl_images(images = images[..., :3]batch_size, height, width, channels = images.shape- device = images.device- # dtype = images.dtypeimages = images.permute(0, 3, 1, 2)- grid_thw_list = []img = images[0]factor = patch_size * merge_size@@ -56,17 +53,40 @@ def process_qwen2vl_images(for c in range(3):normalized[c] = (img_resized[c] - image_mean[c]) / image_std[c]- grid_h = h_bar // patch_size- grid_w = w_bar // patch_size- grid_thw = torch.tensor([1, grid_h, grid_w], device=device, dtype=torch.long)+ return qwen2vl_patchify(normalized.unsqueeze(0).repeat(temporal_patch_size, 1, 1, 1), patch_size, temporal_patch_size, merge_size)- pixel_values = normalized- grid_thw_list.append(grid_thw)- image_grid_thw = torch.stack(grid_thw_list)- grid_t = 1- channel = pixel_values.shape[0]- pixel_values = pixel_values.unsqueeze(0).repeat(2, 1, 1, 1)+def qwen2vl_video_smart_resize(num_frames, height, width, factor=32, temporal_factor=2, min_pixels=128 * 128, max_pixels=32 * 32 * 768):+ # Qwen3-VL video budget: total t*h*w pixels, t rounded up to the temporal patch.+ h_bar = round(height / factor) * factor+ w_bar = round(width / factor) * factor+ t_bar = math.ceil(num_frames / temporal_factor) * temporal_factor+ if t_bar * h_bar * w_bar > max_pixels:+ beta = math.sqrt((num_frames * height * width) / max_pixels)+ h_bar = max(factor, math.floor(height / beta / factor) * factor)+ w_bar = max(factor, math.floor(width / beta / factor) * factor)+ elif t_bar * h_bar * w_bar < min_pixels:+ beta = math.sqrt(min_pixels / (num_frames * height * width))+ h_bar = math.ceil(height * beta / factor) * factor+ w_bar = math.ceil(width * beta / factor) * factor+ return h_bar, w_bar+++def process_qwen2vl_video(frames, height, width, patch_size=14, temporal_patch_size=2, merge_size=2, image_mean=None, image_std=None, interpolation="bilinear"):+ # frames: [T, H, W, C] in [0, 1], T a multiple of temporal_patch_size, already sized by qwen2vl_video_smart_resize.+ frames = frames[..., :3].permute(0, 3, 1, 2)+ frames = F.interpolate(frames, size=(height, width), mode=interpolation, align_corners=False)+ mean = torch.tensor(image_mean, device=frames.device, dtype=frames.dtype).view(1, 3, 1, 1)+ std = torch.tensor(image_std, device=frames.device, dtype=frames.dtype).view(1, 3, 1, 1)+ return qwen2vl_patchify((frames - mean) / std, patch_size, temporal_patch_size, merge_size)+++def qwen2vl_patchify(pixel_values, patch_size, temporal_patch_size, merge_size):+ # pixel_values: [T, C, H, W] -> (flattened patches, grid_thw [1, 3])+ grid_t = pixel_values.shape[0] // temporal_patch_size+ channel = pixel_values.shape[1]+ grid_h = pixel_values.shape[2] // patch_size+ grid_w = pixel_values.shape[3] // patch_sizepatches = pixel_values.reshape(grid_t,@@ -86,16 +106,17 @@ def process_qwen2vl_images(channel * temporal_patch_size * patch_size * patch_size)- return flatten_patches, image_grid_thw+ return flatten_patches, torch.tensor([[grid_t, grid_h, grid_w]], device=pixel_values.device, dtype=torch.long)def qwen2vl_mrope_position_ids(embeds_info, seq_len, device):# (3, seq_len) T/H/W MRoPE position ids: text runs sequentially, each image span gets its grid positions.+ # Video frame pairs arrive as separate t=1 spans split by their timestamp text, so they index like images.# Returns None when there are no image embeds. `extra` is the image grid_thw, or a dict carrying it under "grid".position_ids = Noneoffset = 0for e in embeds_info:- if e.get("type") == "image":+ if e.get("type") in ("image", "video"):extra = e.get("extra", None)grid = extra["grid"] if isinstance(extra, dict) else extrastart = e.get("index")diff --git a/comfy_extras/nodes_textgen.py b/comfy_extras/nodes_textgen.pyindex 832346a09..8d1368c29 100644--- a/comfy_extras/nodes_textgen.py+++ b/comfy_extras/nodes_textgen.py@@ -1,7 +1,7 @@from comfy_api.latest import ComfyExtension, iofrom typing_extensions import override-VIDEO_TOOLTIP = "Video frames as image batch. Assumed to be 24 FPS; subsampled to 1 FPS internally."+VIDEO_TOOLTIP = "Video frames as image batch. Assumed to be 24 FPS; subsampled to the model's frame rate internally."class TextGenerateV2(io.ComfyNode):@classmethod