The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add SeedVR2 support (CORE-6) (#14424)
John Pollock8e2e54e2b8fd
diff --git a/comfy/latent_formats.py b/comfy/latent_formats.pyindex bbdfd4bc2..8a16cfe55 100644--- a/comfy/latent_formats.py+++ b/comfy/latent_formats.py@@ -779,6 +779,10 @@ class ACEAudio(LatentFormat):latent_channels = 8latent_dimensions = 2+class SeedVR2(LatentFormat):+ latent_channels = 16+ latent_dimensions = 3+class ACEAudio15(LatentFormat):latent_channels = 64latent_dimensions = 1diff --git a/comfy/ldm/modules/diffusionmodules/model.py b/comfy/ldm/modules/diffusionmodules/model.pyindex fcbaa074f..e752d0ecb 100644--- a/comfy/ldm/modules/diffusionmodules/model.py+++ b/comfy/ldm/modules/diffusionmodules/model.py@@ -22,7 +22,7 @@ def torch_cat_if_needed(xl, dim):else:return None-def get_timestep_embedding(timesteps, embedding_dim):+def get_timestep_embedding(timesteps, embedding_dim, flip_sin_to_cos=False, downscale_freq_shift=1):"""This matches the implementation in Denoising Diffusion Probabilistic Models:From Fairseq.@@ -33,11 +33,13 @@ def get_timestep_embedding(timesteps, embedding_dim):assert len(timesteps.shape) == 1half_dim = embedding_dim // 2- emb = math.log(10000) / (half_dim - 1)+ emb = math.log(10000) / (half_dim - downscale_freq_shift)emb = torch.exp(torch.arange(half_dim, dtype=torch.float32) * -emb)emb = emb.to(device=timesteps.device)emb = timesteps.float()[:, None] * emb[None, :]emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)+ if flip_sin_to_cos:+ emb = torch.cat([emb[:, half_dim:], emb[:, :half_dim]], dim=-1)if embedding_dim % 2 == 1: # zero pademb = torch.nn.functional.pad(emb, (0,1,0,0))return embdiff --git a/comfy/ldm/seedvr/attention.py b/comfy/ldm/seedvr/attention.pynew file mode 100644index 000000000..11b4c1e4a--- /dev/null+++ b/comfy/ldm/seedvr/attention.py@@ -0,0 +1,51 @@+import torch++from comfy.ldm.modules import attention as _attention+++def _var_attention_qkv(q, k, v, heads, skip_reshape):+ if skip_reshape:+ return q, k, v, q.shape[-1]+ total_tokens, embed_dim = q.shape+ head_dim = embed_dim // heads+ return (+ q.view(total_tokens, heads, head_dim),+ k.view(k.shape[0], heads, head_dim),+ v.view(v.shape[0], heads, head_dim),+ head_dim,+ )+++def _var_attention_output(out, heads, head_dim, skip_output_reshape):+ if skip_output_reshape:+ return out+ return out.reshape(-1, heads * head_dim)+++def var_attention_optimized_split(q, k, v, heads, cu_seqlens_q, cu_seqlens_k, *args, skip_reshape=False, skip_output_reshape=False, **kwargs):+ q, k, v, head_dim = _var_attention_qkv(q, k, v, heads, skip_reshape)++ q_split_indices = cu_seqlens_q[1:-1]+ k_split_indices = cu_seqlens_k[1:-1]+ if k.shape[0] != v.shape[0]:+ raise ValueError("cu_seqlens_k does not match v token count")++ q_splits = torch.tensor_split(q, q_split_indices, dim=0)+ k_splits = torch.tensor_split(k, k_split_indices, dim=0)+ v_splits = torch.tensor_split(v, k_split_indices, dim=0)+ if len(q_splits) != len(k_splits) or len(q_splits) != len(v_splits):+ raise ValueError("cu_seqlens_q and cu_seqlens_k must describe the same sequence count")++ out = []+ for q_i, k_i, v_i in zip(q_splits, k_splits, v_splits):+ q_i = q_i.permute(1, 0, 2).unsqueeze(0)+ k_i = k_i.permute(1, 0, 2).unsqueeze(0)+ v_i = v_i.permute(1, 0, 2).unsqueeze(0)+ out_i = _attention.optimized_attention(q_i, k_i, v_i, heads, skip_reshape=True, skip_output_reshape=True)+ out.append(out_i.squeeze(0).permute(1, 0, 2))++ out = torch.cat(out, dim=0)+ return _var_attention_output(out, heads, head_dim, skip_output_reshape)+++optimized_var_attention = var_attention_optimized_splitdiff --git a/comfy/ldm/seedvr/color_fix.py b/comfy/ldm/seedvr/color_fix.pynew file mode 100644index 000000000..a43cb5270--- /dev/null+++ b/comfy/ldm/seedvr/color_fix.py@@ -0,0 +1,301 @@+import torch+import torch.nn.functional as F+from torch import Tensor++from comfy.ldm.seedvr.constants import (+ CIELAB_DELTA,+ CIELAB_KAPPA,+ D65_WHITE_X,+ D65_WHITE_Z,+ WAVELET_DECOMP_LEVELS,+)+++def wavelet_blur(image: Tensor, radius):+ max_safe_radius = max(1, min(image.shape[-2:]) // 8)+ if radius > max_safe_radius:+ radius = max_safe_radius++ num_channels = image.shape[1]++ kernel_vals = [+ [0.0625, 0.125, 0.0625],+ [0.125, 0.25, 0.125],+ [0.0625, 0.125, 0.0625],+ ]+ kernel = torch.tensor(kernel_vals, dtype=image.dtype, device=image.device)+ kernel = kernel[None, None].repeat(num_channels, 1, 1, 1)++ image = F.pad(image, (radius, radius, radius, radius), mode='replicate')+ output = F.conv2d(image, kernel, groups=num_channels, dilation=radius)++ return output++def wavelet_decomposition(image: Tensor, levels: int = WAVELET_DECOMP_LEVELS):+ high_freq = torch.zeros_like(image)++ for i in range(levels):+ radius = 2 ** i+ low_freq = wavelet_blur(image, radius)+ high_freq.add_(image).sub_(low_freq)+ image = low_freq++ return high_freq, low_freq++def wavelet_reconstruction(content_feat: Tensor, style_feat: Tensor) -> Tensor:++ if content_feat.shape != style_feat.shape:+ if len(content_feat.shape) >= 3:+ style_feat = F.interpolate(+ style_feat,+ size=content_feat.shape[-2:],+ mode='bilinear',+ align_corners=False+ )++ content_high_freq, content_low_freq = wavelet_decomposition(content_feat)+ del content_low_freq++ style_high_freq, style_low_freq = wavelet_decomposition(style_feat)+ del style_high_freq++ if content_high_freq.shape != style_low_freq.shape:+ style_low_freq = F.interpolate(+ style_low_freq,+ size=content_high_freq.shape[-2:],+ mode='bilinear',+ align_corners=False+ )++ content_high_freq.add_(style_low_freq)++ return content_high_freq.clamp_(-1.0, 1.0)++def _histogram_matching_channel(source: Tensor, reference: Tensor) -> Tensor:+ original_shape = source.shape++ source_flat = source.flatten()+ reference_flat = reference.flatten()++ source_sorted, source_indices = torch.sort(source_flat)+ reference_sorted, _ = torch.sort(reference_flat)+ del reference_flat++ n_source = len(source_sorted)+ n_reference = len(reference_sorted)++ if n_source == n_reference:+ matched_sorted = reference_sorted+ else:+ source_quantiles = torch.linspace(0, 1, n_source, device=source.device)+ ref_indices = (source_quantiles * (n_reference - 1)).long()+ ref_indices.clamp_(0, n_reference - 1)+ matched_sorted = reference_sorted[ref_indices]+ del source_quantiles, ref_indices, reference_sorted++ del source_sorted, source_flat++ inverse_indices = torch.argsort(source_indices)+ del source_indices+ matched_flat = matched_sorted[inverse_indices]+ del matched_sorted, inverse_indices++ return matched_flat.reshape(original_shape)++def _lab_to_rgb_batch(lab: Tensor, matrix_inv: Tensor, epsilon: float, kappa: float) -> Tensor:+ L, a, b = lab[:, 0], lab[:, 1], lab[:, 2]++ fy = (L + 16.0) / 116.0+ fx = a.div(500.0).add_(fy)+ fz = fy - b / 200.0+ del L, a, b++ x = torch.where(+ fx > epsilon,+ torch.pow(fx, 3.0),+ fx.mul(116.0).sub_(16.0).div_(kappa)+ )+ y = torch.where(+ fy > epsilon,+ torch.pow(fy, 3.0),+ fy.mul(116.0).sub_(16.0).div_(kappa)+ )+ z = torch.where(+ fz > epsilon,+ torch.pow(fz, 3.0),+ fz.mul(116.0).sub_(16.0).div_(kappa)+ )+ del fx, fy, fz++ x.mul_(D65_WHITE_X)+ z.mul_(D65_WHITE_Z)++ xyz = torch.stack([x, y, z], dim=1)+ del x, y, z++ B, _, H, W = xyz.shape+ xyz_flat = xyz.permute(0, 2, 3, 1).reshape(-1, 3)+ del xyz++ xyz_flat = xyz_flat.to(dtype=matrix_inv.dtype)+ rgb_linear_flat = torch.matmul(xyz_flat, matrix_inv.T)+ del xyz_flat++ rgb_linear = rgb_linear_flat.reshape(B, H, W, 3).permute(0, 3, 1, 2)+ del rgb_linear_flat++ mask = rgb_linear > 0.0031308+ rgb = torch.where(+ mask,+ torch.pow(torch.clamp(rgb_linear, min=0.0), 1.0 / 2.4).mul_(1.055).sub_(0.055),+ rgb_linear * 12.92+ )+ del mask, rgb_linear++ return torch.clamp(rgb, 0.0, 1.0)++def _rgb_to_lab_batch(rgb: Tensor, matrix: Tensor, epsilon: float, kappa: float) -> Tensor:+ mask = rgb > 0.04045+ rgb_linear = torch.where(+ mask,+ torch.pow((rgb + 0.055) / 1.055, 2.4),+ rgb / 12.92+ )+ del mask++ B, _, H, W = rgb_linear.shape+ rgb_flat = rgb_linear.permute(0, 2, 3, 1).reshape(-1, 3)+ del rgb_linear++ rgb_flat = rgb_flat.to(dtype=matrix.dtype)+ xyz_flat = torch.matmul(rgb_flat, matrix.T)+ del rgb_flat++ xyz = xyz_flat.reshape(B, H, W, 3).permute(0, 3, 1, 2)+ del xyz_flat++ xyz[:, 0].div_(D65_WHITE_X)+ xyz[:, 2].div_(D65_WHITE_Z)++ epsilon_cubed = epsilon ** 3+ mask = xyz > epsilon_cubed+ f_xyz = torch.where(+ mask,+ torch.pow(xyz, 1.0 / 3.0),+ xyz.mul(kappa).add_(16.0).div_(116.0)+ )+ del xyz, mask++ L = f_xyz[:, 1].mul(116.0).sub_(16.0)+ a = (f_xyz[:, 0] - f_xyz[:, 1]).mul_(500.0)+ b = (f_xyz[:, 1] - f_xyz[:, 2]).mul_(200.0)+ del f_xyz++ return torch.stack([L, a, b], dim=1)++def lab_color_transfer(+ content_feat: Tensor,+ style_feat: Tensor,+ luminance_weight: float = 0.8+) -> Tensor:+ content_feat = wavelet_reconstruction(content_feat, style_feat)++ if content_feat.shape != style_feat.shape:+ style_feat = F.interpolate(+ style_feat,+ size=content_feat.shape[-2:],+ mode='bilinear',+ align_corners=False+ )++ device = content_feat.device+ original_dtype = content_feat.dtype+ content_feat = content_feat.float()+ style_feat = style_feat.float()++ rgb_to_xyz_matrix = torch.tensor([+ [0.4124564, 0.3575761, 0.1804375],+ [0.2126729, 0.7151522, 0.0721750],+ [0.0193339, 0.1191920, 0.9503041]+ ], dtype=torch.float32, device=device)++ xyz_to_rgb_matrix = torch.tensor([+ [ 3.2404542, -1.5371385, -0.4985314],+ [-0.9692660, 1.8760108, 0.0415560],+ [ 0.0556434, -0.2040259, 1.0572252]+ ], dtype=torch.float32, device=device)++ epsilon = CIELAB_DELTA+ kappa = CIELAB_KAPPA++ content_feat.add_(1.0).mul_(0.5).clamp_(0.0, 1.0)+ style_feat.add_(1.0).mul_(0.5).clamp_(0.0, 1.0)++ content_lab = _rgb_to_lab_batch(content_feat, rgb_to_xyz_matrix, epsilon, kappa)+ del content_feat++ style_lab = _rgb_to_lab_batch(style_feat, rgb_to_xyz_matrix, epsilon, kappa)+ del style_feat, rgb_to_xyz_matrix++ matched_a = _histogram_matching_channel(content_lab[:, 1], style_lab[:, 1])+ matched_b = _histogram_matching_channel(content_lab[:, 2], style_lab[:, 2])++ if luminance_weight < 1.0:+ matched_L = _histogram_matching_channel(content_lab[:, 0], style_lab[:, 0])+ result_L = content_lab[:, 0].mul(luminance_weight).add_(matched_L.mul(1.0 - luminance_weight))+ del matched_L+ else:+ result_L = content_lab[:, 0]++ del content_lab, style_lab++ result_lab = torch.stack([result_L, matched_a, matched_b], dim=1)+ del result_L, matched_a, matched_b++ result_rgb = _lab_to_rgb_batch(result_lab, xyz_to_rgb_matrix, epsilon, kappa)+ del result_lab, xyz_to_rgb_matrix++ result = result_rgb.mul_(2.0).sub_(1.0)+ del result_rgb++ result = result.to(original_dtype)++ return result+++def wavelet_color_transfer(content_feat: Tensor, style_feat: Tensor) -> Tensor:+ return wavelet_reconstruction(content_feat, style_feat)+++def adain_color_transfer(content_feat: Tensor, style_feat: Tensor, eps: float = 1e-5) -> Tensor:+ if content_feat.shape != style_feat.shape:+ style_feat = F.interpolate(+ style_feat,+ size=content_feat.shape[-2:],+ mode='bilinear',+ align_corners=False,+ )++ original_dtype = content_feat.dtype+ content_feat = content_feat.float()+ style_feat = style_feat.float()++ b, c = content_feat.shape[:2]+ content_flat = content_feat.reshape(b, c, -1)+ style_flat = style_feat.reshape(b, c, -1)++ content_mean = content_flat.mean(dim=2).reshape(b, c, 1, 1)+ content_std = (content_flat.var(dim=2, correction=0) + eps).sqrt().reshape(b, c, 1, 1)+ style_mean = style_flat.mean(dim=2).reshape(b, c, 1, 1)+ style_std = (style_flat.var(dim=2, correction=0) + eps).sqrt().reshape(b, c, 1, 1)+ del content_flat, style_flat++ normalized = (content_feat - content_mean) / content_std+ del content_mean, content_std+ result = normalized * style_std + style_mean+ del normalized, style_mean, style_std++ result = result.clamp_(-1.0, 1.0)+ if result.dtype != original_dtype:+ result = result.to(original_dtype)+ return resultdiff --git a/comfy/ldm/seedvr/constants.py b/comfy/ldm/seedvr/constants.pynew file mode 100644index 000000000..12c4b4bef--- /dev/null+++ b/comfy/ldm/seedvr/constants.py@@ -0,0 +1,48 @@+"""SeedVR2 constants."""++# Temporal chunk-size law: the sampler's activation wall is linear in+# T_latent * pixel area (17-cell resolution sweep + T bisection, RTX 5090, 3b fp16):+# max_latent_frames = (free_GiB - RESERVED - K*SIGMA) / (GIB_PER_MPX_FRAME * megapixels)+# RESERVED covers model staging plus fixed CUDA/torch overhead; SIGMA is the measured+# run-to-run spread of the wall; K=4 trades ~10% smaller chunks for ~1e-5 OOM odds.+SEEDVR2_CHUNK_GIB_PER_MPX_FRAME = 0.55+SEEDVR2_CHUNK_RESERVED_GIB = 8.5+SEEDVR2_CHUNK_SIGMA_GIB = 0.55+SEEDVR2_CHUNK_SIGMA_K = 4++SEEDVR2_7B_VID_DIM = 3072+SEEDVR2_OOM_BACKOFF_DIVISOR = 2+SEEDVR2_DTYPE_BYTES_FLOOR = 4+SEEDVR2_7B_MLP_CHUNK = 8192+SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS = 4096 # partial-RoPE application token-chunk.+SEEDVR2_LATENT_CHANNELS = 16++SEEDVR2_COLOR_MEM_HEADROOM = 0.75+SEEDVR2_LAB_SCALE_MULTIPLIER = 13+SEEDVR2_WAVELET_SCALE_MULTIPLIER = 10 # per-frame byte multiplier, wavelet path.+SEEDVR2_ADAIN_SCALE_MULTIPLIER = 6++BYTEDANCE_VAE_SCALING_FACTOR = 0.9152 # configs_3b/main.yaml:57.+BYTEDANCE_VAE_SHIFTING_FACTOR = 0.0+BYTEDANCE_VAE_CONV_MEM_GIB = 0.5+BYTEDANCE_VAE_NORM_MEM_GIB = 0.5+BYTEDANCE_LOGVAR_CLAMP_MIN = -30.0 # video_vae_v3/modules/types.py:28.+BYTEDANCE_LOGVAR_CLAMP_MAX = 20.0 # video_vae_v3/modules/types.py:28.+BYTEDANCE_GN_CHUNKS_FP16 = 4 # causal_inflation_lib.py:351 (GroupNorm chunk count, fp16).+BYTEDANCE_GN_CHUNKS_FP32 = 2 # causal_inflation_lib.py:351 (GroupNorm chunk count, fp32).+BYTEDANCE_BLOCK_OUT_CHANNELS = (128, 256, 512, 512) # s8_c16_t4_inflation_sd3.yaml:7-11.+BYTEDANCE_SLICING_SAMPLE_MIN = 4 # s8_c16_t4_inflation_sd3.yaml:22 (slicing_sample_min_size).+BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE = 4 # infer.py:230 (temporal_downsample_factor); the 4n+1 factor.+BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE = 8 # infer.py:231 (spatial_downsample_factor).+BYTEDANCE_720P_REF_AREA = 45 * 80 # dit_v2/window.py:32 (720p reference area for window scaling).+BYTEDANCE_MAX_TEMPORAL_WINDOW = 30 # dit_v2/window.py:35 (max temporal window frames).+BYTEDANCE_ROPE_MAX_FREQ = 256 # dit_v2/rope.py:31 (pixel-RoPE max frequency).+BYTEDANCE_SINUSOIDAL_DIM = 256 # dit_3b/nadit.py:120 (timestep sinusoidal embed dim).++ROPE_THETA = 10000 # RoPE base; Su et al., "RoFormer", arXiv:2104.09864.++CIELAB_DELTA = 6.0 / 29.0 # CIE 15 (delta).+CIELAB_KAPPA = (29.0 / 3.0) ** 3 # CIE 15 (kappa).+D65_WHITE_X = 0.95047 # CIE D65 standard illuminant Xn (Yn = 1).+D65_WHITE_Z = 1.08883 # CIE D65 standard illuminant Zn.+WAVELET_DECOMP_LEVELS = 5 # wavelet color-fix decomposition depth (GIMP/Krita; StableSR).diff --git a/comfy/ldm/seedvr/model.py b/comfy/ldm/seedvr/model.pynew file mode 100644index 000000000..a978698d5--- /dev/null+++ b/comfy/ldm/seedvr/model.py@@ -0,0 +1,1361 @@+from dataclasses import dataclass+from typing import Optional, Tuple, Union, List, Dict, Any, Callable+import torch.nn.functional as F+from math import ceil, pi+import torch+from itertools import accumulate, chain+from comfy.ldm.modules.diffusionmodules.model import get_timestep_embedding+from comfy.ldm.seedvr.attention import optimized_var_attention+from torch.nn.modules.utils import _triple+from torch import nn+import math+from comfy.ldm.flux.math import apply_rope1+from comfy.ldm.seedvr.constants import (+ BYTEDANCE_720P_REF_AREA,+ BYTEDANCE_MAX_TEMPORAL_WINDOW,+ BYTEDANCE_ROPE_MAX_FREQ,+ BYTEDANCE_SINUSOIDAL_DIM,+ ROPE_THETA,+ SEEDVR2_7B_MLP_CHUNK,+ SEEDVR2_7B_VID_DIM,+ SEEDVR2_LATENT_CHANNELS,+ SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS,+)+import comfy.model_management+import comfy.ops++class Cache:+ def __init__(self, disable=False, prefix="", cache=None):+ self.cache = cache if cache is not None else {}+ self.disable = disable+ self.prefix = prefix++ def __call__(self, key: str, fn: Callable):+ if self.disable:+ return fn()++ key = self.prefix + key+ if key not in self.cache:+ result = fn()+ self.cache[key] = result+ return self.cache[key]++ def namespace(self, namespace: str):+ return Cache(+ disable=self.disable,+ prefix=self.prefix + namespace + ".",+ cache=self.cache,+ )++def repeat_concat(+ vid: torch.FloatTensor, # (VL ... c)+ txt: torch.FloatTensor, # (TL ... c)+ vid_len: torch.LongTensor, # (n*b)+ txt_len: torch.LongTensor, # (b)+ txt_repeat: List, # (n)+) -> torch.FloatTensor: # (L ... c)+ vid = torch.split(vid, vid_len.tolist())+ txt = torch.split(txt, txt_len.tolist())+ txt = [[x] * n for x, n in zip(txt, txt_repeat)]+ txt = list(chain(*txt))+ return torch.cat(list(chain(*zip(vid, txt))))++def repeat_concat_idx(+ vid_len: torch.LongTensor, # (n*b)+ txt_len: torch.LongTensor, # (b)+ txt_repeat: torch.LongTensor, # (n)+) -> Tuple[+ Callable,+ Callable,+]:+ device = vid_len.device+ vid_idx = torch.arange(vid_len.sum(), device=device)+ txt_idx = torch.arange(len(vid_idx), len(vid_idx) + txt_len.sum(), device=device)+ txt_repeat_list = txt_repeat.tolist()+ tgt_idx = repeat_concat(vid_idx, txt_idx, vid_len, txt_len, txt_repeat_list)+ src_idx = torch.argsort(tgt_idx)+ txt_idx_len = len(tgt_idx) - len(vid_idx)+ repeat_txt_len = (txt_len * txt_repeat).tolist()++ def unconcat_coalesce(all):+ vid_out, txt_out = all[src_idx].split([len(vid_idx), txt_idx_len])+ txt_out_coalesced = []+ for txt, repeat_time in zip(txt_out.split(repeat_txt_len), txt_repeat_list):+ txt = txt.reshape(-1, repeat_time, *txt.shape[1:]).mean(1)+ txt_out_coalesced.append(txt)+ return vid_out, torch.cat(txt_out_coalesced)++ return (+ lambda vid, txt: torch.cat([vid, txt])[tgt_idx],+ lambda all: unconcat_coalesce(all),+ )++def cumulative_lengths(lengths):+ return [0, *accumulate(lengths)]+++@dataclass+class MMArg:+ vid: Any+ txt: Any++def get_args(key: str, args: List[Any]) -> List[Any]:+ return [getattr(v, key) if isinstance(v, MMArg) else v for v in args]+++def get_kwargs(key: str, kwargs: Dict[str, Any]) -> Dict[str, Any]:+ return {k: getattr(v, key) if isinstance(v, MMArg) else v for k, v in kwargs.items()}+++def get_window_op(name: str):+ if name == "720pwin_by_size_bysize":+ return make_720Pwindows_bysize+ if name == "720pswin_by_size_bysize":+ return make_shifted_720Pwindows_bysize+ raise ValueError(f"Unknown windowing method: {name}")+++def make_720Pwindows_bysize(size: Tuple[int, int, int], num_windows: Tuple[int, int, int]):+ t, h, w = size+ resized_nt, resized_nh, resized_nw = num_windows+ scale = math.sqrt(BYTEDANCE_720P_REF_AREA / (h * w))+ resized_h, resized_w = round(h * scale), round(w * scale)+ wh, ww = ceil(resized_h / resized_nh), ceil(resized_w / resized_nw)+ wt = ceil(min(t, BYTEDANCE_MAX_TEMPORAL_WINDOW) / resized_nt)+ nt, nh, nw = ceil(t / wt), ceil(h / wh), ceil(w / ww)+ return [+ (+ slice(it * wt, min((it + 1) * wt, t)),+ slice(ih * wh, min((ih + 1) * wh, h)),+ slice(iw * ww, min((iw + 1) * ww, w)),+ )+ for iw in range(nw)+ if min((iw + 1) * ww, w) > iw * ww+ for ih in range(nh)+ if min((ih + 1) * wh, h) > ih * wh+ for it in range(nt)+ if min((it + 1) * wt, t) > it * wt+ ]++def make_shifted_720Pwindows_bysize(size: Tuple[int, int, int], num_windows: Tuple[int, int, int]):+ t, h, w = size+ resized_nt, resized_nh, resized_nw = num_windows+ scale = math.sqrt(BYTEDANCE_720P_REF_AREA / (h * w))+ resized_h, resized_w = round(h * scale), round(w * scale)+ wh, ww = ceil(resized_h / resized_nh), ceil(resized_w / resized_nw)+ wt = ceil(min(t, BYTEDANCE_MAX_TEMPORAL_WINDOW) / resized_nt)++ st, sh, sw = (+ 0.5 if wt < t else 0,+ 0.5 if wh < h else 0,+ 0.5 if ww < w else 0,+ )+ nt, nh, nw = ceil((t - st) / wt), ceil((h - sh) / wh), ceil((w - sw) / ww)+ nt, nh, nw = (+ nt + 1 if st > 0 else 1,+ nh + 1 if sh > 0 else 1,+ nw + 1 if sw > 0 else 1,+ )+ return [+ (+ slice(max(int((it - st) * wt), 0), min(int((it - st + 1) * wt), t)),+ slice(max(int((ih - sh) * wh), 0), min(int((ih - sh + 1) * wh), h)),+ slice(max(int((iw - sw) * ww), 0), min(int((iw - sw + 1) * ww), w)),+ )+ for iw in range(nw)+ if min(int((iw - sw + 1) * ww), w) > max(int((iw - sw) * ww), 0)+ for ih in range(nh)+ if min(int((ih - sh + 1) * wh), h) > max(int((ih - sh) * wh), 0)+ for it in range(nt)+ if min(int((it - st + 1) * wt), t) > max(int((it - st) * wt), 0)+ ]++class RotaryEmbedding(nn.Module):+ def __init__(+ self,+ dim,+ freqs_for = 'lang',+ theta = 10000,+ max_freq = 10,+ ):+ super().__init__()++ self.freqs_for = freqs_for++ if freqs_for == 'lang':+ freqs = 1. / (theta ** (torch.arange(0, dim, 2)[:(dim // 2)].float() / dim))+ elif freqs_for == 'pixel':+ freqs = torch.linspace(1., max_freq / 2, dim // 2) * pi+ else:+ raise ValueError(f"Unknown rotary frequency type: {freqs_for}")++ self.register_buffer("freqs", freqs)++ @property+ def device(self):+ return self.freqs.device++ def get_axial_freqs(+ self,+ *dims,+ offsets = None+ ):+ Colon = slice(None)+ all_freqs = []++ if exists(offsets):+ if len(offsets) != len(dims):+ raise ValueError(f"SeedVR2 rotary offsets length must match dims length, got {len(offsets)} and {len(dims)}.")++ for ind, dim in enumerate(dims):++ offset = 0+ if exists(offsets):+ offset = offsets[ind]++ if self.freqs_for == 'pixel':+ pos = torch.linspace(-1, 1, steps = dim, device = self.device)+ else:+ pos = torch.arange(dim, device = self.device)++ pos = pos + offset++ freqs = self.forward(pos)++ all_axis = [None] * len(dims)+ all_axis[ind] = Colon++ new_axis_slice = (Ellipsis, *all_axis, Colon)+ all_freqs.append(freqs[new_axis_slice])++ all_freqs = torch.broadcast_tensors(*all_freqs)+ return torch.cat(all_freqs, dim = -1)++ def forward(+ self,+ t,+ ):+ freqs = self.freqs++ freqs = torch.einsum('..., f -> ... f', t.type(freqs.dtype), freqs)+ freqs = freqs.unsqueeze(-1).expand(*freqs.shape, 2).flatten(-2)++ return freqs++class RotaryEmbeddingBase(nn.Module):+ def __init__(self, dim: int, rope_dim: int):+ super().__init__()+ self.rope = RotaryEmbedding(+ dim=dim // rope_dim,+ freqs_for="pixel",+ max_freq=BYTEDANCE_ROPE_MAX_FREQ,+ )++ def get_axial_freqs(self, *dims):+ return self.rope.get_axial_freqs(*dims)+++class RotaryEmbedding3d(RotaryEmbeddingBase):+ def __init__(self, dim: int):+ super().__init__(dim, rope_dim=3)+ self.mm = False+++class NaRotaryEmbedding3d(RotaryEmbedding3d):+ def forward(+ self,+ q: torch.FloatTensor,+ k: torch.FloatTensor,+ shape: torch.LongTensor,+ cache: Cache,+ ) -> Tuple[+ torch.FloatTensor,+ torch.FloatTensor,+ ]:+ freqs = cache("rope_freqs_3d", lambda: self.get_freqs(shape))+ freqs = freqs.to(device=q.device)+ q = q.transpose(0, 1)+ k = k.transpose(0, 1)+ q = _apply_seedvr2_rotary_emb(freqs, q.float()).to(q.dtype)+ k = _apply_seedvr2_rotary_emb(freqs, k.float()).to(k.dtype)+ q = q.transpose(0, 1)+ k = k.transpose(0, 1)+ return q, k++ @torch._dynamo.disable+ def get_freqs(+ self,+ shape: torch.LongTensor,+ ) -> torch.Tensor:+ # Primary provenance: ByteDance-Seed/SeedVR models/dit/rope.py builds+ # 7B pixel RoPE with the interleaved-angle convention, not Comfy's+ # Flux freqs_cis matrix.+ plain_rope = RotaryEmbedding(+ dim=self.rope.freqs.numel() * 2,+ freqs_for="pixel",+ max_freq=BYTEDANCE_ROPE_MAX_FREQ,+ )+ plain_rope = plain_rope.to(self.rope.device)+ freq_list = []+ for f, h, w in shape.tolist():+ freqs = plain_rope.get_axial_freqs(f, h, w)+ freq_list.append(freqs.view(-1, freqs.size(-1)))+ return torch.cat(freq_list, dim=0)+++class MMRotaryEmbeddingBase(RotaryEmbeddingBase):+ def __init__(self, dim: int, rope_dim: int):+ super().__init__(dim, rope_dim)+ self.rope = RotaryEmbedding(+ dim=dim // rope_dim,+ freqs_for="lang",+ theta=ROPE_THETA,+ )+ self.mm = True++def slice_at_dim(t, dim_slice: slice, *, dim):+ dim += (t.ndim if dim < 0 else 0)+ colons = [slice(None)] * t.ndim+ colons[dim] = dim_slice+ return t[tuple(colons)]++def rotate_half(x):+ x = x.reshape(*x.shape[:-1], x.shape[-1] // 2, 2)+ x1, x2 = x.unbind(dim = -1)+ x = torch.stack((-x2, x1), dim = -1)+ return x.flatten(-2)+def exists(val):+ return val is not None++def _apply_seedvr2_rotary_emb(+ freqs: torch.Tensor,+ t: torch.Tensor,+ start_index: int = 0,+ scale: float = 1.0,+ seq_dim: int = -2,+ freqs_seq_dim: int | None = None,+) -> torch.Tensor:+ dtype = t.dtype+ if freqs_seq_dim is None and (freqs.ndim == 2 or t.ndim == 3):+ freqs_seq_dim = 0++ if t.ndim == 3 or freqs_seq_dim is not None:+ seq_len = t.shape[seq_dim]+ freqs = slice_at_dim(freqs, slice(-seq_len, None), dim=freqs_seq_dim)++ rot_feats = freqs.shape[-1]+ end_index = start_index + rot_feats++ t_left = t[..., :start_index]+ t_middle = t[..., start_index:end_index]+ t_right = t[..., end_index:]++ freqs = freqs.to(device=t_middle.device, dtype=t_middle.dtype)+ cos = freqs.cos() * scale+ sin = freqs.sin() * scale+ t_middle = (t_middle * cos) + (rotate_half(t_middle) * sin)+ return torch.cat((t_left, t_middle, t_right), dim=-1).to(dtype)++def _to_flux_freqs_cis(freqs_interleaved: torch.Tensor) -> torch.Tensor:+ angles = freqs_interleaved[..., ::2].float()+ cos = torch.cos(angles)+ sin = torch.sin(angles)+ out = torch.stack([cos, -sin, sin, cos], dim=-1)+ return out.reshape(*out.shape[:-1], 2, 2)+++def _apply_rope1_partial(t: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor:+ out = t.clone() if t.requires_grad or comfy.model_management.in_training else t+ rot_d = 2 * freqs_cis.shape[-3]+ seq_len = out.shape[-2]+ for start in range(0, seq_len, SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS):+ end = min(start + SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS, seq_len)+ freqs_chunk = freqs_cis[start:end]+ if rot_d == out.shape[-1]:+ out[..., start:end, :] = apply_rope1(out[..., start:end, :], freqs_chunk).to(out.dtype)+ else:+ out[..., start:end, :rot_d] = apply_rope1(out[..., start:end, :rot_d], freqs_chunk).to(out.dtype)+ return out+++class NaMMRotaryEmbedding3d(MMRotaryEmbeddingBase):+ def __init__(self, dim: int):+ super().__init__(dim, rope_dim=3)++ def forward(+ self,+ vid_q: torch.FloatTensor, # L h d+ vid_k: torch.FloatTensor, # L h d+ vid_shape: torch.LongTensor, # B 3+ txt_q: torch.FloatTensor, # L h d+ txt_k: torch.FloatTensor, # L h d+ txt_shape: torch.LongTensor, # B 1+ cache: Cache,+ ) -> Tuple[+ torch.FloatTensor,+ torch.FloatTensor,+ torch.FloatTensor,+ torch.FloatTensor,+ ]:+ vid_freqs, txt_freqs = cache(+ "mmrope_freqs_3d",+ lambda: self.get_freqs(vid_shape, txt_shape),+ )+ target_device = vid_q.device+ if vid_freqs.device != target_device:+ vid_freqs = vid_freqs.to(target_device)+ if txt_freqs.device != target_device:+ txt_freqs = txt_freqs.to(target_device)+ vid_q = vid_q.transpose(0, 1)+ vid_k = vid_k.transpose(0, 1)+ vid_q = _apply_rope1_partial(vid_q, vid_freqs)+ vid_k = _apply_rope1_partial(vid_k, vid_freqs)+ vid_q = vid_q.transpose(0, 1)+ vid_k = vid_k.transpose(0, 1)++ txt_q = txt_q.transpose(0, 1)+ txt_k = txt_k.transpose(0, 1)+ txt_q = _apply_rope1_partial(txt_q, txt_freqs)+ txt_k = _apply_rope1_partial(txt_k, txt_freqs)+ txt_q = txt_q.transpose(0, 1)+ txt_k = txt_k.transpose(0, 1)+ return vid_q, vid_k, txt_q, txt_k++ @torch._dynamo.disable # Disable compilation: .tolist() is data-dependent and causes graph breaks+ def get_freqs(+ self,+ vid_shape: torch.LongTensor,+ txt_shape: torch.LongTensor,+ ) -> Tuple[+ torch.Tensor,+ torch.Tensor,+ ]:++ max_temporal = 0+ max_height = 0+ max_width = 0+ max_txt_len = 0++ for (f, h, w), l in zip(vid_shape.tolist(), txt_shape[:, 0].tolist()):+ max_temporal = max(max_temporal, l + f)+ max_height = max(max_height, h)+ max_width = max(max_width, w)+ max_txt_len = max(max_txt_len, l)++ autocast_device = "cuda" if torch.cuda.is_available() else "cpu"+ with torch.amp.autocast(autocast_device, enabled=False):+ vid_freqs = self.get_axial_freqs(+ max_temporal + 16,+ max_height + 4,+ max_width + 4,+ ).float()+ txt_freqs = self.get_axial_freqs(max_txt_len + 16)++ vid_freq_list, txt_freq_list = [], []+ for (f, h, w), l in zip(vid_shape.tolist(), txt_shape[:, 0].tolist()):+ vid_freq = vid_freqs[l : l + f, :h, :w].reshape(-1, vid_freqs.size(-1))+ txt_freq = txt_freqs[:l].repeat(1, 3).reshape(-1, vid_freqs.size(-1))+ vid_freq_list.append(vid_freq)+ txt_freq_list.append(txt_freq)+ vid_freqs_interleaved = torch.cat(vid_freq_list, dim=0)+ txt_freqs_interleaved = torch.cat(txt_freq_list, dim=0)++ return _to_flux_freqs_cis(vid_freqs_interleaved), _to_flux_freqs_cis(txt_freqs_interleaved)++class MMModule(nn.Module):+ def __init__(+ self,+ module: Callable[..., nn.Module],+ *args,+ shared_weights: bool = False,+ vid_only: bool = False,+ **kwargs,+ ):+ super().__init__()+ self.shared_weights = shared_weights+ self.vid_only = vid_only+ if self.shared_weights:+ if get_args("vid", args) != get_args("txt", args):+ raise ValueError("SeedVR2 shared MMModule requires matching vid/txt args.")+ if get_kwargs("vid", kwargs) != get_kwargs("txt", kwargs):+ raise ValueError("SeedVR2 shared MMModule requires matching vid/txt kwargs.")+ self.all = module(*get_args("vid", args), **get_kwargs("vid", kwargs))+ else:+ self.vid = module(*get_args("vid", args), **get_kwargs("vid", kwargs))+ self.txt = (+ module(*get_args("txt", args), **get_kwargs("txt", kwargs))+ if not vid_only+ else None+ )++ def forward(+ self,+ vid: torch.FloatTensor,+ txt: torch.FloatTensor,+ *args,+ **kwargs,+ ) -> Tuple[+ torch.FloatTensor,+ torch.FloatTensor,+ ]:+ vid_module = self.vid if not self.shared_weights else self.all+ vid = vid_module(vid, *get_args("vid", args), **get_kwargs("vid", kwargs))+ if not self.vid_only:+ txt_module = self.txt if not self.shared_weights else self.all+ txt = txt.to(device=vid.device, dtype=vid.dtype)+ txt = txt_module(txt, *get_args("txt", args), **get_kwargs("txt", kwargs))+ return vid, txt++def get_na_rope(rope_type: Optional[str], dim: int):+ if rope_type is None:+ return None+ if rope_type == "rope3d":+ return NaRotaryEmbedding3d(dim=dim)+ if rope_type == "mmrope3d":+ return NaMMRotaryEmbedding3d(dim=dim)+ raise ValueError(f"Unknown SeedVR2 rope type: {rope_type}")++class NaMMAttention(nn.Module):+ def __init__(+ self,+ vid_dim: int,+ txt_dim: int,+ heads: int,+ head_dim: int,+ qk_bias: bool,+ qk_norm,+ qk_norm_eps: float,+ rope_type: Optional[str],+ rope_dim: int,+ shared_weights: bool,+ device, dtype, operations,+ ):+ super().__init__()+ dim = MMArg(vid_dim, txt_dim)+ self.heads = heads+ inner_dim = heads * head_dim+ qkv_dim = inner_dim * 3+ self.head_dim = head_dim+ self.proj_qkv = MMModule(+ operations.Linear, dim, qkv_dim, bias=qk_bias, shared_weights=shared_weights, device=device, dtype=dtype+ )+ self.proj_out = MMModule(operations.Linear, inner_dim, dim, shared_weights=shared_weights, device=device, dtype=dtype)+ self.norm_q = MMModule(+ qk_norm,+ normalized_shape=head_dim,+ eps=qk_norm_eps,+ elementwise_affine=True,+ shared_weights=shared_weights,+ device=device, dtype=dtype+ )+ self.norm_k = MMModule(+ qk_norm,+ normalized_shape=head_dim,+ eps=qk_norm_eps,+ elementwise_affine=True,+ shared_weights=shared_weights,+ device=device, dtype=dtype+ )+++ self.rope = get_na_rope(rope_type=rope_type, dim=rope_dim)++def window(+ hid: torch.FloatTensor, # (L c)+ hid_shape: torch.LongTensor, # (b n)+ window_fn: Callable[[torch.Tensor], List[torch.Tensor]],+):+ hid = unflatten(hid, hid_shape)+ hid = list(map(window_fn, hid))+ hid_windows_list = [len(x) for x in hid]+ hid_windows = torch.as_tensor(hid_windows_list, device=hid_shape.device)+ hid = list(chain(*hid))+ hid_len_list = [math.prod(x.shape[:-1]) for x in hid]+ hid, hid_shape = flatten(hid)+ return hid, hid_shape, hid_windows, hid_len_list, hid_windows_list++def window_idx(+ hid_shape: torch.LongTensor, # (b n)+ window_fn: Callable[[torch.Tensor], List[torch.Tensor]],+):+ hid_idx = torch.arange(hid_shape.prod(-1).sum(), device=hid_shape.device).unsqueeze(-1)+ tgt_idx, tgt_shape, tgt_windows, tgt_len_list, tgt_windows_list = window(hid_idx, hid_shape, window_fn)+ tgt_idx = tgt_idx.squeeze(-1)+ src_idx = torch.argsort(tgt_idx)+ return (+ lambda hid: torch.index_select(hid, 0, tgt_idx),+ lambda hid: torch.index_select(hid, 0, src_idx),+ tgt_shape,+ tgt_windows,+ tgt_len_list,+ tgt_windows_list,+ )++class NaSwinAttention(NaMMAttention):+ def __init__(+ self,+ *args,+ window: Union[int, Tuple[int, int, int]],+ window_method: str,+ version: bool = False,+ **kwargs,+ ):+ super().__init__(*args, **kwargs)+ self.version_7b = version+ self.window = _triple(window)+ self.window_method = window_method+ if not all(isinstance(v, int) and v >= 0 for v in self.window):+ raise ValueError(f"SeedVR2 window must contain non-negative integers, got {self.window}.")++ self.window_op = get_window_op(window_method)++ def forward(+ self,+ vid: torch.FloatTensor, # l c+ txt: torch.FloatTensor, # l c+ vid_shape: torch.LongTensor, # b 3+ txt_shape: torch.LongTensor, # b 1+ cache: Cache,+ ) -> Tuple[+ torch.FloatTensor,+ torch.FloatTensor,+ ]:++ vid_qkv, txt_qkv = self.proj_qkv(vid, txt)++ cache_win = cache.namespace(f"{self.window_method}_{self.window}_sd3")++ def make_window(x: torch.Tensor):+ t, h, w, _ = x.shape+ window_slices = self.window_op((t, h, w), self.window)+ return [x[st, sh, sw] for (st, sh, sw) in window_slices]++ window_partition, window_reverse, window_shape, window_count, vid_len_win_list, window_count_list = cache_win(+ "win_transform",+ lambda: window_idx(vid_shape, make_window),+ )+ vid_qkv_win = window_partition(vid_qkv)++ vid_qkv_win = vid_qkv_win.reshape(vid_qkv_win.shape[0], 3, self.heads, self.head_dim)+ txt_qkv = txt_qkv.reshape(txt_qkv.shape[0], 3, self.heads, self.head_dim)++ vid_q, vid_k, vid_v = vid_qkv_win.unbind(1)+ txt_q, txt_k, txt_v = txt_qkv.unbind(1)++ vid_q, txt_q = self.norm_q(vid_q, txt_q)+ vid_k, txt_k = self.norm_k(vid_k, txt_k)++ txt_len = cache("txt_len", lambda: txt_shape.prod(-1))++ vid_len_win = cache_win("vid_len", lambda: window_shape.prod(-1))+ txt_len = txt_len.to(window_count.device)++ if self.rope:+ if self.version_7b:+ vid_q, vid_k = self.rope(vid_q, vid_k, window_shape, cache_win)+ elif self.rope.mm:+ _, num_h, _ = txt_q.shape+ txt_q_repeat = txt_q.flatten(1, 2)+ txt_q_repeat = unflatten(txt_q_repeat, txt_shape)+ txt_q_repeat = [[x] * n for x, n in zip(txt_q_repeat, window_count_list)]+ txt_q_repeat = list(chain(*txt_q_repeat))+ txt_q_repeat, txt_shape_repeat = flatten(txt_q_repeat)+ txt_q_repeat = txt_q_repeat.reshape(txt_q_repeat.shape[0], num_h, self.head_dim)++ txt_k_repeat = txt_k.flatten(1, 2)+ txt_k_repeat = unflatten(txt_k_repeat, txt_shape)+ txt_k_repeat = [[x] * n for x, n in zip(txt_k_repeat, window_count_list)]+ txt_k_repeat = list(chain(*txt_k_repeat))+ txt_k_repeat, _ = flatten(txt_k_repeat)+ txt_k_repeat = txt_k_repeat.reshape(txt_k_repeat.shape[0], num_h, self.head_dim)++ vid_q, vid_k, txt_q, txt_k = self.rope(+ vid_q, vid_k, window_shape, txt_q_repeat, txt_k_repeat, txt_shape_repeat, cache_win+ )+ else:+ vid_q, vid_k = self.rope(vid_q, vid_k, window_shape, cache_win)++ txt_len_win_list = cache_win(+ "txt_len_list",+ lambda: [txt_len for txt_len, window_count in zip(txt_len.tolist(), window_count_list) for _ in range(window_count)],+ )+ all_len_win = cache_win("all_len", lambda: [vid_len + txt_len for vid_len, txt_len in zip(vid_len_win_list, txt_len_win_list)])+ concat_win, unconcat_win = cache_win(+ "mm_pnp", lambda: repeat_concat_idx(vid_len_win, txt_len, window_count)+ )+ out = optimized_var_attention(+ q=concat_win(vid_q, txt_q),+ k=concat_win(vid_k, txt_k),+ v=concat_win(vid_v, txt_v),+ heads=self.heads, skip_reshape=True, skip_output_reshape=True,+ cu_seqlens_q=cache_win("vid_seqlens_q", lambda: cumulative_lengths(all_len_win)),+ cu_seqlens_k=cache_win("vid_seqlens_k", lambda: cumulative_lengths(all_len_win)),+ )+ vid_out, txt_out = unconcat_win(out)++ vid_out = vid_out.flatten(1, 2)+ txt_out = txt_out.flatten(1, 2)+ vid_out = window_reverse(vid_out)++ vid_out, txt_out = self.proj_out(vid_out, txt_out)++ return vid_out, txt_out++class MLP(nn.Module):+ def __init__(+ self,+ dim: int,+ expand_ratio: int,+ device, dtype, operations+ ):+ super().__init__()+ self.proj_in = operations.Linear(dim, dim * expand_ratio, device=device, dtype=dtype)+ self.act = nn.GELU("tanh")+ self.proj_out = operations.Linear(dim * expand_ratio, dim, device=device, dtype=dtype)++ def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:+ x = self.proj_in(x)+ x = self.act(x)+ x = self.proj_out(x)+ return x+++class SwiGLUMLP(nn.Module):+ def __init__(+ self,+ dim: int,+ expand_ratio: int,+ multiple_of: int = 256,+ device=None, dtype=None, operations=None+ ):+ super().__init__()+ hidden_dim = int(2 * dim * expand_ratio / 3)+ hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)+ self.proj_in_gate = operations.Linear(dim, hidden_dim, bias=False, device=device, dtype=dtype)+ self.proj_out = operations.Linear(hidden_dim, dim, bias=False, device=device, dtype=dtype)+ self.proj_in = operations.Linear(dim, hidden_dim, bias=False, device=device, dtype=dtype)++ def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:+ return self.proj_out(F.silu(self.proj_in_gate(x)) * self.proj_in(x))++def get_mlp(mlp_type: Optional[str] = "normal"):+ if mlp_type == "normal":+ return MLP+ if mlp_type == "swiglu":+ return SwiGLUMLP+ raise ValueError(f"Unknown SeedVR2 MLP type: {mlp_type}")++class NaMMSRTransformerBlock(nn.Module):+ def __init__(+ self,+ *,+ vid_dim: int,+ txt_dim: int,+ emb_dim: int,+ heads: int,+ head_dim: int,+ expand_ratio: int,+ norm,+ norm_eps: float,+ ada,+ qk_bias: bool,+ qk_norm,+ mlp_type: str,+ shared_weights: bool,+ rope_type: str,+ rope_dim: int,+ is_last_layer: bool,+ window: Union[int, Tuple[int, int, int]],+ window_method: str,+ version: bool,+ device, dtype, operations,+ ):+ super().__init__()+ dim = MMArg(vid_dim, txt_dim)+ self.attn_norm = MMModule(norm, normalized_shape=dim, eps=norm_eps, elementwise_affine=False, shared_weights=shared_weights, device=device, dtype=dtype)++ self.attn = NaSwinAttention(+ vid_dim=vid_dim,+ txt_dim=txt_dim,+ heads=heads,+ head_dim=head_dim,+ qk_bias=qk_bias,+ qk_norm=qk_norm,+ qk_norm_eps=norm_eps,+ rope_type=rope_type,+ rope_dim=rope_dim,+ shared_weights=shared_weights,+ window=window,+ window_method=window_method,+ version=version,+ device=device, dtype=dtype, operations=operations+ )++ self.mlp_norm = MMModule(norm, normalized_shape=dim, eps=norm_eps, elementwise_affine=False, shared_weights=shared_weights, vid_only=is_last_layer, device=device, dtype=dtype)+ self.mlp = MMModule(+ get_mlp(mlp_type),+ dim=dim,+ expand_ratio=expand_ratio,+ shared_weights=shared_weights,+ vid_only=is_last_layer,+ device=device, dtype=dtype, operations=operations+ )+ self.ada = MMModule(ada, dim=dim, emb_dim=emb_dim, layers=["attn", "mlp"], shared_weights=shared_weights, vid_only=is_last_layer, device=device, dtype=dtype)+ self.is_last_layer = is_last_layer+ self.version = version++ def _seedvr2_7b_mlp(+ self,+ vid: torch.FloatTensor,+ txt: torch.FloatTensor,+ ) -> Tuple[+ torch.FloatTensor,+ torch.FloatTensor,+ ]:+ vid_module = self.mlp.vid if not self.mlp.shared_weights else self.mlp.all+ if comfy.model_management.in_training or vid.requires_grad:+ vid = torch.cat([vid_module(chunk) for chunk in vid.split(SEEDVR2_7B_MLP_CHUNK, dim=0)], dim=0)+ else:+ vid_out = None+ offset = 0+ for chunk in vid.split(SEEDVR2_7B_MLP_CHUNK, dim=0):+ chunk_out = vid_module(chunk)+ if vid_out is None:+ vid_out = chunk_out.new_empty((vid.shape[0], *chunk_out.shape[1:]))+ vid_out[offset:offset + chunk_out.shape[0]] = chunk_out+ offset += chunk_out.shape[0]+ vid = vid_out+ if not self.mlp.vid_only:+ txt_module = self.mlp.txt if not self.mlp.shared_weights else self.mlp.all+ txt = txt.to(device=vid.device, dtype=vid.dtype)+ txt = txt_module(txt)+ return vid, txt++ def forward(+ self,+ vid: torch.FloatTensor, # l c+ txt: torch.FloatTensor, # l c+ vid_shape: torch.LongTensor, # b 3+ txt_shape: torch.LongTensor, # b 1+ emb: torch.FloatTensor,+ cache: Cache,+ ) -> Tuple[+ torch.FloatTensor,+ torch.FloatTensor,+ torch.LongTensor,+ torch.LongTensor,+ ]:+ hid_len = MMArg(+ cache("vid_len", lambda: vid_shape.prod(-1)),+ cache("txt_len", lambda: txt_shape.prod(-1)),+ )+ ada_kwargs = {+ "emb": emb,+ "hid_len": hid_len,+ "cache": cache,+ "branch_tag": MMArg("vid", "txt"),+ }++ vid_attn, txt_attn = self.attn_norm(vid, txt)+ vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="in", **ada_kwargs)+ vid_attn, txt_attn = self.attn(vid_attn, txt_attn, vid_shape, txt_shape, cache)+ vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="out", **ada_kwargs)+ vid_attn, txt_attn = (vid_attn + vid), (txt_attn + txt)++ vid_mlp, txt_mlp = self.mlp_norm(vid_attn, txt_attn)+ vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, layer="mlp", mode="in", **ada_kwargs)+ if self.version:+ vid_mlp, txt_mlp = self._seedvr2_7b_mlp(vid_mlp, txt_mlp)+ else:+ vid_mlp, txt_mlp = self.mlp(vid_mlp, txt_mlp)+ vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, layer="mlp", mode="out", **ada_kwargs)+ vid_mlp, txt_mlp = (vid_mlp + vid_attn), (txt_mlp + txt_attn)++ return vid_mlp, txt_mlp, vid_shape, txt_shape++class PatchOut(nn.Module):+ def __init__(+ self,+ out_channels: int,+ patch_size: Union[int, Tuple[int, int, int]],+ dim: int,+ device, dtype, operations+ ):+ super().__init__()+ t, h, w = _triple(patch_size)+ self.patch_size = t, h, w+ self.proj = operations.Linear(dim, out_channels * t * h * w, device=device, dtype=dtype)++ def forward(+ self,+ vid: torch.Tensor,+ ) -> torch.Tensor:+ t, h, w = self.patch_size+ vid = self.proj(vid)+ b, T, H, W, channels = vid.shape+ c = channels // (t * h * w)+ vid = vid.view(b, T, H, W, t, h, w, c).permute(0, 7, 1, 4, 2, 5, 3, 6).reshape(b, c, T * t, H * h, W * w)+ if t > 1:+ vid = vid[:, :, (t - 1) :]+ return vid++class NaPatchOut(PatchOut):+ def forward(+ self,+ vid: torch.FloatTensor, # l c+ vid_shape: torch.LongTensor,+ cache: Optional[Cache] = None,+ vid_shape_before_patchify = None+ ) -> Tuple[+ torch.FloatTensor,+ torch.LongTensor,+ ]:+ if cache is None:+ cache = Cache(disable=True)++ t, h, w = self.patch_size+ vid = self.proj(vid)++ if not (t == h == w == 1):+ vid = unflatten(vid, vid_shape)+ for i in range(len(vid)):+ T, H, W, channels = vid[i].shape+ c = channels // (t * h * w)+ vid[i] = vid[i].view(T, H, W, t, h, w, c).permute(0, 3, 1, 4, 2, 5, 6).reshape(T * t, H * h, W * w, c)+ if t > 1 and vid_shape_before_patchify[i, 0] % t != 0:+ vid[i] = vid[i][(t - vid_shape_before_patchify[i, 0] % t) :]+ vid, vid_shape = flatten(vid)++ return vid, vid_shape++class PatchIn(nn.Module):+ def __init__(+ self,+ in_channels: int,+ patch_size: Union[int, Tuple[int, int, int]],+ dim: int,+ device, dtype, operations+ ):+ super().__init__()+ t, h, w = _triple(patch_size)+ self.patch_size = t, h, w+ self.proj = operations.Linear(in_channels * t * h * w, dim, device=device, dtype=dtype)++ def forward(+ self,+ vid: torch.Tensor,+ ) -> torch.Tensor:+ t, h, w = self.patch_size+ if t > 1:+ if vid.size(2) % t != 1:+ raise ValueError(+ f"SeedVR2 patch input temporal size must satisfy T % {t} == 1, got {vid.size(2)}."+ )+ vid = torch.cat([vid[:, :, :1]] * (t - 1) + [vid], dim=2)+ b, c, Tt, Hh, Ww = vid.shape+ vid = vid.view(b, c, Tt // t, t, Hh // h, h, Ww // w, w).permute(0, 2, 4, 6, 3, 5, 7, 1).reshape(b, Tt // t, Hh // h, Ww // w, t * h * w * c)+ vid = self.proj(vid)+ return vid++class NaPatchIn(PatchIn):+ def forward(+ self,+ vid: torch.Tensor, # l c+ vid_shape: torch.LongTensor,+ cache: Optional[Cache] = None,+ ) -> torch.Tensor:+ if cache is None:+ cache = Cache(disable=True)+ cache = cache.namespace("patch")+ vid_shape_before_patchify = cache("vid_shape_before_patchify", lambda: vid_shape)+ t, h, w = self.patch_size+ if not (t == h == w == 1):+ vid = unflatten(vid, vid_shape)+ for i in range(len(vid)):+ if t > 1 and vid_shape_before_patchify[i, 0] % t != 0:+ vid[i] = torch.cat([vid[i][:1]] * (t - vid[i].size(0) % t) + [vid[i]], dim=0)+ Tt, Hh, Ww, c = vid[i].shape+ vid[i] = vid[i].view(Tt // t, t, Hh // h, h, Ww // w, w, c).permute(0, 2, 4, 1, 3, 5, 6).reshape(Tt // t, Hh // h, Ww // w, t * h * w * c)+ vid, vid_shape = flatten(vid)++ vid = self.proj(vid)+ return vid, vid_shape++def expand_dims(x: torch.Tensor, dim: int, ndim: int):+ shape = x.shape+ shape = shape[:dim] + (1,) * (ndim - len(shape)) + shape[dim:]+ return x.reshape(shape)+++class AdaSingle(nn.Module):+ def __init__(+ self,+ dim: int,+ emb_dim: int,+ layers: List[str],+ modes: Tuple[str, ...] = ("in", "out"),+ device = None, dtype = None,+ ):+ if emb_dim != 6 * dim:+ raise ValueError(f"SeedVR2 AdaSingle requires emb_dim == 6 * dim, got emb_dim={emb_dim}, dim={dim}.")+ super().__init__()+ self.dim = dim+ self.emb_dim = emb_dim+ self.layers = layers++ param_kwargs = {"device": device, "dtype": dtype}++ for l in layers:+ if "in" in modes:+ self.register_parameter(f"{l}_shift", nn.Parameter(torch.empty(dim, **param_kwargs)))+ self.register_parameter(f"{l}_scale", nn.Parameter(torch.empty(dim, **param_kwargs)))+ if "out" in modes:+ self.register_parameter(f"{l}_gate", nn.Parameter(torch.empty(dim, **param_kwargs)))++ def forward(+ self,+ hid: torch.FloatTensor, # b ... c+ emb: torch.FloatTensor, # b d+ layer: str,+ mode: str,+ cache: Optional[Cache] = None,+ branch_tag: str = "",+ hid_len: Optional[torch.LongTensor] = None, # b+ ) -> torch.FloatTensor:+ if cache is None:+ cache = Cache(disable=True)+ idx = self.layers.index(layer)+ emb = emb.reshape(emb.shape[0], -1, len(self.layers), 3)[:, :, idx, :]+ emb = expand_dims(emb, 1, hid.ndim + 1)++ if hid_len is not None:+ emb = cache(+ f"emb_repeat_{idx}_{branch_tag}",+ lambda: torch.repeat_interleave(emb, hid_len, dim=0),+ )++ shiftA, scaleA, gateA = emb.unbind(-1)+ shiftB, scaleB, gateB = (+ getattr(self, f"{layer}_shift", None),+ getattr(self, f"{layer}_scale", None),+ getattr(self, f"{layer}_gate", None),+ )++ if mode == "in":+ shiftB = comfy.ops.cast_to_input(shiftB, hid)+ scaleB = comfy.ops.cast_to_input(scaleB, hid)+ return hid.mul_(scaleA + scaleB).add_(shiftA + shiftB)+ if mode == "out":+ if gateB is not None:+ gateB = comfy.ops.cast_to_input(gateB, hid)+ return hid.mul_(gateA + gateB)+ else:+ return hid.mul_(gateA)++ raise ValueError(f"Unknown AdaSingle mode: {mode}")+++class TimeEmbedding(nn.Module):+ def __init__(+ self,+ sinusoidal_dim: int,+ hidden_dim: int,+ output_dim: int,+ device, dtype, operations+ ):+ super().__init__()+ self.sinusoidal_dim = sinusoidal_dim+ self.proj_in = operations.Linear(sinusoidal_dim, hidden_dim, device=device, dtype=dtype)+ self.proj_hid = operations.Linear(hidden_dim, hidden_dim, device=device, dtype=dtype)+ self.proj_out = operations.Linear(hidden_dim, output_dim, device=device, dtype=dtype)+ self.act = nn.SiLU()++ def forward(+ self,+ timestep: Union[int, float, torch.IntTensor, torch.FloatTensor],+ device: torch.device,+ dtype: torch.dtype,+ ) -> torch.FloatTensor:+ if not torch.is_tensor(timestep):+ timestep = torch.tensor([timestep], device=device, dtype=dtype)+ if timestep.ndim == 0:+ timestep = timestep[None]++ emb = get_timestep_embedding(+ timesteps=timestep,+ embedding_dim=self.sinusoidal_dim,+ flip_sin_to_cos=False,+ downscale_freq_shift=0,+ ).to(dtype)+ emb = self.proj_in(emb)+ emb = self.act(emb)+ emb = self.proj_hid(emb)+ emb = self.act(emb)+ emb = self.proj_out(emb)+ return emb++def flatten(+ hid: List[torch.FloatTensor], # List of (*** c)+) -> Tuple[+ torch.FloatTensor, # (L c)+ torch.LongTensor, # (b n)+]:+ if len(hid) == 0:+ raise ValueError("SeedVR2 flatten requires at least one tensor.")+ shape = torch.as_tensor([x.shape[:-1] for x in hid], device=hid[0].device)+ hid = torch.cat([x.flatten(0, -2) for x in hid])+ return hid, shape+++def unflatten(+ hid: torch.FloatTensor, # (L c) or (L ... c)+ hid_shape: torch.LongTensor, # (b n)+) -> List[torch.Tensor]: # List of (*** c) or (*** ... c)+ hid_len = hid_shape.prod(-1)+ hid = hid.split(hid_len.tolist())+ hid = [x.unflatten(0, s.tolist()) for x, s in zip(hid, hid_shape)]+ return hid++class NaDiT(nn.Module):++ def __init__(+ self,+ norm_eps,+ num_layers,+ mlp_type,+ vid_in_channels = 33,+ vid_out_channels = SEEDVR2_LATENT_CHANNELS,+ vid_dim = 2560,+ txt_in_dim = 5120,+ heads = 20,+ head_dim = 128,+ mm_layers = 10,+ expand_ratio = 4,+ qk_bias = False,+ patch_size = (1, 2, 2),+ rope_dim = 128,+ rope_type = "mmrope3d",+ vid_out_norm: Optional[str] = None,+ image_model = None,+ device = None,+ dtype = None,+ operations = None,+ ):+ if image_model not in (None, "seedvr2"):+ raise ValueError(f"SeedVR2 NaDiT expected image_model='seedvr2', got {image_model!r}.")+ self._7b_version = vid_dim == SEEDVR2_7B_VID_DIM+ if self._7b_version:+ rope_type = "rope3d"+ self.dtype = dtype+ factory_kwargs = {"device": device, "dtype": dtype}+ window_method = num_layers // 2 * ["720pwin_by_size_bysize","720pswin_by_size_bysize"]+ txt_dim = vid_dim+ emb_dim = vid_dim * 6+ window = num_layers * [(4,3,3)]+ ada = AdaSingle+ norm = operations.RMSNorm+ qk_norm = operations.RMSNorm+ super().__init__()+ self.register_buffer("positive_conditioning", torch.empty((58, 5120), device=device, dtype=dtype))+ self.register_buffer("negative_conditioning", torch.empty((64, 5120), device=device, dtype=dtype))+ self.vid_in = NaPatchIn(+ in_channels=vid_in_channels,+ patch_size=patch_size,+ dim=vid_dim,+ device=device, dtype=dtype, operations=operations+ )+ self.txt_in = (+ operations.Linear(txt_in_dim, txt_dim, **factory_kwargs)+ if txt_in_dim and txt_in_dim != txt_dim+ else nn.Identity()+ )+ self.emb_in = TimeEmbedding(+ sinusoidal_dim=BYTEDANCE_SINUSOIDAL_DIM,+ hidden_dim=max(vid_dim, txt_dim),+ output_dim=emb_dim,+ device=device, dtype=dtype, operations=operations+ )++ if window is None or isinstance(window[0], int):+ window = [window] * num_layers++ rope_dim = rope_dim if rope_dim is not None else head_dim // 2+ self.blocks = nn.ModuleList(+ [+ NaMMSRTransformerBlock(+ vid_dim=vid_dim,+ txt_dim=txt_dim,+ emb_dim=emb_dim,+ heads=heads,+ head_dim=head_dim,+ expand_ratio=expand_ratio,+ norm=norm,+ norm_eps=norm_eps,+ ada=ada,+ qk_bias=qk_bias,+ qk_norm=qk_norm,+ mlp_type=mlp_type,+ rope_dim = rope_dim,+ window=window[i],+ window_method=window_method[i],+ version = self._7b_version,+ is_last_layer=(i == num_layers - 1) and not self._7b_version,+ rope_type = rope_type,+ shared_weights=not (+ (i < mm_layers) if isinstance(mm_layers, int) else mm_layers[i]+ ),+ operations = operations,+ **factory_kwargs+ )+ for i in range(num_layers)+ ]+ )+ self.vid_out = NaPatchOut(+ out_channels=vid_out_channels,+ patch_size=patch_size,+ dim=vid_dim,+ device=device, dtype=dtype, operations=operations+ )++ self.vid_out_norm = None+ if vid_out_norm is not None:+ self.vid_out_norm = operations.RMSNorm(+ normalized_shape=vid_dim,+ eps=norm_eps,+ elementwise_affine=True,+ device=device, dtype=dtype+ )+ self.vid_out_ada = ada(+ dim=vid_dim,+ emb_dim=emb_dim,+ layers=["out"],+ modes=["in"],+ device=device, dtype=dtype+ )++ def _resolve_text_conditioning(self, context, cond_or_uncond=None):+ if context is None or context.numel() == 0:+ context = self.positive_conditioning+ return flatten([context])+ if NaDiT._seedvr2_is_single_conditioning_branch(cond_or_uncond):+ if context.shape[0] == 1:+ context = context.squeeze(0)+ return flatten([context])+ return flatten(context.unbind(0))+ if context.shape[0] % 2 != 0:+ raise ValueError(f"SeedVR2 expected an even text-conditioning batch, got shape {tuple(context.shape)}")+ neg_cond, pos_cond = context.chunk(2, dim=0)+ if pos_cond.shape[0] == 1:+ pos_cond, neg_cond = pos_cond.squeeze(0), neg_cond.squeeze(0)+ return flatten([pos_cond, neg_cond])+ return flatten((*pos_cond.unbind(0), *neg_cond.unbind(0)))++ @staticmethod+ def _seedvr2_is_single_conditioning_branch(cond_or_uncond):+ if cond_or_uncond is None or len(cond_or_uncond) == 0:+ return False+ first = cond_or_uncond[0]+ return all(entry == first for entry in cond_or_uncond)++ @staticmethod+ def _check_seedvr2_video_latent(x, channels, name):+ if x.ndim != 5:+ raise ValueError(f"SeedVR2 expected {name} to be 5-D native latent, got shape {tuple(x.shape)}.")+ if x.shape[1] != channels:+ raise ValueError(f"SeedVR2 expected {name} channels to be {channels}, got shape {tuple(x.shape)}.")+ return x++ def _swap_pos_neg_halves(self, out, cond_or_uncond=None):+ if NaDiT._seedvr2_is_single_conditioning_branch(cond_or_uncond):+ return out+ pos, neg = out.chunk(2, dim=0)+ return torch.cat([neg, pos], dim=0)++ def forward(+ self,+ x,+ timestep,+ context, # l c+ disable_cache: bool = False,+ **kwargs+ ):+ transformer_options = kwargs.get("transformer_options", {})+ patches_replace = transformer_options.get("patches_replace", {})+ blocks_replace = patches_replace.get("dit", {})+ conditions = kwargs.get("condition")+ if conditions is None:+ raise ValueError("SeedVR2 requires conditioning latents from the SeedVR2Conditioning node.")+ x = self._check_seedvr2_video_latent(x, SEEDVR2_LATENT_CHANNELS, "latent")+ conditions = self._check_seedvr2_video_latent(conditions, SEEDVR2_LATENT_CHANNELS + 1, "conditioning")+ b, _, t, h, w = x.shape+ if conditions.shape[0] != b or conditions.shape[2:] != (t, h, w):+ raise ValueError(+ f"SeedVR2 conditioning shape must match latent batch/temporal/spatial dimensions; got latent {tuple(x.shape)} and conditioning {tuple(conditions.shape)}."+ )+ x = x.movedim(1, -1)+ conditions = conditions.movedim(1, -1)+ cache = Cache(disable=disable_cache)++ txt, txt_shape = self._resolve_text_conditioning(context, transformer_options.get("cond_or_uncond"))++ vid, vid_shape = flatten(x)+ cond_latent, _ = flatten(conditions)++ vid = torch.cat([vid, cond_latent], dim=-1)++ txt = self.txt_in(txt)++ vid_shape_before_patchify = vid_shape+ vid, vid_shape = self.vid_in(vid, vid_shape, cache=cache)++ emb = self.emb_in(timestep, device=vid.device, dtype=vid.dtype)++ for i, block in enumerate(self.blocks):+ if ("block", i) in blocks_replace:+ def block_wrap(args):+ out = {}+ out["vid"], out["txt"], out["vid_shape"], out["txt_shape"] = block(+ vid=args["vid"],+ txt=args["txt"],+ vid_shape=args["vid_shape"],+ txt_shape=args["txt_shape"],+ emb=args["emb"],+ cache=args["cache"],+ )+ return out+ out = blocks_replace[("block", i)]({+ "vid":vid,+ "txt":txt,+ "vid_shape":vid_shape,+ "txt_shape":txt_shape,+ "emb":emb,+ "cache":cache,+ }, {"original_block": block_wrap})+ vid, txt, vid_shape, txt_shape = out["vid"], out["txt"], out["vid_shape"], out["txt_shape"]+ else:+ vid, txt, vid_shape, txt_shape = block(+ vid=vid,+ txt=txt,+ vid_shape=vid_shape,+ txt_shape=txt_shape,+ emb=emb,+ cache=cache,+ )++ if self.vid_out_norm:+ vid = self.vid_out_norm(vid)+ vid = self.vid_out_ada(+ vid,+ emb=emb,+ layer="out",+ mode="in",+ hid_len=cache("vid_len", lambda: vid_shape.prod(-1)),+ cache=cache,+ branch_tag="vid",+ )++ vid, vid_shape = self.vid_out(vid, vid_shape, cache, vid_shape_before_patchify = vid_shape_before_patchify)+ vid = unflatten(vid, vid_shape)+ out = torch.stack(vid)+ out = out.movedim(-1, 1)+ return self._swap_pos_neg_halves(out, transformer_options.get("cond_or_uncond"))diff --git a/comfy/ldm/seedvr/vae.py b/comfy/ldm/seedvr/vae.pynew file mode 100644index 000000000..c9f430184--- /dev/null+++ b/comfy/ldm/seedvr/vae.py@@ -0,0 +1,1612 @@+from typing import Literal, Optional, Tuple+import torch+import torch.nn as nn+import torch.nn.functional as F+from torch import Tensor+from contextlib import contextmanager+from comfy.utils import ProgressBar++from comfy.ldm.seedvr.constants import (+ BYTEDANCE_BLOCK_OUT_CHANNELS,+ BYTEDANCE_GN_CHUNKS_FP16,+ BYTEDANCE_GN_CHUNKS_FP32,+ BYTEDANCE_LOGVAR_CLAMP_MAX,+ BYTEDANCE_LOGVAR_CLAMP_MIN,+ BYTEDANCE_SLICING_SAMPLE_MIN,+ BYTEDANCE_VAE_CONV_MEM_GIB,+ BYTEDANCE_VAE_NORM_MEM_GIB,+ BYTEDANCE_VAE_SCALING_FACTOR,+ BYTEDANCE_VAE_SHIFTING_FACTOR,+ BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE,+ BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE,+ SEEDVR2_LATENT_CHANNELS,+)+from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.diffusionmodules.model import vae_attention++import math+from enum import Enum++import logging+import comfy.model_management+import comfy.ops+ops = comfy.ops.disable_weight_init+++def _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap, temporal_scale=1):+ if temporal_size is None:+ return None++ temporal_size = int(temporal_size)+ if temporal_size <= 0:+ return None++ temporal_overlap = max(0, int(temporal_overlap or 0))+ temporal_overlap = min(temporal_overlap, temporal_size - 1)+ temporal_step = temporal_size - temporal_overlap+ temporal_scale = max(1, int(temporal_scale))+ return max(1, math.ceil(temporal_step / temporal_scale))+++def _seedvr2_clamped_spatial_overlap(overlap, tile_size):+ overlap = max(0, int(overlap))+ tile_size = max(1, int(tile_size))+ return min(overlap, tile_size - 1)+++def tiled_vae(+ x,+ vae_model,+ tile_size=(512, 512),+ tile_overlap=(64, 64),+ temporal_size=16,+ temporal_overlap=0,+ encode=True,+):+ if x.ndim != 5:+ x = x.unsqueeze(2)++ _, _, d, h, w = x.shape++ sf_s = getattr(vae_model, "spatial_downsample_factor", BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE)+ sf_t = getattr(vae_model, "temporal_downsample_factor", BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE)+ if encode:+ slicing_attr = "slicing_sample_min_size"+ slicing_min_size = _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap)+ else:+ slicing_attr = "slicing_latent_min_size"+ slicing_min_size = _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap, sf_t)+ if encode:+ ti_h, ti_w = tile_size+ ov_h = _seedvr2_clamped_spatial_overlap(tile_overlap[0], ti_h)+ ov_w = _seedvr2_clamped_spatial_overlap(tile_overlap[1], ti_w)+ blend_ov_h = max(0, ov_h // sf_s)+ blend_ov_w = max(0, ov_w // sf_s)+ target_d = (d + sf_t - 1) // sf_t+ target_h = (h + sf_s - 1) // sf_s+ target_w = (w + sf_s - 1) // sf_s+ else:+ ti_h = max(1, tile_size[0] // sf_s)+ ti_w = max(1, tile_size[1] // sf_s)+ ov_h = _seedvr2_clamped_spatial_overlap(tile_overlap[0] // sf_s, ti_h)+ ov_w = _seedvr2_clamped_spatial_overlap(tile_overlap[1] // sf_s, ti_w)+ blend_ov_h = ov_h * sf_s+ blend_ov_w = ov_w * sf_s++ target_d = max(1, d * sf_t - (sf_t - 1))+ target_h = h * sf_s+ target_w = w * sf_s++ stride_h = max(1, ti_h - ov_h)+ stride_w = max(1, ti_w - ov_w)++ storage_device = vae_model.device+ result = None+ count = None+ def run_temporal_chunks(spatial_tile, model=vae_model, device=storage_device):+ device = torch.device(device)+ t_chunk = spatial_tile.to(device=device, dtype=next(model.parameters()).dtype, non_blocking=True).contiguous()+ old_device = getattr(model, "device", None)+ model.device = device+ old_slicing_min_size = getattr(model, slicing_attr, None)+ if old_slicing_min_size is not None and slicing_min_size is not None:+ if slicing_min_size <= 0:+ setattr(model, slicing_attr, t_chunk.shape[2])+ else:+ setattr(model, slicing_attr, slicing_min_size)+ try:+ if encode:+ out = model.encode(t_chunk)+ else:+ out = model.decode_(t_chunk)+ finally:+ if old_slicing_min_size is not None and slicing_min_size is not None:+ setattr(model, slicing_attr, old_slicing_min_size)+ if old_device is not None:+ model.device = old_device+ if out.ndim == 4:+ out = out.unsqueeze(2)+ return out.to(storage_device)++ ramp_cache = {}+ def get_ramp(steps):+ if steps not in ramp_cache:+ t = torch.linspace(0, 1, steps=steps, device=storage_device, dtype=torch.float32)+ ramp_cache[steps] = 0.5 - 0.5 * torch.cos(t * torch.pi)+ return ramp_cache[steps]++ tile_ranges = []+ for y_idx in range(0, h, stride_h):+ y_end = min(y_idx + ti_h, h)+ if y_idx > 0 and (y_end - y_idx) <= ov_h:+ continue+ for x_idx in range(0, w, stride_w):+ x_end = min(x_idx + ti_w, w)+ if x_idx > 0 and (x_end - x_idx) <= ov_w:+ continue+ tile_ranges.append((y_idx, y_end, x_idx, x_end))++ total_tiles = len(tile_ranges)+ bar = ProgressBar(total_tiles)+ single_spatial_tile = h <= ti_h and w <= ti_w++ def run_tile(tile_index, tile_range):+ y_idx, y_end, x_idx, x_end = tile_range+ tile_x = x[:, :, :, y_idx:y_end, x_idx:x_end]+ tile_out = run_temporal_chunks(tile_x)+ return tile_index, y_idx, y_end, x_idx, x_end, tile_out++ ordered_tile_outputs = (+ run_tile(tile_index, tile_range)+ for tile_index, tile_range in enumerate(tile_ranges)+ )++ for _, y_idx, y_end, x_idx, x_end, tile_out in ordered_tile_outputs:++ if single_spatial_tile:+ result = tile_out[:, :, :target_d, :target_h, :target_w]+ if result.device != x.device or result.dtype != x.dtype:+ result = result.to(device=x.device, dtype=x.dtype)+ if x.shape[2] == 1 and sf_t == 1:+ result = result.squeeze(2)+ bar.update(1)+ return result++ if result is None:+ b_out, c_out = tile_out.shape[0], tile_out.shape[1]+ result = torch.zeros((b_out, c_out, target_d, target_h, target_w), device=storage_device, dtype=torch.float32)+ count = torch.zeros((1, 1, 1, target_h, target_w), device=storage_device, dtype=torch.float32)++ if encode:+ ys, ye = y_idx // sf_s, (y_idx // sf_s) + tile_out.shape[3]+ xs, xe = x_idx // sf_s, (x_idx // sf_s) + tile_out.shape[4]+ cur_ov_h = max(0, min(blend_ov_h, tile_out.shape[3] // 2))+ cur_ov_w = max(0, min(blend_ov_w, tile_out.shape[4] // 2))+ else:+ ys, ye = y_idx * sf_s, (y_idx * sf_s) + tile_out.shape[3]+ xs, xe = x_idx * sf_s, (x_idx * sf_s) + tile_out.shape[4]+ cur_ov_h = max(0, min(blend_ov_h, tile_out.shape[3] // 2))+ cur_ov_w = max(0, min(blend_ov_w, tile_out.shape[4] // 2))++ w_h = torch.ones((tile_out.shape[3],), device=storage_device)+ w_w = torch.ones((tile_out.shape[4],), device=storage_device)++ if cur_ov_h > 0:+ r = get_ramp(cur_ov_h)+ if y_idx > 0:+ w_h[:cur_ov_h] = r+ if y_end < h:+ w_h[-cur_ov_h:] = 1.0 - r++ if cur_ov_w > 0:+ r = get_ramp(cur_ov_w)+ if x_idx > 0:+ w_w[:cur_ov_w] = r+ if x_end < w:+ w_w[-cur_ov_w:] = 1.0 - r++ final_weight = w_h.view(1,1,1,-1,1) * w_w.view(1,1,1,1,-1)++ valid_d = min(tile_out.shape[2], result.shape[2])+ tile_out = tile_out[:, :, :valid_d, :, :]++ tile_out.mul_(final_weight)++ result[:, :, :valid_d, ys:ye, xs:xe] += tile_out+ count[:, :, :, ys:ye, xs:xe] += final_weight++ del tile_out, final_weight, w_h, w_w+ bar.update(1)++ result.div_(count.clamp(min=1e-6))++ if result.device != x.device or result.dtype != x.dtype:+ result = result.to(device=x.device, dtype=x.dtype)++ if x.shape[2] == 1 and sf_t == 1:+ result = result.squeeze(2)++ return result++_NORM_LIMIT = float("inf")+def get_norm_limit():+ return _NORM_LIMIT+++def set_norm_limit(value: Optional[float] = None):+ global _NORM_LIMIT+ if value is None:+ value = float("inf")+ _NORM_LIMIT = value++@contextmanager+def ignore_padding(model):+ orig_padding = model.padding+ model.padding = (0, 0, 0)+ try:+ yield+ finally:+ model.padding = orig_padding++class MemoryState(Enum):+ DISABLED = 0+ INITIALIZING = 1+ ACTIVE = 2+ UNSET = 3++def get_cache_size(conv_module, input_len, pad_len, dim=0):+ dilated_kernel_size = conv_module.dilation[dim] * (conv_module.kernel_size[dim] - 1) + 1+ output_len = (input_len + pad_len - dilated_kernel_size) // conv_module.stride[dim] + 1+ remain_len = (+ input_len + pad_len - ((output_len - 1) * conv_module.stride[dim] + dilated_kernel_size)+ )+ overlap_len = dilated_kernel_size - conv_module.stride[dim]+ cache_len = overlap_len + remain_len++ if output_len <= 0:+ raise ValueError(+ f"SeedVR2 VAE cache input is too short for convolution: input_len={input_len}, pad_len={pad_len}."+ )+ return cache_len++class DiagonalGaussianDistribution(object):+ def __init__(self, parameters: torch.Tensor):+ self.parameters = parameters+ self.mean, self.logvar = torch.chunk(parameters, 2, dim=1)+ self.logvar = torch.clamp(self.logvar, BYTEDANCE_LOGVAR_CLAMP_MIN, BYTEDANCE_LOGVAR_CLAMP_MAX)++ def mode(self):+ return self.mean++class SpatialNorm(nn.Module):+ def __init__(+ self,+ f_channels: int,+ zq_channels: int,+ ):+ super().__init__()+ self.norm_layer = ops.GroupNorm(num_channels=f_channels, num_groups=32, eps=1e-6, affine=True)+ self.conv_y = ops.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0)+ self.conv_b = ops.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0)++ def forward(self, f: torch.Tensor, zq: torch.Tensor) -> torch.Tensor:+ f_size = f.shape[-2:]+ zq = F.interpolate(zq, size=f_size, mode="nearest")+ norm_f = self.norm_layer(f)+ new_f = norm_f * self.conv_y(zq) + self.conv_b(zq)+ return new_f++class Attention(nn.Module):+ def __init__(+ self,+ query_dim: int,+ heads: int = 8,+ dim_head: int = 64,+ bias: bool = False,+ norm_num_groups: Optional[int] = None,+ spatial_norm_dim: Optional[int] = None,+ out_bias: bool = True,+ eps: float = 1e-5,+ rescale_output_factor: float = 1.0,+ residual_connection: bool = False,+ ):+ super().__init__()++ self.inner_dim = dim_head * heads+ self.rescale_output_factor = rescale_output_factor+ self.residual_connection = residual_connection+ self.out_dim = query_dim+ self.heads = heads++ if norm_num_groups is not None:+ self.group_norm = ops.GroupNorm(num_channels=query_dim, num_groups=norm_num_groups, eps=eps, affine=True)+ else:+ self.group_norm = None++ if spatial_norm_dim is not None:+ self.spatial_norm = SpatialNorm(f_channels=query_dim, zq_channels=spatial_norm_dim)+ else:+ self.spatial_norm = None++ self.to_q = ops.Linear(query_dim, self.inner_dim, bias=bias)+ self.to_k = ops.Linear(query_dim, self.inner_dim, bias=bias)+ self.to_v = ops.Linear(query_dim, self.inner_dim, bias=bias)+ self.to_out = nn.ModuleList([])+ self.to_out.append(ops.Linear(self.inner_dim, self.out_dim, bias=out_bias))+ self.to_out.append(nn.Identity())++ self.optimized_vae_attention = vae_attention()++ def forward(+ self,+ hidden_states: torch.Tensor,+ temb: Optional[torch.Tensor] = None,+ ) -> torch.Tensor:++ residual = hidden_states+ if self.spatial_norm is not None:+ hidden_states = self.spatial_norm(hidden_states, temb)++ input_ndim = hidden_states.ndim++ if input_ndim == 4:+ batch_size, channel, height, width = hidden_states.shape+ hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)++ batch_size = hidden_states.shape[0]++ if self.group_norm is not None:+ hidden_states = self.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)++ query = self.to_q(hidden_states)+ key = self.to_k(hidden_states)+ value = self.to_v(hidden_states)++ inner_dim = key.shape[-1]+ head_dim = inner_dim // self.heads++ query = query.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)++ key = key.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)+ value = value.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)++ if input_ndim == 4 and self.heads == 1:+ query = query.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width)+ key = key.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width)+ value = value.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width)+ hidden_states = self.optimized_vae_attention(query, key, value).reshape(batch_size, self.heads, head_dim, height * width).transpose(2, 3)+ else:+ hidden_states = optimized_attention(query, key, value, heads = self.heads, skip_reshape=True, skip_output_reshape=True)++ hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, self.heads * head_dim)+ hidden_states = hidden_states.to(query.dtype)++ hidden_states = self.to_out[0](hidden_states)+ hidden_states = self.to_out[1](hidden_states)++ if input_ndim == 4:+ hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)++ if self.residual_connection:+ hidden_states = hidden_states + residual++ hidden_states = hidden_states / self.rescale_output_factor++ return hidden_states+++def causal_norm_wrapper(norm_layer: nn.Module, x: torch.Tensor) -> torch.Tensor:+ input_dtype = x.dtype+ if isinstance(norm_layer, (ops.LayerNorm, ops.RMSNorm)):+ if x.ndim == 4:+ x = x.permute(0, 2, 3, 1)+ x = norm_layer(x)+ x = x.permute(0, 3, 1, 2)+ return x.to(input_dtype)+ if x.ndim == 5:+ x = x.permute(0, 2, 3, 4, 1)+ x = norm_layer(x)+ x = x.permute(0, 4, 1, 2, 3)+ return x.to(input_dtype)+ if isinstance(norm_layer, (ops.GroupNorm, nn.BatchNorm2d, nn.SyncBatchNorm)):+ if x.ndim <= 4:+ return norm_layer(x).to(input_dtype)+ if x.ndim == 5:+ b, c, t, h, w = x.shape+ x = x.transpose(1, 2).reshape(b * t, c, h, w)+ memory_occupy = x.numel() * x.element_size() / 1024**3+ if isinstance(norm_layer, ops.GroupNorm) and memory_occupy > get_norm_limit():+ num_chunks = min(BYTEDANCE_GN_CHUNKS_FP16 if x.element_size() == 2 else BYTEDANCE_GN_CHUNKS_FP32, norm_layer.num_groups)+ if norm_layer.num_groups % num_chunks != 0:+ raise ValueError(+ f"SeedVR2 VAE GroupNorm groups must divide chunks: groups={norm_layer.num_groups}, chunks={num_chunks}."+ )+ num_groups_per_chunk = norm_layer.num_groups // num_chunks++ x = list(x.chunk(num_chunks, dim=1))+ weights = norm_layer.weight.chunk(num_chunks, dim=0)+ biases = norm_layer.bias.chunk(num_chunks, dim=0)+ for i, (w, bias) in enumerate(zip(weights, biases)):+ x[i] = F.group_norm(x[i], num_groups_per_chunk, w, bias, norm_layer.eps)+ x[i] = x[i].to(input_dtype)+ x = torch.cat(x, dim=1)+ else:+ x = norm_layer(x)+ x = x.reshape((b, t, x.size(1), x.size(2), x.size(3))).transpose(1, 2)+ return x.to(input_dtype)+ raise TypeError(f"SeedVR2 VAE unsupported norm layer type: {type(norm_layer).__name__}")++_receptive_field_t = Literal["half", "full"]++def extend_head(tensor, times: int = 2, memory = None):+ if memory is not None:+ return torch.cat((memory.to(tensor), tensor), dim=2)+ if times < 0:+ raise ValueError(f"SeedVR2 VAE extend_head expected times >= 0, got {times}.")+ if times == 0:+ return tensor+ else:+ tile_repeat = [1] * tensor.ndim+ tile_repeat[2] = times+ return torch.cat(tensors=(torch.tile(tensor[:, :, :1], tile_repeat), tensor), dim=2)++def cache_send_recv(tensor, cache_size, times, memory=None):+ recv_buffer = None++ if memory is not None:+ recv_buffer = memory.to(tensor[0])+ elif times > 0:+ tile_repeat = [1] * tensor[0].ndim+ tile_repeat[2] = times+ recv_buffer = torch.tile(tensor[0][:, :, :1], tile_repeat)++ return recv_buffer++class InflatedCausalConv3d(ops.Conv3d):+ def __init__(+ self,+ *args,+ inflation_mode,+ **kwargs,+ ):+ self.inflation_mode = inflation_mode+ super().__init__(*args, **kwargs)+ self.temporal_padding = self.padding[0]+ self.padding = (0, *self.padding[1:])+ self.memory_limit = float("inf")+ self.logged_once = False++ def set_memory_limit(self, value: float):+ self.memory_limit = value++ def _conv_forward(self, input, weight, bias, *args, **kwargs):+ try:+ return super()._conv_forward(input, weight, bias, *args, **kwargs)+ except NotImplementedError:+ # for: Could not run 'aten::cudnn_convolution' with arguments from the 'CPU' backend+ if not self.logged_once:+ logging.warning("VAE is on CPU for decoding. This is most likely due to not enough memory")+ self.logged_once = True+ return F.conv3d(input, weight, bias, *args, **kwargs)++ def memory_limit_conv(+ self,+ x,+ *,+ split_dim=3,+ padding=(0, 0, 0, 0, 0, 0),+ prev_cache=None,+ ):+ if math.isinf(self.memory_limit):+ if prev_cache is not None:+ x = torch.cat([prev_cache, x], dim=split_dim - 1)+ return super().forward(x)++ shape = list(x.size())+ if prev_cache is not None:+ shape[split_dim - 1] += prev_cache.size(split_dim - 1)+ for i, pad_sum in enumerate((padding[4] + padding[5], padding[2] + padding[3], padding[0] + padding[1])):+ shape[-3 + i] += pad_sum+ memory_occupy = math.prod(shape) * x.element_size() / 1024**3 # GiB+ if memory_occupy < self.memory_limit or split_dim == x.ndim:+ x_concat = x+ if prev_cache is not None:+ x_concat = torch.cat([prev_cache, x], dim=split_dim - 1)++ def pad_and_forward():+ padded = F.pad(x_concat, padding, mode='constant', value=0.0)+ if not padded.is_contiguous():+ padded = padded.contiguous()+ with ignore_padding(self):+ return torch.nn.Conv3d.forward(self, padded)++ return pad_and_forward()++ num_splits = math.ceil(memory_occupy / self.memory_limit)+ size_per_split = x.size(split_dim) // num_splits+ split_sizes = [size_per_split] * (num_splits - 1)+ split_sizes += [x.size(split_dim) - sum(split_sizes)]++ x = list(x.split(split_sizes, dim=split_dim))+ if prev_cache is not None:+ prev_cache = list(prev_cache.split(split_sizes, dim=split_dim))+ cache = None+ for idx in range(len(x)):+ if prev_cache is not None:+ x[idx] = torch.cat([prev_cache[idx], x[idx]], dim=split_dim - 1)++ lpad_dim = (x[idx].ndim - split_dim - 1) * 2+ rpad_dim = lpad_dim + 1+ padding = list(padding)+ padding[lpad_dim] = self.padding[split_dim - 2] if idx == 0 else 0+ padding[rpad_dim] = self.padding[split_dim - 2] if idx == len(x) - 1 else 0+ pad_len = padding[lpad_dim] + padding[rpad_dim]+ padding = tuple(padding)++ next_cache = None+ cache_len = cache.size(split_dim) if cache is not None else 0+ next_cache_size = get_cache_size(+ conv_module=self,+ input_len=x[idx].size(split_dim) + cache_len,+ pad_len=pad_len,+ dim=split_dim - 2,+ )+ if next_cache_size != 0:+ if next_cache_size > x[idx].size(split_dim):+ raise ValueError(+ f"SeedVR2 VAE cache size {next_cache_size} exceeds split size {x[idx].size(split_dim)}."+ )+ next_cache = (+ x[idx].transpose(0, split_dim)[-next_cache_size:].transpose(0, split_dim)+ )++ x[idx] = self.memory_limit_conv(+ x[idx],+ split_dim=split_dim + 1,+ padding=padding,+ prev_cache=cache+ )++ cache = next_cache++ output = torch.cat(x, dim=split_dim)+ return output++ def forward(+ self,+ input,+ memory_state: MemoryState = MemoryState.UNSET,+ memory_cache = None,+ ) -> Tensor:+ if memory_state == MemoryState.UNSET:+ raise ValueError("SeedVR2 VAE convolution requires an explicit MemoryState.")+ if memory_cache is None:+ memory_cache = {}+ if memory_state != MemoryState.ACTIVE:+ memory_cache.pop(self, None)+ if (+ math.isinf(self.memory_limit)+ and torch.is_tensor(input)+ ):+ return self.basic_forward(input, memory_state, memory_cache)+ return self.slicing_forward(input, memory_state, memory_cache)++ def basic_forward(self, input: Tensor, memory_state: MemoryState = MemoryState.UNSET, memory_cache = None):+ mem_size = self.stride[0] - self.kernel_size[0]+ memory = memory_cache.get(self) if memory_cache is not None else None+ if (memory is not None) and (memory_state == MemoryState.ACTIVE):+ input = extend_head(input, memory=memory, times=-1)+ else:+ input = extend_head(input, times=self.temporal_padding * 2)+ next_memory = (+ input[:, :, mem_size:].detach()+ if (mem_size != 0 and memory_state != MemoryState.DISABLED)+ else None+ )+ if memory_cache is not None and memory_state != MemoryState.DISABLED:+ if next_memory is None:+ memory_cache.pop(self, None)+ else:+ memory_cache[self] = next_memory+ return super().forward(input)++ def slicing_forward(+ self,+ input,+ memory_state: MemoryState = MemoryState.UNSET,+ memory_cache = None,+ ) -> Tensor:+ if memory_cache is None:+ memory_cache = {}+ squeeze_out = False+ if torch.is_tensor(input):+ input = [input]+ squeeze_out = True++ cache_size = self.kernel_size[0] - self.stride[0]+ memory = memory_cache.get(self) if memory_cache is not None else None+ cache = cache_send_recv(+ input, cache_size=cache_size, memory=memory, times=self.temporal_padding * 2+ )++ if (+ memory_state in [MemoryState.INITIALIZING, MemoryState.ACTIVE]+ and cache_size != 0+ ):+ if cache_size > input[-1].size(2) and cache is not None and len(input) == 1:+ input[0] = torch.cat([cache, input[0]], dim=2)+ cache = None+ if cache_size <= input[-1].size(2):+ memory_cache[self] = input[-1][:, :, -cache_size:].detach().contiguous()++ padding = tuple(x for x in reversed(self.padding) for _ in range(2))+ for i in range(len(input)):+ next_cache = None+ cache_size = 0+ if i < len(input) - 1:+ cache_len = cache.size(2) if cache is not None else 0+ cache_size = get_cache_size(self, input[i].size(2) + cache_len, pad_len=0)+ if cache_size != 0:+ if cache_size > input[i].size(2) and cache is not None:+ input[i] = torch.cat([cache, input[i]], dim=2)+ cache = None+ if cache_size > input[i].size(2):+ raise ValueError(f"SeedVR2 VAE cache size {cache_size} exceeds input length {input[i].size(2)}.")+ next_cache = input[i][:, :, -cache_size:]++ input[i] = self.memory_limit_conv(+ input[i],+ padding=padding,+ prev_cache=cache+ )++ cache = next_cache++ return input[0] if squeeze_out else input++def remove_head(tensor: Tensor, times: int = 1) -> Tensor:+ if times == 0:+ return tensor+ return torch.cat(tensors=(tensor[:, :, :1], tensor[:, :, times + 1 :]), dim=2)++class Upsample3D(nn.Module):++ def __init__(+ self,+ channels,+ out_channels = None,+ inflation_mode = "tail",+ temporal_up: bool = False,+ spatial_up: bool = True,+ ):+ super().__init__()+ self.channels = channels+ self.out_channels = out_channels or channels++ conv = InflatedCausalConv3d(+ self.channels,+ self.out_channels,+ 3,+ padding=1,+ inflation_mode=inflation_mode,+ )++ self.temporal_up = temporal_up+ self.spatial_up = spatial_up+ self.temporal_ratio = 2 if temporal_up else 1+ self.spatial_ratio = 2 if spatial_up else 1++ upscale_ratio = (self.spatial_ratio**2) * self.temporal_ratio+ self.upscale_conv = ops.Conv3d(+ self.channels, self.channels * upscale_ratio, kernel_size=1, padding=0+ )++ self.conv = conv++ def forward(+ self,+ hidden_states: torch.FloatTensor,+ memory_state=None,+ memory_cache=None,+ ) -> torch.FloatTensor:+ if hidden_states.shape[1] != self.channels:+ raise ValueError(f"SeedVR2 upsample expected {self.channels} channels, got {hidden_states.shape[1]}.")++ hidden_states = self.upscale_conv(hidden_states)+ b, channels, f, h, w = hidden_states.shape+ c = channels // (self.spatial_ratio * self.spatial_ratio * self.temporal_ratio)+ hidden_states = hidden_states.view(b, self.spatial_ratio, self.spatial_ratio, self.temporal_ratio, c, f, h, w)+ hidden_states = hidden_states.permute(0, 4, 5, 3, 6, 1, 7, 2).reshape(+ b,+ c,+ f * self.temporal_ratio,+ h * self.spatial_ratio,+ w * self.spatial_ratio,+ )++ if self.temporal_up and memory_state != MemoryState.ACTIVE:+ hidden_states = remove_head(hidden_states)++ hidden_states = self.conv(hidden_states, memory_state=memory_state, memory_cache=memory_cache)++ return hidden_states+++class Downsample3D(nn.Module):+ def __init__(+ self,+ channels,+ out_channels = None,+ inflation_mode = "tail",+ spatial_down: bool = False,+ temporal_down: bool = False,+ ):+ super().__init__()+ self.channels = channels+ self.out_channels = out_channels or channels+ self.temporal_down = temporal_down+ self.spatial_down = spatial_down++ self.temporal_ratio = 2 if temporal_down else 1+ self.spatial_ratio = 2 if spatial_down else 1++ self.temporal_kernel = 3 if temporal_down else 1+ self.spatial_kernel = 3 if spatial_down else 1++ self.conv = InflatedCausalConv3d(+ self.channels,+ self.out_channels,+ kernel_size=(self.temporal_kernel, self.spatial_kernel, self.spatial_kernel),+ stride=(self.temporal_ratio, self.spatial_ratio, self.spatial_ratio),+ padding=(1 if self.temporal_down else 0, 0, 0),+ inflation_mode=inflation_mode,+ )+++ def forward(+ self,+ hidden_states: torch.FloatTensor,+ memory_state = None,+ memory_cache = None,+ ) -> torch.FloatTensor:++ if hidden_states.shape[1] != self.channels:+ raise ValueError(f"SeedVR2 downsample expected {self.channels} channels, got {hidden_states.shape[1]}.")++ if self.spatial_down:+ pad = (0, 1, 0, 1)+ hidden_states = F.pad(hidden_states, pad, mode="constant", value=0)++ if hidden_states.shape[1] != self.channels:+ raise ValueError(f"SeedVR2 downsample expected {self.channels} channels after padding, got {hidden_states.shape[1]}.")++ hidden_states = self.conv(hidden_states, memory_state=memory_state, memory_cache=memory_cache)++ return hidden_states+++class ResnetBlock3D(nn.Module):+ def __init__(+ self,+ in_channels: int,+ out_channels: Optional[int] = None,+ temb_channels: int = 512,+ groups: int = 32,+ groups_out: Optional[int] = None,+ eps: float = 1e-6,+ output_scale_factor: float = 1.0,+ skip_time_act: bool = False,+ inflation_mode = "tail",+ time_receptive_field: _receptive_field_t = "half",+ ):+ super().__init__()+ self.in_channels = in_channels+ self.out_channels = in_channels if out_channels is None else out_channels+ self.output_scale_factor = output_scale_factor+ self.skip_time_act = skip_time_act+ self.nonlinearity = nn.SiLU()+ if temb_channels is not None:+ self.time_emb_proj = ops.Linear(temb_channels, self.out_channels)+ else:+ self.time_emb_proj = None+ self.norm1 = ops.GroupNorm(num_groups=groups, num_channels=in_channels, eps=eps, affine=True)+ if groups_out is None:+ groups_out = groups+ self.norm2 = ops.GroupNorm(num_groups=groups_out, num_channels=self.out_channels, eps=eps, affine=True)+ self.use_in_shortcut = self.in_channels != self.out_channels+ self.conv1 = InflatedCausalConv3d(+ self.in_channels,+ self.out_channels,+ kernel_size=(1, 3, 3) if time_receptive_field == "half" else (3, 3, 3),+ stride=1,+ padding=(0, 1, 1) if time_receptive_field == "half" else (1, 1, 1),+ inflation_mode=inflation_mode,+ )++ self.conv2 = InflatedCausalConv3d(+ self.out_channels,+ self.out_channels,+ kernel_size=3,+ stride=1,+ padding=1,+ inflation_mode=inflation_mode,+ )++ self.conv_shortcut = None+ if self.use_in_shortcut:+ self.conv_shortcut = InflatedCausalConv3d(+ self.in_channels,+ self.out_channels,+ kernel_size=1,+ stride=1,+ padding=0,+ bias=True,+ inflation_mode=inflation_mode,+ )++ def forward(self, input_tensor, temb, memory_state = None, memory_cache = None):+ hidden_states = input_tensor++ hidden_states = causal_norm_wrapper(self.norm1, hidden_states)++ hidden_states = self.nonlinearity(hidden_states)++ hidden_states = self.conv1(hidden_states, memory_state=memory_state, memory_cache=memory_cache)++ if self.time_emb_proj is not None:+ if not self.skip_time_act:+ temb = self.nonlinearity(temb)+ temb = self.time_emb_proj(temb)[:, :, None, None]++ if temb is not None:+ hidden_states = hidden_states + temb++ hidden_states = causal_norm_wrapper(self.norm2, hidden_states)++ hidden_states = self.nonlinearity(hidden_states)++ hidden_states = self.conv2(hidden_states, memory_state=memory_state, memory_cache=memory_cache)++ if self.conv_shortcut is not None:+ input_tensor = self.conv_shortcut(input_tensor, memory_state=memory_state, memory_cache=memory_cache)++ output_tensor = (input_tensor + hidden_states) / self.output_scale_factor++ return output_tensor+++class DownEncoderBlock3D(nn.Module):+ def __init__(+ self,+ in_channels: int,+ out_channels: int,+ num_layers: int = 1,+ resnet_eps: float = 1e-6,+ resnet_groups: int = 32,+ output_scale_factor: float = 1.0,+ add_downsample: bool = True,+ inflation_mode = "tail",+ time_receptive_field: _receptive_field_t = "half",+ temporal_down: bool = True,+ spatial_down: bool = True,+ ):+ super().__init__()+ resnets = []++ for i in range(num_layers):+ in_channels = in_channels if i == 0 else out_channels+ resnets.append(+ ResnetBlock3D(+ in_channels=in_channels,+ out_channels=out_channels,+ temb_channels=None,+ eps=resnet_eps,+ groups=resnet_groups,+ output_scale_factor=output_scale_factor,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )+ )++ self.resnets = nn.ModuleList(resnets)++ if add_downsample:+ self.downsamplers = nn.ModuleList(+ [+ Downsample3D(+ out_channels,+ out_channels=out_channels,+ temporal_down=temporal_down,+ spatial_down=spatial_down,+ inflation_mode=inflation_mode,+ )+ ]+ )+ else:+ self.downsamplers = None++ def forward(+ self,+ hidden_states: torch.FloatTensor,+ memory_state = None,+ memory_cache = None,+ ) -> torch.FloatTensor:+ for resnet in self.resnets:+ hidden_states = resnet(hidden_states, temb=None, memory_state=memory_state, memory_cache=memory_cache)++ if self.downsamplers is not None:+ for downsampler in self.downsamplers:+ hidden_states = downsampler(hidden_states, memory_state=memory_state, memory_cache=memory_cache)++ return hidden_states+++class UpDecoderBlock3D(nn.Module):+ def __init__(+ self,+ in_channels: int,+ out_channels: int,+ num_layers: int = 1,+ resnet_eps: float = 1e-6,+ resnet_groups: int = 32,+ output_scale_factor: float = 1.0,+ add_upsample: bool = True,+ temb_channels: Optional[int] = None,+ inflation_mode = "tail",+ time_receptive_field: _receptive_field_t = "half",+ temporal_up: bool = True,+ spatial_up: bool = True,+ ):+ super().__init__()+ resnets = []++ for i in range(num_layers):+ input_channels = in_channels if i == 0 else out_channels++ resnets.append(+ ResnetBlock3D(+ in_channels=input_channels,+ out_channels=out_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ output_scale_factor=output_scale_factor,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )+ )++ self.resnets = nn.ModuleList(resnets)++ if add_upsample:+ self.upsamplers = nn.ModuleList(+ [+ Upsample3D(+ out_channels,+ out_channels=out_channels,+ temporal_up=temporal_up,+ spatial_up=spatial_up,+ inflation_mode=inflation_mode,+ )+ ]+ )+ else:+ self.upsamplers = None++ def forward(+ self,+ hidden_states: torch.FloatTensor,+ temb: Optional[torch.FloatTensor] = None,+ memory_state=None,+ memory_cache=None,+ ) -> torch.FloatTensor:+ for resnet in self.resnets:+ hidden_states = resnet(hidden_states, temb=None, memory_state=memory_state, memory_cache=memory_cache)++ if self.upsamplers is not None:+ for upsampler in self.upsamplers:+ hidden_states = upsampler(hidden_states, memory_state=memory_state, memory_cache=memory_cache)++ return hidden_states+++class UNetMidBlock3D(nn.Module):+ def __init__(+ self,+ in_channels: int,+ temb_channels: int,+ num_layers: int = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default", # default, spatial+ resnet_groups: int = 32,+ add_attention: bool = True,+ attention_head_dim: int = 1,+ output_scale_factor: float = 1.0,+ inflation_mode = "tail",+ time_receptive_field: _receptive_field_t = "half",+ ):+ super().__init__()+ resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)+ self.add_attention = add_attention++ resnets = [+ ResnetBlock3D(+ in_channels=in_channels,+ out_channels=in_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ output_scale_factor=output_scale_factor,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )+ ]+ attentions = []++ if attention_head_dim is None:+ attention_head_dim = in_channels++ for _ in range(num_layers):+ if self.add_attention:+ attentions.append(+ Attention(+ in_channels,+ heads=in_channels // attention_head_dim,+ dim_head=attention_head_dim,+ rescale_output_factor=output_scale_factor,+ eps=resnet_eps,+ norm_num_groups=(+ resnet_groups if resnet_time_scale_shift == "default" else None+ ),+ spatial_norm_dim=(+ temb_channels if resnet_time_scale_shift == "spatial" else None+ ),+ residual_connection=True,+ bias=True,+ )+ )+ else:+ attentions.append(None)++ resnets.append(+ ResnetBlock3D(+ in_channels=in_channels,+ out_channels=in_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ output_scale_factor=output_scale_factor,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )+ )++ self.attentions = nn.ModuleList(attentions)+ self.resnets = nn.ModuleList(resnets)++ def forward(self, hidden_states, temb=None, memory_state=None, memory_cache=None):+ video_length = hidden_states.size(2)+ hidden_states = self.resnets[0](hidden_states, temb, memory_state=memory_state, memory_cache=memory_cache)+ for attn, resnet in zip(self.attentions, self.resnets[1:]):+ if attn is not None:+ b, c, f, h, w = hidden_states.shape+ hidden_states = hidden_states.transpose(1, 2).reshape(b * f, c, h, w)+ hidden_states = attn(hidden_states, temb=temb)+ hidden_states = hidden_states.reshape(b, video_length, c, h, w).transpose(1, 2)+ hidden_states = resnet(hidden_states, temb, memory_state=memory_state, memory_cache=memory_cache)++ return hidden_states+++class Encoder3D(nn.Module):+ def __init__(+ self,+ in_channels: int = 3,+ out_channels: int = 3,+ down_block_types: Tuple[str, ...] = ("DownEncoderBlock3D",),+ block_out_channels: Tuple[int, ...] = (64,),+ layers_per_block: int = 2,+ norm_num_groups: int = 32,+ mid_block_add_attention=True,+ temporal_down_num: int = 2,+ inflation_mode = "tail",+ time_receptive_field: _receptive_field_t = "half",+ ):+ super().__init__()+ self.layers_per_block = layers_per_block+ self.temporal_down_num = temporal_down_num++ self.conv_in = InflatedCausalConv3d(+ in_channels,+ block_out_channels[0],+ kernel_size=3,+ stride=1,+ padding=1,+ inflation_mode=inflation_mode,+ )++ self.mid_block = None+ self.down_blocks = nn.ModuleList([])++ output_channel = block_out_channels[0]+ for i, down_block_type in enumerate(down_block_types):+ input_channel = output_channel+ output_channel = block_out_channels[i]+ is_final_block = i == len(block_out_channels) - 1+ is_temporal_down_block = i >= len(block_out_channels) - self.temporal_down_num - 1++ if down_block_type != "DownEncoderBlock3D":+ raise ValueError(f"SeedVR2 encoder only supports DownEncoderBlock3D, got {down_block_type}.")++ down_block = DownEncoderBlock3D(+ num_layers=self.layers_per_block,+ in_channels=input_channel,+ out_channels=output_channel,+ add_downsample=not is_final_block,+ resnet_eps=1e-6,+ resnet_groups=norm_num_groups,+ temporal_down=is_temporal_down_block,+ spatial_down=True,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )+ self.down_blocks.append(down_block)++ self.mid_block = UNetMidBlock3D(+ in_channels=block_out_channels[-1],+ resnet_eps=1e-6,+ output_scale_factor=1,+ resnet_time_scale_shift="default",+ attention_head_dim=block_out_channels[-1],+ resnet_groups=norm_num_groups,+ temb_channels=None,+ add_attention=mid_block_add_attention,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )++ self.conv_norm_out = ops.GroupNorm(+ num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6+ )+ self.conv_act = nn.SiLU()++ conv_out_channels = 2 * out_channels+ self.conv_out = InflatedCausalConv3d(+ block_out_channels[-1], conv_out_channels, 3, padding=1, inflation_mode=inflation_mode+ )+++ def forward(+ self,+ sample: torch.FloatTensor,+ memory_state = None,+ memory_cache = None,+ ) -> torch.FloatTensor:+ sample = sample.to(next(self.parameters()).device)+ sample = self.conv_in(sample, memory_state=memory_state, memory_cache=memory_cache)+ for down_block in self.down_blocks:+ sample = down_block(sample, memory_state=memory_state, memory_cache=memory_cache)++ sample = self.mid_block(sample, memory_state=memory_state, memory_cache=memory_cache)++ sample = causal_norm_wrapper(self.conv_norm_out, sample)+ sample = self.conv_act(sample)+ sample = self.conv_out(sample, memory_state=memory_state, memory_cache=memory_cache)++ return sample+++class Decoder3D(nn.Module):++ def __init__(+ self,+ in_channels: int = 3,+ out_channels: int = 3,+ up_block_types: Tuple[str, ...] = ("UpDecoderBlock3D",),+ block_out_channels: Tuple[int, ...] = (64,),+ layers_per_block: int = 2,+ norm_num_groups: int = 32,+ mid_block_add_attention=True,+ inflation_mode = "tail",+ time_receptive_field: _receptive_field_t = "half",+ temporal_up_num: int = 2,+ ):+ super().__init__()+ self.layers_per_block = layers_per_block+ self.temporal_up_num = temporal_up_num++ self.conv_in = InflatedCausalConv3d(+ in_channels,+ block_out_channels[-1],+ kernel_size=3,+ stride=1,+ padding=1,+ inflation_mode=inflation_mode,+ )++ self.mid_block = None+ self.up_blocks = nn.ModuleList([])++ temb_channels = None++ self.mid_block = UNetMidBlock3D(+ in_channels=block_out_channels[-1],+ resnet_eps=1e-6,+ output_scale_factor=1,+ resnet_time_scale_shift="default",+ attention_head_dim=block_out_channels[-1],+ resnet_groups=norm_num_groups,+ temb_channels=temb_channels,+ add_attention=mid_block_add_attention,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )++ reversed_block_out_channels = list(reversed(block_out_channels))+ output_channel = reversed_block_out_channels[0]+ for i, up_block_type in enumerate(up_block_types):+ prev_output_channel = output_channel+ output_channel = reversed_block_out_channels[i]++ is_final_block = i == len(block_out_channels) - 1+ is_temporal_up_block = i < self.temporal_up_num+ if up_block_type != "UpDecoderBlock3D":+ raise ValueError(f"SeedVR2 decoder only supports UpDecoderBlock3D, got {up_block_type}.")+ up_block = UpDecoderBlock3D(+ num_layers=self.layers_per_block + 1,+ in_channels=prev_output_channel,+ out_channels=output_channel,+ add_upsample=not is_final_block,+ resnet_eps=1e-6,+ resnet_groups=norm_num_groups,+ temb_channels=temb_channels,+ temporal_up=is_temporal_up_block,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )+ self.up_blocks.append(up_block)+ prev_output_channel = output_channel++ self.conv_norm_out = ops.GroupNorm(+ num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6+ )+ self.conv_act = nn.SiLU()+ self.conv_out = InflatedCausalConv3d(+ block_out_channels[0], out_channels, 3, padding=1, inflation_mode=inflation_mode+ )+++ def forward(+ self,+ sample: torch.FloatTensor,+ latent_embeds: Optional[torch.FloatTensor] = None,+ memory_state = None,+ memory_cache = None,+ ) -> torch.FloatTensor:++ sample = sample.to(next(self.parameters()).device)+ sample = self.conv_in(sample, memory_state=memory_state, memory_cache=memory_cache)++ upscale_dtype = next(iter(self.up_blocks.parameters())).dtype+ sample = self.mid_block(sample, latent_embeds, memory_state=memory_state, memory_cache=memory_cache)+ sample = sample.to(upscale_dtype)++ for up_block in self.up_blocks:+ sample = up_block(sample, latent_embeds, memory_state=memory_state, memory_cache=memory_cache)++ sample = causal_norm_wrapper(self.conv_norm_out, sample)+ sample = self.conv_act(sample)+ sample = self.conv_out(sample, memory_state=memory_state, memory_cache=memory_cache)++ return sample++class VideoAutoencoderKL(nn.Module):+ def __init__(+ self,+ in_channels: int = 3,+ out_channels: int = 3,+ layers_per_block: int = 2,+ latent_channels: int = SEEDVR2_LATENT_CHANNELS,+ norm_num_groups: int = 32,+ temporal_scale_num: int = 2,+ inflation_mode = "pad",+ time_receptive_field: _receptive_field_t = "full",+ slicing_sample_min_size = BYTEDANCE_SLICING_SAMPLE_MIN,+ ):+ self.slicing_sample_min_size = slicing_sample_min_size+ self.slicing_latent_min_size = slicing_sample_min_size // (2**temporal_scale_num)+ block_out_channels = BYTEDANCE_BLOCK_OUT_CHANNELS+ down_block_types = ("DownEncoderBlock3D",) * 4+ up_block_types = ("UpDecoderBlock3D",) * 4+ super().__init__()++ self.encoder = Encoder3D(+ in_channels=in_channels,+ out_channels=latent_channels,+ down_block_types=down_block_types,+ block_out_channels=block_out_channels,+ layers_per_block=layers_per_block,+ norm_num_groups=norm_num_groups,+ temporal_down_num=temporal_scale_num,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )++ self.decoder = Decoder3D(+ in_channels=latent_channels,+ out_channels=out_channels,+ up_block_types=up_block_types,+ block_out_channels=block_out_channels,+ layers_per_block=layers_per_block,+ norm_num_groups=norm_num_groups,+ temporal_up_num=temporal_scale_num,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )++ self.use_slicing = True++ def encode(self, x: torch.FloatTensor, return_dict: bool = True):+ h = self.slicing_encode(x)+ posterior = DiagonalGaussianDistribution(h).mode()++ if not return_dict:+ return (posterior,)++ return posterior++ def decode_(+ self, z: torch.Tensor, return_dict: bool = True+ ):+ decoded = self.slicing_decode(z)++ if not return_dict:+ return (decoded,)++ return decoded++ def _encode(+ self, x, memory_state = MemoryState.DISABLED, memory_cache = None+ ) -> torch.Tensor:+ _x = x.to(self.device)+ h = self.encoder(_x, memory_state=memory_state, memory_cache=memory_cache)+ return h.to(x.device)++ def _decode(+ self, z, memory_state = MemoryState.DISABLED, memory_cache = None+ ) -> torch.Tensor:+ _z = z.to(self.device)+ output = self.decoder(_z, memory_state=memory_state, memory_cache=memory_cache)+ return output.to(z.device)++ def slicing_encode(self, x: torch.Tensor) -> torch.Tensor:+ if self.use_slicing and (x.shape[2] - 1) > self.slicing_sample_min_size:+ memory_cache = {}+ split_size = max(+ self.slicing_sample_min_size,+ getattr(self, "temporal_downsample_factor", 1),+ )+ x_slices = list(x[:, :, 1:].split(split_size=split_size, dim=2))+ min_active_len = getattr(self, "temporal_downsample_factor", 1)+ if len(x_slices) > 1 and x_slices[-1].shape[2] < min_active_len:+ x_slices[-2] = torch.cat((x_slices[-2], x_slices[-1]), dim=2)+ x_slices.pop()+ encoded_slices = [+ self._encode(+ torch.cat((x[:, :, :1], x_slices[0]), dim=2),+ memory_state=MemoryState.INITIALIZING,+ memory_cache=memory_cache,+ )+ ]+ for x_idx in range(1, len(x_slices)):+ encoded_slices.append(+ self._encode(x_slices[x_idx], memory_state=MemoryState.ACTIVE, memory_cache=memory_cache)+ )+ out = torch.cat(encoded_slices, dim=2)+ return out+ else:+ return self._encode(x)++ def slicing_decode(self, z: torch.Tensor) -> torch.Tensor:+ if self.use_slicing and (z.shape[2] - 1) > self.slicing_latent_min_size:+ memory_cache = {}+ z_slices = z[:, :, 1:].split(split_size=self.slicing_latent_min_size, dim=2)+ decoded_slices = [+ self._decode(+ torch.cat((z[:, :, :1], z_slices[0]), dim=2),+ memory_state=MemoryState.INITIALIZING,+ memory_cache=memory_cache,+ )+ ]+ for z_idx in range(1, len(z_slices)):+ decoded_slices.append(+ self._decode(z_slices[z_idx], memory_state=MemoryState.ACTIVE, memory_cache=memory_cache)+ )+ out = torch.cat(decoded_slices, dim=2)+ return out+ else:+ return self._decode(z)++ def forward(self, x: torch.FloatTensor, mode: Literal["encode", "decode", "all"] = "all"):+ def _unwrap(value):+ return value[0] if isinstance(value, tuple) else value++ if mode == "encode":+ return _unwrap(self.encode(x))+ if mode == "decode":+ return _unwrap(self.decode_(x))+ if mode == "all":+ latent = _unwrap(self.encode(x))+ return _unwrap(self.decode_(latent))+ raise ValueError(f"Unknown SeedVR2 VAE forward mode: {mode}")++class VideoAutoencoderKLWrapper(VideoAutoencoderKL):+ def __init__(+ self,+ spatial_downsample_factor = 8,+ temporal_downsample_factor = 4,+ ):+ self.spatial_downsample_factor = spatial_downsample_factor+ self.temporal_downsample_factor = temporal_downsample_factor+ super().__init__()+ self.set_memory_limit(BYTEDANCE_VAE_CONV_MEM_GIB, BYTEDANCE_VAE_NORM_MEM_GIB)++ def forward(self, x: torch.FloatTensor):+ z, p = self._encode_with_raw_latent(x)+ x = self.decode(z)+ return x, z, p++ def _encode_with_raw_latent(self, x):+ if x.ndim == 4:+ x = x.unsqueeze(2)+ x = x.to(dtype=next(self.parameters()).dtype)+ self.device = x.device+ p = super().encode(x)+ z = p.squeeze(2)+ return z, p++ def encode(self, x):+ z, _ = self._encode_with_raw_latent(x)+ return z++ def decode(self, z, seedvr2_tiling=None):+ seedvr2_tiling = {} if seedvr2_tiling is None else seedvr2_tiling+ if not isinstance(seedvr2_tiling, dict):+ raise RuntimeError(+ "SeedVR2 VideoAutoencoderKLWrapper.decode: `seedvr2_tiling` must be a dict; "+ f"got {type(seedvr2_tiling).__name__} with value {seedvr2_tiling!r}."+ )++ if z.ndim == 5:+ _, c, _, _, _ = z.shape+ if c != SEEDVR2_LATENT_CHANNELS:+ raise RuntimeError(+ "SeedVR2 VideoAutoencoderKLWrapper.decode: 5-D latent input must "+ f"have {SEEDVR2_LATENT_CHANNELS} channels; got shape {tuple(z.shape)}."+ )+ latent = z+ elif z.ndim == 4:+ b, tc, h, w = z.shape+ if tc % SEEDVR2_LATENT_CHANNELS != 0:+ raise RuntimeError(+ "SeedVR2 VideoAutoencoderKLWrapper.decode: 4-D latent input must "+ f"use collapsed channel layout (B, {SEEDVR2_LATENT_CHANNELS}*T, H, W); "+ f"got shape {tuple(z.shape)}."+ )+ latent = z.reshape(b, SEEDVR2_LATENT_CHANNELS, -1, h, w)+ else:+ raise RuntimeError(+ "SeedVR2 VideoAutoencoderKLWrapper.decode: latent input must be "+ f"4-D collapsed (B, {SEEDVR2_LATENT_CHANNELS}*T, H, W) or "+ f"5-D (B, {SEEDVR2_LATENT_CHANNELS}, T, H, W); "+ f"got shape {tuple(z.shape)}."+ )+ scale = BYTEDANCE_VAE_SCALING_FACTOR+ shift = BYTEDANCE_VAE_SHIFTING_FACTOR+ latent = latent / scale + shift++ self.device = latent.device+ enable_tiling = seedvr2_tiling.get("enable_tiling", False)++ if enable_tiling:+ decode_seedvr2_args = dict(seedvr2_tiling)+ decode_seedvr2_args.pop("enable_tiling", None)+ tile_h, tile_w = decode_seedvr2_args.get("tile_size", (512, 512))+ ov_h, ov_w = decode_seedvr2_args.get("tile_overlap", (64, 64))+ decode_seedvr2_args["tile_overlap"] = (+ min(ov_h, max(0, tile_h - 8)),+ min(ov_w, max(0, tile_w - 8)),+ )+ x = tiled_vae(latent, self, **decode_seedvr2_args, encode=False)+ if x.ndim == 4:+ # tiled_vae squeezes the temporal axis when+ # temporal_downsample_factor == 1 AND latent T == 1+ # (see tiled_vae line 179-180); re-add it so the post-decode+ # pipeline can keep batch and time distinct on the tiled path.+ x = x.unsqueeze(2)+ else:+ x = super().decode_(latent)++ h, w = x.shape[-2:]+ w2 = w - (w % 2)+ h2 = h - (h % 2)+ x = x[..., :h2, :w2]++ return x++ def decode_tiled(self, z, tile_x=32, tile_y=32, overlap=8, tile_t=None, overlap_t=None):+ # SeedVR2's causal VAE owns temporal via the MemoryState cache; external+ # temporal tiling breaks that continuity, so only spatial tiling is applied.+ sf = self.spatial_downsample_factor+ seedvr2_tiling = {+ "enable_tiling": True,+ "tile_size": (tile_y * sf, tile_x * sf),+ "tile_overlap": (overlap * sf, overlap * sf),+ "temporal_size": None,+ "temporal_overlap": None,+ }+ return self.decode(z, seedvr2_tiling=seedvr2_tiling)++ def encode_tiled(self, x, tile_x=None, tile_y=None, overlap=None, tile_t=None, overlap_t=None):+ # External temporal tiling knobs are discarded; the causal VAE keeps its+ # own internal MemoryState slicing.+ if tile_y is None:+ tile_y = 512+ if tile_x is None:+ tile_x = 512+ if overlap is None:+ overlap_y = 64+ overlap_x = 64+ else:+ overlap_y = overlap+ overlap_x = overlap+ overlap_y = min(overlap_y, max(0, tile_y - 8))+ overlap_x = min(overlap_x, max(0, tile_x - 8))+ self.device = x.device+ return tiled_vae(+ x,+ self,+ tile_size=(tile_y, tile_x),+ tile_overlap=(overlap_y, overlap_x),+ temporal_size=None,+ temporal_overlap=None,+ encode=True,+ )++ def comfy_format_encoded(self, samples):+ if samples.ndim == 4:+ samples = samples.unsqueeze(2)+ samples = samples.contiguous()+ samples = samples * BYTEDANCE_VAE_SCALING_FACTOR+ return samples++ def comfy_memory_used_decode(self, shape):+ bytes_per_output_pixel = 160++ def output_pixels(latent_t, latent_h, latent_w):+ output_t = max(1, (latent_t - 1) * 4 + 1)+ return output_t * latent_h * 8 * latent_w * 8++ # SeedVR2 decode performs full-frame LAB histogram matching: fp32 channels+ # plus int64 sort indices dominate peak memory, not the VAE weight dtype.+ if len(shape) == 5:+ candidates = []+ if shape[1] == SEEDVR2_LATENT_CHANNELS:+ candidates.append((shape[2], shape[3], shape[4]))+ if shape[-1] == SEEDVR2_LATENT_CHANNELS:+ candidates.append((shape[1], shape[2], shape[3]))+ if len(candidates) == 0:+ candidates.append((shape[2], shape[3], shape[4]))+ pixels = max(output_pixels(*candidate) for candidate in candidates)+ elif len(shape) == 4:+ latent_t = max(1, (shape[1] + SEEDVR2_LATENT_CHANNELS - 1) // SEEDVR2_LATENT_CHANNELS)+ pixels = output_pixels(latent_t, shape[2], shape[3])+ else:+ pixels = output_pixels(1, shape[-2], shape[-1])+ return pixels * bytes_per_output_pixel++ def set_memory_limit(self, conv_max_mem: Optional[float], norm_max_mem: Optional[float]):+ set_norm_limit(norm_max_mem)+ for m in self.modules():+ if isinstance(m, InflatedCausalConv3d):+ m.set_memory_limit(conv_max_mem if conv_max_mem is not None else float("inf"))diff --git a/comfy/model_base.py b/comfy/model_base.pyindex dcfa555dc..786a7c127 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -55,6 +55,7 @@ import comfy.ldm.pixeldit.modelimport comfy.ldm.pixeldit.pidimport comfy.ldm.ace.modelimport comfy.ldm.omnigen.omnigen2+import comfy.ldm.seedvr.modelimport comfy.ldm.boogu.modelimport comfy.ldm.qwen_image.modelimport comfy.ldm.ideogram4.model@@ -932,6 +933,17 @@ class HunyuanDiT(BaseModel):out['image_meta_size'] = comfy.conds.CONDRegular(torch.FloatTensor([[height, width, target_height, target_width, 0, 0]]))return out+class SeedVR2(BaseModel):+ def __init__(self, model_config, model_type=ModelType.FLOW, device=None):+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.seedvr.model.NaDiT)++ def extra_conds(self, **kwargs):+ out = super().extra_conds(**kwargs)+ condition = kwargs.get("condition", None)+ if condition is not None:+ out["condition"] = comfy.conds.CONDRegular(condition)+ return out+class PixArt(BaseModel):def __init__(self, model_config, model_type=ModelType.EPS, device=None):super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.pixart.pixartms.PixArtMS)diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex e53d848c9..174bc77cc 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -598,6 +598,44 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):return dit_config+ seedvr2_7b_separate_key = "{}blocks.35.mlp.vid.proj_out.weight".format(key_prefix)+ if seedvr2_7b_separate_key in state_dict_keys and state_dict[seedvr2_7b_separate_key].shape[0] == 3072: # seedvr2 7b+ dit_config = {}+ dit_config["image_model"] = "seedvr2"+ dit_config["vid_dim"] = 3072+ dit_config["heads"] = 24+ dit_config["num_layers"] = 36+ # This checkpoint uses separate vid/txt MMModule keys in every block.+ dit_config["mm_layers"] = 36+ dit_config["norm_eps"] = 1e-5+ dit_config["rope_type"] = "rope3d"+ dit_config["rope_dim"] = 64+ dit_config["mlp_type"] = "normal"+ return dit_config+ if "{}blocks.35.mlp.all.proj_in_gate.weight".format(key_prefix) in state_dict_keys: # seedvr2 7b+ dit_config = {}+ dit_config["image_model"] = "seedvr2"+ dit_config["vid_dim"] = 3072+ dit_config["heads"] = 24+ dit_config["num_layers"] = 36+ # This checkpoint uses shared all.* MMModule keys after the initial blocks.+ dit_config["mm_layers"] = 10+ dit_config["norm_eps"] = 1e-5+ dit_config["rope_type"] = "rope3d"+ dit_config["rope_dim"] = 64+ dit_config["mlp_type"] = "swiglu"+ return dit_config+ if "{}blocks.31.mlp.all.proj_in_gate.weight".format(key_prefix) in state_dict_keys: # seedvr2 3b+ dit_config = {}+ dit_config["image_model"] = "seedvr2"+ dit_config["vid_dim"] = 2560+ dit_config["heads"] = 20+ dit_config["num_layers"] = 32+ dit_config["norm_eps"] = 1.0e-05+ dit_config["mlp_type"] = "swiglu"+ dit_config["vid_out_norm"] = True+ return dit_config+if '{}head.modulation'.format(key_prefix) in state_dict_keys: # Wan 2.1dit_config = {}dit_config["image_model"] = "wan2.1"@@ -1119,9 +1157,10 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):return unet_config-def model_config_from_unet_config(unet_config, state_dict=None):++def model_config_from_unet_config(unet_config, state_dict=None, unet_key_prefix=""):for model_config in comfy.supported_models.models:- if model_config.matches(unet_config, state_dict):+ if model_config.matches(unet_config, state_dict, unet_key_prefix=unet_key_prefix):return model_config(unet_config)logging.error("no match {}".format(unet_config))@@ -1131,7 +1170,7 @@ def model_config_from_unet(state_dict, unet_key_prefix, use_base_if_no_match=Falunet_config = detect_unet_config(state_dict, unet_key_prefix, metadata=metadata)if unet_config is None:return None- model_config = model_config_from_unet_config(unet_config, state_dict)+ model_config = model_config_from_unet_config(unet_config, state_dict, unet_key_prefix)if model_config is None and use_base_if_no_match:model_config = comfy.supported_models_base.BASE(unet_config)diff --git a/comfy/sd.py b/comfy/sd.pyindex 071a3102a..4a0742e7a 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -16,6 +16,7 @@ import comfy.ldm.cosmos.vaeimport comfy.ldm.wan.vaeimport comfy.ldm.wan.vae2_2import comfy.ldm.hunyuan3d.vae+import comfy.ldm.seedvr.vaeimport comfy.ldm.triposplat.vaeimport comfy.ldm.ace.vae.music_dcae_pipelineimport comfy.ldm.cogvideo.vae@@ -473,7 +474,8 @@ class CLIP:class VAE:def __init__(self, sd=None, device=None, config=None, dtype=None, metadata=None):- if 'decoder.up_blocks.0.resnets.0.norm1.weight' in sd.keys(): #diffusers format+ is_seedvr2_vae = "decoder.up_blocks.2.upsamplers.0.upscale_conv.weight" in sd+ if not is_seedvr2_vae and 'decoder.up_blocks.0.resnets.0.norm1.weight' in sd.keys(): #diffusers formatsd = diffusers_convert.convert_vae_state_dict(sd)if model_management.is_amd():@@ -500,6 +502,8 @@ class VAE:self.upscale_index_formula = Noneself.extra_1d_channel = Noneself.crop_input = True+ self.handles_tiling = False+ self.format_encoded = Noneself.audio_sample_rate = 44100@@ -546,6 +550,22 @@ class VAE:self.first_stage_model = StageC_coder()self.downscale_ratio = 32self.latent_channels = 16+ elif "decoder.up_blocks.2.upsamplers.0.upscale_conv.weight" in sd: # seedvr2+ self.first_stage_model = comfy.ldm.seedvr.vae.VideoAutoencoderKLWrapper()+ self.latent_channels = comfy.ldm.seedvr.vae.SEEDVR2_LATENT_CHANNELS+ self.latent_dim = 3+ self.disable_offload = True+ self.memory_used_decode = lambda shape, dtype: self.first_stage_model.comfy_memory_used_decode(shape)+ self.memory_used_encode = lambda shape, dtype: (max(shape[2], 5) * shape[3] * shape[4] * 64) * model_management.dtype_size(dtype)+ self.working_dtypes = [torch.float16, torch.bfloat16, torch.float32]+ self.handles_tiling = True+ self.format_encoded = self.first_stage_model.comfy_format_encoded+ self.downscale_ratio = (lambda a: max(0, math.floor((a + 3) / 4)), 8, 8)+ self.downscale_index_formula = (4, 8, 8)+ self.upscale_ratio = (lambda a: max(0, a * 4 - 3), 8, 8)+ self.upscale_index_formula = (4, 8, 8)+ self.process_input = lambda image: image * 2.0 - 1.0+ self.crop_input = Falseelif "decoder.conv_in.weight" in sd:if sd['decoder.conv_in.weight'].shape[1] == 64:ddconfig = {"block_out_channels": [128, 256, 512, 512, 1024, 1024], "in_channels": 3, "out_channels": 3, "num_res_blocks": 2, "ffactor_spatial": 32, "downsample_match_channel": True, "upsample_match_channel": True}@@ -1012,6 +1032,10 @@ class VAE:decode_fn = lambda a: self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)).to(dtype=self.vae_output_dtype())return self.process_output(comfy.utils.tiled_scale_multidim(samples, decode_fn, tile=(tile_t, tile_x, tile_y), overlap=overlap, upscale_amount=self.upscale_ratio, out_channels=self.output_channels, index_formulas=self.upscale_index_formula, output_device=self.output_device))+ def _decode_tiled_owned(self, samples, **kwargs):+ out = self.first_stage_model.decode_tiled(samples.to(self.vae_dtype).to(self.device), **kwargs)+ return self.process_output(out.to(device=self.output_device, dtype=self.vae_output_dtype(), copy=True))+def encode_tiled_(self, pixel_samples, tile_x=512, tile_y=512, overlap = 64):steps = pixel_samples.shape[0] * comfy.utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x, tile_y, overlap)steps += pixel_samples.shape[0] * comfy.utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x // 2, tile_y * 2, overlap)@@ -1048,6 +1072,25 @@ class VAE:encode_fn = lambda a: self.first_stage_model.encode((self.process_input(a)).to(self.vae_dtype).to(self.device)).to(dtype=self.vae_output_dtype())return comfy.utils.tiled_scale_multidim(samples, encode_fn, tile=(tile_t, tile_x, tile_y), overlap=overlap, upscale_amount=self.downscale_ratio, out_channels=self.latent_channels, downscale=True, index_formulas=self.downscale_index_formula, output_device=self.output_device)+ def _encode_tiled_owned(self, pixel_samples, **kwargs):+ x = self.process_input(pixel_samples).to(self.vae_dtype).to(self.device)+ out = self.first_stage_model.encode_tiled(x, **kwargs)+ return out.to(device=self.output_device, dtype=self.vae_output_dtype())++ def _owned_tiled_args(self, tile_x=None, tile_y=None, overlap=None, tile_t=None, overlap_t=None):+ args = {}+ if tile_x is not None:+ args["tile_x"] = tile_x+ if tile_y is not None:+ args["tile_y"] = tile_y+ if overlap is not None:+ args["overlap"] = overlap+ if tile_t is not None:+ args["tile_t"] = tile_t+ if overlap_t is not None:+ args["overlap_t"] = overlap_t+ return args+def decode(self, samples_in, vae_options={}):self.throw_exception_if_invalid()pixel_samples = None@@ -1095,11 +1138,19 @@ class VAE:if dims == 1 or self.extra_1d_channel is not None:pixel_samples = self.decode_tiled_1d(samples_in)elif dims == 2:- pixel_samples = self.decode_tiled_(samples_in)+ if self.handles_tiling:+ tile = 256 // self.spacial_compression_decode()+ overlap = tile // 4+ pixel_samples = self._decode_tiled_owned(samples_in, tile_x=tile, tile_y=tile, overlap=overlap)+ else:+ pixel_samples = self.decode_tiled_(samples_in)elif dims == 3:tile = 256 // self.spacial_compression_decode()overlap = tile // 4- pixel_samples = self.decode_tiled_3d(samples_in, tile_x=tile, tile_y=tile, overlap=(1, overlap, overlap))+ if self.handles_tiling:+ pixel_samples = self._decode_tiled_owned(samples_in, tile_x=tile, tile_y=tile, overlap=overlap)+ else:+ pixel_samples = self.decode_tiled_3d(samples_in, tile_x=tile, tile_y=tile, overlap=(1, overlap, overlap))pixel_samples = pixel_samples.to(self.output_device).movedim(1,-1)return pixel_samples@@ -1118,7 +1169,9 @@ class VAE:args["overlap"] = overlapwith model_management.cuda_device_context(self.device):- if dims == 1 or self.extra_1d_channel is not None:+ if self.handles_tiling and dims in (2, 3):+ output = self._decode_tiled_owned(samples, **self._owned_tiled_args(tile_x, tile_y, overlap, tile_t, overlap_t))+ elif dims == 1 or self.extra_1d_channel is not None:args.pop("tile_y")output = self.decode_tiled_1d(samples, **args)elif dims == 2:@@ -1179,12 +1232,17 @@ class VAE:if self.latent_dim == 3:tile = 256overlap = tile // 4- samples = self.encode_tiled_3d(pixel_samples, tile_x=tile, tile_y=tile, overlap=(1, overlap, overlap))+ if self.handles_tiling:+ samples = self._encode_tiled_owned(pixel_samples, tile_x=tile, tile_y=tile, overlap=overlap)+ else:+ samples = self.encode_tiled_3d(pixel_samples, tile_x=tile, tile_y=tile, overlap=(1, overlap, overlap))elif self.latent_dim == 1 or self.extra_1d_channel is not None:samples = self.encode_tiled_1d(pixel_samples)else:samples = self.encode_tiled_(pixel_samples)+ if self.format_encoded is not None:+ samples = self.format_encoded(samples)return samplesdef encode_tiled(self, pixel_samples, tile_x=None, tile_y=None, overlap=None, tile_t=None, overlap_t=None):@@ -1192,7 +1250,7 @@ class VAE:pixel_samples = self.vae_encode_crop_pixels(pixel_samples)dims = self.latent_dimpixel_samples = pixel_samples.movedim(-1, 1)- if dims == 3:+ if dims == 3 and pixel_samples.ndim < 5:if not self.not_video:pixel_samples = pixel_samples.movedim(1, 0).unsqueeze(0)else:@@ -1216,21 +1274,27 @@ class VAE:elif dims == 2:samples = self.encode_tiled_(pixel_samples, **args)elif dims == 3:- if tile_t is not None:- tile_t_latent = max(2, self.downscale_ratio[0](tile_t))+ if self.handles_tiling:+ samples = self._encode_tiled_owned(pixel_samples, **self._owned_tiled_args(tile_x, tile_y, overlap, tile_t, overlap_t))else:- tile_t_latent = 9999- args["tile_t"] = self.upscale_ratio[0](tile_t_latent)+ if tile_t is not None:+ tile_t_latent = max(2, self.downscale_ratio[0](tile_t))+ else:+ tile_t_latent = 9999+ args["tile_t"] = self.upscale_ratio[0](tile_t_latent)- if overlap_t is None:- args["overlap"] = (1, overlap, overlap)- else:- args["overlap"] = (self.upscale_ratio[0](max(1, min(tile_t_latent // 2, self.downscale_ratio[0](overlap_t)))), overlap, overlap)- maximum = pixel_samples.shape[2]- maximum = self.upscale_ratio[0](self.downscale_ratio[0](maximum))+ spatial_overlap = overlap if overlap is not None else 64+ if overlap_t is None:+ args["overlap"] = (1, spatial_overlap, spatial_overlap)+ else:+ args["overlap"] = (self.upscale_ratio[0](max(1, min(tile_t_latent // 2, self.downscale_ratio[0](overlap_t)))), spatial_overlap, spatial_overlap)+ maximum = pixel_samples.shape[2]+ maximum = self.upscale_ratio[0](self.downscale_ratio[0](maximum))- samples = self.encode_tiled_3d(pixel_samples[:,:,:maximum], **args)+ samples = self.encode_tiled_3d(pixel_samples[:,:,:maximum], **args)+ if self.format_encoded is not None:+ samples = self.format_encoded(samples)return samplesdef get_sd(self):@@ -1898,7 +1962,7 @@ def load_state_dict_guess_config(sd, output_vae=True, output_clip=True, output_cmanual_cast_dtype = model_management.unet_manual_cast(None, load_device, model_config.supported_inference_dtypes)else:manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device, model_config.supported_inference_dtypes)- model_config.set_inference_dtype(unet_dtype, manual_cast_dtype)+ model_config.set_inference_dtype(unet_dtype, manual_cast_dtype, device=load_device)if model_config.clip_vision_prefix is not None:if output_clipvision:@@ -2039,7 +2103,7 @@ def load_diffusion_model_state_dict(sd, model_options={}, metadata=None, disablemanual_cast_dtype = model_management.unet_manual_cast(None, load_device, model_config.supported_inference_dtypes)else:manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device, model_config.supported_inference_dtypes)- model_config.set_inference_dtype(unet_dtype, manual_cast_dtype)+ model_config.set_inference_dtype(unet_dtype, manual_cast_dtype, device=load_device)if custom_operations is not None:model_config.custom_operations = custom_operationsdiff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex afb66e6f3..b82e4178f 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -1685,6 +1685,40 @@ class Chroma(supported_models_base.BASE):t5_detect = comfy.text_encoders.sd3_clip.t5_xxl_detect(state_dict, "{}t5xxl.transformer.".format(pref))return supported_models_base.ClipTarget(comfy.text_encoders.pixart_t5.PixArtTokenizer, comfy.text_encoders.pixart_t5.pixart_te(**t5_detect))+class SeedVR2(supported_models_base.BASE):+ unet_config = {+ "image_model": "seedvr2"+ }+ unet_extra_config = {}+ required_keys = {+ "{}positive_conditioning",+ "{}negative_conditioning",+ }+ latent_format = comfy.latent_formats.SeedVR2++ vae_key_prefix = ["vae."]+ text_encoder_key_prefix = ["text_encoders."]+ supported_inference_dtypes = [torch.bfloat16, torch.float16, torch.float32]+ sampling_settings = {+ "shift": 1.0,+ }++ def set_inference_dtype(self, dtype, manual_cast_dtype, device=None):+ if (+ dtype == torch.float16+ and manual_cast_dtype is None+ and comfy.model_management.should_use_bf16(device)+ ):+ manual_cast_dtype = torch.bfloat16+ super().set_inference_dtype(dtype, manual_cast_dtype, device=device)++ def get_model(self, state_dict, prefix="", device=None):+ out = model_base.SeedVR2(self, device=device)+ return out++ def clip_target(self, state_dict={}):+ return None+class ChromaRadiance(Chroma):unet_config = {"image_model": "chroma_radiance",@@ -2348,6 +2382,7 @@ models = [HiDream,HiDreamO1,Chroma,+ SeedVR2,ChromaRadiance,ACEStep,ACEStep15,diff --git a/comfy/supported_models_base.py b/comfy/supported_models_base.pyindex 0e7a829ba..e3a8e131f 100644--- a/comfy/supported_models_base.py+++ b/comfy/supported_models_base.py@@ -54,13 +54,13 @@ class BASE:optimizations = {"fp8": False}@classmethod- def matches(s, unet_config, state_dict=None):+ def matches(s, unet_config, state_dict=None, unet_key_prefix=""):for k in s.unet_config:if k not in unet_config or s.unet_config[k] != unet_config[k]:return Falseif state_dict is not None:for k in s.required_keys:- if k not in state_dict:+ if k.format(unet_key_prefix) not in state_dict:return Falsereturn True@@ -115,7 +115,7 @@ class BASE:replace_prefix = {"": self.vae_key_prefix[0]}return utils.state_dict_prefix_replace(state_dict, replace_prefix)- def set_inference_dtype(self, dtype, manual_cast_dtype):+ def set_inference_dtype(self, dtype, manual_cast_dtype, device=None):self.unet_config['dtype'] = dtypeself.manual_cast_dtype = manual_cast_dtypediff --git a/comfy/text_encoders/gemma4.py b/comfy/text_encoders/gemma4.pyindex f050061ed..0bba8341b 100644--- a/comfy/text_encoders/gemma4.py+++ b/comfy/text_encoders/gemma4.py@@ -1088,7 +1088,7 @@ class Gemma4_Tokenizer():h, w = samples.shape[2], samples.shape[3]patch_size = 16pooling_k = 3- max_soft_tokens = 70 if is_video else 280 # video uses smaller token budget per frame+ max_soft_tokens = kwargs.get("max_soft_tokens", 70 if is_video else 280)max_patches = max_soft_tokens * pooling_k * pooling_ktarget_px = max_patches * patch_size * patch_sizefactor = (target_px / (h * w)) ** 0.5diff --git a/comfy_extras/nodes_seedvr.py b/comfy_extras/nodes_seedvr.pynew file mode 100644index 000000000..c4ca3b55c--- /dev/null+++ b/comfy_extras/nodes_seedvr.py@@ -0,0 +1,614 @@+import logging++from typing_extensions import override+from comfy_api.latest import ComfyExtension, io+import torch++import comfy.model_management+from comfy.ldm.seedvr.color_fix import (+ adain_color_transfer,+ lab_color_transfer,+ wavelet_color_transfer,+)+from comfy.ldm.seedvr.constants import (+ BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE,+ SEEDVR2_ADAIN_SCALE_MULTIPLIER,+ SEEDVR2_CHUNK_GIB_PER_MPX_FRAME,+ SEEDVR2_CHUNK_RESERVED_GIB,+ SEEDVR2_CHUNK_SIGMA_GIB,+ SEEDVR2_CHUNK_SIGMA_K,+ SEEDVR2_COLOR_MEM_HEADROOM,+ SEEDVR2_DTYPE_BYTES_FLOOR,+ SEEDVR2_LAB_SCALE_MULTIPLIER,+ SEEDVR2_LATENT_CHANNELS,+ SEEDVR2_OOM_BACKOFF_DIVISOR,+ SEEDVR2_WAVELET_SCALE_MULTIPLIER,+)++from torchvision.transforms import functional as TVF+from torchvision.transforms.functional import InterpolationMode+++_SEEDVR2_INVALID_MODEL_MSG_PREFIX = "SeedVR2Conditioning: model object does not match expected SeedVR2 structure"+_ATTR_MISSING = object()+++def _resolve_seedvr2_diffusion_model(model):+ inner = getattr(model, "model", _ATTR_MISSING)+ if inner is _ATTR_MISSING:+ raise RuntimeError(+ f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: input has no 'model' attribute "+ f"(got type {type(model).__name__})."+ )+ if inner is None:+ raise RuntimeError(+ f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: input.model is None "+ f"(input type {type(model).__name__})."+ )+ diffusion_model = getattr(inner, "diffusion_model", _ATTR_MISSING)+ if diffusion_model is _ATTR_MISSING:+ raise RuntimeError(+ f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: 'model.model' has no "+ f"'diffusion_model' attribute (got type {type(inner).__name__})."+ )+ if diffusion_model is None:+ raise RuntimeError(+ f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: 'model.model.diffusion_model' "+ f"is None (model.model type {type(inner).__name__})."+ )+ return diffusion_model+++def div_pad(image, factor):+ height_factor, width_factor = factor+ height, width = image.shape[-2:]++ pad_height = (height_factor - (height % height_factor)) % height_factor+ pad_width = (width_factor - (width % width_factor)) % width_factor++ if pad_height == 0 and pad_width == 0:+ return image++ padding = (0, pad_width, 0, pad_height)+ return torch.nn.functional.pad(image, padding, mode='constant', value=0.0)++def cut_videos(videos):+ t = videos.size(1)+ if t < 1:+ raise ValueError("SeedVR2Preprocess expected at least one frame.")+ if t == 1:+ return videos+ if t <= 4:+ padding = videos[:, -1:].repeat(1, 4 - t + 1, 1, 1, 1)+ return torch.cat([videos, padding], dim=1)+ if (t - 1) % 4 == 0:+ return videos+ padding = videos[:, -1:].repeat(1, 4 - ((t - 1) % 4), 1, 1, 1)+ videos = torch.cat([videos, padding], dim=1)+ if (videos.size(1) - 1) % 4 != 0:+ raise ValueError(f"SeedVR2Preprocess failed to pad video length to 4n+1; got {videos.size(1)} frames.")+ return videos++def _seedvr2_input_shorter_edge(images, node_name):+ if images.dim() == 4:+ return min(images.shape[1], images.shape[2])+ if images.dim() == 5:+ return min(images.shape[2], images.shape[3])+ raise ValueError(+ f"{node_name}: expected 4-D or 5-D IMAGE tensor, "+ f"got shape {tuple(images.shape)}"+ )+++def _seedvr2_pad(images, upscaled_shorter_edge, node_name):+ if upscaled_shorter_edge < 2:+ raise ValueError(+ f"{node_name}: input shorter edge must be at least 2 pixels; "+ f"got {upscaled_shorter_edge}."+ )+ if images.shape[-1] > 3:+ images = images[..., :3]+ if images.dim() == 4:+ # Comfy video components arrive as a 4-D IMAGE frame sequence:+ # (frames, H, W, C). SeedVR2 consumes that as one video.+ images = images.unsqueeze(0)+ elif images.dim() != 5:+ raise ValueError(+ f"{node_name}: expected 4-D or 5-D IMAGE tensor, "+ f"got shape {tuple(images.shape)}"+ )+ images = images.permute(0, 1, 4, 2, 3)++ b, t, c, h, w = images.shape+ images = images.reshape(b * t, c, h, w)++ images = torch.clamp(images, 0.0, 1.0)+ images = div_pad(images, (16, 16))+ _, _, new_h, new_w = images.shape++ images = images.reshape(b, t, c, new_h, new_w)+ images = cut_videos(images)+ images_bthwc = images.permute(0, 1, 3, 4, 2).contiguous()++ return io.NodeOutput(images_bthwc)+++class SeedVR2Preprocess(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="SeedVR2Preprocess",+ display_name="Pre-Process SeedVR2 Input",+ category="image/pre-processors",+ description="Pad a resized image for SeedVR2 model. Alpha channel is dropped. The node Post-Process SeedVR2 Output re-applies it from the original resized image.",+ search_aliases=["seedvr2", "upscale", "video upscale", "pad", "preprocess"],+ inputs=[+ io.Image.Input("resized_images", tooltip="The resized image to process."),+ ],+ outputs=[+ io.Image.Output("images", tooltip="The padded image for VAE encoding."),+ ]+ )++ @classmethod+ def execute(cls, resized_images):+ upscaled_shorter_edge = _seedvr2_input_shorter_edge(resized_images, "SeedVR2Preprocess")+ return _seedvr2_pad(+ resized_images, upscaled_shorter_edge, "SeedVR2Preprocess",+ )+++class SeedVR2PostProcessing(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="SeedVR2PostProcessing",+ display_name="Post-Process SeedVR2 Output",+ category="image/post-processors",+ description="Align the generated image with the original resized image and apply color correction.",+ search_aliases=["seedvr2", "upscale", "color correction", "color match", "postprocess"],+ inputs=[+ io.Image.Input("images", tooltip="The generated image to process."),+ io.Image.Input("original_resized_images", tooltip="The original resized image before pre-processing, used as reference."),+ io.Combo.Input("color_correction_method", options=["lab", "wavelet", "adain", "none"], default="lab", tooltip="Method to match the generated image colors to the original image. lab: transfer color in CIELAB space, preserving detail (most faithful). wavelet: transfer low-frequency color, keeping upscaled high-frequency detail. adain: match per-channel mean/std (fastest, global tint). none: skip color transfer (geometry alignment only)."),+ ],+ outputs=[io.Image.Output(display_name="images", tooltip="The aligned, color-corrected image.")],+ )++ @classmethod+ def execute(cls, images, original_resized_images, color_correction_method):+ alpha_input = None+ if original_resized_images.shape[-1] == 4:+ alpha_input = original_resized_images[..., 3:4]+ original_resized_images = original_resized_images[..., :3]+ decoded_5d, decoded_was_4d = cls._as_bthwc(images)+ reference_full, _ = cls._as_bthwc(original_resized_images)+ decoded_5d = cls._restore_reference_batch_time(decoded_5d, reference_full)++ b = min(decoded_5d.shape[0], reference_full.shape[0])+ t = min(decoded_5d.shape[1], reference_full.shape[1])+ reference_h = reference_full.shape[2]+ reference_w = reference_full.shape[3]++ decoded_5d = decoded_5d[:b, :t, :, :, :]+ target_h = min(decoded_5d.shape[2], reference_h)+ target_w = min(decoded_5d.shape[3], reference_w)+ decoded_5d = decoded_5d[:, :, :target_h, :target_w, :]+ if color_correction_method in ("lab", "wavelet", "adain"):+ reference_5d = reference_full[:b, :t, :, :, :]+ reference_5d = cls._resize_reference(reference_5d, target_h, target_w)+ output_device = decoded_5d.device+ decoded_raw = cls._to_seedvr2_raw(decoded_5d)+ reference_raw = cls._to_seedvr2_raw(reference_5d)+ decoded_flat = decoded_raw.permute(0, 1, 4, 2, 3).reshape(b * t, decoded_raw.shape[4], target_h, target_w)+ reference_flat = reference_raw.permute(0, 1, 4, 2, 3).reshape(b * t, reference_raw.shape[4], target_h, target_w)+ output = cls._color_transfer_chunked(+ decoded_flat, reference_flat, output_device, color_correction_method,+ )+ output = output.reshape(b, t, output.shape[1], output.shape[2], output.shape[3]).permute(0, 1, 3, 4, 2)+ output = output.add(1.0).div(2.0).clamp(0.0, 1.0)+ elif color_correction_method == "none":+ output = decoded_5d+ else:+ raise ValueError(f"SeedVR2PostProcessing: unknown color_correction_method {color_correction_method!r}")++ if alpha_input is not None:+ alpha_5d, _ = cls._as_bthwc(alpha_input)+ alpha_5d = alpha_5d[:output.shape[0], :output.shape[1], :output.shape[2], :output.shape[3], :]+ output = torch.cat([output, alpha_5d.to(dtype=output.dtype, device=output.device)], dim=-1)+ h2 = output.shape[-3] - (output.shape[-3] % 2)+ w2 = output.shape[-2] - (output.shape[-2] % 2)+ output = output[:, :, :h2, :w2, :]+ if decoded_was_4d:+ output = output.reshape(-1, output.shape[-3], output.shape[-2], output.shape[-1])+ return io.NodeOutput(output)++ @staticmethod+ def _as_bthwc(images):+ if images.ndim == 4:+ return images.unsqueeze(0), True+ if images.ndim == 5:+ return images, False+ raise ValueError(+ f"SeedVR2PostProcessing: expected 4-D or 5-D IMAGE tensor, got shape {tuple(images.shape)}"+ )++ @staticmethod+ def _restore_reference_batch_time(decoded, reference):+ if decoded.shape[0] != 1:+ return decoded+ ref_b, ref_t = reference.shape[:2]+ if ref_b < 1 or decoded.shape[1] % ref_b != 0:+ return decoded+ decoded_t = decoded.shape[1] // ref_b+ if decoded_t < ref_t:+ return decoded+ return decoded.reshape(ref_b, decoded_t, decoded.shape[2], decoded.shape[3], decoded.shape[4])++ @staticmethod+ def _to_seedvr2_raw(images):+ return images.mul(2.0).sub(1.0)++ @staticmethod+ def _color_transfer_on_vae_device(decoded_flat, reference_flat, output_device, transfer_fn):+ color_device = comfy.model_management.vae_device()+ decoded_flat = decoded_flat.to(device=color_device)+ reference_flat = reference_flat.to(device=color_device)+ output = transfer_fn(decoded_flat, reference_flat)+ return output.to(device=output_device)++ @staticmethod+ def _lab_color_transfer_on_vae_device(decoded_flat, reference_flat, output_device):+ color_device = comfy.model_management.vae_device()+ result = None+ for start in range(decoded_flat.shape[0]):+ decoded_frame = decoded_flat[start:start + 1].to(device=color_device).clone()+ reference_frame = reference_flat[start:start + 1].to(device=color_device).clone()+ output = lab_color_transfer(decoded_frame, reference_frame).to(device=output_device)+ if result is None:+ result = torch.empty(+ (decoded_flat.shape[0],) + tuple(output.shape[1:]),+ device=output_device,+ dtype=output.dtype,+ )+ result[start:start + 1].copy_(output)+ if result is None:+ raise ValueError("SeedVR2PostProcessing: LAB color correction requires at least one frame.")+ return result++ @classmethod+ def _color_transfer_chunked(cls, decoded_flat, reference_flat, output_device, color_correction_method):+ chunk_size = cls._estimate_color_correction_chunk_size(decoded_flat, color_correction_method)+ while True:+ try:+ return cls._run_color_transfer_chunks(+ decoded_flat, reference_flat, output_device, color_correction_method, chunk_size,+ )+ except Exception as e:+ comfy.model_management.raise_non_oom(e)+ if chunk_size <= 1:+ raise RuntimeError(+ "SeedVR2PostProcessing: color correction OOM at one frame; "+ f"color_correction_method={color_correction_method}, shape={tuple(decoded_flat.shape)}."+ ) from e+ chunk_size = max(1, chunk_size // SEEDVR2_OOM_BACKOFF_DIVISOR)++ @classmethod+ def _run_color_transfer_chunks(cls, decoded_flat, reference_flat, output_device, color_correction_method, chunk_size):+ result = None+ for start in range(0, decoded_flat.shape[0], chunk_size):+ end = min(start + chunk_size, decoded_flat.shape[0])+ decoded_chunk = decoded_flat[start:end]+ reference_chunk = reference_flat[start:end]+ if color_correction_method == "lab":+ output = cls._lab_color_transfer_on_vae_device(decoded_chunk, reference_chunk, output_device)+ elif color_correction_method == "wavelet":+ output = cls._color_transfer_on_vae_device(+ decoded_chunk, reference_chunk, output_device, wavelet_color_transfer,+ )+ else:+ output = cls._color_transfer_on_vae_device(+ decoded_chunk, reference_chunk, output_device, adain_color_transfer,+ )+ if result is None:+ result = torch.empty(+ (decoded_flat.shape[0],) + tuple(output.shape[1:]),+ device=output_device,+ dtype=output.dtype,+ )+ result[start:end].copy_(output)+ if result is None:+ raise ValueError("SeedVR2PostProcessing: color correction requires at least one frame.")+ return result++ @classmethod+ def _estimate_color_correction_chunk_size(cls, decoded_flat, color_correction_method):+ multiplier = cls._color_correction_memory_multiplier(color_correction_method)+ frames = decoded_flat.shape[0]+ _, channels, height, width = decoded_flat.shape+ dtype_bytes = max(decoded_flat.element_size(), SEEDVR2_DTYPE_BYTES_FLOOR)+ bytes_per_frame = height * width * channels * dtype_bytes * multiplier+ if bytes_per_frame <= 0:+ return frames+ color_device = comfy.model_management.vae_device()+ free_memory = comfy.model_management.get_free_memory(color_device)+ chunk_size = int((free_memory * SEEDVR2_COLOR_MEM_HEADROOM) // bytes_per_frame)+ return max(1, min(frames, chunk_size))++ @staticmethod+ def _color_correction_memory_multiplier(color_correction_method):+ if color_correction_method == "lab":+ return SEEDVR2_LAB_SCALE_MULTIPLIER+ if color_correction_method == "wavelet":+ return SEEDVR2_WAVELET_SCALE_MULTIPLIER+ if color_correction_method == "adain":+ return SEEDVR2_ADAIN_SCALE_MULTIPLIER+ raise ValueError(f"SeedVR2PostProcessing: unknown color_correction_method {color_correction_method!r}")++ @staticmethod+ def _resize_reference(reference, height, width):+ if reference.shape[2] == height and reference.shape[3] == width:+ return reference+ b, t = reference.shape[:2]+ reference_flat = reference.permute(0, 1, 4, 2, 3).reshape(b * t, reference.shape[4], reference.shape[2], reference.shape[3])+ resized = TVF.resize(+ reference_flat,+ size=(height, width),+ interpolation=InterpolationMode.BICUBIC,+ antialias=not (isinstance(reference_flat, torch.Tensor) and reference_flat.device.type == "mps"),+ )+ return resized.reshape(b, t, resized.shape[1], height, width).permute(0, 1, 3, 4, 2)+++class SeedVR2Conditioning(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="SeedVR2Conditioning",+ display_name="Apply SeedVR2 Conditioning",+ category="model/conditioning",+ description="Build SeedVR2 positive/negative conditioning from a VAE latent.",+ search_aliases=["seedvr2", "upscale", "conditioning"],+ inputs=[+ io.Model.Input("model", tooltip="The SeedVR2 model."),+ io.Latent.Input("vae_conditioning", display_name="latent"),+ ],+ outputs=[+ io.Conditioning.Output(display_name="positive", tooltip="The positive conditioning for sampling."),+ io.Conditioning.Output(display_name="negative", tooltip="The negative conditioning for sampling."),+ ],+ )++ @classmethod+ def execute(cls, model, vae_conditioning) -> io.NodeOutput:++ vae_conditioning = vae_conditioning["samples"]+ if vae_conditioning.ndim != 5:+ raise ValueError(+ "SeedVR2Conditioning expects a 5-D VAE latent in Comfy "+ f"channel-first layout; got shape {tuple(vae_conditioning.shape)}."+ )+ if vae_conditioning.shape[1] != SEEDVR2_LATENT_CHANNELS:+ if vae_conditioning.shape[-1] == SEEDVR2_LATENT_CHANNELS:+ raise ValueError(+ "SeedVR2Conditioning expects SeedVR2 VAE latents in Comfy "+ f"channel-first layout (B, {SEEDVR2_LATENT_CHANNELS}, T, H, W); "+ f"got channel-last shape {tuple(vae_conditioning.shape)}."+ )+ raise ValueError(+ "SeedVR2Conditioning expects SeedVR2 VAE latents with "+ f"{SEEDVR2_LATENT_CHANNELS} channels; got shape {tuple(vae_conditioning.shape)}."+ )+ vae_conditioning = vae_conditioning.movedim(1, -1).contiguous()+ model = _resolve_seedvr2_diffusion_model(model)+ pos_cond = model.positive_conditioning+ neg_cond = model.negative_conditioning++ mask = vae_conditioning.new_ones(vae_conditioning.shape[:-1] + (1,))+ condition = torch.cat((vae_conditioning, mask), dim=-1)+ condition = condition.movedim(-1, 1)++ negative = [[neg_cond.unsqueeze(0), {"condition": condition}]]+ positive = [[pos_cond.unsqueeze(0), {"condition": condition}]]++ return io.NodeOutput(positive, negative)++def _seedvr2_chunk_crossfade_weights(overlap, device, dtype):+ """Descending previous-chunk weights across the overlap (next chunk gets ``1 - w``): a Hann fade over the middle third, flat shoulders on the outer thirds."""+ ramp = torch.linspace(0.0, 1.0, steps=overlap, device=device, dtype=dtype)+ ramp = ((ramp - 1.0 / 3.0) / (1.0 / 3.0)).clamp(0.0, 1.0)+ return 0.5 + 0.5 * torch.cos(torch.pi * ramp)+++class SeedVR2TemporalChunk(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="SeedVR2TemporalChunk",+ display_name="Split SeedVR2 Latent",+ category="model/latent/batch",+ description="Split a SeedVR2 video latent into overlapping temporal chunks small enough to sample one at a time within VRAM, wiring latents outputs to both Apply SeedVR2 Conditioning and the sampler latent input before recombining with Merge SeedVR2 Latents.",+ search_aliases=["seedvr2", "split", "chunk", "temporal", "video upscale", "rebatch"],+ inputs=[+ io.Latent.Input("latent", tooltip="The VAE-encoded SeedVR2 latent to split."),+ io.Int.Input("temporal_overlap", default=0, min=0, max=16384,+ tooltip="Latent frames shared between adjacent chunks and crossfaded at merge; 0 = no overlap."),+ io.DynamicCombo.Input("chunking_mode",+ tooltip="manual = use frames_per_chunk exactly; auto = predict the largest chunk that fits free VRAM.",+ options=[+ io.DynamicCombo.Option("auto", []),+ io.DynamicCombo.Option("manual", [+ io.Int.Input("frames_per_chunk", default=21, min=1, max=16384, step=4,+ tooltip="Pixel frames per temporal chunk (4n+1: 1, 5, 9, 13, ...)."),+ ]),+ ]),+ ],+ outputs=[+ io.Latent.Output(display_name="latents", is_output_list=True,+ tooltip="The temporal chunks in sequence order."),+ io.Int.Output(display_name="temporal_overlap",+ tooltip="The effective latent-frame overlap between adjacent chunks, for Merge SeedVR2 Latents."),+ ],+ )++ @classmethod+ def execute(cls, latent, temporal_overlap, chunking_mode) -> io.NodeOutput:+ samples = latent["samples"]+ if samples.ndim != 5:+ raise ValueError(+ f"SeedVR2TemporalChunk: expected a 5-D video latent (B, C, T, H, W); "+ f"got shape {tuple(samples.shape)}."+ )+ if samples.shape[1] != SEEDVR2_LATENT_CHANNELS:+ raise ValueError(+ f"SeedVR2TemporalChunk: expected {SEEDVR2_LATENT_CHANNELS} latent channels; "+ f"got shape {tuple(samples.shape)}."+ )+ if temporal_overlap < 0:+ raise ValueError(+ f"SeedVR2TemporalChunk: temporal_overlap must be >= 0; got {temporal_overlap}."+ )+ mode = chunking_mode["chunking_mode"]+ if mode not in ("auto", "manual"):+ raise ValueError(+ f"SeedVR2TemporalChunk: chunking_mode must be 'auto' or 'manual'; "+ f"got {mode!r}."+ )+ t_latent = samples.shape[2]+ t_pixel = 4 * (t_latent - 1) + 1++ if mode == "auto":+ free_gb = comfy.model_management.get_free_memory(+ comfy.model_management.get_torch_device()) / (1024 ** 3)+ mpx_per_frame = (samples.shape[0] * samples.shape[3] * samples.shape[4]) * (BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE ** 2) / 1e6+ budget_gb = free_gb - SEEDVR2_CHUNK_RESERVED_GIB - SEEDVR2_CHUNK_SIGMA_K * SEEDVR2_CHUNK_SIGMA_GIB+ chunk_latent_max = max(1, int(budget_gb / (SEEDVR2_CHUNK_GIB_PER_MPX_FRAME * mpx_per_frame)))+ frames_per_chunk = min(4 * (chunk_latent_max - 1) + 1, t_pixel)+ logging.info(+ "SeedVR2TemporalChunk auto: free=%.2fGiB, %.2fMpx -> frames_per_chunk=%d (t_pixel=%d).",+ free_gb, mpx_per_frame, frames_per_chunk, t_pixel,+ )+ else:+ frames_per_chunk = chunking_mode["frames_per_chunk"]+ if frames_per_chunk < 1 or (frames_per_chunk - 1) % 4 != 0:+ raise ValueError(+ f"SeedVR2TemporalChunk: frames_per_chunk must be a 4n+1 pixel-frame count "+ f"(1, 5, 9, 13, 17, 21, ...); got {frames_per_chunk}."+ )++ if t_pixel <= frames_per_chunk:+ return io.NodeOutput([latent], 0)++ chunk_latent = (frames_per_chunk - 1) // 4 + 1+ temporal_overlap = min(temporal_overlap, chunk_latent - 1)+ step = chunk_latent - temporal_overlap++ chunks = []+ for start in range(0, t_latent, step):+ end = min(start + chunk_latent, t_latent)+ chunk = latent.copy()+ chunk["samples"] = samples[:, :, start:end].contiguous()+ chunks.append(chunk)+ if end >= t_latent:+ break+ return io.NodeOutput(chunks, temporal_overlap)+++class SeedVR2TemporalMerge(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="SeedVR2TemporalMerge",+ display_name="Merge SeedVR2 Latents",+ category="model/latent/batch",+ is_input_list=True,+ description="Recombine sampled SeedVR2 latent temporal chunks into one latent, crossfading each overlap with a Hann window sized by the temporal_overlap wired from Split SeedVR2 Latent.",+ search_aliases=["seedvr2", "merge", "temporal", "hann", "crossfade"],+ inputs=[+ io.Latent.Input("latents", tooltip="The sampled temporal chunks in sequence order."),+ io.Int.Input("temporal_overlap", default=0, min=0, max=16384, force_input=True,+ tooltip="The temporal_overlap output of Split SeedVR2 Latent. 0 = plain concatenation."),+ ],+ outputs=[+ io.Latent.Output(display_name="latent", tooltip="The recombined full-length latent."),+ ],+ )++ @classmethod+ def execute(cls, latents, temporal_overlap) -> io.NodeOutput:+ temporal_overlap = temporal_overlap[0]+ if temporal_overlap < 0:+ raise ValueError(+ f"SeedVR2TemporalMerge: temporal_overlap must be >= 0; got {temporal_overlap}."+ )+ chunks = [entry["samples"] for entry in latents]+ first = chunks[0]+ if first.ndim != 5:+ raise ValueError(+ f"SeedVR2TemporalMerge: expected 5-D video latents (B, C, T, H, W); "+ f"chunk 0 has shape {tuple(first.shape)}."+ )+ for i, chunk in enumerate(chunks[1:], start=1):+ if chunk.shape[:2] != first.shape[:2] or chunk.shape[3:] != first.shape[3:]:+ raise ValueError(+ f"SeedVR2TemporalMerge: chunk {i} shape {tuple(chunk.shape)} does not "+ f"match chunk 0 shape {tuple(first.shape)} outside the temporal axis."+ )+ if i < len(chunks) - 1 and chunk.shape[2] != first.shape[2]:+ raise ValueError(+ f"SeedVR2TemporalMerge: chunk {i} has {chunk.shape[2]} latent frames but "+ f"chunk 0 has {first.shape[2]}; only the final chunk may be shorter."+ )++ out = latents[0].copy()+ out.pop("noise_mask", None)++ if len(chunks) == 1:+ out["samples"] = first+ return io.NodeOutput(out)+ if temporal_overlap == 0:+ out["samples"] = torch.cat(chunks, dim=2)+ return io.NodeOutput(out)++ chunk_latent = first.shape[2]+ step = chunk_latent - min(temporal_overlap, chunk_latent - 1)+ t_total = step * (len(chunks) - 1) + chunks[-1].shape[2]+ b, c, _, h, w = first.shape+ merged = torch.empty((b, c, t_total, h, w), device=first.device, dtype=first.dtype)++ merged[:, :, :chunk_latent] = first+ filled = chunk_latent+ for i, chunk in enumerate(chunks[1:], start=1):+ start = i * step+ end = start + chunk.shape[2]+ # Crossfade width is bounded by the previous fill frontier and by a runt+ # final chunk shorter than the configured overlap.+ fade = min(filled - start, chunk.shape[2])+ if fade > 0:+ w_prev = _seedvr2_chunk_crossfade_weights(+ fade, chunk.device, chunk.dtype).view(1, 1, fade, 1, 1)+ merged[:, :, start:start + fade] = (+ merged[:, :, start:start + fade] * w_prev + chunk[:, :, :fade] * (1.0 - w_prev)+ )+ merged[:, :, start + fade:end] = chunk[:, :, fade:]+ else:+ merged[:, :, start:end] = chunk+ filled = end++ out["samples"] = merged+ return io.NodeOutput(out)+++class SeedVRExtension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[io.ComfyNode]]:+ return [+ SeedVR2Conditioning,+ SeedVR2Preprocess,+ SeedVR2PostProcessing,+ SeedVR2TemporalChunk,+ SeedVR2TemporalMerge,+ ]++async def comfy_entrypoint() -> SeedVRExtension:+ return SeedVRExtension()diff --git a/nodes.py b/nodes.pyindex e126576fe..474e188fe 100644--- a/nodes.py+++ b/nodes.py@@ -2458,6 +2458,7 @@ async def init_builtin_extra_nodes():"nodes_camera_trajectory.py","nodes_edit_model.py","nodes_tcfg.py",+ "nodes_seedvr.py","nodes_context_windows.py","nodes_qwen.py","nodes_boogu.py",diff --git a/tests-unit/comfy_extras_test/test_seedvr2_conditioning.py b/tests-unit/comfy_extras_test/test_seedvr2_conditioning.pynew file mode 100644index 000000000..045502b5b--- /dev/null+++ b/tests-unit/comfy_extras_test/test_seedvr2_conditioning.py@@ -0,0 +1,186 @@+"""SeedVR2 conditioning node regression tests."""++import importlib+import sys+from unittest.mock import MagicMock++import pytest+import torch+import torch.nn as nn++from comfy.cli_args import args as cli_args+from comfy.ldm.seedvr.constants import SEEDVR2_LATENT_CHANNELS++if not torch.cuda.is_available():+ cli_args.cpu = True+++_SENTINEL = object()+_TARGETS = (+ ("comfy.model_management", "comfy"),+ ("comfy_extras.nodes_seedvr", "comfy_extras"),+)+++def _import_nodes_seedvr_isolated():+ """Import comfy_extras.nodes_seedvr with comfy.model_management mocked."""+ priors = []+ for mod_name, parent_name in _TARGETS:+ prior_mod = sys.modules.get(mod_name, _SENTINEL)+ parent = sys.modules.get(parent_name)+ attr = mod_name.split(".")[-1]+ prior_attr = (+ getattr(parent, attr, _SENTINEL) if parent is not None else _SENTINEL+ )+ priors.append((mod_name, parent_name, attr, prior_mod, prior_attr))++ mock_mm = MagicMock()+ for fn in (+ "xformers_enabled", "xformers_enabled_vae",+ "pytorch_attention_enabled", "pytorch_attention_enabled_vae",+ "sage_attention_enabled", "flash_attention_enabled",+ "is_intel_xpu",+ ):+ getattr(mock_mm, fn).return_value = False+ tv = torch.version.__version__.split(".")+ mock_mm.torch_version_numeric = (int(tv[0]), int(tv[1]))+ mock_mm.WINDOWS = False+ sys.modules["comfy.model_management"] = mock_mm+ if sys.modules.get("comfy") is None:+ importlib.import_module("comfy")+ comfy_pkg = sys.modules.get("comfy")+ if comfy_pkg is not None:+ setattr(comfy_pkg, "model_management", mock_mm)+ nodes_seedvr = sys.modules.get("comfy_extras.nodes_seedvr") or (+ importlib.import_module("comfy_extras.nodes_seedvr")+ )++ def _restore():+ for mod_name, parent_name, attr, prior_mod, prior_attr in priors:+ if prior_mod is _SENTINEL:+ sys.modules.pop(mod_name, None)+ else:+ sys.modules[mod_name] = prior_mod+ parent = sys.modules.get(parent_name)+ if parent is None:+ continue+ if prior_attr is _SENTINEL:+ if hasattr(parent, attr):+ delattr(parent, attr)+ else:+ setattr(parent, attr, prior_attr)++ return nodes_seedvr, _restore+++class _Rope(nn.Module):+ def __init__(self):+ super().__init__()+ self.freqs = nn.Parameter(torch.zeros(4))+++class _Block(nn.Module):+ def __init__(self):+ super().__init__()+ self.rope = _Rope()+++class _DiffusionModel(nn.Module):+ def __init__(self, n_blocks=3, conditioning_dtype=torch.float32):+ super().__init__()+ self.blocks = nn.ModuleList([_Block() for _ in range(n_blocks)])+ self.register_buffer("positive_conditioning", torch.ones((2, 4), dtype=conditioning_dtype))+ self.register_buffer("negative_conditioning", torch.zeros((3, 4), dtype=conditioning_dtype))+++class _ModelInner:+ def __init__(self, diffusion_model):+ self.diffusion_model = diffusion_model+++class _ModelPatcher:+ def __init__(self, diffusion_model):+ self.model = _ModelInner(diffusion_model)+++def test_seedvr2_conditioning_schema_exposes_conditioning_outputs():+ nodes_seedvr, restore = _import_nodes_seedvr_isolated()+ try:+ schema = nodes_seedvr.SeedVR2Conditioning.define_schema()+ assert [input_item.id for input_item in schema.inputs] == [+ "model",+ "vae_conditioning",+ ]+ assert schema.inputs[1].display_name == "latent"+ assert [output.display_name for output in schema.outputs] == [+ "positive",+ "negative",+ ]+ finally:+ restore()+++def test_seedvr2_conditioning_rejects_wrong_latent_channels():+ nodes_seedvr, restore = _import_nodes_seedvr_isolated()+ try:+ patcher = _ModelPatcher(_DiffusionModel())+ vae_conditioning = {"samples": torch.zeros(1, 8, 2, 2, 2)}++ with pytest.raises(ValueError, match=f"{SEEDVR2_LATENT_CHANNELS} channels"):+ nodes_seedvr.SeedVR2Conditioning.execute(patcher, vae_conditioning)+ finally:+ restore()+++def test_seedvr2_conditioning_returns_conditioning_deterministically():+ nodes_seedvr, restore = _import_nodes_seedvr_isolated()+ try:+ diffusion_model = _DiffusionModel()+ patcher = _ModelPatcher(diffusion_model)+ samples = torch.arange(+ 1,+ 1 + SEEDVR2_LATENT_CHANNELS * 3 * 2 * 2,+ dtype=torch.float32,+ ).reshape(1, SEEDVR2_LATENT_CHANNELS, 3, 2, 2)+ vae_conditioning = {"samples": samples}++ first_positive, first_negative = (+ nodes_seedvr.SeedVR2Conditioning.execute(+ patcher,+ vae_conditioning,+ )+ )+ second_positive, second_negative = (+ nodes_seedvr.SeedVR2Conditioning.execute(+ patcher,+ vae_conditioning,+ )+ )++ channel_last = samples.movedim(1, -1).contiguous()+ expected_condition = torch.cat(+ [+ channel_last,+ torch.ones((*channel_last.shape[:-1], 1)),+ ],+ dim=-1,+ ).movedim(-1, 1)++ assert torch.equal(+ first_positive[0][1]["condition"],+ expected_condition,+ )+ assert torch.equal(+ second_positive[0][1]["condition"],+ expected_condition,+ )+ assert torch.equal(+ first_negative[0][1]["condition"],+ expected_condition,+ )+ assert torch.equal(+ second_negative[0][1]["condition"],+ expected_condition,+ )+ finally:+ restore()diff --git a/tests-unit/comfy_extras_test/test_seedvr2_nodes.py b/tests-unit/comfy_extras_test/test_seedvr2_nodes.pynew file mode 100644index 000000000..1c5d20ac9--- /dev/null+++ b/tests-unit/comfy_extras_test/test_seedvr2_nodes.py@@ -0,0 +1,55 @@+import importlib+import inspect+import sys+from unittest.mock import MagicMock, patch++import torch++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True+++def test_seedvr_node_signature_matches_schema():+ mock_mm = MagicMock()+ mock_mm.xformers_enabled.return_value = False+ mock_mm.xformers_enabled_vae.return_value = False+ mock_mm.sage_attention_enabled.return_value = False+ mock_mm.flash_attention_enabled.return_value = False++ sentinel = object()+ prior_cpu = cli_args.cpu+ cli_args.cpu = True+ prior_module = sys.modules.get("comfy_extras.nodes_seedvr", sentinel)+ comfy_pkg = sys.modules.get("comfy")+ prior_mm_attr = getattr(comfy_pkg, "model_management", sentinel) if comfy_pkg else sentinel++ with patch.dict(sys.modules, {"comfy.model_management": mock_mm}):+ if comfy_pkg is not None:+ setattr(comfy_pkg, "model_management", mock_mm)+ sys.modules.pop("comfy_extras.nodes_seedvr", None)+ try:+ nodes_seedvr = importlib.import_module("comfy_extras.nodes_seedvr")+ for node_cls in (nodes_seedvr.SeedVR2Preprocess, nodes_seedvr.SeedVR2PostProcessing, nodes_seedvr.SeedVR2Conditioning):+ schema_ids = [i.id for i in node_cls.define_schema().inputs]+ exec_params = [+ p for p in inspect.signature(node_cls.execute).parameters.keys()+ if p != "cls"+ ]+ assert schema_ids == exec_params, (+ f"{node_cls.__name__} schema/execute drift: "+ f"schema_ids={schema_ids}, exec_params={exec_params}"+ )+ finally:+ cli_args.cpu = prior_cpu+ if prior_module is sentinel:+ sys.modules.pop("comfy_extras.nodes_seedvr", None)+ else:+ sys.modules["comfy_extras.nodes_seedvr"] = prior_module+ if comfy_pkg is not None:+ if prior_mm_attr is sentinel:+ if hasattr(comfy_pkg, "model_management"):+ delattr(comfy_pkg, "model_management")+ else:+ setattr(comfy_pkg, "model_management", prior_mm_attr)diff --git a/tests-unit/comfy_extras_test/test_seedvr2_post_processing.py b/tests-unit/comfy_extras_test/test_seedvr2_post_processing.pynew file mode 100644index 000000000..6c821136d--- /dev/null+++ b/tests-unit/comfy_extras_test/test_seedvr2_post_processing.py@@ -0,0 +1,51 @@+from unittest.mock import patch++import pytest+import torch++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True++from comfy_extras import nodes_seedvr # noqa: E402+++def _schema_ids(items):+ return [item.id for item in items]+++def test_seedvr2_post_processing_schema():+ schema = nodes_seedvr.SeedVR2PostProcessing.define_schema()++ assert _schema_ids(schema.inputs) == ["images", "original_resized_images", "color_correction_method"]+ assert schema.inputs[2].options == ["lab", "wavelet", "adain", "none"]+ assert schema.inputs[2].default == "lab"+ assert schema.outputs[0].get_io_type() == "IMAGE"+++def test_seedvr2_post_processing_oom_error_uses_color_correction_method(monkeypatch):+ decoded = torch.full((1, 3, 4, 4), 0.25)+ reference = torch.full((1, 3, 4, 4), 0.75)++ def _lab(content, style):+ raise torch.cuda.OutOfMemoryError("CUDA out of memory")++ monkeypatch.setattr(nodes_seedvr.comfy.model_management, "vae_device", lambda: torch.device("cpu"))+ monkeypatch.setattr(nodes_seedvr.comfy.model_management, "get_free_memory", lambda device: 1_000_000)++ with patch.object(nodes_seedvr, "lab_color_transfer", _lab):+ with pytest.raises(RuntimeError) as excinfo:+ nodes_seedvr.SeedVR2PostProcessing._color_transfer_chunked(+ decoded, reference, torch.device("cpu"), "lab",+ )+ assert "color_correction_method=lab" in str(excinfo.value)+ assert " method=lab" not in str(excinfo.value)+++def test_seedvr2_post_processing_unknown_color_correction_method_raises():+ decoded = torch.zeros(1, 2, 4, 4, 3)+ original = torch.zeros(1, 2, 4, 4, 3)+ with pytest.raises(ValueError) as excinfo:+ nodes_seedvr.SeedVR2PostProcessing.execute(decoded, original, "bogus")+ assert "color_correction_method" in str(excinfo.value)diff --git a/tests-unit/comfy_extras_test/test_seedvr2_temporal_chunk.py b/tests-unit/comfy_extras_test/test_seedvr2_temporal_chunk.pynew file mode 100644index 000000000..328355b49--- /dev/null+++ b/tests-unit/comfy_extras_test/test_seedvr2_temporal_chunk.py@@ -0,0 +1,77 @@+"""SeedVR2 temporal chunk/merge node regression tests."""++import pytest+import torch++from comfy.cli_args import args as cli_args+from comfy.ldm.seedvr.constants import (+ BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE,+ SEEDVR2_CHUNK_GIB_PER_MPX_FRAME,+ SEEDVR2_CHUNK_RESERVED_GIB,+ SEEDVR2_CHUNK_SIGMA_GIB,+ SEEDVR2_CHUNK_SIGMA_K,+ SEEDVR2_LATENT_CHANNELS,+)++if not torch.cuda.is_available():+ cli_args.cpu = True++import comfy.model_management # noqa: E402+from comfy_extras.nodes_seedvr import SeedVR2TemporalChunk, SeedVR2TemporalMerge, _seedvr2_chunk_crossfade_weights # noqa: E402++def _latent(t_latent, h=8, w=8, b=1):+ g = torch.Generator().manual_seed(7)+ return {"samples": torch.randn(b, SEEDVR2_LATENT_CHANNELS, t_latent, h, w, generator=g)}++def _split(latent, frames_per_chunk, temporal_overlap, chunking_mode="manual"):+ combo = {"chunking_mode": chunking_mode}+ if chunking_mode != "auto":+ combo["frames_per_chunk"] = frames_per_chunk+ return SeedVR2TemporalChunk.execute(latent, temporal_overlap, combo).args++def _merge(chunks, temporal_overlap):+ return SeedVR2TemporalMerge.execute(chunks, [temporal_overlap]).args[0]++def test_chunk_temporal_windows_and_validation():+ with pytest.raises(ValueError, match="4n\\+1"):+ _split(_latent(9), 20, 0)+ with pytest.raises(ValueError, match="5-D"):+ _split({"samples": torch.zeros(1, SEEDVR2_LATENT_CHANNELS * 9, 8, 8)}, 21, 0)+ with pytest.raises(ValueError, match="chunking_mode"):+ _split(_latent(13), 21, 0, "adaptive")+ latent = _latent(13)+ chunks, overlap = _split(latent, 21, 2) # chunk_latent=6, step=4 -> [0:6], [4:10], [8:13]+ assert overlap == 2 and [c["samples"].shape[2] for c in chunks] == [6, 6, 5]+ assert all(torch.equal(c["samples"], latent["samples"][:, :, s:e]) for c, (s, e) in zip(chunks, [(0, 6), (4, 10), (8, 13)]))+ assert len(_split(_latent(13), 21, 999)[0]) == 8 # overlap clamps to chunk_latent-1 -> step=1+ assert (r := _split(_latent(5), 21, 3)) and len(r[0]) == 1 and r[1] == 0 # t_pixel <= 21: passthrough++def test_chunk_auto_mode_applies_vram_law(monkeypatch):+ mpx_per_frame = (32 * 32) * (BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE ** 2) / 1e6+ free_gb = (+ SEEDVR2_CHUNK_RESERVED_GIB+ + SEEDVR2_CHUNK_SIGMA_K * SEEDVR2_CHUNK_SIGMA_GIB+ + 5.1 * SEEDVR2_CHUNK_GIB_PER_MPX_FRAME * mpx_per_frame+ )+ monkeypatch.setattr(comfy.model_management, "get_free_memory", lambda dev=None: free_gb * (1024 ** 3))+ assert [c["samples"].shape[2] for c in _split(_latent(13, h=32, w=32), 1, 0, "auto")[0]] == [5, 5, 3]+ assert _split(_latent(13, h=32, w=32, b=2), 1, 0, "auto")[0][0]["samples"].shape[2] == 2 # batch halves the chunk++def test_merge_crossfade_and_reassembly():+ latent = _latent(13)+ latent["noise_mask"] = torch.rand(1, 1, 13, 8, 8)+ latent["batch_index"] = [0]+ merged = _merge(_split(latent, 21, 0)[0], 0)+ assert torch.equal(merged["samples"], latent["samples"])+ assert "noise_mask" not in merged and merged["batch_index"] == [0]+ assert torch.allclose(_merge(_split(latent, 21, 3)[0], 3)["samples"], latent["samples"], atol=1e-6)+ w = _seedvr2_chunk_crossfade_weights(3, merged["samples"].device, merged["samples"].dtype)+ assert w[0] == 1.0 and w[-1] == 0.0 and torch.all(w[:-1] >= w[1:])+ ones, zeros = {"samples": torch.ones(1, SEEDVR2_LATENT_CHANNELS, 6, 8, 8)}, {"samples": torch.zeros(1, SEEDVR2_LATENT_CHANNELS, 6, 8, 8)}+ fused = _merge([ones, zeros], 3)["samples"] # overlap equals w: prev fades out, next fades in+ assert torch.equal(fused[:, :, 3:6], w.view(1, 1, 3, 1, 1).expand(1, SEEDVR2_LATENT_CHANNELS, 3, 8, 8))+ assert torch.equal(fused[:, :, :3], ones["samples"][:, :, :3]) and torch.equal(fused[:, :, 6:], zeros["samples"][:, :, :3])+ short = _split(latent, 21, 2)[0]+ short[0]["samples"] = short[0]["samples"][:, :, :4]+ with pytest.raises(ValueError, match="only the final chunk may be shorter"):+ _merge(short, 2)diff --git a/tests-unit/comfy_test/model_detection_test.py b/tests-unit/comfy_test/model_detection_test.pyindex 4e9350602..6e7d71f79 100644--- a/tests-unit/comfy_test/model_detection_test.py+++ b/tests-unit/comfy_test/model_detection_test.py@@ -2,7 +2,7 @@ from collections import defaultdictimport torch-from comfy.model_detection import detect_unet_config, model_config_from_unet_config+from comfy.model_detection import detect_unet_config, model_config_from_unet, model_config_from_unet_configimport comfy.supported_models@@ -73,6 +73,34 @@ def _make_flux_schnell_comfyui_sd():return sd+def _make_seedvr2_7b_separate_mm_sd():+ return {+ "blocks.35.mlp.vid.proj_out.weight": torch.empty(3072, 1),+ "positive_conditioning": torch.empty(58, 5120),+ "negative_conditioning": torch.empty(64, 5120),+ }+++def _make_seedvr2_7b_shared_mm_sd():+ return {+ "blocks.35.mlp.all.proj_in_gate.weight": torch.empty(1, 1),+ "positive_conditioning": torch.empty(58, 5120),+ "negative_conditioning": torch.empty(64, 5120),+ }+++def _make_seedvr2_3b_shared_mm_sd():+ return {+ "blocks.31.mlp.all.proj_in_gate.weight": torch.empty(1, 1),+ "positive_conditioning": torch.empty(58, 5120),+ "negative_conditioning": torch.empty(64, 5120),+ }+++def _add_model_diffusion_prefix(sd):+ return {f"model.diffusion_model.{k}": v for k, v in sd.items()}++class TestModelDetection:"""Verify that first-match model detection selects the correct modelbased on list ordering and unet_config specificity."""@@ -125,6 +153,59 @@ class TestModelDetection:assert model_config is not Noneassert type(model_config).__name__ == "FluxSchnell"+ def test_seedvr2_7b_separate_mm_detection_config(self):+ sd = _make_seedvr2_7b_separate_mm_sd()+ unet_config = detect_unet_config(sd, "")++ assert unet_config is not None+ assert unet_config["image_model"] == "seedvr2"+ assert unet_config["vid_dim"] == 3072+ assert unet_config["heads"] == 24+ assert unet_config["num_layers"] == 36+ assert unet_config["mm_layers"] == 36+ assert unet_config["mlp_type"] == "normal"+ assert unet_config["rope_type"] == "rope3d"+ assert unet_config["rope_dim"] == 64++ def test_seedvr2_7b_shared_mm_detection_config(self):+ sd = _make_seedvr2_7b_shared_mm_sd()+ unet_config = detect_unet_config(sd, "")++ assert unet_config is not None+ assert unet_config["image_model"] == "seedvr2"+ assert unet_config["vid_dim"] == 3072+ assert unet_config["heads"] == 24+ assert unet_config["num_layers"] == 36+ assert unet_config["mm_layers"] == 10+ assert unet_config["mlp_type"] == "swiglu"+ assert unet_config["rope_type"] == "rope3d"+ assert unet_config["rope_dim"] == 64++ def test_seedvr2_3b_shared_mm_detection_config(self):+ sd = _make_seedvr2_3b_shared_mm_sd()+ unet_config = detect_unet_config(sd, "")++ assert unet_config is not None+ assert unet_config["image_model"] == "seedvr2"+ assert unet_config["vid_dim"] == 2560+ assert unet_config["heads"] == 20+ assert unet_config["num_layers"] == 32+ assert unet_config["mlp_type"] == "swiglu"++ def test_seedvr2_model_match_requires_conditioning_tensors(self):+ sd = _make_seedvr2_7b_shared_mm_sd()+ unet_config = detect_unet_config(sd, "")++ assert type(model_config_from_unet_config(unet_config, sd)).__name__ == "SeedVR2"++ del sd["positive_conditioning"]+ assert model_config_from_unet_config(unet_config, sd) is None++ def test_seedvr2_model_match_accepts_full_checkpoint_prefix(self):+ sd = _add_model_diffusion_prefix(_make_seedvr2_7b_shared_mm_sd())++ assert type(model_config_from_unet(sd, "model.diffusion_model.")).__name__ == "SeedVR2"+def test_unet_config_and_required_keys_combination_is_unique(self):"""Each model in the registry must have a unique combination of``unet_config`` and ``required_keys``. If two models share the samediff --git a/tests-unit/comfy_test/seedvr_vae_forward_test.py b/tests-unit/comfy_test/seedvr_vae_forward_test.pynew file mode 100644index 000000000..7ea7a143e--- /dev/null+++ b/tests-unit/comfy_test/seedvr_vae_forward_test.py@@ -0,0 +1,74 @@+"""Regression tests for the SeedVR2 VAE forward return contract."""++import pytest+import torch+import torch.nn as nn++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True++from comfy.ldm.seedvr.vae import SEEDVR2_LATENT_CHANNELS, VideoAutoencoderKL # noqa: E402+++_LATENT_SHAPE = (1, SEEDVR2_LATENT_CHANNELS, 2, 2, 2)+_DECODED_SHAPE = (1, 3, 5, 16, 16)+_INPUT_ENCODE_SHAPE = (1, 3, 5, 16, 16)+_INPUT_DECODE_SHAPE = _LATENT_SHAPE+++class _StubVAE(VideoAutoencoderKL):+ def __init__(self):+ nn.Module.__init__(self)+ self._encode_out = torch.zeros(*_LATENT_SHAPE)+ self._decode_out = torch.zeros(*_DECODED_SHAPE)++ def encode(self, x, return_dict=True):+ return self._encode_out++ def decode_(self, z, return_dict=True):+ return self._decode_out+++def test_forward_encode_returns_tensor():+ vae = _StubVAE()+ x = torch.zeros(*_INPUT_ENCODE_SHAPE)+ result = vae.forward(x, mode="encode")+ assert type(result) is torch.Tensor+ assert result.shape == torch.Size(_LATENT_SHAPE)+++def test_forward_decode_returns_tensor():+ vae = _StubVAE()+ z = torch.zeros(*_INPUT_DECODE_SHAPE)+ result = vae.forward(z, mode="decode")+ assert type(result) is torch.Tensor+ assert result.shape == torch.Size(_DECODED_SHAPE)+++class _TupleReturningStubVAE(VideoAutoencoderKL):+ def __init__(self):+ nn.Module.__init__(self)+ self._encode_tensor = torch.zeros(*_LATENT_SHAPE)+ self._decode_tensor = torch.zeros(*_DECODED_SHAPE)++ def encode(self, x, return_dict=True):+ return (self._encode_tensor,)++ def decode_(self, z, return_dict=True):+ return (self._decode_tensor,)+++def test_forward_all_unwraps_one_tuple_at_each_step():+ vae = _TupleReturningStubVAE()+ x = torch.zeros(*_INPUT_ENCODE_SHAPE)+ result = vae.forward(x, mode="all")+ assert type(result) is torch.Tensor+ assert result.shape == torch.Size(_DECODED_SHAPE)+++def test_forward_rejects_unknown_mode():+ vae = _StubVAE()+ with pytest.raises(ValueError, match="Unknown SeedVR2 VAE forward mode"):+ vae.forward(torch.zeros(*_INPUT_ENCODE_SHAPE), mode="bogus")diff --git a/tests-unit/comfy_test/test_seedvr2_dtype.py b/tests-unit/comfy_test/test_seedvr2_dtype.pynew file mode 100644index 000000000..8e08b6dde--- /dev/null+++ b/tests-unit/comfy_test/test_seedvr2_dtype.py@@ -0,0 +1,50 @@+import torch++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True++import comfy.sd+import comfy.supported_models+import comfy.ldm.seedvr.model as seedvr_model+import comfy.ldm.seedvr.vae as seedvr_vae+++def test_seedvr2_fp16_manual_cast_only_for_bf16_device(monkeypatch):+ bf16_device = object()+ fp16_device = object()++ monkeypatch.setattr(+ comfy.supported_models.comfy.model_management,+ "should_use_bf16",+ lambda device=None: device is bf16_device,+ )++ bf16_config = comfy.supported_models.SeedVR2({"image_model": "seedvr2"})+ bf16_config.set_inference_dtype(torch.float16, None, device=bf16_device)+ assert bf16_config.manual_cast_dtype is torch.bfloat16++ fp16_config = comfy.supported_models.SeedVR2({"image_model": "seedvr2"})+ fp16_config.set_inference_dtype(torch.float16, None, device=fp16_device)+ assert fp16_config.manual_cast_dtype is None+++def test_seedvr2_text_conditioning_accepts_cfg1_single_branch():+ context = torch.arange(6, dtype=torch.float32).reshape(1, 3, 2)++ txt, txt_shape = seedvr_model.NaDiT._resolve_text_conditioning(object(), context, [0])++ torch.testing.assert_close(txt, context.squeeze(0))+ torch.testing.assert_close(txt_shape, torch.tensor([[3]], device=context.device))+++def test_seedvr2_vae_decode_memory_covers_full_frame_lab_transfer():+ wrapper = seedvr_vae.VideoAutoencoderKLWrapper.__new__(seedvr_vae.VideoAutoencoderKLWrapper)+ latent_channels = seedvr_vae.SEEDVR2_LATENT_CHANNELS+ estimate = wrapper.comfy_memory_used_decode((1, latent_channels, 26, 120, 160))+ old_estimate = latent_channels * 120 * 160 * (4 * 8 * 8) * 2++ assert estimate == 101 * 960 * 1280 * 160+ assert estimate > 15 * 1024 ** 3+ assert estimate > old_estimate * 100diff --git a/tests-unit/comfy_test/test_seedvr2_internals.py b/tests-unit/comfy_test/test_seedvr2_internals.pynew file mode 100644index 000000000..fe4bde1c4--- /dev/null+++ b/tests-unit/comfy_test/test_seedvr2_internals.py@@ -0,0 +1,169 @@+"""SeedVR2 internals regression tests."""++from __future__ import annotations++from unittest.mock import patch++import pytest+import torch++from comfy.cli_args import args++if not torch.cuda.is_available():+ args.cpu = True++import comfy.ldm.seedvr.model as seedvr_model # noqa: E402+import comfy.ldm.seedvr.vae as vae_mod # noqa: E402+import comfy.ldm.modules.attention as attention # noqa: E402+import comfy.ops as comfy_ops # noqa: E402+from comfy.ldm.seedvr.vae import ( # noqa: E402+ causal_norm_wrapper,+ set_norm_limit,+)+from comfy.ldm.seedvr.attention import var_attention_optimized_split # noqa: E402+++_NUM_CHANNELS = 8+_NUM_GROUPS = 4+_TENSOR_SHAPE = (1, 8, 2, 4, 4)++_GROUPNORM_SUBCLASSES = [+ pytest.param(comfy_ops.disable_weight_init.GroupNorm, id="disable_weight_init"),+ pytest.param(comfy_ops.manual_cast.GroupNorm, id="manual_cast"),+]++[email protected]("groupnorm_cls", _GROUPNORM_SUBCLASSES)+def test_seedvr_groupnorm_low_limit_uses_chunked_groupnorm_path(groupnorm_cls):+ real_group_norm = vae_mod.F.group_norm+ set_norm_limit(1e-9)+ try:+ gn = groupnorm_cls(num_channels=_NUM_CHANNELS, num_groups=_NUM_GROUPS)+ gn.eval()++ forward_hook_calls = []++ def _hook(module, inputs, output):+ forward_hook_calls.append(tuple(inputs[0].shape))++ spy_calls = []++ def _group_norm_spy(input_tensor, num_groups_arg, *args, **kwargs):+ spy_calls.append({"num_groups": int(num_groups_arg)})+ return real_group_norm(input_tensor, num_groups_arg, *args, **kwargs)++ handle = gn.register_forward_hook(_hook)+ try:+ with patch.object(vae_mod.F, "group_norm", side_effect=_group_norm_spy):+ out_tensor = causal_norm_wrapper(gn, torch.randn(*_TENSOR_SHAPE))+ finally:+ handle.remove()++ full_calls = len(forward_hook_calls)+ chunked_calls = sum(1 for entry in spy_calls if entry["num_groups"] < _NUM_GROUPS)++ assert tuple(int(s) for s in out_tensor.shape) == _TENSOR_SHAPE+ assert full_calls == 0, (+ f"low-limit GroupNorm gate must NOT take the full-forward path; got full_calls={full_calls}"+ )+ assert chunked_calls > 0, (+ f"low-limit GroupNorm gate must take the chunked path; got chunked_calls={chunked_calls}"+ )+ finally:+ set_norm_limit(None)+++def test_seedvr2_7b_swin_attention_forward_uses_optimized_var_attention(monkeypatch):+ dim = 8+ heads = 2+ head_dim = 4+ attn = seedvr_model.NaSwinAttention(+ vid_dim=dim,+ txt_dim=dim,+ heads=heads,+ head_dim=head_dim,+ qk_bias=False,+ qk_norm=comfy_ops.disable_weight_init.RMSNorm,+ qk_norm_eps=1e-6,+ rope_type=None,+ rope_dim=head_dim,+ shared_weights=False,+ window=(2, 1, 1),+ window_method="720pwin_by_size_bysize",+ version=True,+ device="cpu",+ dtype=torch.float32,+ operations=comfy_ops.disable_weight_init,+ )+ generator = torch.Generator(device="cpu").manual_seed(11)+ vid = torch.randn(8, dim, generator=generator)+ txt = torch.randn(3, dim, generator=generator)+ vid_shape = torch.tensor([[2, 2, 2]], dtype=torch.long)+ txt_shape = torch.tensor([[3]], dtype=torch.long)+ calls = []++ def fake_optimized_var_attention(**kwargs):+ calls.append(kwargs)+ return kwargs["q"]++ monkeypatch.setattr(seedvr_model, "optimized_var_attention", fake_optimized_var_attention)++ vid_out, txt_out = attn(vid, txt, vid_shape, txt_shape, seedvr_model.Cache(disable=True))++ assert tuple(vid_out.shape) == (8, dim)+ assert tuple(txt_out.shape) == (3, dim)+ assert len(calls) == 1+ call = calls[0]+ assert tuple(call["q"].shape) == (14, heads, head_dim)+ assert tuple(call["k"].shape) == (14, heads, head_dim)+ assert tuple(call["v"].shape) == (14, heads, head_dim)+ assert call["heads"] == heads+ assert call["skip_reshape"] is True+ assert call["skip_output_reshape"] is True+ assert call["cu_seqlens_q"] == [0, 7, 14]+ assert call["cu_seqlens_k"] == [0, 7, 14]+++def test_var_attention_optimized_split_calls_dense_backend_per_window(monkeypatch):+ heads = 2+ head_dim = 3+ q = torch.arange(30, dtype=torch.float32).reshape(5, heads, head_dim)+ k = q + 100+ v = q + 200+ cu = [0, 2, 5]+ calls = []++ def fake_optimized_attention(q_arg, k_arg, v_arg, heads_arg, **kwargs):+ calls.append(+ {+ "q_shape": tuple(q_arg.shape),+ "k_shape": tuple(k_arg.shape),+ "v_shape": tuple(v_arg.shape),+ "heads": heads_arg,+ "kwargs": kwargs,+ }+ )+ return q_arg + v_arg++ monkeypatch.setattr(attention, "optimized_attention", fake_optimized_attention)++ out = var_attention_optimized_split(+ q,+ k,+ v,+ heads,+ cu,+ cu,+ skip_reshape=True,+ skip_output_reshape=True,+ )++ assert tuple(out.shape) == (5, heads, head_dim)+ assert len(calls) == 2+ assert calls[0]["q_shape"] == (1, heads, 2, head_dim)+ assert calls[1]["q_shape"] == (1, heads, 3, head_dim)+ assert all(call["heads"] == heads for call in calls)+ assert all(call["kwargs"]["skip_reshape"] is True for call in calls)+ assert all(call["kwargs"]["skip_output_reshape"] is True for call in calls)+ torch.testing.assert_close(out, q + v, rtol=0, atol=0)+diff --git a/tests-unit/comfy_test/test_seedvr2_model.py b/tests-unit/comfy_test/test_seedvr2_model.pynew file mode 100644index 000000000..1d454aaf1--- /dev/null+++ b/tests-unit/comfy_test/test_seedvr2_model.py@@ -0,0 +1,320 @@+"""SeedVR2 model, latent-format, and VAE graph regression tests."""++from __future__ import annotations++from unittest.mock import MagicMock++import pytest+import torch+from torch import nn++from comfy.cli_args import args++if not torch.cuda.is_available():+ args.cpu = True++import comfy # noqa: E402+import comfy.latent_formats # noqa: E402+import comfy.ldm.seedvr.model as seedvr_model # noqa: E402+import comfy.ldm.seedvr.vae as seedvr_vae_mod # noqa: E402+import comfy.model_management # noqa: E402+import comfy.ops as comfy_ops # noqa: E402+import comfy.sample # noqa: E402+import comfy.sd as sd_mod # noqa: E402+import nodes as nodes_mod # noqa: E402+from comfy.ldm.seedvr.model import NaDiT # noqa: E402+++_LATENT_CHANNELS = seedvr_vae_mod.SEEDVR2_LATENT_CHANNELS+++def _make_standin(positive_conditioning):+ class _StandIn(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.register_buffer(+ "positive_conditioning", positive_conditioning+ )++ _resolve_text_conditioning = NaDiT._resolve_text_conditioning++ return _StandIn()+++class _StubModule(nn.Module):+ def __init__(self, *args, **kwargs):+ super().__init__()+++def _capture_last_layer_flags(monkeypatch, vid_dim: int, txt_in_dim: int) -> list[bool]:+ flags = []++ class _Block(_StubModule):+ def __init__(self, *args, **kwargs):+ flags.append(kwargs["is_last_layer"])+ super().__init__()++ monkeypatch.setattr(seedvr_model, "NaPatchIn", _StubModule)+ monkeypatch.setattr(seedvr_model, "NaPatchOut", _StubModule)+ monkeypatch.setattr(seedvr_model, "TimeEmbedding", _StubModule)+ monkeypatch.setattr(seedvr_model, "NaMMSRTransformerBlock", _Block)++ seedvr_model.NaDiT(+ norm_eps=1e-5,+ num_layers=4,+ mlp_type="normal",+ vid_dim=vid_dim,+ txt_in_dim=txt_in_dim,+ heads=24,+ mm_layers=3,+ operations=comfy_ops.disable_weight_init,+ )++ return flags+++class _Model:+ def __init__(self, latent_format):+ self._latent_format = latent_format++ def get_model_object(self, name):+ assert name == "latent_format"+ return self._latent_format+++class _Patcher:+ def get_free_memory(self, device):+ return 1024 * 1024 * 1024+++class _EncodeWrapper(seedvr_vae_mod.VideoAutoencoderKLWrapper):+ def __init__(self, encoded):+ nn.Module.__init__(self)+ self.encoded = encoded+ self.spatial_downsample_factor = 8+ self.temporal_downsample_factor = 4+ self.seen = []++ def encode(self, x):+ self.seen.append(tuple(x.shape))+ return self.encoded.to(device=x.device, dtype=x.dtype)+++class _DecodeWrapper(seedvr_vae_mod.VideoAutoencoderKLWrapper):+ def __init__(self):+ nn.Module.__init__(self)+ self.spatial_downsample_factor = 8+ self.temporal_downsample_factor = 4+ self.calls = []++ def decode(self, z, seedvr2_tiling=None):+ self.calls.append({"shape": tuple(z.shape), "seedvr2_tiling": seedvr2_tiling})+ if z.ndim == 4:+ b, tc, h, w = z.shape+ t = tc // _LATENT_CHANNELS+ else:+ b, _, t, h, w = z.shape+ return torch.zeros(b, 3, t, h * 8, w * 8, dtype=z.dtype, device=z.device)+++def test_seedvr2_wrapper_public_encode_returns_tensor(monkeypatch):+ raw_latent = torch.full((1, _LATENT_CHANNELS, 1, 4, 5), 2.0)+ seen_shapes = []++ def base_encode(self, x):+ seen_shapes.append(tuple(x.shape))+ return raw_latent.to(device=x.device, dtype=x.dtype)++ monkeypatch.setattr(seedvr_vae_mod.VideoAutoencoderKL, "encode", base_encode)++ vae = seedvr_vae_mod.VideoAutoencoderKLWrapper.__new__(seedvr_vae_mod.VideoAutoencoderKLWrapper)+ nn.Module.__init__(vae)+ vae._dummy = nn.Parameter(torch.zeros((), dtype=torch.float32))++ latent = vae.encode(torch.zeros(1, 3, 32, 40))++ assert type(latent) is torch.Tensor+ assert tuple(latent.shape) == (1, _LATENT_CHANNELS, 4, 5)+ assert seen_shapes == [(1, 3, 1, 32, 40)]+++def test_seedvr2_wrapper_private_encode_helper_keeps_raw_latent(monkeypatch):+ raw_latent = torch.full((1, _LATENT_CHANNELS, 1, 4, 5), 3.0)++ def base_encode(self, x):+ return raw_latent.to(device=x.device, dtype=x.dtype)++ monkeypatch.setattr(seedvr_vae_mod.VideoAutoencoderKL, "encode", base_encode)++ vae = seedvr_vae_mod.VideoAutoencoderKLWrapper.__new__(seedvr_vae_mod.VideoAutoencoderKLWrapper)+ nn.Module.__init__(vae)+ vae._dummy = nn.Parameter(torch.zeros((), dtype=torch.float32))++ latent, raw = vae._encode_with_raw_latent(torch.zeros(1, 3, 32, 40))++ assert tuple(latent.shape) == (1, _LATENT_CHANNELS, 4, 5)+ assert tuple(raw.shape) == (1, _LATENT_CHANNELS, 1, 4, 5)+ assert torch.equal(raw, raw_latent)+++def _make_vae(wrapper):+ vae = sd_mod.VAE.__new__(sd_mod.VAE)+ vae.first_stage_model = wrapper+ vae.device = torch.device("cpu")+ vae.output_device = torch.device("cpu")+ vae.vae_dtype = torch.float32+ vae.latent_channels = _LATENT_CHANNELS+ vae.latent_dim = 3+ vae.downscale_ratio = (lambda a: max(0, (a + 3) // 4), 8, 8)+ vae.upscale_ratio = (lambda a: max(0, a * 4 - 3), 8, 8)+ vae.output_channels = 3+ vae.disable_offload = True+ vae.extra_1d_channel = None+ vae.crop_input = False+ vae.not_video = False+ vae.handles_tiling = isinstance(wrapper, seedvr_vae_mod.VideoAutoencoderKLWrapper)+ vae.format_encoded = wrapper.comfy_format_encoded+ vae.patcher = _Patcher()+ vae.process_input = lambda image: image+ vae.process_output = lambda image: image.add(1.0).div(2.0).clamp(0.0, 1.0)+ vae.vae_output_dtype = lambda: torch.float32+ vae.memory_used_encode = lambda shape, dtype: 1+ vae.memory_used_decode = lambda shape, dtype: 1+ vae.throw_exception_if_invalid = lambda: None+ vae.vae_encode_crop_pixels = lambda pixels: pixels+ vae.spacial_compression_decode = lambda: 8+ vae.temporal_compression_decode = lambda: 4+ return vae+++def test_missing_context_falls_back_to_positive_buffer():+ pos_buffer = torch.full((58, 5120), 7.0)+ standin = _make_standin(pos_buffer)+ txt, txt_shape = standin._resolve_text_conditioning(None)+ assert txt.shape == (58, 5120)+ assert (txt == 7.0).all(), (+ "fallback path must use the positive_conditioning buffer "+ "verbatim, not a zero tensor"+ )+ assert txt_shape.shape == (1, 1)+ assert txt_shape[0, 0].item() == 58+++def test_seedvr2_7b_keeps_final_block_text_path(monkeypatch):+ assert _capture_last_layer_flags(monkeypatch, vid_dim=3072, txt_in_dim=3072) == [+ False,+ False,+ False,+ False,+ ]+++def test_seedvr2_7b_rope3d_matches_wrapper_oracle():+ rope = seedvr_model.get_na_rope("rope3d", dim=64)+ generator = torch.Generator(device="cpu").manual_seed(0)+ q = torch.randn(4, 2, 128, generator=generator)+ k = torch.randn(4, 2, 128, generator=generator)+ shape = torch.tensor([[1, 2, 2]], dtype=torch.long)+ freqs = rope.get_axial_freqs(1, 2, 2).reshape(4, -1)++ expected_q = seedvr_model._apply_seedvr2_rotary_emb(+ freqs,+ q.permute(1, 0, 2).float(),+ ).to(q.dtype).permute(1, 0, 2)+ expected_k = seedvr_model._apply_seedvr2_rotary_emb(+ freqs,+ k.permute(1, 0, 2).float(),+ ).to(k.dtype).permute(1, 0, 2)++ actual_q, actual_k = rope(q.clone(), k.clone(), shape, seedvr_model.Cache(disable=True))++ torch.testing.assert_close(actual_q, expected_q, rtol=0, atol=0)+ torch.testing.assert_close(actual_k, expected_k, rtol=0, atol=0)+++def test_seedvr2_forward_requires_conditioning_latents():+ model = NaDiT.__new__(NaDiT)+ x = torch.zeros(1, _LATENT_CHANNELS, 1, 4, 5)++ with pytest.raises(ValueError, match="requires conditioning latents"):+ NaDiT.forward(model, x, timestep=torch.tensor([1.0]), context=None)+++def test_seedvr2_latent_format_uses_native_video_latent_shape():+ latent_format = comfy.latent_formats.SeedVR2()+ latent_image = torch.zeros(1, 1, 4, 5)++ fixed = comfy.sample.fix_empty_latent_channels(_Model(latent_format), latent_image)++ assert latent_format.latent_channels == _LATENT_CHANNELS+ assert latent_format.latent_dimensions == 3+ assert fixed.shape == (1, _LATENT_CHANNELS, 1, 4, 5)+++def test_seedvr2_model_requires_native_5d_latent():+ latent = torch.zeros(1, _LATENT_CHANNELS, 2, 4, 5)+ assert NaDiT._check_seedvr2_video_latent(latent, _LATENT_CHANNELS, "latent") is latent++ with pytest.raises(ValueError, match="5-D native latent"):+ NaDiT._check_seedvr2_video_latent(torch.zeros(1, _LATENT_CHANNELS * 2, 4, 5), _LATENT_CHANNELS, "latent")+++def test_seedvr2_encode_and_encode_tiled_preserve_native_latent_contract(monkeypatch):+ monkeypatch.setattr(sd_mod.model_management, "load_models_gpu", lambda *a, **k: None)++ encoded = torch.full((1, _LATENT_CHANNELS, 2, 4, 5), 2.0)+ vae = _make_vae(_EncodeWrapper(encoded))+ pixels = torch.zeros(1, 5, 32, 40, 3)++ node_output = nodes_mod.VAEEncode().encode(vae, pixels)[0]+ node_latent = node_output["samples"]+ assert set(node_output) == {"samples"}+ assert tuple(node_latent.shape) == (1, _LATENT_CHANNELS, 2, 4, 5)+ assert node_latent.dtype == torch.float32+ assert node_latent.stride()[-1] == 1+ assert torch.equal(node_latent, torch.full_like(node_latent, 2.0 * seedvr_vae_mod.BYTEDANCE_VAE_SCALING_FACTOR))++ tiled = torch.full((1, _LATENT_CHANNELS, 2, 4, 5), 3.0)+ monkeypatch.setattr(seedvr_vae_mod, "tiled_vae", MagicMock(return_value=tiled))+ tiled_output = nodes_mod.VAEEncodeTiled().encode(+ vae,+ pixels,+ tile_size=512,+ overlap=64,+ temporal_size=16,+ temporal_overlap=4,+ )[0]+ tiled_latent = tiled_output["samples"]+ assert set(tiled_output) == {"samples"}+ assert tuple(tiled_latent.shape) == (1, _LATENT_CHANNELS, 2, 4, 5)+ assert tiled_latent.dtype == torch.float32+ assert torch.equal(tiled_latent, torch.full_like(tiled_latent, 3.0 * seedvr_vae_mod.BYTEDANCE_VAE_SCALING_FACTOR))+++def test_vaedecode_tiled_spatial_applies_temporal_discarded(monkeypatch):+ monkeypatch.setattr(sd_mod.model_management, "load_models_gpu", lambda *a, **k: None)+ vae = _make_vae(_DecodeWrapper())++ nodes_mod.VAEDecodeTiled().decode(+ vae,+ {"samples": torch.zeros(1, _LATENT_CHANNELS, 2, 4, 5)},+ tile_size=512,+ overlap=64,+ temporal_size=16,+ temporal_overlap=4,+ )++ # Spatial inputs flow through; temporal inputs are discarded as public tiling+ # knobs, but SeedVR2's internal MemoryState causal slicing is left intact.+ assert vae.first_stage_model.calls == [+ {+ "shape": (1, _LATENT_CHANNELS, 2, 4, 5),+ "seedvr2_tiling": {+ "enable_tiling": True,+ "tile_size": (512, 512),+ "tile_overlap": (64, 64),+ "temporal_size": None,+ "temporal_overlap": None,+ },+ }+ ]diff --git a/tests-unit/comfy_test/test_seedvr2_vae_decode.py b/tests-unit/comfy_test/test_seedvr2_vae_decode.pynew file mode 100644index 000000000..c486b9195--- /dev/null+++ b/tests-unit/comfy_test/test_seedvr2_vae_decode.py@@ -0,0 +1,94 @@+from unittest.mock import patch++import pytest+import torch+import torch.nn as nn++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True++import comfy.ldm.seedvr.vae as vae_mod # noqa: E402+from comfy_extras import nodes_seedvr # noqa: E402+++_LATENT_CHANNELS = vae_mod.SEEDVR2_LATENT_CHANNELS+++def _make_wrapper() -> vae_mod.VideoAutoencoderKLWrapper:+ wrapper = vae_mod.VideoAutoencoderKLWrapper.__new__(+ vae_mod.VideoAutoencoderKLWrapper+ )+ nn.Module.__init__(wrapper)+ return wrapper+++def _fingerprint_decode_(self, z, return_dict=True):+ b = int(z.shape[0])+ t = int(z.shape[2])+ h = int(z.shape[3])+ w = int(z.shape[4])+ out = torch.empty(b, 3, t, h * 8, w * 8)+ for batch_idx in range(b):+ out[batch_idx].fill_(float(batch_idx + 1))+ return out+++def _decode_with_patches(wrapper, z):+ with patch.object(vae_mod.VideoAutoencoderKL, "decode_", _fingerprint_decode_):+ return wrapper.decode(z)+++def test_decode_b2_t3_multi_frame_batch_unchanged():+ wrapper = _make_wrapper()++ out = _decode_with_patches(wrapper, torch.zeros(2, _LATENT_CHANNELS * 3, 2, 2))++ assert tuple(out.shape) == (2, 3, 3, 16, 16)+++class _Wrapper(vae_mod.VideoAutoencoderKLWrapper):+ def __init__(self):+ nn.Module.__init__(self)+ self.calls = []++ def parameters(self):+ return iter([torch.nn.Parameter(torch.zeros(()))])++def _decode_stub(self, latent):+ self.calls.append(tuple(latent.shape))+ return torch.zeros(latent.shape[0], 3, latent.shape[2], latent.shape[3] * 8, latent.shape[4] * 8)+++def test_seedvr2_wrapper_decode_accepts_5d_channel_first_latents_without_preprocessor_state():+ wrapper = _Wrapper()++ with patch.object(vae_mod.VideoAutoencoderKL, "decode_", _decode_stub):+ out = wrapper.decode(torch.zeros(1, _LATENT_CHANNELS, 2, 4, 5))++ assert tuple(out.shape) == (1, 3, 2, 32, 40)+ assert wrapper.calls == [(1, _LATENT_CHANNELS, 2, 4, 5)]+++def test_seedvr2_wrapper_decode_rejects_wrong_rank_latents():+ wrapper = _Wrapper()++ with pytest.raises(RuntimeError, match=r"latent input must be 4-D collapsed .* or 5-D"):+ wrapper.decode(torch.zeros(1, _LATENT_CHANNELS, 4))+++def _t_padded(t_in: int) -> int:+ if t_in == 1:+ return 1+ if t_in <= 4:+ return 5+ if (t_in - 1) % 4 == 0:+ return t_in+ return t_in + (4 - ((t_in - 1) % 4))++[email protected]("t_in", [1, 5, 9])+def test_t_padded_matches_cut_videos(t_in):+ dummy = torch.zeros(1, t_in, 1, 1, 1)+ assert nodes_seedvr.cut_videos(dummy).shape[1] == _t_padded(t_in)diff --git a/tests-unit/comfy_test/test_seedvr2_vae_tiled.py b/tests-unit/comfy_test/test_seedvr2_vae_tiled.pynew file mode 100644index 000000000..a2866b609--- /dev/null+++ b/tests-unit/comfy_test/test_seedvr2_vae_tiled.py@@ -0,0 +1,382 @@+from contextlib import ExitStack+from unittest.mock import MagicMock, patch++import pytest+import torch+import torch.nn as nn++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True++import comfy.ldm.seedvr.vae as vae_mod # noqa: E402+import comfy.ldm.seedvr.vae as seedvr_vae_mod # noqa: E402+import comfy.sd as sd_mod # noqa: E402+from comfy.ldm.seedvr.vae import MemoryState, tiled_vae # noqa: E402+++_LATENT_CHANNELS = seedvr_vae_mod.SEEDVR2_LATENT_CHANNELS+++def test_runtime_decode_zero_temporal_size_preserves_model_slicing():+ class StubVAEModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.slicing_latent_min_size = 2+ self.spatial_downsample_factor = 8+ self.temporal_downsample_factor = 4+ self.device = torch.device("cpu")+ self.use_slicing = True+ self._dummy = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32))+ self.decode_min_sizes = []+ self.memory_states = []++ def decode_(self, t_chunk):+ self.decode_min_sizes.append(self.slicing_latent_min_size)+ return vae_mod.VideoAutoencoderKL.slicing_decode(self, t_chunk)++ def _decode(self, z, memory_state=MemoryState.DISABLED, memory_cache=None):+ self.memory_states.append(memory_state)+ b, c, d, h, w = z.shape+ return torch.zeros((b, 3, d, h * 8, w * 8), dtype=z.dtype)++ vae = StubVAEModel()+ z = torch.zeros((1, _LATENT_CHANNELS, 5, 8, 8), dtype=torch.float32)++ tiled_vae(+ z,+ vae,+ tile_size=(64, 64),+ tile_overlap=(0, 0),+ temporal_size=0,+ temporal_overlap=0,+ encode=False,+ )++ assert vae.decode_min_sizes == [2]+ assert vae.memory_states == [MemoryState.INITIALIZING, MemoryState.ACTIVE]+ assert vae.slicing_latent_min_size == 2+++def test_zero_temporal_size_preserves_min_size_when_encode_raises():+ class RaisingVAEModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.slicing_sample_min_size = 4+ self.spatial_downsample_factor = 8+ self.temporal_downsample_factor = 4+ self.device = torch.device("cpu")+ self._dummy = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32))++ def encode(self, t_chunk):+ raise RuntimeError("simulated encode failure")++ vae = RaisingVAEModel()+ x = torch.zeros((1, 3, 12, 64, 64), dtype=torch.float32)++ with pytest.raises(RuntimeError, match="simulated encode failure"):+ tiled_vae(+ x,+ vae,+ tile_size=(64, 64),+ tile_overlap=(0, 0),+ temporal_size=0,+ temporal_overlap=0,+ encode=True,+ )++ assert vae.slicing_sample_min_size == 4+++def test_tiled_vae_encode_uses_tensor_return_without_indexing():+ class TensorEncodeVAEModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.slicing_sample_min_size = 4+ self.spatial_downsample_factor = 8+ self.temporal_downsample_factor = 4+ self.device = torch.device("cpu")+ self._dummy = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32))+ self.calls = []++ def encode(self, t_chunk):+ self.calls.append(tuple(t_chunk.shape))+ b, _, _, h, w = t_chunk.shape+ return torch.ones((b, _LATENT_CHANNELS, 1, h // 8, w // 8), dtype=t_chunk.dtype)++ vae = TensorEncodeVAEModel()+ x = torch.zeros((2, 3, 1, 64, 64), dtype=torch.float32)++ out = tiled_vae(+ x,+ vae,+ tile_size=(64, 64),+ tile_overlap=(0, 0),+ temporal_size=0,+ temporal_overlap=0,+ encode=True,+ )++ assert vae.calls == [(2, 3, 1, 64, 64)]+ assert tuple(out.shape) == (2, _LATENT_CHANNELS, 1, 8, 8)+++def test_tiled_vae_preserves_input_dtype_on_single_tile():+ class FloatOutputVAEModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.slicing_sample_min_size = 4+ self.spatial_downsample_factor = 8+ self.temporal_downsample_factor = 4+ self.device = torch.device("cpu")+ self._dummy = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32))++ def encode(self, t_chunk):+ b, _, _, h, w = t_chunk.shape+ return torch.ones((b, _LATENT_CHANNELS, 1, h // 8, w // 8), dtype=torch.float32)++ out = tiled_vae(+ torch.zeros((1, 3, 1, 64, 64), dtype=torch.float16),+ FloatOutputVAEModel(),+ tile_size=(64, 64),+ tile_overlap=(0, 0),+ temporal_size=0,+ temporal_overlap=0,+ encode=True,+ )++ assert out.dtype == torch.float16+++class _SlicingDecodeVAE(nn.Module):+ def __init__(self, slicing_latent_min_size):+ super().__init__()+ self.slicing_latent_min_size = slicing_latent_min_size+ self.spatial_downsample_factor = 8+ self.temporal_downsample_factor = 4+ self.device = torch.device("cpu")+ self.use_slicing = True+ self._dummy = nn.Parameter(torch.zeros(1, dtype=torch.float32))+ self.decode_min_sizes = []+ self.memory_states = []++ def decode_(self, z):+ self.decode_min_sizes.append(self.slicing_latent_min_size)+ return vae_mod.VideoAutoencoderKL.slicing_decode(self, z)++ def _decode(self, z, memory_state=MemoryState.DISABLED, memory_cache=None):+ self.memory_states.append(memory_state)+ x = z[:, :1].repeat(+ 1,+ 3,+ 1,+ self.spatial_downsample_factor,+ self.spatial_downsample_factor,+ )+ return x+++def test_decode_tiled_vae_maps_temporal_args_to_latent_slicing_min_size():+ vae = _SlicingDecodeVAE(slicing_latent_min_size=2)+ z = torch.arange(+ _LATENT_CHANNELS * 5 * 8 * 8,+ dtype=torch.float32,+ ).reshape(1, _LATENT_CHANNELS, 5, 8, 8)++ tiled_vae(+ z,+ vae,+ tile_size=(64, 64),+ tile_overlap=(0, 0),+ temporal_size=12,+ temporal_overlap=4,+ encode=False,+ )++ assert vae.decode_min_sizes == [2]+ assert vae.memory_states == [MemoryState.INITIALIZING, MemoryState.ACTIVE]+ assert vae.slicing_latent_min_size == 2++ wrapper = vae_mod.VideoAutoencoderKLWrapper.__new__(+ vae_mod.VideoAutoencoderKLWrapper+ )+ nn.Module.__init__(wrapper)+ seedvr2_tiling = {+ "enable_tiling": True,+ "tile_size": (64, 64),+ "tile_overlap": (0, 0),+ "temporal_size": 8,+ "temporal_overlap": 7,+ }++ captured = {}++ def _fake_tiled_vae(latent, model, **kwargs):+ captured.update(kwargs)+ return torch.zeros(1, 3, 1, 16, 16)++ with patch.object(vae_mod, "tiled_vae", side_effect=_fake_tiled_vae):+ wrapper.decode(torch.zeros(1, _LATENT_CHANNELS, 2, 2), seedvr2_tiling=seedvr2_tiling)++ assert captured["temporal_overlap"] == 7+++def _force_oom(*a, **k):+ raise torch.cuda.OutOfMemoryError("forced OOM for dispatcher test")+++def _make_vae(first_stage_model, latent_channels, latent_dim):+ vae = sd_mod.VAE.__new__(sd_mod.VAE)+ vae.first_stage_model = first_stage_model+ vae.patcher = MagicMock()+ vae.patcher.get_free_memory = MagicMock(return_value=8 * 1024 * 1024 * 1024)+ vae.device = vae.output_device = torch.device("cpu")+ vae.vae_dtype = torch.float32+ vae.disable_offload = True+ vae.extra_1d_channel = None+ vae.upscale_ratio = vae.downscale_ratio = 8+ vae.upscale_index_formula = vae.downscale_index_formula = None+ vae.output_channels = 3+ vae.latent_channels = latent_channels+ vae.latent_dim = latent_dim+ vae.vae_output_dtype = lambda: torch.float32+ vae.spacial_compression_decode = lambda: 8+ vae.handles_tiling = isinstance(first_stage_model, seedvr_vae_mod.VideoAutoencoderKLWrapper)+ vae.format_encoded = None+ vae.process_input = lambda x: x+ vae.process_output = lambda x: x+ vae.throw_exception_if_invalid = lambda: None+ vae.memory_used_decode = lambda *a, **k: 1+ return vae+++def _dispatch(vae, samples, seedvr2_call, generic_call, patch_wrapper_decode):+ mm = sd_mod.model_management+ with ExitStack() as stack:+ stack.enter_context(patch.object(mm, "raise_non_oom", lambda e: None))+ stack.enter_context(patch.object(mm, "load_models_gpu", lambda *a, **k: None))+ stack.enter_context(patch.object(mm, "soft_empty_cache", lambda: None))+ stack.enter_context(patch.object(sd_mod.VAE, "_decode_tiled_owned", seedvr2_call))+ stack.enter_context(patch.object(sd_mod.VAE, "decode_tiled_", generic_call))+ if patch_wrapper_decode:+ stack.enter_context(patch.object(+ seedvr_vae_mod.VideoAutoencoderKLWrapper, "decode",+ side_effect=_force_oom))+ vae.decode(samples)+++def test_4d_seedvr2_latent_routes_to_owned_decode_tiled():+ wrapper = seedvr_vae_mod.VideoAutoencoderKLWrapper.__new__(+ seedvr_vae_mod.VideoAutoencoderKLWrapper)+ vae = _make_vae(wrapper, latent_channels=_LATENT_CHANNELS, latent_dim=3)+ seedvr2_call = MagicMock(return_value=torch.zeros(1, 3, 9, 64, 64))+ generic_call = MagicMock(return_value=torch.zeros(1, 3, 64, 64))+ _dispatch(vae, torch.zeros(1, _LATENT_CHANNELS * 3, 8, 8), seedvr2_call, generic_call, True)+ assert seedvr2_call.call_count == 1+ assert generic_call.call_count == 0+++def test_4d_non_seedvr2_latent_still_routes_to_generic_decode_tiled():+ first_stage = MagicMock()+ first_stage.decode = MagicMock(side_effect=_force_oom)+ vae = _make_vae(first_stage, latent_channels=4, latent_dim=2)+ seedvr2_call = MagicMock(return_value=torch.zeros(1, 3, 9, 64, 64))+ generic_call = MagicMock(return_value=torch.zeros(1, 3, 64, 64))+ _dispatch(vae, torch.zeros(1, 4, 8, 8), seedvr2_call, generic_call, False)+ assert generic_call.call_count == 1+ assert seedvr2_call.call_count == 0+++def _populate_common_vae_attrs_fallback(vae):+ vae.patcher = MagicMock()+ vae.patcher.get_free_memory = MagicMock(return_value=8 * 1024 * 1024 * 1024)+ vae.device = torch.device("cpu")+ vae.output_device = torch.device("cpu")+ vae.vae_dtype = torch.float32+ vae.disable_offload = True+ vae.extra_1d_channel = None+ vae.upscale_ratio = 8+ vae.upscale_index_formula = None+ vae.output_channels = 3+ vae.latent_channels = _LATENT_CHANNELS+ vae.latent_dim = 3+ vae.downscale_ratio = 8+ vae.downscale_index_formula = None+ vae.not_video = False+ vae.crop_input = False+ vae.pad_channel_value = None+ vae.handles_tiling = isinstance(vae.first_stage_model, seedvr_vae_mod.VideoAutoencoderKLWrapper)+ vae.format_encoded = None++ vae.vae_output_dtype = lambda: torch.float32+ vae.spacial_compression_encode = lambda: 8+ vae.process_input = lambda x: x+ vae.process_output = lambda x: x+ vae.throw_exception_if_invalid = lambda: None+ vae.memory_used_encode = lambda *a, **k: 1+++def _make_seedvr2_vae_fallback():+ vae = sd_mod.VAE.__new__(sd_mod.VAE)+ wrapper = seedvr_vae_mod.VideoAutoencoderKLWrapper.__new__(+ seedvr_vae_mod.VideoAutoencoderKLWrapper+ )+ vae.first_stage_model = wrapper+ _populate_common_vae_attrs_fallback(vae)+ return vae+++def _make_non_seedvr2_vae_fallback():+ vae = sd_mod.VAE.__new__(sd_mod.VAE)+ vae.first_stage_model = MagicMock()+ _populate_common_vae_attrs_fallback(vae)+ return vae+++def _force_regular_encode_oom(*args, **kwargs):+ raise torch.cuda.OutOfMemoryError("forced OOM for dispatcher test")+++def test_seedvr2_3d_routes_to_owned_encode_tiled_on_oom():+ vae = _make_seedvr2_vae_fallback()+ pixel_samples = torch.zeros((1, 8, 64, 64, 3))++ seedvr2_call = MagicMock(return_value=torch.zeros(1, _LATENT_CHANNELS, 2, 8, 8))+ generic_call = MagicMock(return_value=torch.zeros(1, _LATENT_CHANNELS, 2, 8, 8))++ with patch.object(sd_mod.model_management, "raise_non_oom",+ lambda e: None), \+ patch.object(sd_mod.model_management, "load_models_gpu",+ lambda *a, **k: None), \+ patch.object(sd_mod.model_management, "soft_empty_cache",+ lambda: None), \+ patch.object(seedvr_vae_mod.VideoAutoencoderKLWrapper, "encode",+ side_effect=_force_regular_encode_oom), \+ patch.object(sd_mod.VAE, "_encode_tiled_owned", seedvr2_call), \+ patch.object(sd_mod.VAE, "encode_tiled_3d", generic_call):+ vae.encode(pixel_samples)++ assert seedvr2_call.call_count == 1, (+ f"Expected _encode_tiled_owned to be called once for a SeedVR2 3D "+ f"input under OOM fallback; got {seedvr2_call.call_count} calls."+ )+ assert generic_call.call_count == 0, (+ f"encode_tiled_3d must NOT be called for a SeedVR2 input; got "+ f"{generic_call.call_count} calls."+ )+++def test_non_seedvr2_encode_tiled_3d_default_overlap_is_concrete():+ vae = _make_non_seedvr2_vae_fallback()+ vae.downscale_ratio = (lambda a: max(1, a // 4), 8, 8)+ vae.upscale_ratio = (lambda a: a * 4, 8, 8)+ generic_call = MagicMock(return_value=torch.zeros(1, _LATENT_CHANNELS, 2, 8, 8))+ pixel_samples = torch.zeros((1, 8, 64, 64, 3))++ with patch.object(sd_mod.model_management, "load_models_gpu",+ lambda *a, **k: None), \+ patch.object(sd_mod.VAE, "encode_tiled_3d", generic_call):+ vae.encode_tiled(pixel_samples)++ assert generic_call.call_args.kwargs["overlap"] == (1, 64, 64)