Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Add SeedVR2 support (CORE-6) (#14424)

John Pollock8e2e54e2b8fd
diff --git a/comfy/latent_formats.py b/comfy/latent_formats.py
index bbdfd4bc2..8a16cfe55 100644
--- a/comfy/latent_formats.py
+++ b/comfy/latent_formats.py
@@ -779,6 +779,10 @@ class ACEAudio(LatentFormat):
latent_channels = 8
latent_dimensions = 2
+class SeedVR2(LatentFormat):
+ latent_channels = 16
+ latent_dimensions = 3
+
class ACEAudio15(LatentFormat):
latent_channels = 64
latent_dimensions = 1
diff --git a/comfy/ldm/modules/diffusionmodules/model.py b/comfy/ldm/modules/diffusionmodules/model.py
index fcbaa074f..e752d0ecb 100644
--- a/comfy/ldm/modules/diffusionmodules/model.py
+++ b/comfy/ldm/modules/diffusionmodules/model.py
@@ -22,7 +22,7 @@ def torch_cat_if_needed(xl, dim):
else:
return None
-def get_timestep_embedding(timesteps, embedding_dim):
+def get_timestep_embedding(timesteps, embedding_dim, flip_sin_to_cos=False, downscale_freq_shift=1):
"""
This matches the implementation in Denoising Diffusion Probabilistic Models:
From Fairseq.
@@ -33,11 +33,13 @@ def get_timestep_embedding(timesteps, embedding_dim):
assert len(timesteps.shape) == 1
half_dim = embedding_dim // 2
- emb = math.log(10000) / (half_dim - 1)
+ emb = math.log(10000) / (half_dim - downscale_freq_shift)
emb = torch.exp(torch.arange(half_dim, dtype=torch.float32) * -emb)
emb = emb.to(device=timesteps.device)
emb = timesteps.float()[:, None] * emb[None, :]
emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)
+ if flip_sin_to_cos:
+ emb = torch.cat([emb[:, half_dim:], emb[:, :half_dim]], dim=-1)
if embedding_dim % 2 == 1: # zero pad
emb = torch.nn.functional.pad(emb, (0,1,0,0))
return emb
diff --git a/comfy/ldm/seedvr/attention.py b/comfy/ldm/seedvr/attention.py
new file mode 100644
index 000000000..11b4c1e4a
--- /dev/null
+++ b/comfy/ldm/seedvr/attention.py
@@ -0,0 +1,51 @@
+import torch
+
+from comfy.ldm.modules import attention as _attention
+
+
+def _var_attention_qkv(q, k, v, heads, skip_reshape):
+ if skip_reshape:
+ return q, k, v, q.shape[-1]
+ total_tokens, embed_dim = q.shape
+ head_dim = embed_dim // heads
+ return (
+ q.view(total_tokens, heads, head_dim),
+ k.view(k.shape[0], heads, head_dim),
+ v.view(v.shape[0], heads, head_dim),
+ head_dim,
+ )
+
+
+def _var_attention_output(out, heads, head_dim, skip_output_reshape):
+ if skip_output_reshape:
+ return out
+ return out.reshape(-1, heads * head_dim)
+
+
+def var_attention_optimized_split(q, k, v, heads, cu_seqlens_q, cu_seqlens_k, *args, skip_reshape=False, skip_output_reshape=False, **kwargs):
+ q, k, v, head_dim = _var_attention_qkv(q, k, v, heads, skip_reshape)
+
+ q_split_indices = cu_seqlens_q[1:-1]
+ k_split_indices = cu_seqlens_k[1:-1]
+ if k.shape[0] != v.shape[0]:
+ raise ValueError("cu_seqlens_k does not match v token count")
+
+ q_splits = torch.tensor_split(q, q_split_indices, dim=0)
+ k_splits = torch.tensor_split(k, k_split_indices, dim=0)
+ v_splits = torch.tensor_split(v, k_split_indices, dim=0)
+ if len(q_splits) != len(k_splits) or len(q_splits) != len(v_splits):
+ raise ValueError("cu_seqlens_q and cu_seqlens_k must describe the same sequence count")
+
+ out = []
+ for q_i, k_i, v_i in zip(q_splits, k_splits, v_splits):
+ q_i = q_i.permute(1, 0, 2).unsqueeze(0)
+ k_i = k_i.permute(1, 0, 2).unsqueeze(0)
+ v_i = v_i.permute(1, 0, 2).unsqueeze(0)
+ out_i = _attention.optimized_attention(q_i, k_i, v_i, heads, skip_reshape=True, skip_output_reshape=True)
+ out.append(out_i.squeeze(0).permute(1, 0, 2))
+
+ out = torch.cat(out, dim=0)
+ return _var_attention_output(out, heads, head_dim, skip_output_reshape)
+
+
+optimized_var_attention = var_attention_optimized_split
diff --git a/comfy/ldm/seedvr/color_fix.py b/comfy/ldm/seedvr/color_fix.py
new file mode 100644
index 000000000..a43cb5270
--- /dev/null
+++ b/comfy/ldm/seedvr/color_fix.py
@@ -0,0 +1,301 @@
+import torch
+import torch.nn.functional as F
+from torch import Tensor
+
+from comfy.ldm.seedvr.constants import (
+ CIELAB_DELTA,
+ CIELAB_KAPPA,
+ D65_WHITE_X,
+ D65_WHITE_Z,
+ WAVELET_DECOMP_LEVELS,
+)
+
+
+def wavelet_blur(image: Tensor, radius):
+ max_safe_radius = max(1, min(image.shape[-2:]) // 8)
+ if radius > max_safe_radius:
+ radius = max_safe_radius
+
+ num_channels = image.shape[1]
+
+ kernel_vals = [
+ [0.0625, 0.125, 0.0625],
+ [0.125, 0.25, 0.125],
+ [0.0625, 0.125, 0.0625],
+ ]
+ kernel = torch.tensor(kernel_vals, dtype=image.dtype, device=image.device)
+ kernel = kernel[None, None].repeat(num_channels, 1, 1, 1)
+
+ image = F.pad(image, (radius, radius, radius, radius), mode='replicate')
+ output = F.conv2d(image, kernel, groups=num_channels, dilation=radius)
+
+ return output
+
+def wavelet_decomposition(image: Tensor, levels: int = WAVELET_DECOMP_LEVELS):
+ high_freq = torch.zeros_like(image)
+
+ for i in range(levels):
+ radius = 2 ** i
+ low_freq = wavelet_blur(image, radius)
+ high_freq.add_(image).sub_(low_freq)
+ image = low_freq
+
+ return high_freq, low_freq
+
+def wavelet_reconstruction(content_feat: Tensor, style_feat: Tensor) -> Tensor:
+
+ if content_feat.shape != style_feat.shape:
+ if len(content_feat.shape) >= 3:
+ style_feat = F.interpolate(
+ style_feat,
+ size=content_feat.shape[-2:],
+ mode='bilinear',
+ align_corners=False
+ )
+
+ content_high_freq, content_low_freq = wavelet_decomposition(content_feat)
+ del content_low_freq
+
+ style_high_freq, style_low_freq = wavelet_decomposition(style_feat)
+ del style_high_freq
+
+ if content_high_freq.shape != style_low_freq.shape:
+ style_low_freq = F.interpolate(
+ style_low_freq,
+ size=content_high_freq.shape[-2:],
+ mode='bilinear',
+ align_corners=False
+ )
+
+ content_high_freq.add_(style_low_freq)
+
+ return content_high_freq.clamp_(-1.0, 1.0)
+
+def _histogram_matching_channel(source: Tensor, reference: Tensor) -> Tensor:
+ original_shape = source.shape
+
+ source_flat = source.flatten()
+ reference_flat = reference.flatten()
+
+ source_sorted, source_indices = torch.sort(source_flat)
+ reference_sorted, _ = torch.sort(reference_flat)
+ del reference_flat
+
+ n_source = len(source_sorted)
+ n_reference = len(reference_sorted)
+
+ if n_source == n_reference:
+ matched_sorted = reference_sorted
+ else:
+ source_quantiles = torch.linspace(0, 1, n_source, device=source.device)
+ ref_indices = (source_quantiles * (n_reference - 1)).long()
+ ref_indices.clamp_(0, n_reference - 1)
+ matched_sorted = reference_sorted[ref_indices]
+ del source_quantiles, ref_indices, reference_sorted
+
+ del source_sorted, source_flat
+
+ inverse_indices = torch.argsort(source_indices)
+ del source_indices
+ matched_flat = matched_sorted[inverse_indices]
+ del matched_sorted, inverse_indices
+
+ return matched_flat.reshape(original_shape)
+
+def _lab_to_rgb_batch(lab: Tensor, matrix_inv: Tensor, epsilon: float, kappa: float) -> Tensor:
+ L, a, b = lab[:, 0], lab[:, 1], lab[:, 2]
+
+ fy = (L + 16.0) / 116.0
+ fx = a.div(500.0).add_(fy)
+ fz = fy - b / 200.0
+ del L, a, b
+
+ x = torch.where(
+ fx > epsilon,
+ torch.pow(fx, 3.0),
+ fx.mul(116.0).sub_(16.0).div_(kappa)
+ )
+ y = torch.where(
+ fy > epsilon,
+ torch.pow(fy, 3.0),
+ fy.mul(116.0).sub_(16.0).div_(kappa)
+ )
+ z = torch.where(
+ fz > epsilon,
+ torch.pow(fz, 3.0),
+ fz.mul(116.0).sub_(16.0).div_(kappa)
+ )
+ del fx, fy, fz
+
+ x.mul_(D65_WHITE_X)
+ z.mul_(D65_WHITE_Z)
+
+ xyz = torch.stack([x, y, z], dim=1)
+ del x, y, z
+
+ B, _, H, W = xyz.shape
+ xyz_flat = xyz.permute(0, 2, 3, 1).reshape(-1, 3)
+ del xyz
+
+ xyz_flat = xyz_flat.to(dtype=matrix_inv.dtype)
+ rgb_linear_flat = torch.matmul(xyz_flat, matrix_inv.T)
+ del xyz_flat
+
+ rgb_linear = rgb_linear_flat.reshape(B, H, W, 3).permute(0, 3, 1, 2)
+ del rgb_linear_flat
+
+ mask = rgb_linear > 0.0031308
+ rgb = torch.where(
+ mask,
+ torch.pow(torch.clamp(rgb_linear, min=0.0), 1.0 / 2.4).mul_(1.055).sub_(0.055),
+ rgb_linear * 12.92
+ )
+ del mask, rgb_linear
+
+ return torch.clamp(rgb, 0.0, 1.0)
+
+def _rgb_to_lab_batch(rgb: Tensor, matrix: Tensor, epsilon: float, kappa: float) -> Tensor:
+ mask = rgb > 0.04045
+ rgb_linear = torch.where(
+ mask,
+ torch.pow((rgb + 0.055) / 1.055, 2.4),
+ rgb / 12.92
+ )
+ del mask
+
+ B, _, H, W = rgb_linear.shape
+ rgb_flat = rgb_linear.permute(0, 2, 3, 1).reshape(-1, 3)
+ del rgb_linear
+
+ rgb_flat = rgb_flat.to(dtype=matrix.dtype)
+ xyz_flat = torch.matmul(rgb_flat, matrix.T)
+ del rgb_flat
+
+ xyz = xyz_flat.reshape(B, H, W, 3).permute(0, 3, 1, 2)
+ del xyz_flat
+
+ xyz[:, 0].div_(D65_WHITE_X)
+ xyz[:, 2].div_(D65_WHITE_Z)
+
+ epsilon_cubed = epsilon ** 3
+ mask = xyz > epsilon_cubed
+ f_xyz = torch.where(
+ mask,
+ torch.pow(xyz, 1.0 / 3.0),
+ xyz.mul(kappa).add_(16.0).div_(116.0)
+ )
+ del xyz, mask
+
+ L = f_xyz[:, 1].mul(116.0).sub_(16.0)
+ a = (f_xyz[:, 0] - f_xyz[:, 1]).mul_(500.0)
+ b = (f_xyz[:, 1] - f_xyz[:, 2]).mul_(200.0)
+ del f_xyz
+
+ return torch.stack([L, a, b], dim=1)
+
+def lab_color_transfer(
+ content_feat: Tensor,
+ style_feat: Tensor,
+ luminance_weight: float = 0.8
+) -> Tensor:
+ content_feat = wavelet_reconstruction(content_feat, style_feat)
+
+ if content_feat.shape != style_feat.shape:
+ style_feat = F.interpolate(
+ style_feat,
+ size=content_feat.shape[-2:],
+ mode='bilinear',
+ align_corners=False
+ )
+
+ device = content_feat.device
+ original_dtype = content_feat.dtype
+ content_feat = content_feat.float()
+ style_feat = style_feat.float()
+
+ rgb_to_xyz_matrix = torch.tensor([
+ [0.4124564, 0.3575761, 0.1804375],
+ [0.2126729, 0.7151522, 0.0721750],
+ [0.0193339, 0.1191920, 0.9503041]
+ ], dtype=torch.float32, device=device)
+
+ xyz_to_rgb_matrix = torch.tensor([
+ [ 3.2404542, -1.5371385, -0.4985314],
+ [-0.9692660, 1.8760108, 0.0415560],
+ [ 0.0556434, -0.2040259, 1.0572252]
+ ], dtype=torch.float32, device=device)
+
+ epsilon = CIELAB_DELTA
+ kappa = CIELAB_KAPPA
+
+ content_feat.add_(1.0).mul_(0.5).clamp_(0.0, 1.0)
+ style_feat.add_(1.0).mul_(0.5).clamp_(0.0, 1.0)
+
+ content_lab = _rgb_to_lab_batch(content_feat, rgb_to_xyz_matrix, epsilon, kappa)
+ del content_feat
+
+ style_lab = _rgb_to_lab_batch(style_feat, rgb_to_xyz_matrix, epsilon, kappa)
+ del style_feat, rgb_to_xyz_matrix
+
+ matched_a = _histogram_matching_channel(content_lab[:, 1], style_lab[:, 1])
+ matched_b = _histogram_matching_channel(content_lab[:, 2], style_lab[:, 2])
+
+ if luminance_weight < 1.0:
+ matched_L = _histogram_matching_channel(content_lab[:, 0], style_lab[:, 0])
+ result_L = content_lab[:, 0].mul(luminance_weight).add_(matched_L.mul(1.0 - luminance_weight))
+ del matched_L
+ else:
+ result_L = content_lab[:, 0]
+
+ del content_lab, style_lab
+
+ result_lab = torch.stack([result_L, matched_a, matched_b], dim=1)
+ del result_L, matched_a, matched_b
+
+ result_rgb = _lab_to_rgb_batch(result_lab, xyz_to_rgb_matrix, epsilon, kappa)
+ del result_lab, xyz_to_rgb_matrix
+
+ result = result_rgb.mul_(2.0).sub_(1.0)
+ del result_rgb
+
+ result = result.to(original_dtype)
+
+ return result
+
+
+def wavelet_color_transfer(content_feat: Tensor, style_feat: Tensor) -> Tensor:
+ return wavelet_reconstruction(content_feat, style_feat)
+
+
+def adain_color_transfer(content_feat: Tensor, style_feat: Tensor, eps: float = 1e-5) -> Tensor:
+ if content_feat.shape != style_feat.shape:
+ style_feat = F.interpolate(
+ style_feat,
+ size=content_feat.shape[-2:],
+ mode='bilinear',
+ align_corners=False,
+ )
+
+ original_dtype = content_feat.dtype
+ content_feat = content_feat.float()
+ style_feat = style_feat.float()
+
+ b, c = content_feat.shape[:2]
+ content_flat = content_feat.reshape(b, c, -1)
+ style_flat = style_feat.reshape(b, c, -1)
+
+ content_mean = content_flat.mean(dim=2).reshape(b, c, 1, 1)
+ content_std = (content_flat.var(dim=2, correction=0) + eps).sqrt().reshape(b, c, 1, 1)
+ style_mean = style_flat.mean(dim=2).reshape(b, c, 1, 1)
+ style_std = (style_flat.var(dim=2, correction=0) + eps).sqrt().reshape(b, c, 1, 1)
+ del content_flat, style_flat
+
+ normalized = (content_feat - content_mean) / content_std
+ del content_mean, content_std
+ result = normalized * style_std + style_mean
+ del normalized, style_mean, style_std
+
+ result = result.clamp_(-1.0, 1.0)
+ if result.dtype != original_dtype:
+ result = result.to(original_dtype)
+ return result
diff --git a/comfy/ldm/seedvr/constants.py b/comfy/ldm/seedvr/constants.py
new file mode 100644
index 000000000..12c4b4bef
--- /dev/null
+++ b/comfy/ldm/seedvr/constants.py
@@ -0,0 +1,48 @@
+"""SeedVR2 constants."""
+
+# Temporal chunk-size law: the sampler's activation wall is linear in
+# T_latent * pixel area (17-cell resolution sweep + T bisection, RTX 5090, 3b fp16):
+# max_latent_frames = (free_GiB - RESERVED - K*SIGMA) / (GIB_PER_MPX_FRAME * megapixels)
+# RESERVED covers model staging plus fixed CUDA/torch overhead; SIGMA is the measured
+# run-to-run spread of the wall; K=4 trades ~10% smaller chunks for ~1e-5 OOM odds.
+SEEDVR2_CHUNK_GIB_PER_MPX_FRAME = 0.55
+SEEDVR2_CHUNK_RESERVED_GIB = 8.5
+SEEDVR2_CHUNK_SIGMA_GIB = 0.55
+SEEDVR2_CHUNK_SIGMA_K = 4
+
+SEEDVR2_7B_VID_DIM = 3072
+SEEDVR2_OOM_BACKOFF_DIVISOR = 2
+SEEDVR2_DTYPE_BYTES_FLOOR = 4
+SEEDVR2_7B_MLP_CHUNK = 8192
+SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS = 4096 # partial-RoPE application token-chunk.
+SEEDVR2_LATENT_CHANNELS = 16
+
+SEEDVR2_COLOR_MEM_HEADROOM = 0.75
+SEEDVR2_LAB_SCALE_MULTIPLIER = 13
+SEEDVR2_WAVELET_SCALE_MULTIPLIER = 10 # per-frame byte multiplier, wavelet path.
+SEEDVR2_ADAIN_SCALE_MULTIPLIER = 6
+
+BYTEDANCE_VAE_SCALING_FACTOR = 0.9152 # configs_3b/main.yaml:57.
+BYTEDANCE_VAE_SHIFTING_FACTOR = 0.0
+BYTEDANCE_VAE_CONV_MEM_GIB = 0.5
+BYTEDANCE_VAE_NORM_MEM_GIB = 0.5
+BYTEDANCE_LOGVAR_CLAMP_MIN = -30.0 # video_vae_v3/modules/types.py:28.
+BYTEDANCE_LOGVAR_CLAMP_MAX = 20.0 # video_vae_v3/modules/types.py:28.
+BYTEDANCE_GN_CHUNKS_FP16 = 4 # causal_inflation_lib.py:351 (GroupNorm chunk count, fp16).
+BYTEDANCE_GN_CHUNKS_FP32 = 2 # causal_inflation_lib.py:351 (GroupNorm chunk count, fp32).
+BYTEDANCE_BLOCK_OUT_CHANNELS = (128, 256, 512, 512) # s8_c16_t4_inflation_sd3.yaml:7-11.
+BYTEDANCE_SLICING_SAMPLE_MIN = 4 # s8_c16_t4_inflation_sd3.yaml:22 (slicing_sample_min_size).
+BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE = 4 # infer.py:230 (temporal_downsample_factor); the 4n+1 factor.
+BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE = 8 # infer.py:231 (spatial_downsample_factor).
+BYTEDANCE_720P_REF_AREA = 45 * 80 # dit_v2/window.py:32 (720p reference area for window scaling).
+BYTEDANCE_MAX_TEMPORAL_WINDOW = 30 # dit_v2/window.py:35 (max temporal window frames).
+BYTEDANCE_ROPE_MAX_FREQ = 256 # dit_v2/rope.py:31 (pixel-RoPE max frequency).
+BYTEDANCE_SINUSOIDAL_DIM = 256 # dit_3b/nadit.py:120 (timestep sinusoidal embed dim).
+
+ROPE_THETA = 10000 # RoPE base; Su et al., "RoFormer", arXiv:2104.09864.
+
+CIELAB_DELTA = 6.0 / 29.0 # CIE 15 (delta).
+CIELAB_KAPPA = (29.0 / 3.0) ** 3 # CIE 15 (kappa).
+D65_WHITE_X = 0.95047 # CIE D65 standard illuminant Xn (Yn = 1).
+D65_WHITE_Z = 1.08883 # CIE D65 standard illuminant Zn.
+WAVELET_DECOMP_LEVELS = 5 # wavelet color-fix decomposition depth (GIMP/Krita; StableSR).
diff --git a/comfy/ldm/seedvr/model.py b/comfy/ldm/seedvr/model.py
new file mode 100644
index 000000000..a978698d5
--- /dev/null
+++ b/comfy/ldm/seedvr/model.py
@@ -0,0 +1,1361 @@
+from dataclasses import dataclass
+from typing import Optional, Tuple, Union, List, Dict, Any, Callable
+import torch.nn.functional as F
+from math import ceil, pi
+import torch
+from itertools import accumulate, chain
+from comfy.ldm.modules.diffusionmodules.model import get_timestep_embedding
+from comfy.ldm.seedvr.attention import optimized_var_attention
+from torch.nn.modules.utils import _triple
+from torch import nn
+import math
+from comfy.ldm.flux.math import apply_rope1
+from comfy.ldm.seedvr.constants import (
+ BYTEDANCE_720P_REF_AREA,
+ BYTEDANCE_MAX_TEMPORAL_WINDOW,
+ BYTEDANCE_ROPE_MAX_FREQ,
+ BYTEDANCE_SINUSOIDAL_DIM,
+ ROPE_THETA,
+ SEEDVR2_7B_MLP_CHUNK,
+ SEEDVR2_7B_VID_DIM,
+ SEEDVR2_LATENT_CHANNELS,
+ SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS,
+)
+import comfy.model_management
+import comfy.ops
+
+class Cache:
+ def __init__(self, disable=False, prefix="", cache=None):
+ self.cache = cache if cache is not None else {}
+ self.disable = disable
+ self.prefix = prefix
+
+ def __call__(self, key: str, fn: Callable):
+ if self.disable:
+ return fn()
+
+ key = self.prefix + key
+ if key not in self.cache:
+ result = fn()
+ self.cache[key] = result
+ return self.cache[key]
+
+ def namespace(self, namespace: str):
+ return Cache(
+ disable=self.disable,
+ prefix=self.prefix + namespace + ".",
+ cache=self.cache,
+ )
+
+def repeat_concat(
+ vid: torch.FloatTensor, # (VL ... c)
+ txt: torch.FloatTensor, # (TL ... c)
+ vid_len: torch.LongTensor, # (n*b)
+ txt_len: torch.LongTensor, # (b)
+ txt_repeat: List, # (n)
+) -> torch.FloatTensor: # (L ... c)
+ vid = torch.split(vid, vid_len.tolist())
+ txt = torch.split(txt, txt_len.tolist())
+ txt = [[x] * n for x, n in zip(txt, txt_repeat)]
+ txt = list(chain(*txt))
+ return torch.cat(list(chain(*zip(vid, txt))))
+
+def repeat_concat_idx(
+ vid_len: torch.LongTensor, # (n*b)
+ txt_len: torch.LongTensor, # (b)
+ txt_repeat: torch.LongTensor, # (n)
+) -> Tuple[
+ Callable,
+ Callable,
+]:
+ device = vid_len.device
+ vid_idx = torch.arange(vid_len.sum(), device=device)
+ txt_idx = torch.arange(len(vid_idx), len(vid_idx) + txt_len.sum(), device=device)
+ txt_repeat_list = txt_repeat.tolist()
+ tgt_idx = repeat_concat(vid_idx, txt_idx, vid_len, txt_len, txt_repeat_list)
+ src_idx = torch.argsort(tgt_idx)
+ txt_idx_len = len(tgt_idx) - len(vid_idx)
+ repeat_txt_len = (txt_len * txt_repeat).tolist()
+
+ def unconcat_coalesce(all):
+ vid_out, txt_out = all[src_idx].split([len(vid_idx), txt_idx_len])
+ txt_out_coalesced = []
+ for txt, repeat_time in zip(txt_out.split(repeat_txt_len), txt_repeat_list):
+ txt = txt.reshape(-1, repeat_time, *txt.shape[1:]).mean(1)
+ txt_out_coalesced.append(txt)
+ return vid_out, torch.cat(txt_out_coalesced)
+
+ return (
+ lambda vid, txt: torch.cat([vid, txt])[tgt_idx],
+ lambda all: unconcat_coalesce(all),
+ )
+
+def cumulative_lengths(lengths):
+ return [0, *accumulate(lengths)]
+
+
+@dataclass
+class MMArg:
+ vid: Any
+ txt: Any
+
+def get_args(key: str, args: List[Any]) -> List[Any]:
+ return [getattr(v, key) if isinstance(v, MMArg) else v for v in args]
+
+
+def get_kwargs(key: str, kwargs: Dict[str, Any]) -> Dict[str, Any]:
+ return {k: getattr(v, key) if isinstance(v, MMArg) else v for k, v in kwargs.items()}
+
+
+def get_window_op(name: str):
+ if name == "720pwin_by_size_bysize":
+ return make_720Pwindows_bysize
+ if name == "720pswin_by_size_bysize":
+ return make_shifted_720Pwindows_bysize
+ raise ValueError(f"Unknown windowing method: {name}")
+
+
+def make_720Pwindows_bysize(size: Tuple[int, int, int], num_windows: Tuple[int, int, int]):
+ t, h, w = size
+ resized_nt, resized_nh, resized_nw = num_windows
+ scale = math.sqrt(BYTEDANCE_720P_REF_AREA / (h * w))
+ resized_h, resized_w = round(h * scale), round(w * scale)
+ wh, ww = ceil(resized_h / resized_nh), ceil(resized_w / resized_nw)
+ wt = ceil(min(t, BYTEDANCE_MAX_TEMPORAL_WINDOW) / resized_nt)
+ nt, nh, nw = ceil(t / wt), ceil(h / wh), ceil(w / ww)
+ return [
+ (
+ slice(it * wt, min((it + 1) * wt, t)),
+ slice(ih * wh, min((ih + 1) * wh, h)),
+ slice(iw * ww, min((iw + 1) * ww, w)),
+ )
+ for iw in range(nw)
+ if min((iw + 1) * ww, w) > iw * ww
+ for ih in range(nh)
+ if min((ih + 1) * wh, h) > ih * wh
+ for it in range(nt)
+ if min((it + 1) * wt, t) > it * wt
+ ]
+
+def make_shifted_720Pwindows_bysize(size: Tuple[int, int, int], num_windows: Tuple[int, int, int]):
+ t, h, w = size
+ resized_nt, resized_nh, resized_nw = num_windows
+ scale = math.sqrt(BYTEDANCE_720P_REF_AREA / (h * w))
+ resized_h, resized_w = round(h * scale), round(w * scale)
+ wh, ww = ceil(resized_h / resized_nh), ceil(resized_w / resized_nw)
+ wt = ceil(min(t, BYTEDANCE_MAX_TEMPORAL_WINDOW) / resized_nt)
+
+ st, sh, sw = (
+ 0.5 if wt < t else 0,
+ 0.5 if wh < h else 0,
+ 0.5 if ww < w else 0,
+ )
+ nt, nh, nw = ceil((t - st) / wt), ceil((h - sh) / wh), ceil((w - sw) / ww)
+ nt, nh, nw = (
+ nt + 1 if st > 0 else 1,
+ nh + 1 if sh > 0 else 1,
+ nw + 1 if sw > 0 else 1,
+ )
+ return [
+ (
+ slice(max(int((it - st) * wt), 0), min(int((it - st + 1) * wt), t)),
+ slice(max(int((ih - sh) * wh), 0), min(int((ih - sh + 1) * wh), h)),
+ slice(max(int((iw - sw) * ww), 0), min(int((iw - sw + 1) * ww), w)),
+ )
+ for iw in range(nw)
+ if min(int((iw - sw + 1) * ww), w) > max(int((iw - sw) * ww), 0)
+ for ih in range(nh)
+ if min(int((ih - sh + 1) * wh), h) > max(int((ih - sh) * wh), 0)
+ for it in range(nt)
+ if min(int((it - st + 1) * wt), t) > max(int((it - st) * wt), 0)
+ ]
+
+class RotaryEmbedding(nn.Module):
+ def __init__(
+ self,
+ dim,
+ freqs_for = 'lang',
+ theta = 10000,
+ max_freq = 10,
+ ):
+ super().__init__()
+
+ self.freqs_for = freqs_for
+
+ if freqs_for == 'lang':
+ freqs = 1. / (theta ** (torch.arange(0, dim, 2)[:(dim // 2)].float() / dim))
+ elif freqs_for == 'pixel':
+ freqs = torch.linspace(1., max_freq / 2, dim // 2) * pi
+ else:
+ raise ValueError(f"Unknown rotary frequency type: {freqs_for}")
+
+ self.register_buffer("freqs", freqs)
+
+ @property
+ def device(self):
+ return self.freqs.device
+
+ def get_axial_freqs(
+ self,
+ *dims,
+ offsets = None
+ ):
+ Colon = slice(None)
+ all_freqs = []
+
+ if exists(offsets):
+ if len(offsets) != len(dims):
+ raise ValueError(f"SeedVR2 rotary offsets length must match dims length, got {len(offsets)} and {len(dims)}.")
+
+ for ind, dim in enumerate(dims):
+
+ offset = 0
+ if exists(offsets):
+ offset = offsets[ind]
+
+ if self.freqs_for == 'pixel':
+ pos = torch.linspace(-1, 1, steps = dim, device = self.device)
+ else:
+ pos = torch.arange(dim, device = self.device)
+
+ pos = pos + offset
+
+ freqs = self.forward(pos)
+
+ all_axis = [None] * len(dims)
+ all_axis[ind] = Colon
+
+ new_axis_slice = (Ellipsis, *all_axis, Colon)
+ all_freqs.append(freqs[new_axis_slice])
+
+ all_freqs = torch.broadcast_tensors(*all_freqs)
+ return torch.cat(all_freqs, dim = -1)
+
+ def forward(
+ self,
+ t,
+ ):
+ freqs = self.freqs
+
+ freqs = torch.einsum('..., f -> ... f', t.type(freqs.dtype), freqs)
+ freqs = freqs.unsqueeze(-1).expand(*freqs.shape, 2).flatten(-2)
+
+ return freqs
+
+class RotaryEmbeddingBase(nn.Module):
+ def __init__(self, dim: int, rope_dim: int):
+ super().__init__()
+ self.rope = RotaryEmbedding(
+ dim=dim // rope_dim,
+ freqs_for="pixel",
+ max_freq=BYTEDANCE_ROPE_MAX_FREQ,
+ )
+
+ def get_axial_freqs(self, *dims):
+ return self.rope.get_axial_freqs(*dims)
+
+
+class RotaryEmbedding3d(RotaryEmbeddingBase):
+ def __init__(self, dim: int):
+ super().__init__(dim, rope_dim=3)
+ self.mm = False
+
+
+class NaRotaryEmbedding3d(RotaryEmbedding3d):
+ def forward(
+ self,
+ q: torch.FloatTensor,
+ k: torch.FloatTensor,
+ shape: torch.LongTensor,
+ cache: Cache,
+ ) -> Tuple[
+ torch.FloatTensor,
+ torch.FloatTensor,
+ ]:
+ freqs = cache("rope_freqs_3d", lambda: self.get_freqs(shape))
+ freqs = freqs.to(device=q.device)
+ q = q.transpose(0, 1)
+ k = k.transpose(0, 1)
+ q = _apply_seedvr2_rotary_emb(freqs, q.float()).to(q.dtype)
+ k = _apply_seedvr2_rotary_emb(freqs, k.float()).to(k.dtype)
+ q = q.transpose(0, 1)
+ k = k.transpose(0, 1)
+ return q, k
+
+ @torch._dynamo.disable
+ def get_freqs(
+ self,
+ shape: torch.LongTensor,
+ ) -> torch.Tensor:
+ # Primary provenance: ByteDance-Seed/SeedVR models/dit/rope.py builds
+ # 7B pixel RoPE with the interleaved-angle convention, not Comfy's
+ # Flux freqs_cis matrix.
+ plain_rope = RotaryEmbedding(
+ dim=self.rope.freqs.numel() * 2,
+ freqs_for="pixel",
+ max_freq=BYTEDANCE_ROPE_MAX_FREQ,
+ )
+ plain_rope = plain_rope.to(self.rope.device)
+ freq_list = []
+ for f, h, w in shape.tolist():
+ freqs = plain_rope.get_axial_freqs(f, h, w)
+ freq_list.append(freqs.view(-1, freqs.size(-1)))
+ return torch.cat(freq_list, dim=0)
+
+
+class MMRotaryEmbeddingBase(RotaryEmbeddingBase):
+ def __init__(self, dim: int, rope_dim: int):
+ super().__init__(dim, rope_dim)
+ self.rope = RotaryEmbedding(
+ dim=dim // rope_dim,
+ freqs_for="lang",
+ theta=ROPE_THETA,
+ )
+ self.mm = True
+
+def slice_at_dim(t, dim_slice: slice, *, dim):
+ dim += (t.ndim if dim < 0 else 0)
+ colons = [slice(None)] * t.ndim
+ colons[dim] = dim_slice
+ return t[tuple(colons)]
+
+def rotate_half(x):
+ x = x.reshape(*x.shape[:-1], x.shape[-1] // 2, 2)
+ x1, x2 = x.unbind(dim = -1)
+ x = torch.stack((-x2, x1), dim = -1)
+ return x.flatten(-2)
+def exists(val):
+ return val is not None
+
+def _apply_seedvr2_rotary_emb(
+ freqs: torch.Tensor,
+ t: torch.Tensor,
+ start_index: int = 0,
+ scale: float = 1.0,
+ seq_dim: int = -2,
+ freqs_seq_dim: int | None = None,
+) -> torch.Tensor:
+ dtype = t.dtype
+ if freqs_seq_dim is None and (freqs.ndim == 2 or t.ndim == 3):
+ freqs_seq_dim = 0
+
+ if t.ndim == 3 or freqs_seq_dim is not None:
+ seq_len = t.shape[seq_dim]
+ freqs = slice_at_dim(freqs, slice(-seq_len, None), dim=freqs_seq_dim)
+
+ rot_feats = freqs.shape[-1]
+ end_index = start_index + rot_feats
+
+ t_left = t[..., :start_index]
+ t_middle = t[..., start_index:end_index]
+ t_right = t[..., end_index:]
+
+ freqs = freqs.to(device=t_middle.device, dtype=t_middle.dtype)
+ cos = freqs.cos() * scale
+ sin = freqs.sin() * scale
+ t_middle = (t_middle * cos) + (rotate_half(t_middle) * sin)
+ return torch.cat((t_left, t_middle, t_right), dim=-1).to(dtype)
+
+def _to_flux_freqs_cis(freqs_interleaved: torch.Tensor) -> torch.Tensor:
+ angles = freqs_interleaved[..., ::2].float()
+ cos = torch.cos(angles)
+ sin = torch.sin(angles)
+ out = torch.stack([cos, -sin, sin, cos], dim=-1)
+ return out.reshape(*out.shape[:-1], 2, 2)
+
+
+def _apply_rope1_partial(t: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor:
+ out = t.clone() if t.requires_grad or comfy.model_management.in_training else t
+ rot_d = 2 * freqs_cis.shape[-3]
+ seq_len = out.shape[-2]
+ for start in range(0, seq_len, SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS):
+ end = min(start + SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS, seq_len)
+ freqs_chunk = freqs_cis[start:end]
+ if rot_d == out.shape[-1]:
+ out[..., start:end, :] = apply_rope1(out[..., start:end, :], freqs_chunk).to(out.dtype)
+ else:
+ out[..., start:end, :rot_d] = apply_rope1(out[..., start:end, :rot_d], freqs_chunk).to(out.dtype)
+ return out
+
+
+class NaMMRotaryEmbedding3d(MMRotaryEmbeddingBase):
+ def __init__(self, dim: int):
+ super().__init__(dim, rope_dim=3)
+
+ def forward(
+ self,
+ vid_q: torch.FloatTensor, # L h d
+ vid_k: torch.FloatTensor, # L h d
+ vid_shape: torch.LongTensor, # B 3
+ txt_q: torch.FloatTensor, # L h d
+ txt_k: torch.FloatTensor, # L h d
+ txt_shape: torch.LongTensor, # B 1
+ cache: Cache,
+ ) -> Tuple[
+ torch.FloatTensor,
+ torch.FloatTensor,
+ torch.FloatTensor,
+ torch.FloatTensor,
+ ]:
+ vid_freqs, txt_freqs = cache(
+ "mmrope_freqs_3d",
+ lambda: self.get_freqs(vid_shape, txt_shape),
+ )
+ target_device = vid_q.device
+ if vid_freqs.device != target_device:
+ vid_freqs = vid_freqs.to(target_device)
+ if txt_freqs.device != target_device:
+ txt_freqs = txt_freqs.to(target_device)
+ vid_q = vid_q.transpose(0, 1)
+ vid_k = vid_k.transpose(0, 1)
+ vid_q = _apply_rope1_partial(vid_q, vid_freqs)
+ vid_k = _apply_rope1_partial(vid_k, vid_freqs)
+ vid_q = vid_q.transpose(0, 1)
+ vid_k = vid_k.transpose(0, 1)
+
+ txt_q = txt_q.transpose(0, 1)
+ txt_k = txt_k.transpose(0, 1)
+ txt_q = _apply_rope1_partial(txt_q, txt_freqs)
+ txt_k = _apply_rope1_partial(txt_k, txt_freqs)
+ txt_q = txt_q.transpose(0, 1)
+ txt_k = txt_k.transpose(0, 1)
+ return vid_q, vid_k, txt_q, txt_k
+
+ @torch._dynamo.disable # Disable compilation: .tolist() is data-dependent and causes graph breaks
+ def get_freqs(
+ self,
+ vid_shape: torch.LongTensor,
+ txt_shape: torch.LongTensor,
+ ) -> Tuple[
+ torch.Tensor,
+ torch.Tensor,
+ ]:
+
+ max_temporal = 0
+ max_height = 0
+ max_width = 0
+ max_txt_len = 0
+
+ for (f, h, w), l in zip(vid_shape.tolist(), txt_shape[:, 0].tolist()):
+ max_temporal = max(max_temporal, l + f)
+ max_height = max(max_height, h)
+ max_width = max(max_width, w)
+ max_txt_len = max(max_txt_len, l)
+
+ autocast_device = "cuda" if torch.cuda.is_available() else "cpu"
+ with torch.amp.autocast(autocast_device, enabled=False):
+ vid_freqs = self.get_axial_freqs(
+ max_temporal + 16,
+ max_height + 4,
+ max_width + 4,
+ ).float()
+ txt_freqs = self.get_axial_freqs(max_txt_len + 16)
+
+ vid_freq_list, txt_freq_list = [], []
+ for (f, h, w), l in zip(vid_shape.tolist(), txt_shape[:, 0].tolist()):
+ vid_freq = vid_freqs[l : l + f, :h, :w].reshape(-1, vid_freqs.size(-1))
+ txt_freq = txt_freqs[:l].repeat(1, 3).reshape(-1, vid_freqs.size(-1))
+ vid_freq_list.append(vid_freq)
+ txt_freq_list.append(txt_freq)
+ vid_freqs_interleaved = torch.cat(vid_freq_list, dim=0)
+ txt_freqs_interleaved = torch.cat(txt_freq_list, dim=0)
+
+ return _to_flux_freqs_cis(vid_freqs_interleaved), _to_flux_freqs_cis(txt_freqs_interleaved)
+
+class MMModule(nn.Module):
+ def __init__(
+ self,
+ module: Callable[..., nn.Module],
+ *args,
+ shared_weights: bool = False,
+ vid_only: bool = False,
+ **kwargs,
+ ):
+ super().__init__()
+ self.shared_weights = shared_weights
+ self.vid_only = vid_only
+ if self.shared_weights:
+ if get_args("vid", args) != get_args("txt", args):
+ raise ValueError("SeedVR2 shared MMModule requires matching vid/txt args.")
+ if get_kwargs("vid", kwargs) != get_kwargs("txt", kwargs):
+ raise ValueError("SeedVR2 shared MMModule requires matching vid/txt kwargs.")
+ self.all = module(*get_args("vid", args), **get_kwargs("vid", kwargs))
+ else:
+ self.vid = module(*get_args("vid", args), **get_kwargs("vid", kwargs))
+ self.txt = (
+ module(*get_args("txt", args), **get_kwargs("txt", kwargs))
+ if not vid_only
+ else None
+ )
+
+ def forward(
+ self,
+ vid: torch.FloatTensor,
+ txt: torch.FloatTensor,
+ *args,
+ **kwargs,
+ ) -> Tuple[
+ torch.FloatTensor,
+ torch.FloatTensor,
+ ]:
+ vid_module = self.vid if not self.shared_weights else self.all
+ vid = vid_module(vid, *get_args("vid", args), **get_kwargs("vid", kwargs))
+ if not self.vid_only:
+ txt_module = self.txt if not self.shared_weights else self.all
+ txt = txt.to(device=vid.device, dtype=vid.dtype)
+ txt = txt_module(txt, *get_args("txt", args), **get_kwargs("txt", kwargs))
+ return vid, txt
+
+def get_na_rope(rope_type: Optional[str], dim: int):
+ if rope_type is None:
+ return None
+ if rope_type == "rope3d":
+ return NaRotaryEmbedding3d(dim=dim)
+ if rope_type == "mmrope3d":
+ return NaMMRotaryEmbedding3d(dim=dim)
+ raise ValueError(f"Unknown SeedVR2 rope type: {rope_type}")
+
+class NaMMAttention(nn.Module):
+ def __init__(
+ self,
+ vid_dim: int,
+ txt_dim: int,
+ heads: int,
+ head_dim: int,
+ qk_bias: bool,
+ qk_norm,
+ qk_norm_eps: float,
+ rope_type: Optional[str],
+ rope_dim: int,
+ shared_weights: bool,
+ device, dtype, operations,
+ ):
+ super().__init__()
+ dim = MMArg(vid_dim, txt_dim)
+ self.heads = heads
+ inner_dim = heads * head_dim
+ qkv_dim = inner_dim * 3
+ self.head_dim = head_dim
+ self.proj_qkv = MMModule(
+ operations.Linear, dim, qkv_dim, bias=qk_bias, shared_weights=shared_weights, device=device, dtype=dtype
+ )
+ self.proj_out = MMModule(operations.Linear, inner_dim, dim, shared_weights=shared_weights, device=device, dtype=dtype)
+ self.norm_q = MMModule(
+ qk_norm,
+ normalized_shape=head_dim,
+ eps=qk_norm_eps,
+ elementwise_affine=True,
+ shared_weights=shared_weights,
+ device=device, dtype=dtype
+ )
+ self.norm_k = MMModule(
+ qk_norm,
+ normalized_shape=head_dim,
+ eps=qk_norm_eps,
+ elementwise_affine=True,
+ shared_weights=shared_weights,
+ device=device, dtype=dtype
+ )
+
+
+ self.rope = get_na_rope(rope_type=rope_type, dim=rope_dim)
+
+def window(
+ hid: torch.FloatTensor, # (L c)
+ hid_shape: torch.LongTensor, # (b n)
+ window_fn: Callable[[torch.Tensor], List[torch.Tensor]],
+):
+ hid = unflatten(hid, hid_shape)
+ hid = list(map(window_fn, hid))
+ hid_windows_list = [len(x) for x in hid]
+ hid_windows = torch.as_tensor(hid_windows_list, device=hid_shape.device)
+ hid = list(chain(*hid))
+ hid_len_list = [math.prod(x.shape[:-1]) for x in hid]
+ hid, hid_shape = flatten(hid)
+ return hid, hid_shape, hid_windows, hid_len_list, hid_windows_list
+
+def window_idx(
+ hid_shape: torch.LongTensor, # (b n)
+ window_fn: Callable[[torch.Tensor], List[torch.Tensor]],
+):
+ hid_idx = torch.arange(hid_shape.prod(-1).sum(), device=hid_shape.device).unsqueeze(-1)
+ tgt_idx, tgt_shape, tgt_windows, tgt_len_list, tgt_windows_list = window(hid_idx, hid_shape, window_fn)
+ tgt_idx = tgt_idx.squeeze(-1)
+ src_idx = torch.argsort(tgt_idx)
+ return (
+ lambda hid: torch.index_select(hid, 0, tgt_idx),
+ lambda hid: torch.index_select(hid, 0, src_idx),
+ tgt_shape,
+ tgt_windows,
+ tgt_len_list,
+ tgt_windows_list,
+ )
+
+class NaSwinAttention(NaMMAttention):
+ def __init__(
+ self,
+ *args,
+ window: Union[int, Tuple[int, int, int]],
+ window_method: str,
+ version: bool = False,
+ **kwargs,
+ ):
+ super().__init__(*args, **kwargs)
+ self.version_7b = version
+ self.window = _triple(window)
+ self.window_method = window_method
+ if not all(isinstance(v, int) and v >= 0 for v in self.window):
+ raise ValueError(f"SeedVR2 window must contain non-negative integers, got {self.window}.")
+
+ self.window_op = get_window_op(window_method)
+
+ def forward(
+ self,
+ vid: torch.FloatTensor, # l c
+ txt: torch.FloatTensor, # l c
+ vid_shape: torch.LongTensor, # b 3
+ txt_shape: torch.LongTensor, # b 1
+ cache: Cache,
+ ) -> Tuple[
+ torch.FloatTensor,
+ torch.FloatTensor,
+ ]:
+
+ vid_qkv, txt_qkv = self.proj_qkv(vid, txt)
+
+ cache_win = cache.namespace(f"{self.window_method}_{self.window}_sd3")
+
+ def make_window(x: torch.Tensor):
+ t, h, w, _ = x.shape
+ window_slices = self.window_op((t, h, w), self.window)
+ return [x[st, sh, sw] for (st, sh, sw) in window_slices]
+
+ window_partition, window_reverse, window_shape, window_count, vid_len_win_list, window_count_list = cache_win(
+ "win_transform",
+ lambda: window_idx(vid_shape, make_window),
+ )
+ vid_qkv_win = window_partition(vid_qkv)
+
+ vid_qkv_win = vid_qkv_win.reshape(vid_qkv_win.shape[0], 3, self.heads, self.head_dim)
+ txt_qkv = txt_qkv.reshape(txt_qkv.shape[0], 3, self.heads, self.head_dim)
+
+ vid_q, vid_k, vid_v = vid_qkv_win.unbind(1)
+ txt_q, txt_k, txt_v = txt_qkv.unbind(1)
+
+ vid_q, txt_q = self.norm_q(vid_q, txt_q)
+ vid_k, txt_k = self.norm_k(vid_k, txt_k)
+
+ txt_len = cache("txt_len", lambda: txt_shape.prod(-1))
+
+ vid_len_win = cache_win("vid_len", lambda: window_shape.prod(-1))
+ txt_len = txt_len.to(window_count.device)
+
+ if self.rope:
+ if self.version_7b:
+ vid_q, vid_k = self.rope(vid_q, vid_k, window_shape, cache_win)
+ elif self.rope.mm:
+ _, num_h, _ = txt_q.shape
+ txt_q_repeat = txt_q.flatten(1, 2)
+ txt_q_repeat = unflatten(txt_q_repeat, txt_shape)
+ txt_q_repeat = [[x] * n for x, n in zip(txt_q_repeat, window_count_list)]
+ txt_q_repeat = list(chain(*txt_q_repeat))
+ txt_q_repeat, txt_shape_repeat = flatten(txt_q_repeat)
+ txt_q_repeat = txt_q_repeat.reshape(txt_q_repeat.shape[0], num_h, self.head_dim)
+
+ txt_k_repeat = txt_k.flatten(1, 2)
+ txt_k_repeat = unflatten(txt_k_repeat, txt_shape)
+ txt_k_repeat = [[x] * n for x, n in zip(txt_k_repeat, window_count_list)]
+ txt_k_repeat = list(chain(*txt_k_repeat))
+ txt_k_repeat, _ = flatten(txt_k_repeat)
+ txt_k_repeat = txt_k_repeat.reshape(txt_k_repeat.shape[0], num_h, self.head_dim)
+
+ vid_q, vid_k, txt_q, txt_k = self.rope(
+ vid_q, vid_k, window_shape, txt_q_repeat, txt_k_repeat, txt_shape_repeat, cache_win
+ )
+ else:
+ vid_q, vid_k = self.rope(vid_q, vid_k, window_shape, cache_win)
+
+ txt_len_win_list = cache_win(
+ "txt_len_list",
+ lambda: [txt_len for txt_len, window_count in zip(txt_len.tolist(), window_count_list) for _ in range(window_count)],
+ )
+ all_len_win = cache_win("all_len", lambda: [vid_len + txt_len for vid_len, txt_len in zip(vid_len_win_list, txt_len_win_list)])
+ concat_win, unconcat_win = cache_win(
+ "mm_pnp", lambda: repeat_concat_idx(vid_len_win, txt_len, window_count)
+ )
+ out = optimized_var_attention(
+ q=concat_win(vid_q, txt_q),
+ k=concat_win(vid_k, txt_k),
+ v=concat_win(vid_v, txt_v),
+ heads=self.heads, skip_reshape=True, skip_output_reshape=True,
+ cu_seqlens_q=cache_win("vid_seqlens_q", lambda: cumulative_lengths(all_len_win)),
+ cu_seqlens_k=cache_win("vid_seqlens_k", lambda: cumulative_lengths(all_len_win)),
+ )
+ vid_out, txt_out = unconcat_win(out)
+
+ vid_out = vid_out.flatten(1, 2)
+ txt_out = txt_out.flatten(1, 2)
+ vid_out = window_reverse(vid_out)
+
+ vid_out, txt_out = self.proj_out(vid_out, txt_out)
+
+ return vid_out, txt_out
+
+class MLP(nn.Module):
+ def __init__(
+ self,
+ dim: int,
+ expand_ratio: int,
+ device, dtype, operations
+ ):
+ super().__init__()
+ self.proj_in = operations.Linear(dim, dim * expand_ratio, device=device, dtype=dtype)
+ self.act = nn.GELU("tanh")
+ self.proj_out = operations.Linear(dim * expand_ratio, dim, device=device, dtype=dtype)
+
+ def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:
+ x = self.proj_in(x)
+ x = self.act(x)
+ x = self.proj_out(x)
+ return x
+
+
+class SwiGLUMLP(nn.Module):
+ def __init__(
+ self,
+ dim: int,
+ expand_ratio: int,
+ multiple_of: int = 256,
+ device=None, dtype=None, operations=None
+ ):
+ super().__init__()
+ hidden_dim = int(2 * dim * expand_ratio / 3)
+ hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)
+ self.proj_in_gate = operations.Linear(dim, hidden_dim, bias=False, device=device, dtype=dtype)
+ self.proj_out = operations.Linear(hidden_dim, dim, bias=False, device=device, dtype=dtype)
+ self.proj_in = operations.Linear(dim, hidden_dim, bias=False, device=device, dtype=dtype)
+
+ def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:
+ return self.proj_out(F.silu(self.proj_in_gate(x)) * self.proj_in(x))
+
+def get_mlp(mlp_type: Optional[str] = "normal"):
+ if mlp_type == "normal":
+ return MLP
+ if mlp_type == "swiglu":
+ return SwiGLUMLP
+ raise ValueError(f"Unknown SeedVR2 MLP type: {mlp_type}")
+
+class NaMMSRTransformerBlock(nn.Module):
+ def __init__(
+ self,
+ *,
+ vid_dim: int,
+ txt_dim: int,
+ emb_dim: int,
+ heads: int,
+ head_dim: int,
+ expand_ratio: int,
+ norm,
+ norm_eps: float,
+ ada,
+ qk_bias: bool,
+ qk_norm,
+ mlp_type: str,
+ shared_weights: bool,
+ rope_type: str,
+ rope_dim: int,
+ is_last_layer: bool,
+ window: Union[int, Tuple[int, int, int]],
+ window_method: str,
+ version: bool,
+ device, dtype, operations,
+ ):
+ super().__init__()
+ dim = MMArg(vid_dim, txt_dim)
+ self.attn_norm = MMModule(norm, normalized_shape=dim, eps=norm_eps, elementwise_affine=False, shared_weights=shared_weights, device=device, dtype=dtype)
+
+ self.attn = NaSwinAttention(
+ vid_dim=vid_dim,
+ txt_dim=txt_dim,
+ heads=heads,
+ head_dim=head_dim,
+ qk_bias=qk_bias,
+ qk_norm=qk_norm,
+ qk_norm_eps=norm_eps,
+ rope_type=rope_type,
+ rope_dim=rope_dim,
+ shared_weights=shared_weights,
+ window=window,
+ window_method=window_method,
+ version=version,
+ device=device, dtype=dtype, operations=operations
+ )
+
+ self.mlp_norm = MMModule(norm, normalized_shape=dim, eps=norm_eps, elementwise_affine=False, shared_weights=shared_weights, vid_only=is_last_layer, device=device, dtype=dtype)
+ self.mlp = MMModule(
+ get_mlp(mlp_type),
+ dim=dim,
+ expand_ratio=expand_ratio,
+ shared_weights=shared_weights,
+ vid_only=is_last_layer,
+ device=device, dtype=dtype, operations=operations
+ )
+ self.ada = MMModule(ada, dim=dim, emb_dim=emb_dim, layers=["attn", "mlp"], shared_weights=shared_weights, vid_only=is_last_layer, device=device, dtype=dtype)
+ self.is_last_layer = is_last_layer
+ self.version = version
+
+ def _seedvr2_7b_mlp(
+ self,
+ vid: torch.FloatTensor,
+ txt: torch.FloatTensor,
+ ) -> Tuple[
+ torch.FloatTensor,
+ torch.FloatTensor,
+ ]:
+ vid_module = self.mlp.vid if not self.mlp.shared_weights else self.mlp.all
+ if comfy.model_management.in_training or vid.requires_grad:
+ vid = torch.cat([vid_module(chunk) for chunk in vid.split(SEEDVR2_7B_MLP_CHUNK, dim=0)], dim=0)
+ else:
+ vid_out = None
+ offset = 0
+ for chunk in vid.split(SEEDVR2_7B_MLP_CHUNK, dim=0):
+ chunk_out = vid_module(chunk)
+ if vid_out is None:
+ vid_out = chunk_out.new_empty((vid.shape[0], *chunk_out.shape[1:]))
+ vid_out[offset:offset + chunk_out.shape[0]] = chunk_out
+ offset += chunk_out.shape[0]
+ vid = vid_out
+ if not self.mlp.vid_only:
+ txt_module = self.mlp.txt if not self.mlp.shared_weights else self.mlp.all
+ txt = txt.to(device=vid.device, dtype=vid.dtype)
+ txt = txt_module(txt)
+ return vid, txt
+
+ def forward(
+ self,
+ vid: torch.FloatTensor, # l c
+ txt: torch.FloatTensor, # l c
+ vid_shape: torch.LongTensor, # b 3
+ txt_shape: torch.LongTensor, # b 1
+ emb: torch.FloatTensor,
+ cache: Cache,
+ ) -> Tuple[
+ torch.FloatTensor,
+ torch.FloatTensor,
+ torch.LongTensor,
+ torch.LongTensor,
+ ]:
+ hid_len = MMArg(
+ cache("vid_len", lambda: vid_shape.prod(-1)),
+ cache("txt_len", lambda: txt_shape.prod(-1)),
+ )
+ ada_kwargs = {
+ "emb": emb,
+ "hid_len": hid_len,
+ "cache": cache,
+ "branch_tag": MMArg("vid", "txt"),
+ }
+
+ vid_attn, txt_attn = self.attn_norm(vid, txt)
+ vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="in", **ada_kwargs)
+ vid_attn, txt_attn = self.attn(vid_attn, txt_attn, vid_shape, txt_shape, cache)
+ vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="out", **ada_kwargs)
+ vid_attn, txt_attn = (vid_attn + vid), (txt_attn + txt)
+
+ vid_mlp, txt_mlp = self.mlp_norm(vid_attn, txt_attn)
+ vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, layer="mlp", mode="in", **ada_kwargs)
+ if self.version:
+ vid_mlp, txt_mlp = self._seedvr2_7b_mlp(vid_mlp, txt_mlp)
+ else:
+ vid_mlp, txt_mlp = self.mlp(vid_mlp, txt_mlp)
+ vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, layer="mlp", mode="out", **ada_kwargs)
+ vid_mlp, txt_mlp = (vid_mlp + vid_attn), (txt_mlp + txt_attn)
+
+ return vid_mlp, txt_mlp, vid_shape, txt_shape
+
+class PatchOut(nn.Module):
+ def __init__(
+ self,
+ out_channels: int,
+ patch_size: Union[int, Tuple[int, int, int]],
+ dim: int,
+ device, dtype, operations
+ ):
+ super().__init__()
+ t, h, w = _triple(patch_size)
+ self.patch_size = t, h, w
+ self.proj = operations.Linear(dim, out_channels * t * h * w, device=device, dtype=dtype)
+
+ def forward(
+ self,
+ vid: torch.Tensor,
+ ) -> torch.Tensor:
+ t, h, w = self.patch_size
+ vid = self.proj(vid)
+ b, T, H, W, channels = vid.shape
+ c = channels // (t * h * w)
+ vid = vid.view(b, T, H, W, t, h, w, c).permute(0, 7, 1, 4, 2, 5, 3, 6).reshape(b, c, T * t, H * h, W * w)
+ if t > 1:
+ vid = vid[:, :, (t - 1) :]
+ return vid
+
+class NaPatchOut(PatchOut):
+ def forward(
+ self,
+ vid: torch.FloatTensor, # l c
+ vid_shape: torch.LongTensor,
+ cache: Optional[Cache] = None,
+ vid_shape_before_patchify = None
+ ) -> Tuple[
+ torch.FloatTensor,
+ torch.LongTensor,
+ ]:
+ if cache is None:
+ cache = Cache(disable=True)
+
+ t, h, w = self.patch_size
+ vid = self.proj(vid)
+
+ if not (t == h == w == 1):
+ vid = unflatten(vid, vid_shape)
+ for i in range(len(vid)):
+ T, H, W, channels = vid[i].shape
+ c = channels // (t * h * w)
+ vid[i] = vid[i].view(T, H, W, t, h, w, c).permute(0, 3, 1, 4, 2, 5, 6).reshape(T * t, H * h, W * w, c)
+ if t > 1 and vid_shape_before_patchify[i, 0] % t != 0:
+ vid[i] = vid[i][(t - vid_shape_before_patchify[i, 0] % t) :]
+ vid, vid_shape = flatten(vid)
+
+ return vid, vid_shape
+
+class PatchIn(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ patch_size: Union[int, Tuple[int, int, int]],
+ dim: int,
+ device, dtype, operations
+ ):
+ super().__init__()
+ t, h, w = _triple(patch_size)
+ self.patch_size = t, h, w
+ self.proj = operations.Linear(in_channels * t * h * w, dim, device=device, dtype=dtype)
+
+ def forward(
+ self,
+ vid: torch.Tensor,
+ ) -> torch.Tensor:
+ t, h, w = self.patch_size
+ if t > 1:
+ if vid.size(2) % t != 1:
+ raise ValueError(
+ f"SeedVR2 patch input temporal size must satisfy T % {t} == 1, got {vid.size(2)}."
+ )
+ vid = torch.cat([vid[:, :, :1]] * (t - 1) + [vid], dim=2)
+ b, c, Tt, Hh, Ww = vid.shape
+ vid = vid.view(b, c, Tt // t, t, Hh // h, h, Ww // w, w).permute(0, 2, 4, 6, 3, 5, 7, 1).reshape(b, Tt // t, Hh // h, Ww // w, t * h * w * c)
+ vid = self.proj(vid)
+ return vid
+
+class NaPatchIn(PatchIn):
+ def forward(
+ self,
+ vid: torch.Tensor, # l c
+ vid_shape: torch.LongTensor,
+ cache: Optional[Cache] = None,
+ ) -> torch.Tensor:
+ if cache is None:
+ cache = Cache(disable=True)
+ cache = cache.namespace("patch")
+ vid_shape_before_patchify = cache("vid_shape_before_patchify", lambda: vid_shape)
+ t, h, w = self.patch_size
+ if not (t == h == w == 1):
+ vid = unflatten(vid, vid_shape)
+ for i in range(len(vid)):
+ if t > 1 and vid_shape_before_patchify[i, 0] % t != 0:
+ vid[i] = torch.cat([vid[i][:1]] * (t - vid[i].size(0) % t) + [vid[i]], dim=0)
+ Tt, Hh, Ww, c = vid[i].shape
+ vid[i] = vid[i].view(Tt // t, t, Hh // h, h, Ww // w, w, c).permute(0, 2, 4, 1, 3, 5, 6).reshape(Tt // t, Hh // h, Ww // w, t * h * w * c)
+ vid, vid_shape = flatten(vid)
+
+ vid = self.proj(vid)
+ return vid, vid_shape
+
+def expand_dims(x: torch.Tensor, dim: int, ndim: int):
+ shape = x.shape
+ shape = shape[:dim] + (1,) * (ndim - len(shape)) + shape[dim:]
+ return x.reshape(shape)
+
+
+class AdaSingle(nn.Module):
+ def __init__(
+ self,
+ dim: int,
+ emb_dim: int,
+ layers: List[str],
+ modes: Tuple[str, ...] = ("in", "out"),
+ device = None, dtype = None,
+ ):
+ if emb_dim != 6 * dim:
+ raise ValueError(f"SeedVR2 AdaSingle requires emb_dim == 6 * dim, got emb_dim={emb_dim}, dim={dim}.")
+ super().__init__()
+ self.dim = dim
+ self.emb_dim = emb_dim
+ self.layers = layers
+
+ param_kwargs = {"device": device, "dtype": dtype}
+
+ for l in layers:
+ if "in" in modes:
+ self.register_parameter(f"{l}_shift", nn.Parameter(torch.empty(dim, **param_kwargs)))
+ self.register_parameter(f"{l}_scale", nn.Parameter(torch.empty(dim, **param_kwargs)))
+ if "out" in modes:
+ self.register_parameter(f"{l}_gate", nn.Parameter(torch.empty(dim, **param_kwargs)))
+
+ def forward(
+ self,
+ hid: torch.FloatTensor, # b ... c
+ emb: torch.FloatTensor, # b d
+ layer: str,
+ mode: str,
+ cache: Optional[Cache] = None,
+ branch_tag: str = "",
+ hid_len: Optional[torch.LongTensor] = None, # b
+ ) -> torch.FloatTensor:
+ if cache is None:
+ cache = Cache(disable=True)
+ idx = self.layers.index(layer)
+ emb = emb.reshape(emb.shape[0], -1, len(self.layers), 3)[:, :, idx, :]
+ emb = expand_dims(emb, 1, hid.ndim + 1)
+
+ if hid_len is not None:
+ emb = cache(
+ f"emb_repeat_{idx}_{branch_tag}",
+ lambda: torch.repeat_interleave(emb, hid_len, dim=0),
+ )
+
+ shiftA, scaleA, gateA = emb.unbind(-1)
+ shiftB, scaleB, gateB = (
+ getattr(self, f"{layer}_shift", None),
+ getattr(self, f"{layer}_scale", None),
+ getattr(self, f"{layer}_gate", None),
+ )
+
+ if mode == "in":
+ shiftB = comfy.ops.cast_to_input(shiftB, hid)
+ scaleB = comfy.ops.cast_to_input(scaleB, hid)
+ return hid.mul_(scaleA + scaleB).add_(shiftA + shiftB)
+ if mode == "out":
+ if gateB is not None:
+ gateB = comfy.ops.cast_to_input(gateB, hid)
+ return hid.mul_(gateA + gateB)
+ else:
+ return hid.mul_(gateA)
+
+ raise ValueError(f"Unknown AdaSingle mode: {mode}")
+
+
+class TimeEmbedding(nn.Module):
+ def __init__(
+ self,
+ sinusoidal_dim: int,
+ hidden_dim: int,
+ output_dim: int,
+ device, dtype, operations
+ ):
+ super().__init__()
+ self.sinusoidal_dim = sinusoidal_dim
+ self.proj_in = operations.Linear(sinusoidal_dim, hidden_dim, device=device, dtype=dtype)
+ self.proj_hid = operations.Linear(hidden_dim, hidden_dim, device=device, dtype=dtype)
+ self.proj_out = operations.Linear(hidden_dim, output_dim, device=device, dtype=dtype)
+ self.act = nn.SiLU()
+
+ def forward(
+ self,
+ timestep: Union[int, float, torch.IntTensor, torch.FloatTensor],
+ device: torch.device,
+ dtype: torch.dtype,
+ ) -> torch.FloatTensor:
+ if not torch.is_tensor(timestep):
+ timestep = torch.tensor([timestep], device=device, dtype=dtype)
+ if timestep.ndim == 0:
+ timestep = timestep[None]
+
+ emb = get_timestep_embedding(
+ timesteps=timestep,
+ embedding_dim=self.sinusoidal_dim,
+ flip_sin_to_cos=False,
+ downscale_freq_shift=0,
+ ).to(dtype)
+ emb = self.proj_in(emb)
+ emb = self.act(emb)
+ emb = self.proj_hid(emb)
+ emb = self.act(emb)
+ emb = self.proj_out(emb)
+ return emb
+
+def flatten(
+ hid: List[torch.FloatTensor], # List of (*** c)
+) -> Tuple[
+ torch.FloatTensor, # (L c)
+ torch.LongTensor, # (b n)
+]:
+ if len(hid) == 0:
+ raise ValueError("SeedVR2 flatten requires at least one tensor.")
+ shape = torch.as_tensor([x.shape[:-1] for x in hid], device=hid[0].device)
+ hid = torch.cat([x.flatten(0, -2) for x in hid])
+ return hid, shape
+
+
+def unflatten(
+ hid: torch.FloatTensor, # (L c) or (L ... c)
+ hid_shape: torch.LongTensor, # (b n)
+) -> List[torch.Tensor]: # List of (*** c) or (*** ... c)
+ hid_len = hid_shape.prod(-1)
+ hid = hid.split(hid_len.tolist())
+ hid = [x.unflatten(0, s.tolist()) for x, s in zip(hid, hid_shape)]
+ return hid
+
+class NaDiT(nn.Module):
+
+ def __init__(
+ self,
+ norm_eps,
+ num_layers,
+ mlp_type,
+ vid_in_channels = 33,
+ vid_out_channels = SEEDVR2_LATENT_CHANNELS,
+ vid_dim = 2560,
+ txt_in_dim = 5120,
+ heads = 20,
+ head_dim = 128,
+ mm_layers = 10,
+ expand_ratio = 4,
+ qk_bias = False,
+ patch_size = (1, 2, 2),
+ rope_dim = 128,
+ rope_type = "mmrope3d",
+ vid_out_norm: Optional[str] = None,
+ image_model = None,
+ device = None,
+ dtype = None,
+ operations = None,
+ ):
+ if image_model not in (None, "seedvr2"):
+ raise ValueError(f"SeedVR2 NaDiT expected image_model='seedvr2', got {image_model!r}.")
+ self._7b_version = vid_dim == SEEDVR2_7B_VID_DIM
+ if self._7b_version:
+ rope_type = "rope3d"
+ self.dtype = dtype
+ factory_kwargs = {"device": device, "dtype": dtype}
+ window_method = num_layers // 2 * ["720pwin_by_size_bysize","720pswin_by_size_bysize"]
+ txt_dim = vid_dim
+ emb_dim = vid_dim * 6
+ window = num_layers * [(4,3,3)]
+ ada = AdaSingle
+ norm = operations.RMSNorm
+ qk_norm = operations.RMSNorm
+ super().__init__()
+ self.register_buffer("positive_conditioning", torch.empty((58, 5120), device=device, dtype=dtype))
+ self.register_buffer("negative_conditioning", torch.empty((64, 5120), device=device, dtype=dtype))
+ self.vid_in = NaPatchIn(
+ in_channels=vid_in_channels,
+ patch_size=patch_size,
+ dim=vid_dim,
+ device=device, dtype=dtype, operations=operations
+ )
+ self.txt_in = (
+ operations.Linear(txt_in_dim, txt_dim, **factory_kwargs)
+ if txt_in_dim and txt_in_dim != txt_dim
+ else nn.Identity()
+ )
+ self.emb_in = TimeEmbedding(
+ sinusoidal_dim=BYTEDANCE_SINUSOIDAL_DIM,
+ hidden_dim=max(vid_dim, txt_dim),
+ output_dim=emb_dim,
+ device=device, dtype=dtype, operations=operations
+ )
+
+ if window is None or isinstance(window[0], int):
+ window = [window] * num_layers
+
+ rope_dim = rope_dim if rope_dim is not None else head_dim // 2
+ self.blocks = nn.ModuleList(
+ [
+ NaMMSRTransformerBlock(
+ vid_dim=vid_dim,
+ txt_dim=txt_dim,
+ emb_dim=emb_dim,
+ heads=heads,
+ head_dim=head_dim,
+ expand_ratio=expand_ratio,
+ norm=norm,
+ norm_eps=norm_eps,
+ ada=ada,
+ qk_bias=qk_bias,
+ qk_norm=qk_norm,
+ mlp_type=mlp_type,
+ rope_dim = rope_dim,
+ window=window[i],
+ window_method=window_method[i],
+ version = self._7b_version,
+ is_last_layer=(i == num_layers - 1) and not self._7b_version,
+ rope_type = rope_type,
+ shared_weights=not (
+ (i < mm_layers) if isinstance(mm_layers, int) else mm_layers[i]
+ ),
+ operations = operations,
+ **factory_kwargs
+ )
+ for i in range(num_layers)
+ ]
+ )
+ self.vid_out = NaPatchOut(
+ out_channels=vid_out_channels,
+ patch_size=patch_size,
+ dim=vid_dim,
+ device=device, dtype=dtype, operations=operations
+ )
+
+ self.vid_out_norm = None
+ if vid_out_norm is not None:
+ self.vid_out_norm = operations.RMSNorm(
+ normalized_shape=vid_dim,
+ eps=norm_eps,
+ elementwise_affine=True,
+ device=device, dtype=dtype
+ )
+ self.vid_out_ada = ada(
+ dim=vid_dim,
+ emb_dim=emb_dim,
+ layers=["out"],
+ modes=["in"],
+ device=device, dtype=dtype
+ )
+
+ def _resolve_text_conditioning(self, context, cond_or_uncond=None):
+ if context is None or context.numel() == 0:
+ context = self.positive_conditioning
+ return flatten([context])
+ if NaDiT._seedvr2_is_single_conditioning_branch(cond_or_uncond):
+ if context.shape[0] == 1:
+ context = context.squeeze(0)
+ return flatten([context])
+ return flatten(context.unbind(0))
+ if context.shape[0] % 2 != 0:
+ raise ValueError(f"SeedVR2 expected an even text-conditioning batch, got shape {tuple(context.shape)}")
+ neg_cond, pos_cond = context.chunk(2, dim=0)
+ if pos_cond.shape[0] == 1:
+ pos_cond, neg_cond = pos_cond.squeeze(0), neg_cond.squeeze(0)
+ return flatten([pos_cond, neg_cond])
+ return flatten((*pos_cond.unbind(0), *neg_cond.unbind(0)))
+
+ @staticmethod
+ def _seedvr2_is_single_conditioning_branch(cond_or_uncond):
+ if cond_or_uncond is None or len(cond_or_uncond) == 0:
+ return False
+ first = cond_or_uncond[0]
+ return all(entry == first for entry in cond_or_uncond)
+
+ @staticmethod
+ def _check_seedvr2_video_latent(x, channels, name):
+ if x.ndim != 5:
+ raise ValueError(f"SeedVR2 expected {name} to be 5-D native latent, got shape {tuple(x.shape)}.")
+ if x.shape[1] != channels:
+ raise ValueError(f"SeedVR2 expected {name} channels to be {channels}, got shape {tuple(x.shape)}.")
+ return x
+
+ def _swap_pos_neg_halves(self, out, cond_or_uncond=None):
+ if NaDiT._seedvr2_is_single_conditioning_branch(cond_or_uncond):
+ return out
+ pos, neg = out.chunk(2, dim=0)
+ return torch.cat([neg, pos], dim=0)
+
+ def forward(
+ self,
+ x,
+ timestep,
+ context, # l c
+ disable_cache: bool = False,
+ **kwargs
+ ):
+ transformer_options = kwargs.get("transformer_options", {})
+ patches_replace = transformer_options.get("patches_replace", {})
+ blocks_replace = patches_replace.get("dit", {})
+ conditions = kwargs.get("condition")
+ if conditions is None:
+ raise ValueError("SeedVR2 requires conditioning latents from the SeedVR2Conditioning node.")
+ x = self._check_seedvr2_video_latent(x, SEEDVR2_LATENT_CHANNELS, "latent")
+ conditions = self._check_seedvr2_video_latent(conditions, SEEDVR2_LATENT_CHANNELS + 1, "conditioning")
+ b, _, t, h, w = x.shape
+ if conditions.shape[0] != b or conditions.shape[2:] != (t, h, w):
+ raise ValueError(
+ f"SeedVR2 conditioning shape must match latent batch/temporal/spatial dimensions; got latent {tuple(x.shape)} and conditioning {tuple(conditions.shape)}."
+ )
+ x = x.movedim(1, -1)
+ conditions = conditions.movedim(1, -1)
+ cache = Cache(disable=disable_cache)
+
+ txt, txt_shape = self._resolve_text_conditioning(context, transformer_options.get("cond_or_uncond"))
+
+ vid, vid_shape = flatten(x)
+ cond_latent, _ = flatten(conditions)
+
+ vid = torch.cat([vid, cond_latent], dim=-1)
+
+ txt = self.txt_in(txt)
+
+ vid_shape_before_patchify = vid_shape
+ vid, vid_shape = self.vid_in(vid, vid_shape, cache=cache)
+
+ emb = self.emb_in(timestep, device=vid.device, dtype=vid.dtype)
+
+ for i, block in enumerate(self.blocks):
+ if ("block", i) in blocks_replace:
+ def block_wrap(args):
+ out = {}
+ out["vid"], out["txt"], out["vid_shape"], out["txt_shape"] = block(
+ vid=args["vid"],
+ txt=args["txt"],
+ vid_shape=args["vid_shape"],
+ txt_shape=args["txt_shape"],
+ emb=args["emb"],
+ cache=args["cache"],
+ )
+ return out
+ out = blocks_replace[("block", i)]({
+ "vid":vid,
+ "txt":txt,
+ "vid_shape":vid_shape,
+ "txt_shape":txt_shape,
+ "emb":emb,
+ "cache":cache,
+ }, {"original_block": block_wrap})
+ vid, txt, vid_shape, txt_shape = out["vid"], out["txt"], out["vid_shape"], out["txt_shape"]
+ else:
+ vid, txt, vid_shape, txt_shape = block(
+ vid=vid,
+ txt=txt,
+ vid_shape=vid_shape,
+ txt_shape=txt_shape,
+ emb=emb,
+ cache=cache,
+ )
+
+ if self.vid_out_norm:
+ vid = self.vid_out_norm(vid)
+ vid = self.vid_out_ada(
+ vid,
+ emb=emb,
+ layer="out",
+ mode="in",
+ hid_len=cache("vid_len", lambda: vid_shape.prod(-1)),
+ cache=cache,
+ branch_tag="vid",
+ )
+
+ vid, vid_shape = self.vid_out(vid, vid_shape, cache, vid_shape_before_patchify = vid_shape_before_patchify)
+ vid = unflatten(vid, vid_shape)
+ out = torch.stack(vid)
+ out = out.movedim(-1, 1)
+ return self._swap_pos_neg_halves(out, transformer_options.get("cond_or_uncond"))
diff --git a/comfy/ldm/seedvr/vae.py b/comfy/ldm/seedvr/vae.py
new file mode 100644
index 000000000..c9f430184
--- /dev/null
+++ b/comfy/ldm/seedvr/vae.py
@@ -0,0 +1,1612 @@
+from typing import Literal, Optional, Tuple
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch import Tensor
+from contextlib import contextmanager
+from comfy.utils import ProgressBar
+
+from comfy.ldm.seedvr.constants import (
+ BYTEDANCE_BLOCK_OUT_CHANNELS,
+ BYTEDANCE_GN_CHUNKS_FP16,
+ BYTEDANCE_GN_CHUNKS_FP32,
+ BYTEDANCE_LOGVAR_CLAMP_MAX,
+ BYTEDANCE_LOGVAR_CLAMP_MIN,
+ BYTEDANCE_SLICING_SAMPLE_MIN,
+ BYTEDANCE_VAE_CONV_MEM_GIB,
+ BYTEDANCE_VAE_NORM_MEM_GIB,
+ BYTEDANCE_VAE_SCALING_FACTOR,
+ BYTEDANCE_VAE_SHIFTING_FACTOR,
+ BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE,
+ BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE,
+ SEEDVR2_LATENT_CHANNELS,
+)
+from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.diffusionmodules.model import vae_attention
+
+import math
+from enum import Enum
+
+import logging
+import comfy.model_management
+import comfy.ops
+ops = comfy.ops.disable_weight_init
+
+
+def _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap, temporal_scale=1):
+ if temporal_size is None:
+ return None
+
+ temporal_size = int(temporal_size)
+ if temporal_size <= 0:
+ return None
+
+ temporal_overlap = max(0, int(temporal_overlap or 0))
+ temporal_overlap = min(temporal_overlap, temporal_size - 1)
+ temporal_step = temporal_size - temporal_overlap
+ temporal_scale = max(1, int(temporal_scale))
+ return max(1, math.ceil(temporal_step / temporal_scale))
+
+
+def _seedvr2_clamped_spatial_overlap(overlap, tile_size):
+ overlap = max(0, int(overlap))
+ tile_size = max(1, int(tile_size))
+ return min(overlap, tile_size - 1)
+
+
+def tiled_vae(
+ x,
+ vae_model,
+ tile_size=(512, 512),
+ tile_overlap=(64, 64),
+ temporal_size=16,
+ temporal_overlap=0,
+ encode=True,
+):
+ if x.ndim != 5:
+ x = x.unsqueeze(2)
+
+ _, _, d, h, w = x.shape
+
+ sf_s = getattr(vae_model, "spatial_downsample_factor", BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE)
+ sf_t = getattr(vae_model, "temporal_downsample_factor", BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE)
+ if encode:
+ slicing_attr = "slicing_sample_min_size"
+ slicing_min_size = _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap)
+ else:
+ slicing_attr = "slicing_latent_min_size"
+ slicing_min_size = _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap, sf_t)
+ if encode:
+ ti_h, ti_w = tile_size
+ ov_h = _seedvr2_clamped_spatial_overlap(tile_overlap[0], ti_h)
+ ov_w = _seedvr2_clamped_spatial_overlap(tile_overlap[1], ti_w)
+ blend_ov_h = max(0, ov_h // sf_s)
+ blend_ov_w = max(0, ov_w // sf_s)
+ target_d = (d + sf_t - 1) // sf_t
+ target_h = (h + sf_s - 1) // sf_s
+ target_w = (w + sf_s - 1) // sf_s
+ else:
+ ti_h = max(1, tile_size[0] // sf_s)
+ ti_w = max(1, tile_size[1] // sf_s)
+ ov_h = _seedvr2_clamped_spatial_overlap(tile_overlap[0] // sf_s, ti_h)
+ ov_w = _seedvr2_clamped_spatial_overlap(tile_overlap[1] // sf_s, ti_w)
+ blend_ov_h = ov_h * sf_s
+ blend_ov_w = ov_w * sf_s
+
+ target_d = max(1, d * sf_t - (sf_t - 1))
+ target_h = h * sf_s
+ target_w = w * sf_s
+
+ stride_h = max(1, ti_h - ov_h)
+ stride_w = max(1, ti_w - ov_w)
+
+ storage_device = vae_model.device
+ result = None
+ count = None
+ def run_temporal_chunks(spatial_tile, model=vae_model, device=storage_device):
+ device = torch.device(device)
+ t_chunk = spatial_tile.to(device=device, dtype=next(model.parameters()).dtype, non_blocking=True).contiguous()
+ old_device = getattr(model, "device", None)
+ model.device = device
+ old_slicing_min_size = getattr(model, slicing_attr, None)
+ if old_slicing_min_size is not None and slicing_min_size is not None:
+ if slicing_min_size <= 0:
+ setattr(model, slicing_attr, t_chunk.shape[2])
+ else:
+ setattr(model, slicing_attr, slicing_min_size)
+ try:
+ if encode:
+ out = model.encode(t_chunk)
+ else:
+ out = model.decode_(t_chunk)
+ finally:
+ if old_slicing_min_size is not None and slicing_min_size is not None:
+ setattr(model, slicing_attr, old_slicing_min_size)
+ if old_device is not None:
+ model.device = old_device
+ if out.ndim == 4:
+ out = out.unsqueeze(2)
+ return out.to(storage_device)
+
+ ramp_cache = {}
+ def get_ramp(steps):
+ if steps not in ramp_cache:
+ t = torch.linspace(0, 1, steps=steps, device=storage_device, dtype=torch.float32)
+ ramp_cache[steps] = 0.5 - 0.5 * torch.cos(t * torch.pi)
+ return ramp_cache[steps]
+
+ tile_ranges = []
+ for y_idx in range(0, h, stride_h):
+ y_end = min(y_idx + ti_h, h)
+ if y_idx > 0 and (y_end - y_idx) <= ov_h:
+ continue
+ for x_idx in range(0, w, stride_w):
+ x_end = min(x_idx + ti_w, w)
+ if x_idx > 0 and (x_end - x_idx) <= ov_w:
+ continue
+ tile_ranges.append((y_idx, y_end, x_idx, x_end))
+
+ total_tiles = len(tile_ranges)
+ bar = ProgressBar(total_tiles)
+ single_spatial_tile = h <= ti_h and w <= ti_w
+
+ def run_tile(tile_index, tile_range):
+ y_idx, y_end, x_idx, x_end = tile_range
+ tile_x = x[:, :, :, y_idx:y_end, x_idx:x_end]
+ tile_out = run_temporal_chunks(tile_x)
+ return tile_index, y_idx, y_end, x_idx, x_end, tile_out
+
+ ordered_tile_outputs = (
+ run_tile(tile_index, tile_range)
+ for tile_index, tile_range in enumerate(tile_ranges)
+ )
+
+ for _, y_idx, y_end, x_idx, x_end, tile_out in ordered_tile_outputs:
+
+ if single_spatial_tile:
+ result = tile_out[:, :, :target_d, :target_h, :target_w]
+ if result.device != x.device or result.dtype != x.dtype:
+ result = result.to(device=x.device, dtype=x.dtype)
+ if x.shape[2] == 1 and sf_t == 1:
+ result = result.squeeze(2)
+ bar.update(1)
+ return result
+
+ if result is None:
+ b_out, c_out = tile_out.shape[0], tile_out.shape[1]
+ result = torch.zeros((b_out, c_out, target_d, target_h, target_w), device=storage_device, dtype=torch.float32)
+ count = torch.zeros((1, 1, 1, target_h, target_w), device=storage_device, dtype=torch.float32)
+
+ if encode:
+ ys, ye = y_idx // sf_s, (y_idx // sf_s) + tile_out.shape[3]
+ xs, xe = x_idx // sf_s, (x_idx // sf_s) + tile_out.shape[4]
+ cur_ov_h = max(0, min(blend_ov_h, tile_out.shape[3] // 2))
+ cur_ov_w = max(0, min(blend_ov_w, tile_out.shape[4] // 2))
+ else:
+ ys, ye = y_idx * sf_s, (y_idx * sf_s) + tile_out.shape[3]
+ xs, xe = x_idx * sf_s, (x_idx * sf_s) + tile_out.shape[4]
+ cur_ov_h = max(0, min(blend_ov_h, tile_out.shape[3] // 2))
+ cur_ov_w = max(0, min(blend_ov_w, tile_out.shape[4] // 2))
+
+ w_h = torch.ones((tile_out.shape[3],), device=storage_device)
+ w_w = torch.ones((tile_out.shape[4],), device=storage_device)
+
+ if cur_ov_h > 0:
+ r = get_ramp(cur_ov_h)
+ if y_idx > 0:
+ w_h[:cur_ov_h] = r
+ if y_end < h:
+ w_h[-cur_ov_h:] = 1.0 - r
+
+ if cur_ov_w > 0:
+ r = get_ramp(cur_ov_w)
+ if x_idx > 0:
+ w_w[:cur_ov_w] = r
+ if x_end < w:
+ w_w[-cur_ov_w:] = 1.0 - r
+
+ final_weight = w_h.view(1,1,1,-1,1) * w_w.view(1,1,1,1,-1)
+
+ valid_d = min(tile_out.shape[2], result.shape[2])
+ tile_out = tile_out[:, :, :valid_d, :, :]
+
+ tile_out.mul_(final_weight)
+
+ result[:, :, :valid_d, ys:ye, xs:xe] += tile_out
+ count[:, :, :, ys:ye, xs:xe] += final_weight
+
+ del tile_out, final_weight, w_h, w_w
+ bar.update(1)
+
+ result.div_(count.clamp(min=1e-6))
+
+ if result.device != x.device or result.dtype != x.dtype:
+ result = result.to(device=x.device, dtype=x.dtype)
+
+ if x.shape[2] == 1 and sf_t == 1:
+ result = result.squeeze(2)
+
+ return result
+
+_NORM_LIMIT = float("inf")
+def get_norm_limit():
+ return _NORM_LIMIT
+
+
+def set_norm_limit(value: Optional[float] = None):
+ global _NORM_LIMIT
+ if value is None:
+ value = float("inf")
+ _NORM_LIMIT = value
+
+@contextmanager
+def ignore_padding(model):
+ orig_padding = model.padding
+ model.padding = (0, 0, 0)
+ try:
+ yield
+ finally:
+ model.padding = orig_padding
+
+class MemoryState(Enum):
+ DISABLED = 0
+ INITIALIZING = 1
+ ACTIVE = 2
+ UNSET = 3
+
+def get_cache_size(conv_module, input_len, pad_len, dim=0):
+ dilated_kernel_size = conv_module.dilation[dim] * (conv_module.kernel_size[dim] - 1) + 1
+ output_len = (input_len + pad_len - dilated_kernel_size) // conv_module.stride[dim] + 1
+ remain_len = (
+ input_len + pad_len - ((output_len - 1) * conv_module.stride[dim] + dilated_kernel_size)
+ )
+ overlap_len = dilated_kernel_size - conv_module.stride[dim]
+ cache_len = overlap_len + remain_len
+
+ if output_len <= 0:
+ raise ValueError(
+ f"SeedVR2 VAE cache input is too short for convolution: input_len={input_len}, pad_len={pad_len}."
+ )
+ return cache_len
+
+class DiagonalGaussianDistribution(object):
+ def __init__(self, parameters: torch.Tensor):
+ self.parameters = parameters
+ self.mean, self.logvar = torch.chunk(parameters, 2, dim=1)
+ self.logvar = torch.clamp(self.logvar, BYTEDANCE_LOGVAR_CLAMP_MIN, BYTEDANCE_LOGVAR_CLAMP_MAX)
+
+ def mode(self):
+ return self.mean
+
+class SpatialNorm(nn.Module):
+ def __init__(
+ self,
+ f_channels: int,
+ zq_channels: int,
+ ):
+ super().__init__()
+ self.norm_layer = ops.GroupNorm(num_channels=f_channels, num_groups=32, eps=1e-6, affine=True)
+ self.conv_y = ops.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0)
+ self.conv_b = ops.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0)
+
+ def forward(self, f: torch.Tensor, zq: torch.Tensor) -> torch.Tensor:
+ f_size = f.shape[-2:]
+ zq = F.interpolate(zq, size=f_size, mode="nearest")
+ norm_f = self.norm_layer(f)
+ new_f = norm_f * self.conv_y(zq) + self.conv_b(zq)
+ return new_f
+
+class Attention(nn.Module):
+ def __init__(
+ self,
+ query_dim: int,
+ heads: int = 8,
+ dim_head: int = 64,
+ bias: bool = False,
+ norm_num_groups: Optional[int] = None,
+ spatial_norm_dim: Optional[int] = None,
+ out_bias: bool = True,
+ eps: float = 1e-5,
+ rescale_output_factor: float = 1.0,
+ residual_connection: bool = False,
+ ):
+ super().__init__()
+
+ self.inner_dim = dim_head * heads
+ self.rescale_output_factor = rescale_output_factor
+ self.residual_connection = residual_connection
+ self.out_dim = query_dim
+ self.heads = heads
+
+ if norm_num_groups is not None:
+ self.group_norm = ops.GroupNorm(num_channels=query_dim, num_groups=norm_num_groups, eps=eps, affine=True)
+ else:
+ self.group_norm = None
+
+ if spatial_norm_dim is not None:
+ self.spatial_norm = SpatialNorm(f_channels=query_dim, zq_channels=spatial_norm_dim)
+ else:
+ self.spatial_norm = None
+
+ self.to_q = ops.Linear(query_dim, self.inner_dim, bias=bias)
+ self.to_k = ops.Linear(query_dim, self.inner_dim, bias=bias)
+ self.to_v = ops.Linear(query_dim, self.inner_dim, bias=bias)
+ self.to_out = nn.ModuleList([])
+ self.to_out.append(ops.Linear(self.inner_dim, self.out_dim, bias=out_bias))
+ self.to_out.append(nn.Identity())
+
+ self.optimized_vae_attention = vae_attention()
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ temb: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
+
+ residual = hidden_states
+ if self.spatial_norm is not None:
+ hidden_states = self.spatial_norm(hidden_states, temb)
+
+ input_ndim = hidden_states.ndim
+
+ if input_ndim == 4:
+ batch_size, channel, height, width = hidden_states.shape
+ hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
+
+ batch_size = hidden_states.shape[0]
+
+ if self.group_norm is not None:
+ hidden_states = self.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
+
+ query = self.to_q(hidden_states)
+ key = self.to_k(hidden_states)
+ value = self.to_v(hidden_states)
+
+ inner_dim = key.shape[-1]
+ head_dim = inner_dim // self.heads
+
+ query = query.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
+
+ key = key.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
+ value = value.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
+
+ if input_ndim == 4 and self.heads == 1:
+ query = query.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width)
+ key = key.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width)
+ value = value.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width)
+ hidden_states = self.optimized_vae_attention(query, key, value).reshape(batch_size, self.heads, head_dim, height * width).transpose(2, 3)
+ else:
+ hidden_states = optimized_attention(query, key, value, heads = self.heads, skip_reshape=True, skip_output_reshape=True)
+
+ hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, self.heads * head_dim)
+ hidden_states = hidden_states.to(query.dtype)
+
+ hidden_states = self.to_out[0](hidden_states)
+ hidden_states = self.to_out[1](hidden_states)
+
+ if input_ndim == 4:
+ hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
+
+ if self.residual_connection:
+ hidden_states = hidden_states + residual
+
+ hidden_states = hidden_states / self.rescale_output_factor
+
+ return hidden_states
+
+
+def causal_norm_wrapper(norm_layer: nn.Module, x: torch.Tensor) -> torch.Tensor:
+ input_dtype = x.dtype
+ if isinstance(norm_layer, (ops.LayerNorm, ops.RMSNorm)):
+ if x.ndim == 4:
+ x = x.permute(0, 2, 3, 1)
+ x = norm_layer(x)
+ x = x.permute(0, 3, 1, 2)
+ return x.to(input_dtype)
+ if x.ndim == 5:
+ x = x.permute(0, 2, 3, 4, 1)
+ x = norm_layer(x)
+ x = x.permute(0, 4, 1, 2, 3)
+ return x.to(input_dtype)
+ if isinstance(norm_layer, (ops.GroupNorm, nn.BatchNorm2d, nn.SyncBatchNorm)):
+ if x.ndim <= 4:
+ return norm_layer(x).to(input_dtype)
+ if x.ndim == 5:
+ b, c, t, h, w = x.shape
+ x = x.transpose(1, 2).reshape(b * t, c, h, w)
+ memory_occupy = x.numel() * x.element_size() / 1024**3
+ if isinstance(norm_layer, ops.GroupNorm) and memory_occupy > get_norm_limit():
+ num_chunks = min(BYTEDANCE_GN_CHUNKS_FP16 if x.element_size() == 2 else BYTEDANCE_GN_CHUNKS_FP32, norm_layer.num_groups)
+ if norm_layer.num_groups % num_chunks != 0:
+ raise ValueError(
+ f"SeedVR2 VAE GroupNorm groups must divide chunks: groups={norm_layer.num_groups}, chunks={num_chunks}."
+ )
+ num_groups_per_chunk = norm_layer.num_groups // num_chunks
+
+ x = list(x.chunk(num_chunks, dim=1))
+ weights = norm_layer.weight.chunk(num_chunks, dim=0)
+ biases = norm_layer.bias.chunk(num_chunks, dim=0)
+ for i, (w, bias) in enumerate(zip(weights, biases)):
+ x[i] = F.group_norm(x[i], num_groups_per_chunk, w, bias, norm_layer.eps)
+ x[i] = x[i].to(input_dtype)
+ x = torch.cat(x, dim=1)
+ else:
+ x = norm_layer(x)
+ x = x.reshape((b, t, x.size(1), x.size(2), x.size(3))).transpose(1, 2)
+ return x.to(input_dtype)
+ raise TypeError(f"SeedVR2 VAE unsupported norm layer type: {type(norm_layer).__name__}")
+
+_receptive_field_t = Literal["half", "full"]
+
+def extend_head(tensor, times: int = 2, memory = None):
+ if memory is not None:
+ return torch.cat((memory.to(tensor), tensor), dim=2)
+ if times < 0:
+ raise ValueError(f"SeedVR2 VAE extend_head expected times >= 0, got {times}.")
+ if times == 0:
+ return tensor
+ else:
+ tile_repeat = [1] * tensor.ndim
+ tile_repeat[2] = times
+ return torch.cat(tensors=(torch.tile(tensor[:, :, :1], tile_repeat), tensor), dim=2)
+
+def cache_send_recv(tensor, cache_size, times, memory=None):
+ recv_buffer = None
+
+ if memory is not None:
+ recv_buffer = memory.to(tensor[0])
+ elif times > 0:
+ tile_repeat = [1] * tensor[0].ndim
+ tile_repeat[2] = times
+ recv_buffer = torch.tile(tensor[0][:, :, :1], tile_repeat)
+
+ return recv_buffer
+
+class InflatedCausalConv3d(ops.Conv3d):
+ def __init__(
+ self,
+ *args,
+ inflation_mode,
+ **kwargs,
+ ):
+ self.inflation_mode = inflation_mode
+ super().__init__(*args, **kwargs)
+ self.temporal_padding = self.padding[0]
+ self.padding = (0, *self.padding[1:])
+ self.memory_limit = float("inf")
+ self.logged_once = False
+
+ def set_memory_limit(self, value: float):
+ self.memory_limit = value
+
+ def _conv_forward(self, input, weight, bias, *args, **kwargs):
+ try:
+ return super()._conv_forward(input, weight, bias, *args, **kwargs)
+ except NotImplementedError:
+ # for: Could not run 'aten::cudnn_convolution' with arguments from the 'CPU' backend
+ if not self.logged_once:
+ logging.warning("VAE is on CPU for decoding. This is most likely due to not enough memory")
+ self.logged_once = True
+ return F.conv3d(input, weight, bias, *args, **kwargs)
+
+ def memory_limit_conv(
+ self,
+ x,
+ *,
+ split_dim=3,
+ padding=(0, 0, 0, 0, 0, 0),
+ prev_cache=None,
+ ):
+ if math.isinf(self.memory_limit):
+ if prev_cache is not None:
+ x = torch.cat([prev_cache, x], dim=split_dim - 1)
+ return super().forward(x)
+
+ shape = list(x.size())
+ if prev_cache is not None:
+ shape[split_dim - 1] += prev_cache.size(split_dim - 1)
+ for i, pad_sum in enumerate((padding[4] + padding[5], padding[2] + padding[3], padding[0] + padding[1])):
+ shape[-3 + i] += pad_sum
+ memory_occupy = math.prod(shape) * x.element_size() / 1024**3 # GiB
+ if memory_occupy < self.memory_limit or split_dim == x.ndim:
+ x_concat = x
+ if prev_cache is not None:
+ x_concat = torch.cat([prev_cache, x], dim=split_dim - 1)
+
+ def pad_and_forward():
+ padded = F.pad(x_concat, padding, mode='constant', value=0.0)
+ if not padded.is_contiguous():
+ padded = padded.contiguous()
+ with ignore_padding(self):
+ return torch.nn.Conv3d.forward(self, padded)
+
+ return pad_and_forward()
+
+ num_splits = math.ceil(memory_occupy / self.memory_limit)
+ size_per_split = x.size(split_dim) // num_splits
+ split_sizes = [size_per_split] * (num_splits - 1)
+ split_sizes += [x.size(split_dim) - sum(split_sizes)]
+
+ x = list(x.split(split_sizes, dim=split_dim))
+ if prev_cache is not None:
+ prev_cache = list(prev_cache.split(split_sizes, dim=split_dim))
+ cache = None
+ for idx in range(len(x)):
+ if prev_cache is not None:
+ x[idx] = torch.cat([prev_cache[idx], x[idx]], dim=split_dim - 1)
+
+ lpad_dim = (x[idx].ndim - split_dim - 1) * 2
+ rpad_dim = lpad_dim + 1
+ padding = list(padding)
+ padding[lpad_dim] = self.padding[split_dim - 2] if idx == 0 else 0
+ padding[rpad_dim] = self.padding[split_dim - 2] if idx == len(x) - 1 else 0
+ pad_len = padding[lpad_dim] + padding[rpad_dim]
+ padding = tuple(padding)
+
+ next_cache = None
+ cache_len = cache.size(split_dim) if cache is not None else 0
+ next_cache_size = get_cache_size(
+ conv_module=self,
+ input_len=x[idx].size(split_dim) + cache_len,
+ pad_len=pad_len,
+ dim=split_dim - 2,
+ )
+ if next_cache_size != 0:
+ if next_cache_size > x[idx].size(split_dim):
+ raise ValueError(
+ f"SeedVR2 VAE cache size {next_cache_size} exceeds split size {x[idx].size(split_dim)}."
+ )
+ next_cache = (
+ x[idx].transpose(0, split_dim)[-next_cache_size:].transpose(0, split_dim)
+ )
+
+ x[idx] = self.memory_limit_conv(
+ x[idx],
+ split_dim=split_dim + 1,
+ padding=padding,
+ prev_cache=cache
+ )
+
+ cache = next_cache
+
+ output = torch.cat(x, dim=split_dim)
+ return output
+
+ def forward(
+ self,
+ input,
+ memory_state: MemoryState = MemoryState.UNSET,
+ memory_cache = None,
+ ) -> Tensor:
+ if memory_state == MemoryState.UNSET:
+ raise ValueError("SeedVR2 VAE convolution requires an explicit MemoryState.")
+ if memory_cache is None:
+ memory_cache = {}
+ if memory_state != MemoryState.ACTIVE:
+ memory_cache.pop(self, None)
+ if (
+ math.isinf(self.memory_limit)
+ and torch.is_tensor(input)
+ ):
+ return self.basic_forward(input, memory_state, memory_cache)
+ return self.slicing_forward(input, memory_state, memory_cache)
+
+ def basic_forward(self, input: Tensor, memory_state: MemoryState = MemoryState.UNSET, memory_cache = None):
+ mem_size = self.stride[0] - self.kernel_size[0]
+ memory = memory_cache.get(self) if memory_cache is not None else None
+ if (memory is not None) and (memory_state == MemoryState.ACTIVE):
+ input = extend_head(input, memory=memory, times=-1)
+ else:
+ input = extend_head(input, times=self.temporal_padding * 2)
+ next_memory = (
+ input[:, :, mem_size:].detach()
+ if (mem_size != 0 and memory_state != MemoryState.DISABLED)
+ else None
+ )
+ if memory_cache is not None and memory_state != MemoryState.DISABLED:
+ if next_memory is None:
+ memory_cache.pop(self, None)
+ else:
+ memory_cache[self] = next_memory
+ return super().forward(input)
+
+ def slicing_forward(
+ self,
+ input,
+ memory_state: MemoryState = MemoryState.UNSET,
+ memory_cache = None,
+ ) -> Tensor:
+ if memory_cache is None:
+ memory_cache = {}
+ squeeze_out = False
+ if torch.is_tensor(input):
+ input = [input]
+ squeeze_out = True
+
+ cache_size = self.kernel_size[0] - self.stride[0]
+ memory = memory_cache.get(self) if memory_cache is not None else None
+ cache = cache_send_recv(
+ input, cache_size=cache_size, memory=memory, times=self.temporal_padding * 2
+ )
+
+ if (
+ memory_state in [MemoryState.INITIALIZING, MemoryState.ACTIVE]
+ and cache_size != 0
+ ):
+ if cache_size > input[-1].size(2) and cache is not None and len(input) == 1:
+ input[0] = torch.cat([cache, input[0]], dim=2)
+ cache = None
+ if cache_size <= input[-1].size(2):
+ memory_cache[self] = input[-1][:, :, -cache_size:].detach().contiguous()
+
+ padding = tuple(x for x in reversed(self.padding) for _ in range(2))
+ for i in range(len(input)):
+ next_cache = None
+ cache_size = 0
+ if i < len(input) - 1:
+ cache_len = cache.size(2) if cache is not None else 0
+ cache_size = get_cache_size(self, input[i].size(2) + cache_len, pad_len=0)
+ if cache_size != 0:
+ if cache_size > input[i].size(2) and cache is not None:
+ input[i] = torch.cat([cache, input[i]], dim=2)
+ cache = None
+ if cache_size > input[i].size(2):
+ raise ValueError(f"SeedVR2 VAE cache size {cache_size} exceeds input length {input[i].size(2)}.")
+ next_cache = input[i][:, :, -cache_size:]
+
+ input[i] = self.memory_limit_conv(
+ input[i],
+ padding=padding,
+ prev_cache=cache
+ )
+
+ cache = next_cache
+
+ return input[0] if squeeze_out else input
+
+def remove_head(tensor: Tensor, times: int = 1) -> Tensor:
+ if times == 0:
+ return tensor
+ return torch.cat(tensors=(tensor[:, :, :1], tensor[:, :, times + 1 :]), dim=2)
+
+class Upsample3D(nn.Module):
+
+ def __init__(
+ self,
+ channels,
+ out_channels = None,
+ inflation_mode = "tail",
+ temporal_up: bool = False,
+ spatial_up: bool = True,
+ ):
+ super().__init__()
+ self.channels = channels
+ self.out_channels = out_channels or channels
+
+ conv = InflatedCausalConv3d(
+ self.channels,
+ self.out_channels,
+ 3,
+ padding=1,
+ inflation_mode=inflation_mode,
+ )
+
+ self.temporal_up = temporal_up
+ self.spatial_up = spatial_up
+ self.temporal_ratio = 2 if temporal_up else 1
+ self.spatial_ratio = 2 if spatial_up else 1
+
+ upscale_ratio = (self.spatial_ratio**2) * self.temporal_ratio
+ self.upscale_conv = ops.Conv3d(
+ self.channels, self.channels * upscale_ratio, kernel_size=1, padding=0
+ )
+
+ self.conv = conv
+
+ def forward(
+ self,
+ hidden_states: torch.FloatTensor,
+ memory_state=None,
+ memory_cache=None,
+ ) -> torch.FloatTensor:
+ if hidden_states.shape[1] != self.channels:
+ raise ValueError(f"SeedVR2 upsample expected {self.channels} channels, got {hidden_states.shape[1]}.")
+
+ hidden_states = self.upscale_conv(hidden_states)
+ b, channels, f, h, w = hidden_states.shape
+ c = channels // (self.spatial_ratio * self.spatial_ratio * self.temporal_ratio)
+ hidden_states = hidden_states.view(b, self.spatial_ratio, self.spatial_ratio, self.temporal_ratio, c, f, h, w)
+ hidden_states = hidden_states.permute(0, 4, 5, 3, 6, 1, 7, 2).reshape(
+ b,
+ c,
+ f * self.temporal_ratio,
+ h * self.spatial_ratio,
+ w * self.spatial_ratio,
+ )
+
+ if self.temporal_up and memory_state != MemoryState.ACTIVE:
+ hidden_states = remove_head(hidden_states)
+
+ hidden_states = self.conv(hidden_states, memory_state=memory_state, memory_cache=memory_cache)
+
+ return hidden_states
+
+
+class Downsample3D(nn.Module):
+ def __init__(
+ self,
+ channels,
+ out_channels = None,
+ inflation_mode = "tail",
+ spatial_down: bool = False,
+ temporal_down: bool = False,
+ ):
+ super().__init__()
+ self.channels = channels
+ self.out_channels = out_channels or channels
+ self.temporal_down = temporal_down
+ self.spatial_down = spatial_down
+
+ self.temporal_ratio = 2 if temporal_down else 1
+ self.spatial_ratio = 2 if spatial_down else 1
+
+ self.temporal_kernel = 3 if temporal_down else 1
+ self.spatial_kernel = 3 if spatial_down else 1
+
+ self.conv = InflatedCausalConv3d(
+ self.channels,
+ self.out_channels,
+ kernel_size=(self.temporal_kernel, self.spatial_kernel, self.spatial_kernel),
+ stride=(self.temporal_ratio, self.spatial_ratio, self.spatial_ratio),
+ padding=(1 if self.temporal_down else 0, 0, 0),
+ inflation_mode=inflation_mode,
+ )
+
+
+ def forward(
+ self,
+ hidden_states: torch.FloatTensor,
+ memory_state = None,
+ memory_cache = None,
+ ) -> torch.FloatTensor:
+
+ if hidden_states.shape[1] != self.channels:
+ raise ValueError(f"SeedVR2 downsample expected {self.channels} channels, got {hidden_states.shape[1]}.")
+
+ if self.spatial_down:
+ pad = (0, 1, 0, 1)
+ hidden_states = F.pad(hidden_states, pad, mode="constant", value=0)
+
+ if hidden_states.shape[1] != self.channels:
+ raise ValueError(f"SeedVR2 downsample expected {self.channels} channels after padding, got {hidden_states.shape[1]}.")
+
+ hidden_states = self.conv(hidden_states, memory_state=memory_state, memory_cache=memory_cache)
+
+ return hidden_states
+
+
+class ResnetBlock3D(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: Optional[int] = None,
+ temb_channels: int = 512,
+ groups: int = 32,
+ groups_out: Optional[int] = None,
+ eps: float = 1e-6,
+ output_scale_factor: float = 1.0,
+ skip_time_act: bool = False,
+ inflation_mode = "tail",
+ time_receptive_field: _receptive_field_t = "half",
+ ):
+ super().__init__()
+ self.in_channels = in_channels
+ self.out_channels = in_channels if out_channels is None else out_channels
+ self.output_scale_factor = output_scale_factor
+ self.skip_time_act = skip_time_act
+ self.nonlinearity = nn.SiLU()
+ if temb_channels is not None:
+ self.time_emb_proj = ops.Linear(temb_channels, self.out_channels)
+ else:
+ self.time_emb_proj = None
+ self.norm1 = ops.GroupNorm(num_groups=groups, num_channels=in_channels, eps=eps, affine=True)
+ if groups_out is None:
+ groups_out = groups
+ self.norm2 = ops.GroupNorm(num_groups=groups_out, num_channels=self.out_channels, eps=eps, affine=True)
+ self.use_in_shortcut = self.in_channels != self.out_channels
+ self.conv1 = InflatedCausalConv3d(
+ self.in_channels,
+ self.out_channels,
+ kernel_size=(1, 3, 3) if time_receptive_field == "half" else (3, 3, 3),
+ stride=1,
+ padding=(0, 1, 1) if time_receptive_field == "half" else (1, 1, 1),
+ inflation_mode=inflation_mode,
+ )
+
+ self.conv2 = InflatedCausalConv3d(
+ self.out_channels,
+ self.out_channels,
+ kernel_size=3,
+ stride=1,
+ padding=1,
+ inflation_mode=inflation_mode,
+ )
+
+ self.conv_shortcut = None
+ if self.use_in_shortcut:
+ self.conv_shortcut = InflatedCausalConv3d(
+ self.in_channels,
+ self.out_channels,
+ kernel_size=1,
+ stride=1,
+ padding=0,
+ bias=True,
+ inflation_mode=inflation_mode,
+ )
+
+ def forward(self, input_tensor, temb, memory_state = None, memory_cache = None):
+ hidden_states = input_tensor
+
+ hidden_states = causal_norm_wrapper(self.norm1, hidden_states)
+
+ hidden_states = self.nonlinearity(hidden_states)
+
+ hidden_states = self.conv1(hidden_states, memory_state=memory_state, memory_cache=memory_cache)
+
+ if self.time_emb_proj is not None:
+ if not self.skip_time_act:
+ temb = self.nonlinearity(temb)
+ temb = self.time_emb_proj(temb)[:, :, None, None]
+
+ if temb is not None:
+ hidden_states = hidden_states + temb
+
+ hidden_states = causal_norm_wrapper(self.norm2, hidden_states)
+
+ hidden_states = self.nonlinearity(hidden_states)
+
+ hidden_states = self.conv2(hidden_states, memory_state=memory_state, memory_cache=memory_cache)
+
+ if self.conv_shortcut is not None:
+ input_tensor = self.conv_shortcut(input_tensor, memory_state=memory_state, memory_cache=memory_cache)
+
+ output_tensor = (input_tensor + hidden_states) / self.output_scale_factor
+
+ return output_tensor
+
+
+class DownEncoderBlock3D(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ num_layers: int = 1,
+ resnet_eps: float = 1e-6,
+ resnet_groups: int = 32,
+ output_scale_factor: float = 1.0,
+ add_downsample: bool = True,
+ inflation_mode = "tail",
+ time_receptive_field: _receptive_field_t = "half",
+ temporal_down: bool = True,
+ spatial_down: bool = True,
+ ):
+ super().__init__()
+ resnets = []
+
+ for i in range(num_layers):
+ in_channels = in_channels if i == 0 else out_channels
+ resnets.append(
+ ResnetBlock3D(
+ in_channels=in_channels,
+ out_channels=out_channels,
+ temb_channels=None,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ output_scale_factor=output_scale_factor,
+ inflation_mode=inflation_mode,
+ time_receptive_field=time_receptive_field,
+ )
+ )
+
+ self.resnets = nn.ModuleList(resnets)
+
+ if add_downsample:
+ self.downsamplers = nn.ModuleList(
+ [
+ Downsample3D(
+ out_channels,
+ out_channels=out_channels,
+ temporal_down=temporal_down,
+ spatial_down=spatial_down,
+ inflation_mode=inflation_mode,
+ )
+ ]
+ )
+ else:
+ self.downsamplers = None
+
+ def forward(
+ self,
+ hidden_states: torch.FloatTensor,
+ memory_state = None,
+ memory_cache = None,
+ ) -> torch.FloatTensor:
+ for resnet in self.resnets:
+ hidden_states = resnet(hidden_states, temb=None, memory_state=memory_state, memory_cache=memory_cache)
+
+ if self.downsamplers is not None:
+ for downsampler in self.downsamplers:
+ hidden_states = downsampler(hidden_states, memory_state=memory_state, memory_cache=memory_cache)
+
+ return hidden_states
+
+
+class UpDecoderBlock3D(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ out_channels: int,
+ num_layers: int = 1,
+ resnet_eps: float = 1e-6,
+ resnet_groups: int = 32,
+ output_scale_factor: float = 1.0,
+ add_upsample: bool = True,
+ temb_channels: Optional[int] = None,
+ inflation_mode = "tail",
+ time_receptive_field: _receptive_field_t = "half",
+ temporal_up: bool = True,
+ spatial_up: bool = True,
+ ):
+ super().__init__()
+ resnets = []
+
+ for i in range(num_layers):
+ input_channels = in_channels if i == 0 else out_channels
+
+ resnets.append(
+ ResnetBlock3D(
+ in_channels=input_channels,
+ out_channels=out_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ output_scale_factor=output_scale_factor,
+ inflation_mode=inflation_mode,
+ time_receptive_field=time_receptive_field,
+ )
+ )
+
+ self.resnets = nn.ModuleList(resnets)
+
+ if add_upsample:
+ self.upsamplers = nn.ModuleList(
+ [
+ Upsample3D(
+ out_channels,
+ out_channels=out_channels,
+ temporal_up=temporal_up,
+ spatial_up=spatial_up,
+ inflation_mode=inflation_mode,
+ )
+ ]
+ )
+ else:
+ self.upsamplers = None
+
+ def forward(
+ self,
+ hidden_states: torch.FloatTensor,
+ temb: Optional[torch.FloatTensor] = None,
+ memory_state=None,
+ memory_cache=None,
+ ) -> torch.FloatTensor:
+ for resnet in self.resnets:
+ hidden_states = resnet(hidden_states, temb=None, memory_state=memory_state, memory_cache=memory_cache)
+
+ if self.upsamplers is not None:
+ for upsampler in self.upsamplers:
+ hidden_states = upsampler(hidden_states, memory_state=memory_state, memory_cache=memory_cache)
+
+ return hidden_states
+
+
+class UNetMidBlock3D(nn.Module):
+ def __init__(
+ self,
+ in_channels: int,
+ temb_channels: int,
+ num_layers: int = 1,
+ resnet_eps: float = 1e-6,
+ resnet_time_scale_shift: str = "default", # default, spatial
+ resnet_groups: int = 32,
+ add_attention: bool = True,
+ attention_head_dim: int = 1,
+ output_scale_factor: float = 1.0,
+ inflation_mode = "tail",
+ time_receptive_field: _receptive_field_t = "half",
+ ):
+ super().__init__()
+ resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)
+ self.add_attention = add_attention
+
+ resnets = [
+ ResnetBlock3D(
+ in_channels=in_channels,
+ out_channels=in_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ output_scale_factor=output_scale_factor,
+ inflation_mode=inflation_mode,
+ time_receptive_field=time_receptive_field,
+ )
+ ]
+ attentions = []
+
+ if attention_head_dim is None:
+ attention_head_dim = in_channels
+
+ for _ in range(num_layers):
+ if self.add_attention:
+ attentions.append(
+ Attention(
+ in_channels,
+ heads=in_channels // attention_head_dim,
+ dim_head=attention_head_dim,
+ rescale_output_factor=output_scale_factor,
+ eps=resnet_eps,
+ norm_num_groups=(
+ resnet_groups if resnet_time_scale_shift == "default" else None
+ ),
+ spatial_norm_dim=(
+ temb_channels if resnet_time_scale_shift == "spatial" else None
+ ),
+ residual_connection=True,
+ bias=True,
+ )
+ )
+ else:
+ attentions.append(None)
+
+ resnets.append(
+ ResnetBlock3D(
+ in_channels=in_channels,
+ out_channels=in_channels,
+ temb_channels=temb_channels,
+ eps=resnet_eps,
+ groups=resnet_groups,
+ output_scale_factor=output_scale_factor,
+ inflation_mode=inflation_mode,
+ time_receptive_field=time_receptive_field,
+ )
+ )
+
+ self.attentions = nn.ModuleList(attentions)
+ self.resnets = nn.ModuleList(resnets)
+
+ def forward(self, hidden_states, temb=None, memory_state=None, memory_cache=None):
+ video_length = hidden_states.size(2)
+ hidden_states = self.resnets[0](hidden_states, temb, memory_state=memory_state, memory_cache=memory_cache)
+ for attn, resnet in zip(self.attentions, self.resnets[1:]):
+ if attn is not None:
+ b, c, f, h, w = hidden_states.shape
+ hidden_states = hidden_states.transpose(1, 2).reshape(b * f, c, h, w)
+ hidden_states = attn(hidden_states, temb=temb)
+ hidden_states = hidden_states.reshape(b, video_length, c, h, w).transpose(1, 2)
+ hidden_states = resnet(hidden_states, temb, memory_state=memory_state, memory_cache=memory_cache)
+
+ return hidden_states
+
+
+class Encoder3D(nn.Module):
+ def __init__(
+ self,
+ in_channels: int = 3,
+ out_channels: int = 3,
+ down_block_types: Tuple[str, ...] = ("DownEncoderBlock3D",),
+ block_out_channels: Tuple[int, ...] = (64,),
+ layers_per_block: int = 2,
+ norm_num_groups: int = 32,
+ mid_block_add_attention=True,
+ temporal_down_num: int = 2,
+ inflation_mode = "tail",
+ time_receptive_field: _receptive_field_t = "half",
+ ):
+ super().__init__()
+ self.layers_per_block = layers_per_block
+ self.temporal_down_num = temporal_down_num
+
+ self.conv_in = InflatedCausalConv3d(
+ in_channels,
+ block_out_channels[0],
+ kernel_size=3,
+ stride=1,
+ padding=1,
+ inflation_mode=inflation_mode,
+ )
+
+ self.mid_block = None
+ self.down_blocks = nn.ModuleList([])
+
+ output_channel = block_out_channels[0]
+ for i, down_block_type in enumerate(down_block_types):
+ input_channel = output_channel
+ output_channel = block_out_channels[i]
+ is_final_block = i == len(block_out_channels) - 1
+ is_temporal_down_block = i >= len(block_out_channels) - self.temporal_down_num - 1
+
+ if down_block_type != "DownEncoderBlock3D":
+ raise ValueError(f"SeedVR2 encoder only supports DownEncoderBlock3D, got {down_block_type}.")
+
+ down_block = DownEncoderBlock3D(
+ num_layers=self.layers_per_block,
+ in_channels=input_channel,
+ out_channels=output_channel,
+ add_downsample=not is_final_block,
+ resnet_eps=1e-6,
+ resnet_groups=norm_num_groups,
+ temporal_down=is_temporal_down_block,
+ spatial_down=True,
+ inflation_mode=inflation_mode,
+ time_receptive_field=time_receptive_field,
+ )
+ self.down_blocks.append(down_block)
+
+ self.mid_block = UNetMidBlock3D(
+ in_channels=block_out_channels[-1],
+ resnet_eps=1e-6,
+ output_scale_factor=1,
+ resnet_time_scale_shift="default",
+ attention_head_dim=block_out_channels[-1],
+ resnet_groups=norm_num_groups,
+ temb_channels=None,
+ add_attention=mid_block_add_attention,
+ inflation_mode=inflation_mode,
+ time_receptive_field=time_receptive_field,
+ )
+
+ self.conv_norm_out = ops.GroupNorm(
+ num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6
+ )
+ self.conv_act = nn.SiLU()
+
+ conv_out_channels = 2 * out_channels
+ self.conv_out = InflatedCausalConv3d(
+ block_out_channels[-1], conv_out_channels, 3, padding=1, inflation_mode=inflation_mode
+ )
+
+
+ def forward(
+ self,
+ sample: torch.FloatTensor,
+ memory_state = None,
+ memory_cache = None,
+ ) -> torch.FloatTensor:
+ sample = sample.to(next(self.parameters()).device)
+ sample = self.conv_in(sample, memory_state=memory_state, memory_cache=memory_cache)
+ for down_block in self.down_blocks:
+ sample = down_block(sample, memory_state=memory_state, memory_cache=memory_cache)
+
+ sample = self.mid_block(sample, memory_state=memory_state, memory_cache=memory_cache)
+
+ sample = causal_norm_wrapper(self.conv_norm_out, sample)
+ sample = self.conv_act(sample)
+ sample = self.conv_out(sample, memory_state=memory_state, memory_cache=memory_cache)
+
+ return sample
+
+
+class Decoder3D(nn.Module):
+
+ def __init__(
+ self,
+ in_channels: int = 3,
+ out_channels: int = 3,
+ up_block_types: Tuple[str, ...] = ("UpDecoderBlock3D",),
+ block_out_channels: Tuple[int, ...] = (64,),
+ layers_per_block: int = 2,
+ norm_num_groups: int = 32,
+ mid_block_add_attention=True,
+ inflation_mode = "tail",
+ time_receptive_field: _receptive_field_t = "half",
+ temporal_up_num: int = 2,
+ ):
+ super().__init__()
+ self.layers_per_block = layers_per_block
+ self.temporal_up_num = temporal_up_num
+
+ self.conv_in = InflatedCausalConv3d(
+ in_channels,
+ block_out_channels[-1],
+ kernel_size=3,
+ stride=1,
+ padding=1,
+ inflation_mode=inflation_mode,
+ )
+
+ self.mid_block = None
+ self.up_blocks = nn.ModuleList([])
+
+ temb_channels = None
+
+ self.mid_block = UNetMidBlock3D(
+ in_channels=block_out_channels[-1],
+ resnet_eps=1e-6,
+ output_scale_factor=1,
+ resnet_time_scale_shift="default",
+ attention_head_dim=block_out_channels[-1],
+ resnet_groups=norm_num_groups,
+ temb_channels=temb_channels,
+ add_attention=mid_block_add_attention,
+ inflation_mode=inflation_mode,
+ time_receptive_field=time_receptive_field,
+ )
+
+ reversed_block_out_channels = list(reversed(block_out_channels))
+ output_channel = reversed_block_out_channels[0]
+ for i, up_block_type in enumerate(up_block_types):
+ prev_output_channel = output_channel
+ output_channel = reversed_block_out_channels[i]
+
+ is_final_block = i == len(block_out_channels) - 1
+ is_temporal_up_block = i < self.temporal_up_num
+ if up_block_type != "UpDecoderBlock3D":
+ raise ValueError(f"SeedVR2 decoder only supports UpDecoderBlock3D, got {up_block_type}.")
+ up_block = UpDecoderBlock3D(
+ num_layers=self.layers_per_block + 1,
+ in_channels=prev_output_channel,
+ out_channels=output_channel,
+ add_upsample=not is_final_block,
+ resnet_eps=1e-6,
+ resnet_groups=norm_num_groups,
+ temb_channels=temb_channels,
+ temporal_up=is_temporal_up_block,
+ inflation_mode=inflation_mode,
+ time_receptive_field=time_receptive_field,
+ )
+ self.up_blocks.append(up_block)
+ prev_output_channel = output_channel
+
+ self.conv_norm_out = ops.GroupNorm(
+ num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6
+ )
+ self.conv_act = nn.SiLU()
+ self.conv_out = InflatedCausalConv3d(
+ block_out_channels[0], out_channels, 3, padding=1, inflation_mode=inflation_mode
+ )
+
+
+ def forward(
+ self,
+ sample: torch.FloatTensor,
+ latent_embeds: Optional[torch.FloatTensor] = None,
+ memory_state = None,
+ memory_cache = None,
+ ) -> torch.FloatTensor:
+
+ sample = sample.to(next(self.parameters()).device)
+ sample = self.conv_in(sample, memory_state=memory_state, memory_cache=memory_cache)
+
+ upscale_dtype = next(iter(self.up_blocks.parameters())).dtype
+ sample = self.mid_block(sample, latent_embeds, memory_state=memory_state, memory_cache=memory_cache)
+ sample = sample.to(upscale_dtype)
+
+ for up_block in self.up_blocks:
+ sample = up_block(sample, latent_embeds, memory_state=memory_state, memory_cache=memory_cache)
+
+ sample = causal_norm_wrapper(self.conv_norm_out, sample)
+ sample = self.conv_act(sample)
+ sample = self.conv_out(sample, memory_state=memory_state, memory_cache=memory_cache)
+
+ return sample
+
+class VideoAutoencoderKL(nn.Module):
+ def __init__(
+ self,
+ in_channels: int = 3,
+ out_channels: int = 3,
+ layers_per_block: int = 2,
+ latent_channels: int = SEEDVR2_LATENT_CHANNELS,
+ norm_num_groups: int = 32,
+ temporal_scale_num: int = 2,
+ inflation_mode = "pad",
+ time_receptive_field: _receptive_field_t = "full",
+ slicing_sample_min_size = BYTEDANCE_SLICING_SAMPLE_MIN,
+ ):
+ self.slicing_sample_min_size = slicing_sample_min_size
+ self.slicing_latent_min_size = slicing_sample_min_size // (2**temporal_scale_num)
+ block_out_channels = BYTEDANCE_BLOCK_OUT_CHANNELS
+ down_block_types = ("DownEncoderBlock3D",) * 4
+ up_block_types = ("UpDecoderBlock3D",) * 4
+ super().__init__()
+
+ self.encoder = Encoder3D(
+ in_channels=in_channels,
+ out_channels=latent_channels,
+ down_block_types=down_block_types,
+ block_out_channels=block_out_channels,
+ layers_per_block=layers_per_block,
+ norm_num_groups=norm_num_groups,
+ temporal_down_num=temporal_scale_num,
+ inflation_mode=inflation_mode,
+ time_receptive_field=time_receptive_field,
+ )
+
+ self.decoder = Decoder3D(
+ in_channels=latent_channels,
+ out_channels=out_channels,
+ up_block_types=up_block_types,
+ block_out_channels=block_out_channels,
+ layers_per_block=layers_per_block,
+ norm_num_groups=norm_num_groups,
+ temporal_up_num=temporal_scale_num,
+ inflation_mode=inflation_mode,
+ time_receptive_field=time_receptive_field,
+ )
+
+ self.use_slicing = True
+
+ def encode(self, x: torch.FloatTensor, return_dict: bool = True):
+ h = self.slicing_encode(x)
+ posterior = DiagonalGaussianDistribution(h).mode()
+
+ if not return_dict:
+ return (posterior,)
+
+ return posterior
+
+ def decode_(
+ self, z: torch.Tensor, return_dict: bool = True
+ ):
+ decoded = self.slicing_decode(z)
+
+ if not return_dict:
+ return (decoded,)
+
+ return decoded
+
+ def _encode(
+ self, x, memory_state = MemoryState.DISABLED, memory_cache = None
+ ) -> torch.Tensor:
+ _x = x.to(self.device)
+ h = self.encoder(_x, memory_state=memory_state, memory_cache=memory_cache)
+ return h.to(x.device)
+
+ def _decode(
+ self, z, memory_state = MemoryState.DISABLED, memory_cache = None
+ ) -> torch.Tensor:
+ _z = z.to(self.device)
+ output = self.decoder(_z, memory_state=memory_state, memory_cache=memory_cache)
+ return output.to(z.device)
+
+ def slicing_encode(self, x: torch.Tensor) -> torch.Tensor:
+ if self.use_slicing and (x.shape[2] - 1) > self.slicing_sample_min_size:
+ memory_cache = {}
+ split_size = max(
+ self.slicing_sample_min_size,
+ getattr(self, "temporal_downsample_factor", 1),
+ )
+ x_slices = list(x[:, :, 1:].split(split_size=split_size, dim=2))
+ min_active_len = getattr(self, "temporal_downsample_factor", 1)
+ if len(x_slices) > 1 and x_slices[-1].shape[2] < min_active_len:
+ x_slices[-2] = torch.cat((x_slices[-2], x_slices[-1]), dim=2)
+ x_slices.pop()
+ encoded_slices = [
+ self._encode(
+ torch.cat((x[:, :, :1], x_slices[0]), dim=2),
+ memory_state=MemoryState.INITIALIZING,
+ memory_cache=memory_cache,
+ )
+ ]
+ for x_idx in range(1, len(x_slices)):
+ encoded_slices.append(
+ self._encode(x_slices[x_idx], memory_state=MemoryState.ACTIVE, memory_cache=memory_cache)
+ )
+ out = torch.cat(encoded_slices, dim=2)
+ return out
+ else:
+ return self._encode(x)
+
+ def slicing_decode(self, z: torch.Tensor) -> torch.Tensor:
+ if self.use_slicing and (z.shape[2] - 1) > self.slicing_latent_min_size:
+ memory_cache = {}
+ z_slices = z[:, :, 1:].split(split_size=self.slicing_latent_min_size, dim=2)
+ decoded_slices = [
+ self._decode(
+ torch.cat((z[:, :, :1], z_slices[0]), dim=2),
+ memory_state=MemoryState.INITIALIZING,
+ memory_cache=memory_cache,
+ )
+ ]
+ for z_idx in range(1, len(z_slices)):
+ decoded_slices.append(
+ self._decode(z_slices[z_idx], memory_state=MemoryState.ACTIVE, memory_cache=memory_cache)
+ )
+ out = torch.cat(decoded_slices, dim=2)
+ return out
+ else:
+ return self._decode(z)
+
+ def forward(self, x: torch.FloatTensor, mode: Literal["encode", "decode", "all"] = "all"):
+ def _unwrap(value):
+ return value[0] if isinstance(value, tuple) else value
+
+ if mode == "encode":
+ return _unwrap(self.encode(x))
+ if mode == "decode":
+ return _unwrap(self.decode_(x))
+ if mode == "all":
+ latent = _unwrap(self.encode(x))
+ return _unwrap(self.decode_(latent))
+ raise ValueError(f"Unknown SeedVR2 VAE forward mode: {mode}")
+
+class VideoAutoencoderKLWrapper(VideoAutoencoderKL):
+ def __init__(
+ self,
+ spatial_downsample_factor = 8,
+ temporal_downsample_factor = 4,
+ ):
+ self.spatial_downsample_factor = spatial_downsample_factor
+ self.temporal_downsample_factor = temporal_downsample_factor
+ super().__init__()
+ self.set_memory_limit(BYTEDANCE_VAE_CONV_MEM_GIB, BYTEDANCE_VAE_NORM_MEM_GIB)
+
+ def forward(self, x: torch.FloatTensor):
+ z, p = self._encode_with_raw_latent(x)
+ x = self.decode(z)
+ return x, z, p
+
+ def _encode_with_raw_latent(self, x):
+ if x.ndim == 4:
+ x = x.unsqueeze(2)
+ x = x.to(dtype=next(self.parameters()).dtype)
+ self.device = x.device
+ p = super().encode(x)
+ z = p.squeeze(2)
+ return z, p
+
+ def encode(self, x):
+ z, _ = self._encode_with_raw_latent(x)
+ return z
+
+ def decode(self, z, seedvr2_tiling=None):
+ seedvr2_tiling = {} if seedvr2_tiling is None else seedvr2_tiling
+ if not isinstance(seedvr2_tiling, dict):
+ raise RuntimeError(
+ "SeedVR2 VideoAutoencoderKLWrapper.decode: `seedvr2_tiling` must be a dict; "
+ f"got {type(seedvr2_tiling).__name__} with value {seedvr2_tiling!r}."
+ )
+
+ if z.ndim == 5:
+ _, c, _, _, _ = z.shape
+ if c != SEEDVR2_LATENT_CHANNELS:
+ raise RuntimeError(
+ "SeedVR2 VideoAutoencoderKLWrapper.decode: 5-D latent input must "
+ f"have {SEEDVR2_LATENT_CHANNELS} channels; got shape {tuple(z.shape)}."
+ )
+ latent = z
+ elif z.ndim == 4:
+ b, tc, h, w = z.shape
+ if tc % SEEDVR2_LATENT_CHANNELS != 0:
+ raise RuntimeError(
+ "SeedVR2 VideoAutoencoderKLWrapper.decode: 4-D latent input must "
+ f"use collapsed channel layout (B, {SEEDVR2_LATENT_CHANNELS}*T, H, W); "
+ f"got shape {tuple(z.shape)}."
+ )
+ latent = z.reshape(b, SEEDVR2_LATENT_CHANNELS, -1, h, w)
+ else:
+ raise RuntimeError(
+ "SeedVR2 VideoAutoencoderKLWrapper.decode: latent input must be "
+ f"4-D collapsed (B, {SEEDVR2_LATENT_CHANNELS}*T, H, W) or "
+ f"5-D (B, {SEEDVR2_LATENT_CHANNELS}, T, H, W); "
+ f"got shape {tuple(z.shape)}."
+ )
+ scale = BYTEDANCE_VAE_SCALING_FACTOR
+ shift = BYTEDANCE_VAE_SHIFTING_FACTOR
+ latent = latent / scale + shift
+
+ self.device = latent.device
+ enable_tiling = seedvr2_tiling.get("enable_tiling", False)
+
+ if enable_tiling:
+ decode_seedvr2_args = dict(seedvr2_tiling)
+ decode_seedvr2_args.pop("enable_tiling", None)
+ tile_h, tile_w = decode_seedvr2_args.get("tile_size", (512, 512))
+ ov_h, ov_w = decode_seedvr2_args.get("tile_overlap", (64, 64))
+ decode_seedvr2_args["tile_overlap"] = (
+ min(ov_h, max(0, tile_h - 8)),
+ min(ov_w, max(0, tile_w - 8)),
+ )
+ x = tiled_vae(latent, self, **decode_seedvr2_args, encode=False)
+ if x.ndim == 4:
+ # tiled_vae squeezes the temporal axis when
+ # temporal_downsample_factor == 1 AND latent T == 1
+ # (see tiled_vae line 179-180); re-add it so the post-decode
+ # pipeline can keep batch and time distinct on the tiled path.
+ x = x.unsqueeze(2)
+ else:
+ x = super().decode_(latent)
+
+ h, w = x.shape[-2:]
+ w2 = w - (w % 2)
+ h2 = h - (h % 2)
+ x = x[..., :h2, :w2]
+
+ return x
+
+ def decode_tiled(self, z, tile_x=32, tile_y=32, overlap=8, tile_t=None, overlap_t=None):
+ # SeedVR2's causal VAE owns temporal via the MemoryState cache; external
+ # temporal tiling breaks that continuity, so only spatial tiling is applied.
+ sf = self.spatial_downsample_factor
+ seedvr2_tiling = {
+ "enable_tiling": True,
+ "tile_size": (tile_y * sf, tile_x * sf),
+ "tile_overlap": (overlap * sf, overlap * sf),
+ "temporal_size": None,
+ "temporal_overlap": None,
+ }
+ return self.decode(z, seedvr2_tiling=seedvr2_tiling)
+
+ def encode_tiled(self, x, tile_x=None, tile_y=None, overlap=None, tile_t=None, overlap_t=None):
+ # External temporal tiling knobs are discarded; the causal VAE keeps its
+ # own internal MemoryState slicing.
+ if tile_y is None:
+ tile_y = 512
+ if tile_x is None:
+ tile_x = 512
+ if overlap is None:
+ overlap_y = 64
+ overlap_x = 64
+ else:
+ overlap_y = overlap
+ overlap_x = overlap
+ overlap_y = min(overlap_y, max(0, tile_y - 8))
+ overlap_x = min(overlap_x, max(0, tile_x - 8))
+ self.device = x.device
+ return tiled_vae(
+ x,
+ self,
+ tile_size=(tile_y, tile_x),
+ tile_overlap=(overlap_y, overlap_x),
+ temporal_size=None,
+ temporal_overlap=None,
+ encode=True,
+ )
+
+ def comfy_format_encoded(self, samples):
+ if samples.ndim == 4:
+ samples = samples.unsqueeze(2)
+ samples = samples.contiguous()
+ samples = samples * BYTEDANCE_VAE_SCALING_FACTOR
+ return samples
+
+ def comfy_memory_used_decode(self, shape):
+ bytes_per_output_pixel = 160
+
+ def output_pixels(latent_t, latent_h, latent_w):
+ output_t = max(1, (latent_t - 1) * 4 + 1)
+ return output_t * latent_h * 8 * latent_w * 8
+
+ # SeedVR2 decode performs full-frame LAB histogram matching: fp32 channels
+ # plus int64 sort indices dominate peak memory, not the VAE weight dtype.
+ if len(shape) == 5:
+ candidates = []
+ if shape[1] == SEEDVR2_LATENT_CHANNELS:
+ candidates.append((shape[2], shape[3], shape[4]))
+ if shape[-1] == SEEDVR2_LATENT_CHANNELS:
+ candidates.append((shape[1], shape[2], shape[3]))
+ if len(candidates) == 0:
+ candidates.append((shape[2], shape[3], shape[4]))
+ pixels = max(output_pixels(*candidate) for candidate in candidates)
+ elif len(shape) == 4:
+ latent_t = max(1, (shape[1] + SEEDVR2_LATENT_CHANNELS - 1) // SEEDVR2_LATENT_CHANNELS)
+ pixels = output_pixels(latent_t, shape[2], shape[3])
+ else:
+ pixels = output_pixels(1, shape[-2], shape[-1])
+ return pixels * bytes_per_output_pixel
+
+ def set_memory_limit(self, conv_max_mem: Optional[float], norm_max_mem: Optional[float]):
+ set_norm_limit(norm_max_mem)
+ for m in self.modules():
+ if isinstance(m, InflatedCausalConv3d):
+ m.set_memory_limit(conv_max_mem if conv_max_mem is not None else float("inf"))
diff --git a/comfy/model_base.py b/comfy/model_base.py
index dcfa555dc..786a7c127 100644
--- a/comfy/model_base.py
+++ b/comfy/model_base.py
@@ -55,6 +55,7 @@ import comfy.ldm.pixeldit.model
import comfy.ldm.pixeldit.pid
import comfy.ldm.ace.model
import comfy.ldm.omnigen.omnigen2
+import comfy.ldm.seedvr.model
import comfy.ldm.boogu.model
import comfy.ldm.qwen_image.model
import comfy.ldm.ideogram4.model
@@ -932,6 +933,17 @@ class HunyuanDiT(BaseModel):
out['image_meta_size'] = comfy.conds.CONDRegular(torch.FloatTensor([[height, width, target_height, target_width, 0, 0]]))
return out
+class SeedVR2(BaseModel):
+ def __init__(self, model_config, model_type=ModelType.FLOW, device=None):
+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.seedvr.model.NaDiT)
+
+ def extra_conds(self, **kwargs):
+ out = super().extra_conds(**kwargs)
+ condition = kwargs.get("condition", None)
+ if condition is not None:
+ out["condition"] = comfy.conds.CONDRegular(condition)
+ return out
+
class PixArt(BaseModel):
def __init__(self, model_config, model_type=ModelType.EPS, device=None):
super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.pixart.pixartms.PixArtMS)
diff --git a/comfy/model_detection.py b/comfy/model_detection.py
index e53d848c9..174bc77cc 100644
--- a/comfy/model_detection.py
+++ b/comfy/model_detection.py
@@ -598,6 +598,44 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
return dit_config
+ seedvr2_7b_separate_key = "{}blocks.35.mlp.vid.proj_out.weight".format(key_prefix)
+ if seedvr2_7b_separate_key in state_dict_keys and state_dict[seedvr2_7b_separate_key].shape[0] == 3072: # seedvr2 7b
+ dit_config = {}
+ dit_config["image_model"] = "seedvr2"
+ dit_config["vid_dim"] = 3072
+ dit_config["heads"] = 24
+ dit_config["num_layers"] = 36
+ # This checkpoint uses separate vid/txt MMModule keys in every block.
+ dit_config["mm_layers"] = 36
+ dit_config["norm_eps"] = 1e-5
+ dit_config["rope_type"] = "rope3d"
+ dit_config["rope_dim"] = 64
+ dit_config["mlp_type"] = "normal"
+ return dit_config
+ if "{}blocks.35.mlp.all.proj_in_gate.weight".format(key_prefix) in state_dict_keys: # seedvr2 7b
+ dit_config = {}
+ dit_config["image_model"] = "seedvr2"
+ dit_config["vid_dim"] = 3072
+ dit_config["heads"] = 24
+ dit_config["num_layers"] = 36
+ # This checkpoint uses shared all.* MMModule keys after the initial blocks.
+ dit_config["mm_layers"] = 10
+ dit_config["norm_eps"] = 1e-5
+ dit_config["rope_type"] = "rope3d"
+ dit_config["rope_dim"] = 64
+ dit_config["mlp_type"] = "swiglu"
+ return dit_config
+ if "{}blocks.31.mlp.all.proj_in_gate.weight".format(key_prefix) in state_dict_keys: # seedvr2 3b
+ dit_config = {}
+ dit_config["image_model"] = "seedvr2"
+ dit_config["vid_dim"] = 2560
+ dit_config["heads"] = 20
+ dit_config["num_layers"] = 32
+ dit_config["norm_eps"] = 1.0e-05
+ dit_config["mlp_type"] = "swiglu"
+ dit_config["vid_out_norm"] = True
+ return dit_config
+
if '{}head.modulation'.format(key_prefix) in state_dict_keys: # Wan 2.1
dit_config = {}
dit_config["image_model"] = "wan2.1"
@@ -1119,9 +1157,10 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
return unet_config
-def model_config_from_unet_config(unet_config, state_dict=None):
+
+def model_config_from_unet_config(unet_config, state_dict=None, unet_key_prefix=""):
for model_config in comfy.supported_models.models:
- if model_config.matches(unet_config, state_dict):
+ if model_config.matches(unet_config, state_dict, unet_key_prefix=unet_key_prefix):
return model_config(unet_config)
logging.error("no match {}".format(unet_config))
@@ -1131,7 +1170,7 @@ def model_config_from_unet(state_dict, unet_key_prefix, use_base_if_no_match=Fal
unet_config = detect_unet_config(state_dict, unet_key_prefix, metadata=metadata)
if unet_config is None:
return None
- model_config = model_config_from_unet_config(unet_config, state_dict)
+ model_config = model_config_from_unet_config(unet_config, state_dict, unet_key_prefix)
if model_config is None and use_base_if_no_match:
model_config = comfy.supported_models_base.BASE(unet_config)
diff --git a/comfy/sd.py b/comfy/sd.py
index 071a3102a..4a0742e7a 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -16,6 +16,7 @@ import comfy.ldm.cosmos.vae
import comfy.ldm.wan.vae
import comfy.ldm.wan.vae2_2
import comfy.ldm.hunyuan3d.vae
+import comfy.ldm.seedvr.vae
import comfy.ldm.triposplat.vae
import comfy.ldm.ace.vae.music_dcae_pipeline
import comfy.ldm.cogvideo.vae
@@ -473,7 +474,8 @@ class CLIP:
class VAE:
def __init__(self, sd=None, device=None, config=None, dtype=None, metadata=None):
- if 'decoder.up_blocks.0.resnets.0.norm1.weight' in sd.keys(): #diffusers format
+ is_seedvr2_vae = "decoder.up_blocks.2.upsamplers.0.upscale_conv.weight" in sd
+ if not is_seedvr2_vae and 'decoder.up_blocks.0.resnets.0.norm1.weight' in sd.keys(): #diffusers format
sd = diffusers_convert.convert_vae_state_dict(sd)
if model_management.is_amd():
@@ -500,6 +502,8 @@ class VAE:
self.upscale_index_formula = None
self.extra_1d_channel = None
self.crop_input = True
+ self.handles_tiling = False
+ self.format_encoded = None
self.audio_sample_rate = 44100
@@ -546,6 +550,22 @@ class VAE:
self.first_stage_model = StageC_coder()
self.downscale_ratio = 32
self.latent_channels = 16
+ elif "decoder.up_blocks.2.upsamplers.0.upscale_conv.weight" in sd: # seedvr2
+ self.first_stage_model = comfy.ldm.seedvr.vae.VideoAutoencoderKLWrapper()
+ self.latent_channels = comfy.ldm.seedvr.vae.SEEDVR2_LATENT_CHANNELS
+ self.latent_dim = 3
+ self.disable_offload = True
+ self.memory_used_decode = lambda shape, dtype: self.first_stage_model.comfy_memory_used_decode(shape)
+ self.memory_used_encode = lambda shape, dtype: (max(shape[2], 5) * shape[3] * shape[4] * 64) * model_management.dtype_size(dtype)
+ self.working_dtypes = [torch.float16, torch.bfloat16, torch.float32]
+ self.handles_tiling = True
+ self.format_encoded = self.first_stage_model.comfy_format_encoded
+ self.downscale_ratio = (lambda a: max(0, math.floor((a + 3) / 4)), 8, 8)
+ self.downscale_index_formula = (4, 8, 8)
+ self.upscale_ratio = (lambda a: max(0, a * 4 - 3), 8, 8)
+ self.upscale_index_formula = (4, 8, 8)
+ self.process_input = lambda image: image * 2.0 - 1.0
+ self.crop_input = False
elif "decoder.conv_in.weight" in sd:
if sd['decoder.conv_in.weight'].shape[1] == 64:
ddconfig = {"block_out_channels": [128, 256, 512, 512, 1024, 1024], "in_channels": 3, "out_channels": 3, "num_res_blocks": 2, "ffactor_spatial": 32, "downsample_match_channel": True, "upsample_match_channel": True}
@@ -1012,6 +1032,10 @@ class VAE:
decode_fn = lambda a: self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)).to(dtype=self.vae_output_dtype())
return self.process_output(comfy.utils.tiled_scale_multidim(samples, decode_fn, tile=(tile_t, tile_x, tile_y), overlap=overlap, upscale_amount=self.upscale_ratio, out_channels=self.output_channels, index_formulas=self.upscale_index_formula, output_device=self.output_device))
+ def _decode_tiled_owned(self, samples, **kwargs):
+ out = self.first_stage_model.decode_tiled(samples.to(self.vae_dtype).to(self.device), **kwargs)
+ return self.process_output(out.to(device=self.output_device, dtype=self.vae_output_dtype(), copy=True))
+
def encode_tiled_(self, pixel_samples, tile_x=512, tile_y=512, overlap = 64):
steps = pixel_samples.shape[0] * comfy.utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x, tile_y, overlap)
steps += pixel_samples.shape[0] * comfy.utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x // 2, tile_y * 2, overlap)
@@ -1048,6 +1072,25 @@ class VAE:
encode_fn = lambda a: self.first_stage_model.encode((self.process_input(a)).to(self.vae_dtype).to(self.device)).to(dtype=self.vae_output_dtype())
return comfy.utils.tiled_scale_multidim(samples, encode_fn, tile=(tile_t, tile_x, tile_y), overlap=overlap, upscale_amount=self.downscale_ratio, out_channels=self.latent_channels, downscale=True, index_formulas=self.downscale_index_formula, output_device=self.output_device)
+ def _encode_tiled_owned(self, pixel_samples, **kwargs):
+ x = self.process_input(pixel_samples).to(self.vae_dtype).to(self.device)
+ out = self.first_stage_model.encode_tiled(x, **kwargs)
+ return out.to(device=self.output_device, dtype=self.vae_output_dtype())
+
+ def _owned_tiled_args(self, tile_x=None, tile_y=None, overlap=None, tile_t=None, overlap_t=None):
+ args = {}
+ if tile_x is not None:
+ args["tile_x"] = tile_x
+ if tile_y is not None:
+ args["tile_y"] = tile_y
+ if overlap is not None:
+ args["overlap"] = overlap
+ if tile_t is not None:
+ args["tile_t"] = tile_t
+ if overlap_t is not None:
+ args["overlap_t"] = overlap_t
+ return args
+
def decode(self, samples_in, vae_options={}):
self.throw_exception_if_invalid()
pixel_samples = None
@@ -1095,11 +1138,19 @@ class VAE:
if dims == 1 or self.extra_1d_channel is not None:
pixel_samples = self.decode_tiled_1d(samples_in)
elif dims == 2:
- pixel_samples = self.decode_tiled_(samples_in)
+ if self.handles_tiling:
+ tile = 256 // self.spacial_compression_decode()
+ overlap = tile // 4
+ pixel_samples = self._decode_tiled_owned(samples_in, tile_x=tile, tile_y=tile, overlap=overlap)
+ else:
+ pixel_samples = self.decode_tiled_(samples_in)
elif dims == 3:
tile = 256 // self.spacial_compression_decode()
overlap = tile // 4
- pixel_samples = self.decode_tiled_3d(samples_in, tile_x=tile, tile_y=tile, overlap=(1, overlap, overlap))
+ if self.handles_tiling:
+ pixel_samples = self._decode_tiled_owned(samples_in, tile_x=tile, tile_y=tile, overlap=overlap)
+ else:
+ pixel_samples = self.decode_tiled_3d(samples_in, tile_x=tile, tile_y=tile, overlap=(1, overlap, overlap))
pixel_samples = pixel_samples.to(self.output_device).movedim(1,-1)
return pixel_samples
@@ -1118,7 +1169,9 @@ class VAE:
args["overlap"] = overlap
with model_management.cuda_device_context(self.device):
- if dims == 1 or self.extra_1d_channel is not None:
+ if self.handles_tiling and dims in (2, 3):
+ output = self._decode_tiled_owned(samples, **self._owned_tiled_args(tile_x, tile_y, overlap, tile_t, overlap_t))
+ elif dims == 1 or self.extra_1d_channel is not None:
args.pop("tile_y")
output = self.decode_tiled_1d(samples, **args)
elif dims == 2:
@@ -1179,12 +1232,17 @@ class VAE:
if self.latent_dim == 3:
tile = 256
overlap = tile // 4
- samples = self.encode_tiled_3d(pixel_samples, tile_x=tile, tile_y=tile, overlap=(1, overlap, overlap))
+ if self.handles_tiling:
+ samples = self._encode_tiled_owned(pixel_samples, tile_x=tile, tile_y=tile, overlap=overlap)
+ else:
+ samples = self.encode_tiled_3d(pixel_samples, tile_x=tile, tile_y=tile, overlap=(1, overlap, overlap))
elif self.latent_dim == 1 or self.extra_1d_channel is not None:
samples = self.encode_tiled_1d(pixel_samples)
else:
samples = self.encode_tiled_(pixel_samples)
+ if self.format_encoded is not None:
+ samples = self.format_encoded(samples)
return samples
def encode_tiled(self, pixel_samples, tile_x=None, tile_y=None, overlap=None, tile_t=None, overlap_t=None):
@@ -1192,7 +1250,7 @@ class VAE:
pixel_samples = self.vae_encode_crop_pixels(pixel_samples)
dims = self.latent_dim
pixel_samples = pixel_samples.movedim(-1, 1)
- if dims == 3:
+ if dims == 3 and pixel_samples.ndim < 5:
if not self.not_video:
pixel_samples = pixel_samples.movedim(1, 0).unsqueeze(0)
else:
@@ -1216,21 +1274,27 @@ class VAE:
elif dims == 2:
samples = self.encode_tiled_(pixel_samples, **args)
elif dims == 3:
- if tile_t is not None:
- tile_t_latent = max(2, self.downscale_ratio[0](tile_t))
+ if self.handles_tiling:
+ samples = self._encode_tiled_owned(pixel_samples, **self._owned_tiled_args(tile_x, tile_y, overlap, tile_t, overlap_t))
else:
- tile_t_latent = 9999
- args["tile_t"] = self.upscale_ratio[0](tile_t_latent)
+ if tile_t is not None:
+ tile_t_latent = max(2, self.downscale_ratio[0](tile_t))
+ else:
+ tile_t_latent = 9999
+ args["tile_t"] = self.upscale_ratio[0](tile_t_latent)
- if overlap_t is None:
- args["overlap"] = (1, overlap, overlap)
- else:
- args["overlap"] = (self.upscale_ratio[0](max(1, min(tile_t_latent // 2, self.downscale_ratio[0](overlap_t)))), overlap, overlap)
- maximum = pixel_samples.shape[2]
- maximum = self.upscale_ratio[0](self.downscale_ratio[0](maximum))
+ spatial_overlap = overlap if overlap is not None else 64
+ if overlap_t is None:
+ args["overlap"] = (1, spatial_overlap, spatial_overlap)
+ else:
+ args["overlap"] = (self.upscale_ratio[0](max(1, min(tile_t_latent // 2, self.downscale_ratio[0](overlap_t)))), spatial_overlap, spatial_overlap)
+ maximum = pixel_samples.shape[2]
+ maximum = self.upscale_ratio[0](self.downscale_ratio[0](maximum))
- samples = self.encode_tiled_3d(pixel_samples[:,:,:maximum], **args)
+ samples = self.encode_tiled_3d(pixel_samples[:,:,:maximum], **args)
+ if self.format_encoded is not None:
+ samples = self.format_encoded(samples)
return samples
def get_sd(self):
@@ -1898,7 +1962,7 @@ def load_state_dict_guess_config(sd, output_vae=True, output_clip=True, output_c
manual_cast_dtype = model_management.unet_manual_cast(None, load_device, model_config.supported_inference_dtypes)
else:
manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device, model_config.supported_inference_dtypes)
- model_config.set_inference_dtype(unet_dtype, manual_cast_dtype)
+ model_config.set_inference_dtype(unet_dtype, manual_cast_dtype, device=load_device)
if model_config.clip_vision_prefix is not None:
if output_clipvision:
@@ -2039,7 +2103,7 @@ def load_diffusion_model_state_dict(sd, model_options={}, metadata=None, disable
manual_cast_dtype = model_management.unet_manual_cast(None, load_device, model_config.supported_inference_dtypes)
else:
manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device, model_config.supported_inference_dtypes)
- model_config.set_inference_dtype(unet_dtype, manual_cast_dtype)
+ model_config.set_inference_dtype(unet_dtype, manual_cast_dtype, device=load_device)
if custom_operations is not None:
model_config.custom_operations = custom_operations
diff --git a/comfy/supported_models.py b/comfy/supported_models.py
index afb66e6f3..b82e4178f 100644
--- a/comfy/supported_models.py
+++ b/comfy/supported_models.py
@@ -1685,6 +1685,40 @@ class Chroma(supported_models_base.BASE):
t5_detect = comfy.text_encoders.sd3_clip.t5_xxl_detect(state_dict, "{}t5xxl.transformer.".format(pref))
return supported_models_base.ClipTarget(comfy.text_encoders.pixart_t5.PixArtTokenizer, comfy.text_encoders.pixart_t5.pixart_te(**t5_detect))
+class SeedVR2(supported_models_base.BASE):
+ unet_config = {
+ "image_model": "seedvr2"
+ }
+ unet_extra_config = {}
+ required_keys = {
+ "{}positive_conditioning",
+ "{}negative_conditioning",
+ }
+ latent_format = comfy.latent_formats.SeedVR2
+
+ vae_key_prefix = ["vae."]
+ text_encoder_key_prefix = ["text_encoders."]
+ supported_inference_dtypes = [torch.bfloat16, torch.float16, torch.float32]
+ sampling_settings = {
+ "shift": 1.0,
+ }
+
+ def set_inference_dtype(self, dtype, manual_cast_dtype, device=None):
+ if (
+ dtype == torch.float16
+ and manual_cast_dtype is None
+ and comfy.model_management.should_use_bf16(device)
+ ):
+ manual_cast_dtype = torch.bfloat16
+ super().set_inference_dtype(dtype, manual_cast_dtype, device=device)
+
+ def get_model(self, state_dict, prefix="", device=None):
+ out = model_base.SeedVR2(self, device=device)
+ return out
+
+ def clip_target(self, state_dict={}):
+ return None
+
class ChromaRadiance(Chroma):
unet_config = {
"image_model": "chroma_radiance",
@@ -2348,6 +2382,7 @@ models = [
HiDream,
HiDreamO1,
Chroma,
+ SeedVR2,
ChromaRadiance,
ACEStep,
ACEStep15,
diff --git a/comfy/supported_models_base.py b/comfy/supported_models_base.py
index 0e7a829ba..e3a8e131f 100644
--- a/comfy/supported_models_base.py
+++ b/comfy/supported_models_base.py
@@ -54,13 +54,13 @@ class BASE:
optimizations = {"fp8": False}
@classmethod
- def matches(s, unet_config, state_dict=None):
+ def matches(s, unet_config, state_dict=None, unet_key_prefix=""):
for k in s.unet_config:
if k not in unet_config or s.unet_config[k] != unet_config[k]:
return False
if state_dict is not None:
for k in s.required_keys:
- if k not in state_dict:
+ if k.format(unet_key_prefix) not in state_dict:
return False
return True
@@ -115,7 +115,7 @@ class BASE:
replace_prefix = {"": self.vae_key_prefix[0]}
return utils.state_dict_prefix_replace(state_dict, replace_prefix)
- def set_inference_dtype(self, dtype, manual_cast_dtype):
+ def set_inference_dtype(self, dtype, manual_cast_dtype, device=None):
self.unet_config['dtype'] = dtype
self.manual_cast_dtype = manual_cast_dtype
diff --git a/comfy/text_encoders/gemma4.py b/comfy/text_encoders/gemma4.py
index f050061ed..0bba8341b 100644
--- a/comfy/text_encoders/gemma4.py
+++ b/comfy/text_encoders/gemma4.py
@@ -1088,7 +1088,7 @@ class Gemma4_Tokenizer():
h, w = samples.shape[2], samples.shape[3]
patch_size = 16
pooling_k = 3
- max_soft_tokens = 70 if is_video else 280 # video uses smaller token budget per frame
+ max_soft_tokens = kwargs.get("max_soft_tokens", 70 if is_video else 280)
max_patches = max_soft_tokens * pooling_k * pooling_k
target_px = max_patches * patch_size * patch_size
factor = (target_px / (h * w)) ** 0.5
diff --git a/comfy_extras/nodes_seedvr.py b/comfy_extras/nodes_seedvr.py
new file mode 100644
index 000000000..c4ca3b55c
--- /dev/null
+++ b/comfy_extras/nodes_seedvr.py
@@ -0,0 +1,614 @@
+import logging
+
+from typing_extensions import override
+from comfy_api.latest import ComfyExtension, io
+import torch
+
+import comfy.model_management
+from comfy.ldm.seedvr.color_fix import (
+ adain_color_transfer,
+ lab_color_transfer,
+ wavelet_color_transfer,
+)
+from comfy.ldm.seedvr.constants import (
+ BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE,
+ SEEDVR2_ADAIN_SCALE_MULTIPLIER,
+ SEEDVR2_CHUNK_GIB_PER_MPX_FRAME,
+ SEEDVR2_CHUNK_RESERVED_GIB,
+ SEEDVR2_CHUNK_SIGMA_GIB,
+ SEEDVR2_CHUNK_SIGMA_K,
+ SEEDVR2_COLOR_MEM_HEADROOM,
+ SEEDVR2_DTYPE_BYTES_FLOOR,
+ SEEDVR2_LAB_SCALE_MULTIPLIER,
+ SEEDVR2_LATENT_CHANNELS,
+ SEEDVR2_OOM_BACKOFF_DIVISOR,
+ SEEDVR2_WAVELET_SCALE_MULTIPLIER,
+)
+
+from torchvision.transforms import functional as TVF
+from torchvision.transforms.functional import InterpolationMode
+
+
+_SEEDVR2_INVALID_MODEL_MSG_PREFIX = "SeedVR2Conditioning: model object does not match expected SeedVR2 structure"
+_ATTR_MISSING = object()
+
+
+def _resolve_seedvr2_diffusion_model(model):
+ inner = getattr(model, "model", _ATTR_MISSING)
+ if inner is _ATTR_MISSING:
+ raise RuntimeError(
+ f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: input has no 'model' attribute "
+ f"(got type {type(model).__name__})."
+ )
+ if inner is None:
+ raise RuntimeError(
+ f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: input.model is None "
+ f"(input type {type(model).__name__})."
+ )
+ diffusion_model = getattr(inner, "diffusion_model", _ATTR_MISSING)
+ if diffusion_model is _ATTR_MISSING:
+ raise RuntimeError(
+ f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: 'model.model' has no "
+ f"'diffusion_model' attribute (got type {type(inner).__name__})."
+ )
+ if diffusion_model is None:
+ raise RuntimeError(
+ f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: 'model.model.diffusion_model' "
+ f"is None (model.model type {type(inner).__name__})."
+ )
+ return diffusion_model
+
+
+def div_pad(image, factor):
+ height_factor, width_factor = factor
+ height, width = image.shape[-2:]
+
+ pad_height = (height_factor - (height % height_factor)) % height_factor
+ pad_width = (width_factor - (width % width_factor)) % width_factor
+
+ if pad_height == 0 and pad_width == 0:
+ return image
+
+ padding = (0, pad_width, 0, pad_height)
+ return torch.nn.functional.pad(image, padding, mode='constant', value=0.0)
+
+def cut_videos(videos):
+ t = videos.size(1)
+ if t < 1:
+ raise ValueError("SeedVR2Preprocess expected at least one frame.")
+ if t == 1:
+ return videos
+ if t <= 4:
+ padding = videos[:, -1:].repeat(1, 4 - t + 1, 1, 1, 1)
+ return torch.cat([videos, padding], dim=1)
+ if (t - 1) % 4 == 0:
+ return videos
+ padding = videos[:, -1:].repeat(1, 4 - ((t - 1) % 4), 1, 1, 1)
+ videos = torch.cat([videos, padding], dim=1)
+ if (videos.size(1) - 1) % 4 != 0:
+ raise ValueError(f"SeedVR2Preprocess failed to pad video length to 4n+1; got {videos.size(1)} frames.")
+ return videos
+
+def _seedvr2_input_shorter_edge(images, node_name):
+ if images.dim() == 4:
+ return min(images.shape[1], images.shape[2])
+ if images.dim() == 5:
+ return min(images.shape[2], images.shape[3])
+ raise ValueError(
+ f"{node_name}: expected 4-D or 5-D IMAGE tensor, "
+ f"got shape {tuple(images.shape)}"
+ )
+
+
+def _seedvr2_pad(images, upscaled_shorter_edge, node_name):
+ if upscaled_shorter_edge < 2:
+ raise ValueError(
+ f"{node_name}: input shorter edge must be at least 2 pixels; "
+ f"got {upscaled_shorter_edge}."
+ )
+ if images.shape[-1] > 3:
+ images = images[..., :3]
+ if images.dim() == 4:
+ # Comfy video components arrive as a 4-D IMAGE frame sequence:
+ # (frames, H, W, C). SeedVR2 consumes that as one video.
+ images = images.unsqueeze(0)
+ elif images.dim() != 5:
+ raise ValueError(
+ f"{node_name}: expected 4-D or 5-D IMAGE tensor, "
+ f"got shape {tuple(images.shape)}"
+ )
+ images = images.permute(0, 1, 4, 2, 3)
+
+ b, t, c, h, w = images.shape
+ images = images.reshape(b * t, c, h, w)
+
+ images = torch.clamp(images, 0.0, 1.0)
+ images = div_pad(images, (16, 16))
+ _, _, new_h, new_w = images.shape
+
+ images = images.reshape(b, t, c, new_h, new_w)
+ images = cut_videos(images)
+ images_bthwc = images.permute(0, 1, 3, 4, 2).contiguous()
+
+ return io.NodeOutput(images_bthwc)
+
+
+class SeedVR2Preprocess(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="SeedVR2Preprocess",
+ display_name="Pre-Process SeedVR2 Input",
+ category="image/pre-processors",
+ description="Pad a resized image for SeedVR2 model. Alpha channel is dropped. The node Post-Process SeedVR2 Output re-applies it from the original resized image.",
+ search_aliases=["seedvr2", "upscale", "video upscale", "pad", "preprocess"],
+ inputs=[
+ io.Image.Input("resized_images", tooltip="The resized image to process."),
+ ],
+ outputs=[
+ io.Image.Output("images", tooltip="The padded image for VAE encoding."),
+ ]
+ )
+
+ @classmethod
+ def execute(cls, resized_images):
+ upscaled_shorter_edge = _seedvr2_input_shorter_edge(resized_images, "SeedVR2Preprocess")
+ return _seedvr2_pad(
+ resized_images, upscaled_shorter_edge, "SeedVR2Preprocess",
+ )
+
+
+class SeedVR2PostProcessing(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="SeedVR2PostProcessing",
+ display_name="Post-Process SeedVR2 Output",
+ category="image/post-processors",
+ description="Align the generated image with the original resized image and apply color correction.",
+ search_aliases=["seedvr2", "upscale", "color correction", "color match", "postprocess"],
+ inputs=[
+ io.Image.Input("images", tooltip="The generated image to process."),
+ io.Image.Input("original_resized_images", tooltip="The original resized image before pre-processing, used as reference."),
+ io.Combo.Input("color_correction_method", options=["lab", "wavelet", "adain", "none"], default="lab", tooltip="Method to match the generated image colors to the original image. lab: transfer color in CIELAB space, preserving detail (most faithful). wavelet: transfer low-frequency color, keeping upscaled high-frequency detail. adain: match per-channel mean/std (fastest, global tint). none: skip color transfer (geometry alignment only)."),
+ ],
+ outputs=[io.Image.Output(display_name="images", tooltip="The aligned, color-corrected image.")],
+ )
+
+ @classmethod
+ def execute(cls, images, original_resized_images, color_correction_method):
+ alpha_input = None
+ if original_resized_images.shape[-1] == 4:
+ alpha_input = original_resized_images[..., 3:4]
+ original_resized_images = original_resized_images[..., :3]
+ decoded_5d, decoded_was_4d = cls._as_bthwc(images)
+ reference_full, _ = cls._as_bthwc(original_resized_images)
+ decoded_5d = cls._restore_reference_batch_time(decoded_5d, reference_full)
+
+ b = min(decoded_5d.shape[0], reference_full.shape[0])
+ t = min(decoded_5d.shape[1], reference_full.shape[1])
+ reference_h = reference_full.shape[2]
+ reference_w = reference_full.shape[3]
+
+ decoded_5d = decoded_5d[:b, :t, :, :, :]
+ target_h = min(decoded_5d.shape[2], reference_h)
+ target_w = min(decoded_5d.shape[3], reference_w)
+ decoded_5d = decoded_5d[:, :, :target_h, :target_w, :]
+ if color_correction_method in ("lab", "wavelet", "adain"):
+ reference_5d = reference_full[:b, :t, :, :, :]
+ reference_5d = cls._resize_reference(reference_5d, target_h, target_w)
+ output_device = decoded_5d.device
+ decoded_raw = cls._to_seedvr2_raw(decoded_5d)
+ reference_raw = cls._to_seedvr2_raw(reference_5d)
+ decoded_flat = decoded_raw.permute(0, 1, 4, 2, 3).reshape(b * t, decoded_raw.shape[4], target_h, target_w)
+ reference_flat = reference_raw.permute(0, 1, 4, 2, 3).reshape(b * t, reference_raw.shape[4], target_h, target_w)
+ output = cls._color_transfer_chunked(
+ decoded_flat, reference_flat, output_device, color_correction_method,
+ )
+ output = output.reshape(b, t, output.shape[1], output.shape[2], output.shape[3]).permute(0, 1, 3, 4, 2)
+ output = output.add(1.0).div(2.0).clamp(0.0, 1.0)
+ elif color_correction_method == "none":
+ output = decoded_5d
+ else:
+ raise ValueError(f"SeedVR2PostProcessing: unknown color_correction_method {color_correction_method!r}")
+
+ if alpha_input is not None:
+ alpha_5d, _ = cls._as_bthwc(alpha_input)
+ alpha_5d = alpha_5d[:output.shape[0], :output.shape[1], :output.shape[2], :output.shape[3], :]
+ output = torch.cat([output, alpha_5d.to(dtype=output.dtype, device=output.device)], dim=-1)
+ h2 = output.shape[-3] - (output.shape[-3] % 2)
+ w2 = output.shape[-2] - (output.shape[-2] % 2)
+ output = output[:, :, :h2, :w2, :]
+ if decoded_was_4d:
+ output = output.reshape(-1, output.shape[-3], output.shape[-2], output.shape[-1])
+ return io.NodeOutput(output)
+
+ @staticmethod
+ def _as_bthwc(images):
+ if images.ndim == 4:
+ return images.unsqueeze(0), True
+ if images.ndim == 5:
+ return images, False
+ raise ValueError(
+ f"SeedVR2PostProcessing: expected 4-D or 5-D IMAGE tensor, got shape {tuple(images.shape)}"
+ )
+
+ @staticmethod
+ def _restore_reference_batch_time(decoded, reference):
+ if decoded.shape[0] != 1:
+ return decoded
+ ref_b, ref_t = reference.shape[:2]
+ if ref_b < 1 or decoded.shape[1] % ref_b != 0:
+ return decoded
+ decoded_t = decoded.shape[1] // ref_b
+ if decoded_t < ref_t:
+ return decoded
+ return decoded.reshape(ref_b, decoded_t, decoded.shape[2], decoded.shape[3], decoded.shape[4])
+
+ @staticmethod
+ def _to_seedvr2_raw(images):
+ return images.mul(2.0).sub(1.0)
+
+ @staticmethod
+ def _color_transfer_on_vae_device(decoded_flat, reference_flat, output_device, transfer_fn):
+ color_device = comfy.model_management.vae_device()
+ decoded_flat = decoded_flat.to(device=color_device)
+ reference_flat = reference_flat.to(device=color_device)
+ output = transfer_fn(decoded_flat, reference_flat)
+ return output.to(device=output_device)
+
+ @staticmethod
+ def _lab_color_transfer_on_vae_device(decoded_flat, reference_flat, output_device):
+ color_device = comfy.model_management.vae_device()
+ result = None
+ for start in range(decoded_flat.shape[0]):
+ decoded_frame = decoded_flat[start:start + 1].to(device=color_device).clone()
+ reference_frame = reference_flat[start:start + 1].to(device=color_device).clone()
+ output = lab_color_transfer(decoded_frame, reference_frame).to(device=output_device)
+ if result is None:
+ result = torch.empty(
+ (decoded_flat.shape[0],) + tuple(output.shape[1:]),
+ device=output_device,
+ dtype=output.dtype,
+ )
+ result[start:start + 1].copy_(output)
+ if result is None:
+ raise ValueError("SeedVR2PostProcessing: LAB color correction requires at least one frame.")
+ return result
+
+ @classmethod
+ def _color_transfer_chunked(cls, decoded_flat, reference_flat, output_device, color_correction_method):
+ chunk_size = cls._estimate_color_correction_chunk_size(decoded_flat, color_correction_method)
+ while True:
+ try:
+ return cls._run_color_transfer_chunks(
+ decoded_flat, reference_flat, output_device, color_correction_method, chunk_size,
+ )
+ except Exception as e:
+ comfy.model_management.raise_non_oom(e)
+ if chunk_size <= 1:
+ raise RuntimeError(
+ "SeedVR2PostProcessing: color correction OOM at one frame; "
+ f"color_correction_method={color_correction_method}, shape={tuple(decoded_flat.shape)}."
+ ) from e
+ chunk_size = max(1, chunk_size // SEEDVR2_OOM_BACKOFF_DIVISOR)
+
+ @classmethod
+ def _run_color_transfer_chunks(cls, decoded_flat, reference_flat, output_device, color_correction_method, chunk_size):
+ result = None
+ for start in range(0, decoded_flat.shape[0], chunk_size):
+ end = min(start + chunk_size, decoded_flat.shape[0])
+ decoded_chunk = decoded_flat[start:end]
+ reference_chunk = reference_flat[start:end]
+ if color_correction_method == "lab":
+ output = cls._lab_color_transfer_on_vae_device(decoded_chunk, reference_chunk, output_device)
+ elif color_correction_method == "wavelet":
+ output = cls._color_transfer_on_vae_device(
+ decoded_chunk, reference_chunk, output_device, wavelet_color_transfer,
+ )
+ else:
+ output = cls._color_transfer_on_vae_device(
+ decoded_chunk, reference_chunk, output_device, adain_color_transfer,
+ )
+ if result is None:
+ result = torch.empty(
+ (decoded_flat.shape[0],) + tuple(output.shape[1:]),
+ device=output_device,
+ dtype=output.dtype,
+ )
+ result[start:end].copy_(output)
+ if result is None:
+ raise ValueError("SeedVR2PostProcessing: color correction requires at least one frame.")
+ return result
+
+ @classmethod
+ def _estimate_color_correction_chunk_size(cls, decoded_flat, color_correction_method):
+ multiplier = cls._color_correction_memory_multiplier(color_correction_method)
+ frames = decoded_flat.shape[0]
+ _, channels, height, width = decoded_flat.shape
+ dtype_bytes = max(decoded_flat.element_size(), SEEDVR2_DTYPE_BYTES_FLOOR)
+ bytes_per_frame = height * width * channels * dtype_bytes * multiplier
+ if bytes_per_frame <= 0:
+ return frames
+ color_device = comfy.model_management.vae_device()
+ free_memory = comfy.model_management.get_free_memory(color_device)
+ chunk_size = int((free_memory * SEEDVR2_COLOR_MEM_HEADROOM) // bytes_per_frame)
+ return max(1, min(frames, chunk_size))
+
+ @staticmethod
+ def _color_correction_memory_multiplier(color_correction_method):
+ if color_correction_method == "lab":
+ return SEEDVR2_LAB_SCALE_MULTIPLIER
+ if color_correction_method == "wavelet":
+ return SEEDVR2_WAVELET_SCALE_MULTIPLIER
+ if color_correction_method == "adain":
+ return SEEDVR2_ADAIN_SCALE_MULTIPLIER
+ raise ValueError(f"SeedVR2PostProcessing: unknown color_correction_method {color_correction_method!r}")
+
+ @staticmethod
+ def _resize_reference(reference, height, width):
+ if reference.shape[2] == height and reference.shape[3] == width:
+ return reference
+ b, t = reference.shape[:2]
+ reference_flat = reference.permute(0, 1, 4, 2, 3).reshape(b * t, reference.shape[4], reference.shape[2], reference.shape[3])
+ resized = TVF.resize(
+ reference_flat,
+ size=(height, width),
+ interpolation=InterpolationMode.BICUBIC,
+ antialias=not (isinstance(reference_flat, torch.Tensor) and reference_flat.device.type == "mps"),
+ )
+ return resized.reshape(b, t, resized.shape[1], height, width).permute(0, 1, 3, 4, 2)
+
+
+class SeedVR2Conditioning(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="SeedVR2Conditioning",
+ display_name="Apply SeedVR2 Conditioning",
+ category="model/conditioning",
+ description="Build SeedVR2 positive/negative conditioning from a VAE latent.",
+ search_aliases=["seedvr2", "upscale", "conditioning"],
+ inputs=[
+ io.Model.Input("model", tooltip="The SeedVR2 model."),
+ io.Latent.Input("vae_conditioning", display_name="latent"),
+ ],
+ outputs=[
+ io.Conditioning.Output(display_name="positive", tooltip="The positive conditioning for sampling."),
+ io.Conditioning.Output(display_name="negative", tooltip="The negative conditioning for sampling."),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, model, vae_conditioning) -> io.NodeOutput:
+
+ vae_conditioning = vae_conditioning["samples"]
+ if vae_conditioning.ndim != 5:
+ raise ValueError(
+ "SeedVR2Conditioning expects a 5-D VAE latent in Comfy "
+ f"channel-first layout; got shape {tuple(vae_conditioning.shape)}."
+ )
+ if vae_conditioning.shape[1] != SEEDVR2_LATENT_CHANNELS:
+ if vae_conditioning.shape[-1] == SEEDVR2_LATENT_CHANNELS:
+ raise ValueError(
+ "SeedVR2Conditioning expects SeedVR2 VAE latents in Comfy "
+ f"channel-first layout (B, {SEEDVR2_LATENT_CHANNELS}, T, H, W); "
+ f"got channel-last shape {tuple(vae_conditioning.shape)}."
+ )
+ raise ValueError(
+ "SeedVR2Conditioning expects SeedVR2 VAE latents with "
+ f"{SEEDVR2_LATENT_CHANNELS} channels; got shape {tuple(vae_conditioning.shape)}."
+ )
+ vae_conditioning = vae_conditioning.movedim(1, -1).contiguous()
+ model = _resolve_seedvr2_diffusion_model(model)
+ pos_cond = model.positive_conditioning
+ neg_cond = model.negative_conditioning
+
+ mask = vae_conditioning.new_ones(vae_conditioning.shape[:-1] + (1,))
+ condition = torch.cat((vae_conditioning, mask), dim=-1)
+ condition = condition.movedim(-1, 1)
+
+ negative = [[neg_cond.unsqueeze(0), {"condition": condition}]]
+ positive = [[pos_cond.unsqueeze(0), {"condition": condition}]]
+
+ return io.NodeOutput(positive, negative)
+
+def _seedvr2_chunk_crossfade_weights(overlap, device, dtype):
+ """Descending previous-chunk weights across the overlap (next chunk gets ``1 - w``): a Hann fade over the middle third, flat shoulders on the outer thirds."""
+ ramp = torch.linspace(0.0, 1.0, steps=overlap, device=device, dtype=dtype)
+ ramp = ((ramp - 1.0 / 3.0) / (1.0 / 3.0)).clamp(0.0, 1.0)
+ return 0.5 + 0.5 * torch.cos(torch.pi * ramp)
+
+
+class SeedVR2TemporalChunk(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="SeedVR2TemporalChunk",
+ display_name="Split SeedVR2 Latent",
+ category="model/latent/batch",
+ description="Split a SeedVR2 video latent into overlapping temporal chunks small enough to sample one at a time within VRAM, wiring latents outputs to both Apply SeedVR2 Conditioning and the sampler latent input before recombining with Merge SeedVR2 Latents.",
+ search_aliases=["seedvr2", "split", "chunk", "temporal", "video upscale", "rebatch"],
+ inputs=[
+ io.Latent.Input("latent", tooltip="The VAE-encoded SeedVR2 latent to split."),
+ io.Int.Input("temporal_overlap", default=0, min=0, max=16384,
+ tooltip="Latent frames shared between adjacent chunks and crossfaded at merge; 0 = no overlap."),
+ io.DynamicCombo.Input("chunking_mode",
+ tooltip="manual = use frames_per_chunk exactly; auto = predict the largest chunk that fits free VRAM.",
+ options=[
+ io.DynamicCombo.Option("auto", []),
+ io.DynamicCombo.Option("manual", [
+ io.Int.Input("frames_per_chunk", default=21, min=1, max=16384, step=4,
+ tooltip="Pixel frames per temporal chunk (4n+1: 1, 5, 9, 13, ...)."),
+ ]),
+ ]),
+ ],
+ outputs=[
+ io.Latent.Output(display_name="latents", is_output_list=True,
+ tooltip="The temporal chunks in sequence order."),
+ io.Int.Output(display_name="temporal_overlap",
+ tooltip="The effective latent-frame overlap between adjacent chunks, for Merge SeedVR2 Latents."),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, latent, temporal_overlap, chunking_mode) -> io.NodeOutput:
+ samples = latent["samples"]
+ if samples.ndim != 5:
+ raise ValueError(
+ f"SeedVR2TemporalChunk: expected a 5-D video latent (B, C, T, H, W); "
+ f"got shape {tuple(samples.shape)}."
+ )
+ if samples.shape[1] != SEEDVR2_LATENT_CHANNELS:
+ raise ValueError(
+ f"SeedVR2TemporalChunk: expected {SEEDVR2_LATENT_CHANNELS} latent channels; "
+ f"got shape {tuple(samples.shape)}."
+ )
+ if temporal_overlap < 0:
+ raise ValueError(
+ f"SeedVR2TemporalChunk: temporal_overlap must be >= 0; got {temporal_overlap}."
+ )
+ mode = chunking_mode["chunking_mode"]
+ if mode not in ("auto", "manual"):
+ raise ValueError(
+ f"SeedVR2TemporalChunk: chunking_mode must be 'auto' or 'manual'; "
+ f"got {mode!r}."
+ )
+ t_latent = samples.shape[2]
+ t_pixel = 4 * (t_latent - 1) + 1
+
+ if mode == "auto":
+ free_gb = comfy.model_management.get_free_memory(
+ comfy.model_management.get_torch_device()) / (1024 ** 3)
+ mpx_per_frame = (samples.shape[0] * samples.shape[3] * samples.shape[4]) * (BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE ** 2) / 1e6
+ budget_gb = free_gb - SEEDVR2_CHUNK_RESERVED_GIB - SEEDVR2_CHUNK_SIGMA_K * SEEDVR2_CHUNK_SIGMA_GIB
+ chunk_latent_max = max(1, int(budget_gb / (SEEDVR2_CHUNK_GIB_PER_MPX_FRAME * mpx_per_frame)))
+ frames_per_chunk = min(4 * (chunk_latent_max - 1) + 1, t_pixel)
+ logging.info(
+ "SeedVR2TemporalChunk auto: free=%.2fGiB, %.2fMpx -> frames_per_chunk=%d (t_pixel=%d).",
+ free_gb, mpx_per_frame, frames_per_chunk, t_pixel,
+ )
+ else:
+ frames_per_chunk = chunking_mode["frames_per_chunk"]
+ if frames_per_chunk < 1 or (frames_per_chunk - 1) % 4 != 0:
+ raise ValueError(
+ f"SeedVR2TemporalChunk: frames_per_chunk must be a 4n+1 pixel-frame count "
+ f"(1, 5, 9, 13, 17, 21, ...); got {frames_per_chunk}."
+ )
+
+ if t_pixel <= frames_per_chunk:
+ return io.NodeOutput([latent], 0)
+
+ chunk_latent = (frames_per_chunk - 1) // 4 + 1
+ temporal_overlap = min(temporal_overlap, chunk_latent - 1)
+ step = chunk_latent - temporal_overlap
+
+ chunks = []
+ for start in range(0, t_latent, step):
+ end = min(start + chunk_latent, t_latent)
+ chunk = latent.copy()
+ chunk["samples"] = samples[:, :, start:end].contiguous()
+ chunks.append(chunk)
+ if end >= t_latent:
+ break
+ return io.NodeOutput(chunks, temporal_overlap)
+
+
+class SeedVR2TemporalMerge(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="SeedVR2TemporalMerge",
+ display_name="Merge SeedVR2 Latents",
+ category="model/latent/batch",
+ is_input_list=True,
+ description="Recombine sampled SeedVR2 latent temporal chunks into one latent, crossfading each overlap with a Hann window sized by the temporal_overlap wired from Split SeedVR2 Latent.",
+ search_aliases=["seedvr2", "merge", "temporal", "hann", "crossfade"],
+ inputs=[
+ io.Latent.Input("latents", tooltip="The sampled temporal chunks in sequence order."),
+ io.Int.Input("temporal_overlap", default=0, min=0, max=16384, force_input=True,
+ tooltip="The temporal_overlap output of Split SeedVR2 Latent. 0 = plain concatenation."),
+ ],
+ outputs=[
+ io.Latent.Output(display_name="latent", tooltip="The recombined full-length latent."),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, latents, temporal_overlap) -> io.NodeOutput:
+ temporal_overlap = temporal_overlap[0]
+ if temporal_overlap < 0:
+ raise ValueError(
+ f"SeedVR2TemporalMerge: temporal_overlap must be >= 0; got {temporal_overlap}."
+ )
+ chunks = [entry["samples"] for entry in latents]
+ first = chunks[0]
+ if first.ndim != 5:
+ raise ValueError(
+ f"SeedVR2TemporalMerge: expected 5-D video latents (B, C, T, H, W); "
+ f"chunk 0 has shape {tuple(first.shape)}."
+ )
+ for i, chunk in enumerate(chunks[1:], start=1):
+ if chunk.shape[:2] != first.shape[:2] or chunk.shape[3:] != first.shape[3:]:
+ raise ValueError(
+ f"SeedVR2TemporalMerge: chunk {i} shape {tuple(chunk.shape)} does not "
+ f"match chunk 0 shape {tuple(first.shape)} outside the temporal axis."
+ )
+ if i < len(chunks) - 1 and chunk.shape[2] != first.shape[2]:
+ raise ValueError(
+ f"SeedVR2TemporalMerge: chunk {i} has {chunk.shape[2]} latent frames but "
+ f"chunk 0 has {first.shape[2]}; only the final chunk may be shorter."
+ )
+
+ out = latents[0].copy()
+ out.pop("noise_mask", None)
+
+ if len(chunks) == 1:
+ out["samples"] = first
+ return io.NodeOutput(out)
+ if temporal_overlap == 0:
+ out["samples"] = torch.cat(chunks, dim=2)
+ return io.NodeOutput(out)
+
+ chunk_latent = first.shape[2]
+ step = chunk_latent - min(temporal_overlap, chunk_latent - 1)
+ t_total = step * (len(chunks) - 1) + chunks[-1].shape[2]
+ b, c, _, h, w = first.shape
+ merged = torch.empty((b, c, t_total, h, w), device=first.device, dtype=first.dtype)
+
+ merged[:, :, :chunk_latent] = first
+ filled = chunk_latent
+ for i, chunk in enumerate(chunks[1:], start=1):
+ start = i * step
+ end = start + chunk.shape[2]
+ # Crossfade width is bounded by the previous fill frontier and by a runt
+ # final chunk shorter than the configured overlap.
+ fade = min(filled - start, chunk.shape[2])
+ if fade > 0:
+ w_prev = _seedvr2_chunk_crossfade_weights(
+ fade, chunk.device, chunk.dtype).view(1, 1, fade, 1, 1)
+ merged[:, :, start:start + fade] = (
+ merged[:, :, start:start + fade] * w_prev + chunk[:, :, :fade] * (1.0 - w_prev)
+ )
+ merged[:, :, start + fade:end] = chunk[:, :, fade:]
+ else:
+ merged[:, :, start:end] = chunk
+ filled = end
+
+ out["samples"] = merged
+ return io.NodeOutput(out)
+
+
+class SeedVRExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ SeedVR2Conditioning,
+ SeedVR2Preprocess,
+ SeedVR2PostProcessing,
+ SeedVR2TemporalChunk,
+ SeedVR2TemporalMerge,
+ ]
+
+async def comfy_entrypoint() -> SeedVRExtension:
+ return SeedVRExtension()
diff --git a/nodes.py b/nodes.py
index e126576fe..474e188fe 100644
--- a/nodes.py
+++ b/nodes.py
@@ -2458,6 +2458,7 @@ async def init_builtin_extra_nodes():
"nodes_camera_trajectory.py",
"nodes_edit_model.py",
"nodes_tcfg.py",
+ "nodes_seedvr.py",
"nodes_context_windows.py",
"nodes_qwen.py",
"nodes_boogu.py",
diff --git a/tests-unit/comfy_extras_test/test_seedvr2_conditioning.py b/tests-unit/comfy_extras_test/test_seedvr2_conditioning.py
new file mode 100644
index 000000000..045502b5b
--- /dev/null
+++ b/tests-unit/comfy_extras_test/test_seedvr2_conditioning.py
@@ -0,0 +1,186 @@
+"""SeedVR2 conditioning node regression tests."""
+
+import importlib
+import sys
+from unittest.mock import MagicMock
+
+import pytest
+import torch
+import torch.nn as nn
+
+from comfy.cli_args import args as cli_args
+from comfy.ldm.seedvr.constants import SEEDVR2_LATENT_CHANNELS
+
+if not torch.cuda.is_available():
+ cli_args.cpu = True
+
+
+_SENTINEL = object()
+_TARGETS = (
+ ("comfy.model_management", "comfy"),
+ ("comfy_extras.nodes_seedvr", "comfy_extras"),
+)
+
+
+def _import_nodes_seedvr_isolated():
+ """Import comfy_extras.nodes_seedvr with comfy.model_management mocked."""
+ priors = []
+ for mod_name, parent_name in _TARGETS:
+ prior_mod = sys.modules.get(mod_name, _SENTINEL)
+ parent = sys.modules.get(parent_name)
+ attr = mod_name.split(".")[-1]
+ prior_attr = (
+ getattr(parent, attr, _SENTINEL) if parent is not None else _SENTINEL
+ )
+ priors.append((mod_name, parent_name, attr, prior_mod, prior_attr))
+
+ mock_mm = MagicMock()
+ for fn in (
+ "xformers_enabled", "xformers_enabled_vae",
+ "pytorch_attention_enabled", "pytorch_attention_enabled_vae",
+ "sage_attention_enabled", "flash_attention_enabled",
+ "is_intel_xpu",
+ ):
+ getattr(mock_mm, fn).return_value = False
+ tv = torch.version.__version__.split(".")
+ mock_mm.torch_version_numeric = (int(tv[0]), int(tv[1]))
+ mock_mm.WINDOWS = False
+ sys.modules["comfy.model_management"] = mock_mm
+ if sys.modules.get("comfy") is None:
+ importlib.import_module("comfy")
+ comfy_pkg = sys.modules.get("comfy")
+ if comfy_pkg is not None:
+ setattr(comfy_pkg, "model_management", mock_mm)
+ nodes_seedvr = sys.modules.get("comfy_extras.nodes_seedvr") or (
+ importlib.import_module("comfy_extras.nodes_seedvr")
+ )
+
+ def _restore():
+ for mod_name, parent_name, attr, prior_mod, prior_attr in priors:
+ if prior_mod is _SENTINEL:
+ sys.modules.pop(mod_name, None)
+ else:
+ sys.modules[mod_name] = prior_mod
+ parent = sys.modules.get(parent_name)
+ if parent is None:
+ continue
+ if prior_attr is _SENTINEL:
+ if hasattr(parent, attr):
+ delattr(parent, attr)
+ else:
+ setattr(parent, attr, prior_attr)
+
+ return nodes_seedvr, _restore
+
+
+class _Rope(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.freqs = nn.Parameter(torch.zeros(4))
+
+
+class _Block(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.rope = _Rope()
+
+
+class _DiffusionModel(nn.Module):
+ def __init__(self, n_blocks=3, conditioning_dtype=torch.float32):
+ super().__init__()
+ self.blocks = nn.ModuleList([_Block() for _ in range(n_blocks)])
+ self.register_buffer("positive_conditioning", torch.ones((2, 4), dtype=conditioning_dtype))
+ self.register_buffer("negative_conditioning", torch.zeros((3, 4), dtype=conditioning_dtype))
+
+
+class _ModelInner:
+ def __init__(self, diffusion_model):
+ self.diffusion_model = diffusion_model
+
+
+class _ModelPatcher:
+ def __init__(self, diffusion_model):
+ self.model = _ModelInner(diffusion_model)
+
+
+def test_seedvr2_conditioning_schema_exposes_conditioning_outputs():
+ nodes_seedvr, restore = _import_nodes_seedvr_isolated()
+ try:
+ schema = nodes_seedvr.SeedVR2Conditioning.define_schema()
+ assert [input_item.id for input_item in schema.inputs] == [
+ "model",
+ "vae_conditioning",
+ ]
+ assert schema.inputs[1].display_name == "latent"
+ assert [output.display_name for output in schema.outputs] == [
+ "positive",
+ "negative",
+ ]
+ finally:
+ restore()
+
+
+def test_seedvr2_conditioning_rejects_wrong_latent_channels():
+ nodes_seedvr, restore = _import_nodes_seedvr_isolated()
+ try:
+ patcher = _ModelPatcher(_DiffusionModel())
+ vae_conditioning = {"samples": torch.zeros(1, 8, 2, 2, 2)}
+
+ with pytest.raises(ValueError, match=f"{SEEDVR2_LATENT_CHANNELS} channels"):
+ nodes_seedvr.SeedVR2Conditioning.execute(patcher, vae_conditioning)
+ finally:
+ restore()
+
+
+def test_seedvr2_conditioning_returns_conditioning_deterministically():
+ nodes_seedvr, restore = _import_nodes_seedvr_isolated()
+ try:
+ diffusion_model = _DiffusionModel()
+ patcher = _ModelPatcher(diffusion_model)
+ samples = torch.arange(
+ 1,
+ 1 + SEEDVR2_LATENT_CHANNELS * 3 * 2 * 2,
+ dtype=torch.float32,
+ ).reshape(1, SEEDVR2_LATENT_CHANNELS, 3, 2, 2)
+ vae_conditioning = {"samples": samples}
+
+ first_positive, first_negative = (
+ nodes_seedvr.SeedVR2Conditioning.execute(
+ patcher,
+ vae_conditioning,
+ )
+ )
+ second_positive, second_negative = (
+ nodes_seedvr.SeedVR2Conditioning.execute(
+ patcher,
+ vae_conditioning,
+ )
+ )
+
+ channel_last = samples.movedim(1, -1).contiguous()
+ expected_condition = torch.cat(
+ [
+ channel_last,
+ torch.ones((*channel_last.shape[:-1], 1)),
+ ],
+ dim=-1,
+ ).movedim(-1, 1)
+
+ assert torch.equal(
+ first_positive[0][1]["condition"],
+ expected_condition,
+ )
+ assert torch.equal(
+ second_positive[0][1]["condition"],
+ expected_condition,
+ )
+ assert torch.equal(
+ first_negative[0][1]["condition"],
+ expected_condition,
+ )
+ assert torch.equal(
+ second_negative[0][1]["condition"],
+ expected_condition,
+ )
+ finally:
+ restore()
diff --git a/tests-unit/comfy_extras_test/test_seedvr2_nodes.py b/tests-unit/comfy_extras_test/test_seedvr2_nodes.py
new file mode 100644
index 000000000..1c5d20ac9
--- /dev/null
+++ b/tests-unit/comfy_extras_test/test_seedvr2_nodes.py
@@ -0,0 +1,55 @@
+import importlib
+import inspect
+import sys
+from unittest.mock import MagicMock, patch
+
+import torch
+
+from comfy.cli_args import args as cli_args
+
+if not torch.cuda.is_available():
+ cli_args.cpu = True
+
+
+def test_seedvr_node_signature_matches_schema():
+ mock_mm = MagicMock()
+ mock_mm.xformers_enabled.return_value = False
+ mock_mm.xformers_enabled_vae.return_value = False
+ mock_mm.sage_attention_enabled.return_value = False
+ mock_mm.flash_attention_enabled.return_value = False
+
+ sentinel = object()
+ prior_cpu = cli_args.cpu
+ cli_args.cpu = True
+ prior_module = sys.modules.get("comfy_extras.nodes_seedvr", sentinel)
+ comfy_pkg = sys.modules.get("comfy")
+ prior_mm_attr = getattr(comfy_pkg, "model_management", sentinel) if comfy_pkg else sentinel
+
+ with patch.dict(sys.modules, {"comfy.model_management": mock_mm}):
+ if comfy_pkg is not None:
+ setattr(comfy_pkg, "model_management", mock_mm)
+ sys.modules.pop("comfy_extras.nodes_seedvr", None)
+ try:
+ nodes_seedvr = importlib.import_module("comfy_extras.nodes_seedvr")
+ for node_cls in (nodes_seedvr.SeedVR2Preprocess, nodes_seedvr.SeedVR2PostProcessing, nodes_seedvr.SeedVR2Conditioning):
+ schema_ids = [i.id for i in node_cls.define_schema().inputs]
+ exec_params = [
+ p for p in inspect.signature(node_cls.execute).parameters.keys()
+ if p != "cls"
+ ]
+ assert schema_ids == exec_params, (
+ f"{node_cls.__name__} schema/execute drift: "
+ f"schema_ids={schema_ids}, exec_params={exec_params}"
+ )
+ finally:
+ cli_args.cpu = prior_cpu
+ if prior_module is sentinel:
+ sys.modules.pop("comfy_extras.nodes_seedvr", None)
+ else:
+ sys.modules["comfy_extras.nodes_seedvr"] = prior_module
+ if comfy_pkg is not None:
+ if prior_mm_attr is sentinel:
+ if hasattr(comfy_pkg, "model_management"):
+ delattr(comfy_pkg, "model_management")
+ else:
+ setattr(comfy_pkg, "model_management", prior_mm_attr)
diff --git a/tests-unit/comfy_extras_test/test_seedvr2_post_processing.py b/tests-unit/comfy_extras_test/test_seedvr2_post_processing.py
new file mode 100644
index 000000000..6c821136d
--- /dev/null
+++ b/tests-unit/comfy_extras_test/test_seedvr2_post_processing.py
@@ -0,0 +1,51 @@
+from unittest.mock import patch
+
+import pytest
+import torch
+
+from comfy.cli_args import args as cli_args
+
+if not torch.cuda.is_available():
+ cli_args.cpu = True
+
+from comfy_extras import nodes_seedvr # noqa: E402
+
+
+def _schema_ids(items):
+ return [item.id for item in items]
+
+
+def test_seedvr2_post_processing_schema():
+ schema = nodes_seedvr.SeedVR2PostProcessing.define_schema()
+
+ assert _schema_ids(schema.inputs) == ["images", "original_resized_images", "color_correction_method"]
+ assert schema.inputs[2].options == ["lab", "wavelet", "adain", "none"]
+ assert schema.inputs[2].default == "lab"
+ assert schema.outputs[0].get_io_type() == "IMAGE"
+
+
+def test_seedvr2_post_processing_oom_error_uses_color_correction_method(monkeypatch):
+ decoded = torch.full((1, 3, 4, 4), 0.25)
+ reference = torch.full((1, 3, 4, 4), 0.75)
+
+ def _lab(content, style):
+ raise torch.cuda.OutOfMemoryError("CUDA out of memory")
+
+ monkeypatch.setattr(nodes_seedvr.comfy.model_management, "vae_device", lambda: torch.device("cpu"))
+ monkeypatch.setattr(nodes_seedvr.comfy.model_management, "get_free_memory", lambda device: 1_000_000)
+
+ with patch.object(nodes_seedvr, "lab_color_transfer", _lab):
+ with pytest.raises(RuntimeError) as excinfo:
+ nodes_seedvr.SeedVR2PostProcessing._color_transfer_chunked(
+ decoded, reference, torch.device("cpu"), "lab",
+ )
+ assert "color_correction_method=lab" in str(excinfo.value)
+ assert " method=lab" not in str(excinfo.value)
+
+
+def test_seedvr2_post_processing_unknown_color_correction_method_raises():
+ decoded = torch.zeros(1, 2, 4, 4, 3)
+ original = torch.zeros(1, 2, 4, 4, 3)
+ with pytest.raises(ValueError) as excinfo:
+ nodes_seedvr.SeedVR2PostProcessing.execute(decoded, original, "bogus")
+ assert "color_correction_method" in str(excinfo.value)
diff --git a/tests-unit/comfy_extras_test/test_seedvr2_temporal_chunk.py b/tests-unit/comfy_extras_test/test_seedvr2_temporal_chunk.py
new file mode 100644
index 000000000..328355b49
--- /dev/null
+++ b/tests-unit/comfy_extras_test/test_seedvr2_temporal_chunk.py
@@ -0,0 +1,77 @@
+"""SeedVR2 temporal chunk/merge node regression tests."""
+
+import pytest
+import torch
+
+from comfy.cli_args import args as cli_args
+from comfy.ldm.seedvr.constants import (
+ BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE,
+ SEEDVR2_CHUNK_GIB_PER_MPX_FRAME,
+ SEEDVR2_CHUNK_RESERVED_GIB,
+ SEEDVR2_CHUNK_SIGMA_GIB,
+ SEEDVR2_CHUNK_SIGMA_K,
+ SEEDVR2_LATENT_CHANNELS,
+)
+
+if not torch.cuda.is_available():
+ cli_args.cpu = True
+
+import comfy.model_management # noqa: E402
+from comfy_extras.nodes_seedvr import SeedVR2TemporalChunk, SeedVR2TemporalMerge, _seedvr2_chunk_crossfade_weights # noqa: E402
+
+def _latent(t_latent, h=8, w=8, b=1):
+ g = torch.Generator().manual_seed(7)
+ return {"samples": torch.randn(b, SEEDVR2_LATENT_CHANNELS, t_latent, h, w, generator=g)}
+
+def _split(latent, frames_per_chunk, temporal_overlap, chunking_mode="manual"):
+ combo = {"chunking_mode": chunking_mode}
+ if chunking_mode != "auto":
+ combo["frames_per_chunk"] = frames_per_chunk
+ return SeedVR2TemporalChunk.execute(latent, temporal_overlap, combo).args
+
+def _merge(chunks, temporal_overlap):
+ return SeedVR2TemporalMerge.execute(chunks, [temporal_overlap]).args[0]
+
+def test_chunk_temporal_windows_and_validation():
+ with pytest.raises(ValueError, match="4n\\+1"):
+ _split(_latent(9), 20, 0)
+ with pytest.raises(ValueError, match="5-D"):
+ _split({"samples": torch.zeros(1, SEEDVR2_LATENT_CHANNELS * 9, 8, 8)}, 21, 0)
+ with pytest.raises(ValueError, match="chunking_mode"):
+ _split(_latent(13), 21, 0, "adaptive")
+ latent = _latent(13)
+ chunks, overlap = _split(latent, 21, 2) # chunk_latent=6, step=4 -> [0:6], [4:10], [8:13]
+ assert overlap == 2 and [c["samples"].shape[2] for c in chunks] == [6, 6, 5]
+ assert all(torch.equal(c["samples"], latent["samples"][:, :, s:e]) for c, (s, e) in zip(chunks, [(0, 6), (4, 10), (8, 13)]))
+ assert len(_split(_latent(13), 21, 999)[0]) == 8 # overlap clamps to chunk_latent-1 -> step=1
+ assert (r := _split(_latent(5), 21, 3)) and len(r[0]) == 1 and r[1] == 0 # t_pixel <= 21: passthrough
+
+def test_chunk_auto_mode_applies_vram_law(monkeypatch):
+ mpx_per_frame = (32 * 32) * (BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE ** 2) / 1e6
+ free_gb = (
+ SEEDVR2_CHUNK_RESERVED_GIB
+ + SEEDVR2_CHUNK_SIGMA_K * SEEDVR2_CHUNK_SIGMA_GIB
+ + 5.1 * SEEDVR2_CHUNK_GIB_PER_MPX_FRAME * mpx_per_frame
+ )
+ monkeypatch.setattr(comfy.model_management, "get_free_memory", lambda dev=None: free_gb * (1024 ** 3))
+ assert [c["samples"].shape[2] for c in _split(_latent(13, h=32, w=32), 1, 0, "auto")[0]] == [5, 5, 3]
+ assert _split(_latent(13, h=32, w=32, b=2), 1, 0, "auto")[0][0]["samples"].shape[2] == 2 # batch halves the chunk
+
+def test_merge_crossfade_and_reassembly():
+ latent = _latent(13)
+ latent["noise_mask"] = torch.rand(1, 1, 13, 8, 8)
+ latent["batch_index"] = [0]
+ merged = _merge(_split(latent, 21, 0)[0], 0)
+ assert torch.equal(merged["samples"], latent["samples"])
+ assert "noise_mask" not in merged and merged["batch_index"] == [0]
+ assert torch.allclose(_merge(_split(latent, 21, 3)[0], 3)["samples"], latent["samples"], atol=1e-6)
+ w = _seedvr2_chunk_crossfade_weights(3, merged["samples"].device, merged["samples"].dtype)
+ assert w[0] == 1.0 and w[-1] == 0.0 and torch.all(w[:-1] >= w[1:])
+ ones, zeros = {"samples": torch.ones(1, SEEDVR2_LATENT_CHANNELS, 6, 8, 8)}, {"samples": torch.zeros(1, SEEDVR2_LATENT_CHANNELS, 6, 8, 8)}
+ fused = _merge([ones, zeros], 3)["samples"] # overlap equals w: prev fades out, next fades in
+ assert torch.equal(fused[:, :, 3:6], w.view(1, 1, 3, 1, 1).expand(1, SEEDVR2_LATENT_CHANNELS, 3, 8, 8))
+ assert torch.equal(fused[:, :, :3], ones["samples"][:, :, :3]) and torch.equal(fused[:, :, 6:], zeros["samples"][:, :, :3])
+ short = _split(latent, 21, 2)[0]
+ short[0]["samples"] = short[0]["samples"][:, :, :4]
+ with pytest.raises(ValueError, match="only the final chunk may be shorter"):
+ _merge(short, 2)
diff --git a/tests-unit/comfy_test/model_detection_test.py b/tests-unit/comfy_test/model_detection_test.py
index 4e9350602..6e7d71f79 100644
--- a/tests-unit/comfy_test/model_detection_test.py
+++ b/tests-unit/comfy_test/model_detection_test.py
@@ -2,7 +2,7 @@ from collections import defaultdict
import torch
-from comfy.model_detection import detect_unet_config, model_config_from_unet_config
+from comfy.model_detection import detect_unet_config, model_config_from_unet, model_config_from_unet_config
import comfy.supported_models
@@ -73,6 +73,34 @@ def _make_flux_schnell_comfyui_sd():
return sd
+def _make_seedvr2_7b_separate_mm_sd():
+ return {
+ "blocks.35.mlp.vid.proj_out.weight": torch.empty(3072, 1),
+ "positive_conditioning": torch.empty(58, 5120),
+ "negative_conditioning": torch.empty(64, 5120),
+ }
+
+
+def _make_seedvr2_7b_shared_mm_sd():
+ return {
+ "blocks.35.mlp.all.proj_in_gate.weight": torch.empty(1, 1),
+ "positive_conditioning": torch.empty(58, 5120),
+ "negative_conditioning": torch.empty(64, 5120),
+ }
+
+
+def _make_seedvr2_3b_shared_mm_sd():
+ return {
+ "blocks.31.mlp.all.proj_in_gate.weight": torch.empty(1, 1),
+ "positive_conditioning": torch.empty(58, 5120),
+ "negative_conditioning": torch.empty(64, 5120),
+ }
+
+
+def _add_model_diffusion_prefix(sd):
+ return {f"model.diffusion_model.{k}": v for k, v in sd.items()}
+
+
class TestModelDetection:
"""Verify that first-match model detection selects the correct model
based on list ordering and unet_config specificity."""
@@ -125,6 +153,59 @@ class TestModelDetection:
assert model_config is not None
assert type(model_config).__name__ == "FluxSchnell"
+ def test_seedvr2_7b_separate_mm_detection_config(self):
+ sd = _make_seedvr2_7b_separate_mm_sd()
+ unet_config = detect_unet_config(sd, "")
+
+ assert unet_config is not None
+ assert unet_config["image_model"] == "seedvr2"
+ assert unet_config["vid_dim"] == 3072
+ assert unet_config["heads"] == 24
+ assert unet_config["num_layers"] == 36
+ assert unet_config["mm_layers"] == 36
+ assert unet_config["mlp_type"] == "normal"
+ assert unet_config["rope_type"] == "rope3d"
+ assert unet_config["rope_dim"] == 64
+
+ def test_seedvr2_7b_shared_mm_detection_config(self):
+ sd = _make_seedvr2_7b_shared_mm_sd()
+ unet_config = detect_unet_config(sd, "")
+
+ assert unet_config is not None
+ assert unet_config["image_model"] == "seedvr2"
+ assert unet_config["vid_dim"] == 3072
+ assert unet_config["heads"] == 24
+ assert unet_config["num_layers"] == 36
+ assert unet_config["mm_layers"] == 10
+ assert unet_config["mlp_type"] == "swiglu"
+ assert unet_config["rope_type"] == "rope3d"
+ assert unet_config["rope_dim"] == 64
+
+ def test_seedvr2_3b_shared_mm_detection_config(self):
+ sd = _make_seedvr2_3b_shared_mm_sd()
+ unet_config = detect_unet_config(sd, "")
+
+ assert unet_config is not None
+ assert unet_config["image_model"] == "seedvr2"
+ assert unet_config["vid_dim"] == 2560
+ assert unet_config["heads"] == 20
+ assert unet_config["num_layers"] == 32
+ assert unet_config["mlp_type"] == "swiglu"
+
+ def test_seedvr2_model_match_requires_conditioning_tensors(self):
+ sd = _make_seedvr2_7b_shared_mm_sd()
+ unet_config = detect_unet_config(sd, "")
+
+ assert type(model_config_from_unet_config(unet_config, sd)).__name__ == "SeedVR2"
+
+ del sd["positive_conditioning"]
+ assert model_config_from_unet_config(unet_config, sd) is None
+
+ def test_seedvr2_model_match_accepts_full_checkpoint_prefix(self):
+ sd = _add_model_diffusion_prefix(_make_seedvr2_7b_shared_mm_sd())
+
+ assert type(model_config_from_unet(sd, "model.diffusion_model.")).__name__ == "SeedVR2"
+
def test_unet_config_and_required_keys_combination_is_unique(self):
"""Each model in the registry must have a unique combination of
``unet_config`` and ``required_keys``. If two models share the same
diff --git a/tests-unit/comfy_test/seedvr_vae_forward_test.py b/tests-unit/comfy_test/seedvr_vae_forward_test.py
new file mode 100644
index 000000000..7ea7a143e
--- /dev/null
+++ b/tests-unit/comfy_test/seedvr_vae_forward_test.py
@@ -0,0 +1,74 @@
+"""Regression tests for the SeedVR2 VAE forward return contract."""
+
+import pytest
+import torch
+import torch.nn as nn
+
+from comfy.cli_args import args as cli_args
+
+if not torch.cuda.is_available():
+ cli_args.cpu = True
+
+from comfy.ldm.seedvr.vae import SEEDVR2_LATENT_CHANNELS, VideoAutoencoderKL # noqa: E402
+
+
+_LATENT_SHAPE = (1, SEEDVR2_LATENT_CHANNELS, 2, 2, 2)
+_DECODED_SHAPE = (1, 3, 5, 16, 16)
+_INPUT_ENCODE_SHAPE = (1, 3, 5, 16, 16)
+_INPUT_DECODE_SHAPE = _LATENT_SHAPE
+
+
+class _StubVAE(VideoAutoencoderKL):
+ def __init__(self):
+ nn.Module.__init__(self)
+ self._encode_out = torch.zeros(*_LATENT_SHAPE)
+ self._decode_out = torch.zeros(*_DECODED_SHAPE)
+
+ def encode(self, x, return_dict=True):
+ return self._encode_out
+
+ def decode_(self, z, return_dict=True):
+ return self._decode_out
+
+
+def test_forward_encode_returns_tensor():
+ vae = _StubVAE()
+ x = torch.zeros(*_INPUT_ENCODE_SHAPE)
+ result = vae.forward(x, mode="encode")
+ assert type(result) is torch.Tensor
+ assert result.shape == torch.Size(_LATENT_SHAPE)
+
+
+def test_forward_decode_returns_tensor():
+ vae = _StubVAE()
+ z = torch.zeros(*_INPUT_DECODE_SHAPE)
+ result = vae.forward(z, mode="decode")
+ assert type(result) is torch.Tensor
+ assert result.shape == torch.Size(_DECODED_SHAPE)
+
+
+class _TupleReturningStubVAE(VideoAutoencoderKL):
+ def __init__(self):
+ nn.Module.__init__(self)
+ self._encode_tensor = torch.zeros(*_LATENT_SHAPE)
+ self._decode_tensor = torch.zeros(*_DECODED_SHAPE)
+
+ def encode(self, x, return_dict=True):
+ return (self._encode_tensor,)
+
+ def decode_(self, z, return_dict=True):
+ return (self._decode_tensor,)
+
+
+def test_forward_all_unwraps_one_tuple_at_each_step():
+ vae = _TupleReturningStubVAE()
+ x = torch.zeros(*_INPUT_ENCODE_SHAPE)
+ result = vae.forward(x, mode="all")
+ assert type(result) is torch.Tensor
+ assert result.shape == torch.Size(_DECODED_SHAPE)
+
+
+def test_forward_rejects_unknown_mode():
+ vae = _StubVAE()
+ with pytest.raises(ValueError, match="Unknown SeedVR2 VAE forward mode"):
+ vae.forward(torch.zeros(*_INPUT_ENCODE_SHAPE), mode="bogus")
diff --git a/tests-unit/comfy_test/test_seedvr2_dtype.py b/tests-unit/comfy_test/test_seedvr2_dtype.py
new file mode 100644
index 000000000..8e08b6dde
--- /dev/null
+++ b/tests-unit/comfy_test/test_seedvr2_dtype.py
@@ -0,0 +1,50 @@
+import torch
+
+from comfy.cli_args import args as cli_args
+
+if not torch.cuda.is_available():
+ cli_args.cpu = True
+
+import comfy.sd
+import comfy.supported_models
+import comfy.ldm.seedvr.model as seedvr_model
+import comfy.ldm.seedvr.vae as seedvr_vae
+
+
+def test_seedvr2_fp16_manual_cast_only_for_bf16_device(monkeypatch):
+ bf16_device = object()
+ fp16_device = object()
+
+ monkeypatch.setattr(
+ comfy.supported_models.comfy.model_management,
+ "should_use_bf16",
+ lambda device=None: device is bf16_device,
+ )
+
+ bf16_config = comfy.supported_models.SeedVR2({"image_model": "seedvr2"})
+ bf16_config.set_inference_dtype(torch.float16, None, device=bf16_device)
+ assert bf16_config.manual_cast_dtype is torch.bfloat16
+
+ fp16_config = comfy.supported_models.SeedVR2({"image_model": "seedvr2"})
+ fp16_config.set_inference_dtype(torch.float16, None, device=fp16_device)
+ assert fp16_config.manual_cast_dtype is None
+
+
+def test_seedvr2_text_conditioning_accepts_cfg1_single_branch():
+ context = torch.arange(6, dtype=torch.float32).reshape(1, 3, 2)
+
+ txt, txt_shape = seedvr_model.NaDiT._resolve_text_conditioning(object(), context, [0])
+
+ torch.testing.assert_close(txt, context.squeeze(0))
+ torch.testing.assert_close(txt_shape, torch.tensor([[3]], device=context.device))
+
+
+def test_seedvr2_vae_decode_memory_covers_full_frame_lab_transfer():
+ wrapper = seedvr_vae.VideoAutoencoderKLWrapper.__new__(seedvr_vae.VideoAutoencoderKLWrapper)
+ latent_channels = seedvr_vae.SEEDVR2_LATENT_CHANNELS
+ estimate = wrapper.comfy_memory_used_decode((1, latent_channels, 26, 120, 160))
+ old_estimate = latent_channels * 120 * 160 * (4 * 8 * 8) * 2
+
+ assert estimate == 101 * 960 * 1280 * 160
+ assert estimate > 15 * 1024 ** 3
+ assert estimate > old_estimate * 100
diff --git a/tests-unit/comfy_test/test_seedvr2_internals.py b/tests-unit/comfy_test/test_seedvr2_internals.py
new file mode 100644
index 000000000..fe4bde1c4
--- /dev/null
+++ b/tests-unit/comfy_test/test_seedvr2_internals.py
@@ -0,0 +1,169 @@
+"""SeedVR2 internals regression tests."""
+
+from __future__ import annotations
+
+from unittest.mock import patch
+
+import pytest
+import torch
+
+from comfy.cli_args import args
+
+if not torch.cuda.is_available():
+ args.cpu = True
+
+import comfy.ldm.seedvr.model as seedvr_model # noqa: E402
+import comfy.ldm.seedvr.vae as vae_mod # noqa: E402
+import comfy.ldm.modules.attention as attention # noqa: E402
+import comfy.ops as comfy_ops # noqa: E402
+from comfy.ldm.seedvr.vae import ( # noqa: E402
+ causal_norm_wrapper,
+ set_norm_limit,
+)
+from comfy.ldm.seedvr.attention import var_attention_optimized_split # noqa: E402
+
+
+_NUM_CHANNELS = 8
+_NUM_GROUPS = 4
+_TENSOR_SHAPE = (1, 8, 2, 4, 4)
+
+_GROUPNORM_SUBCLASSES = [
+ pytest.param(comfy_ops.disable_weight_init.GroupNorm, id="disable_weight_init"),
+ pytest.param(comfy_ops.manual_cast.GroupNorm, id="manual_cast"),
+]
+
+
[email protected]("groupnorm_cls", _GROUPNORM_SUBCLASSES)
+def test_seedvr_groupnorm_low_limit_uses_chunked_groupnorm_path(groupnorm_cls):
+ real_group_norm = vae_mod.F.group_norm
+ set_norm_limit(1e-9)
+ try:
+ gn = groupnorm_cls(num_channels=_NUM_CHANNELS, num_groups=_NUM_GROUPS)
+ gn.eval()
+
+ forward_hook_calls = []
+
+ def _hook(module, inputs, output):
+ forward_hook_calls.append(tuple(inputs[0].shape))
+
+ spy_calls = []
+
+ def _group_norm_spy(input_tensor, num_groups_arg, *args, **kwargs):
+ spy_calls.append({"num_groups": int(num_groups_arg)})
+ return real_group_norm(input_tensor, num_groups_arg, *args, **kwargs)
+
+ handle = gn.register_forward_hook(_hook)
+ try:
+ with patch.object(vae_mod.F, "group_norm", side_effect=_group_norm_spy):
+ out_tensor = causal_norm_wrapper(gn, torch.randn(*_TENSOR_SHAPE))
+ finally:
+ handle.remove()
+
+ full_calls = len(forward_hook_calls)
+ chunked_calls = sum(1 for entry in spy_calls if entry["num_groups"] < _NUM_GROUPS)
+
+ assert tuple(int(s) for s in out_tensor.shape) == _TENSOR_SHAPE
+ assert full_calls == 0, (
+ f"low-limit GroupNorm gate must NOT take the full-forward path; got full_calls={full_calls}"
+ )
+ assert chunked_calls > 0, (
+ f"low-limit GroupNorm gate must take the chunked path; got chunked_calls={chunked_calls}"
+ )
+ finally:
+ set_norm_limit(None)
+
+
+def test_seedvr2_7b_swin_attention_forward_uses_optimized_var_attention(monkeypatch):
+ dim = 8
+ heads = 2
+ head_dim = 4
+ attn = seedvr_model.NaSwinAttention(
+ vid_dim=dim,
+ txt_dim=dim,
+ heads=heads,
+ head_dim=head_dim,
+ qk_bias=False,
+ qk_norm=comfy_ops.disable_weight_init.RMSNorm,
+ qk_norm_eps=1e-6,
+ rope_type=None,
+ rope_dim=head_dim,
+ shared_weights=False,
+ window=(2, 1, 1),
+ window_method="720pwin_by_size_bysize",
+ version=True,
+ device="cpu",
+ dtype=torch.float32,
+ operations=comfy_ops.disable_weight_init,
+ )
+ generator = torch.Generator(device="cpu").manual_seed(11)
+ vid = torch.randn(8, dim, generator=generator)
+ txt = torch.randn(3, dim, generator=generator)
+ vid_shape = torch.tensor([[2, 2, 2]], dtype=torch.long)
+ txt_shape = torch.tensor([[3]], dtype=torch.long)
+ calls = []
+
+ def fake_optimized_var_attention(**kwargs):
+ calls.append(kwargs)
+ return kwargs["q"]
+
+ monkeypatch.setattr(seedvr_model, "optimized_var_attention", fake_optimized_var_attention)
+
+ vid_out, txt_out = attn(vid, txt, vid_shape, txt_shape, seedvr_model.Cache(disable=True))
+
+ assert tuple(vid_out.shape) == (8, dim)
+ assert tuple(txt_out.shape) == (3, dim)
+ assert len(calls) == 1
+ call = calls[0]
+ assert tuple(call["q"].shape) == (14, heads, head_dim)
+ assert tuple(call["k"].shape) == (14, heads, head_dim)
+ assert tuple(call["v"].shape) == (14, heads, head_dim)
+ assert call["heads"] == heads
+ assert call["skip_reshape"] is True
+ assert call["skip_output_reshape"] is True
+ assert call["cu_seqlens_q"] == [0, 7, 14]
+ assert call["cu_seqlens_k"] == [0, 7, 14]
+
+
+def test_var_attention_optimized_split_calls_dense_backend_per_window(monkeypatch):
+ heads = 2
+ head_dim = 3
+ q = torch.arange(30, dtype=torch.float32).reshape(5, heads, head_dim)
+ k = q + 100
+ v = q + 200
+ cu = [0, 2, 5]
+ calls = []
+
+ def fake_optimized_attention(q_arg, k_arg, v_arg, heads_arg, **kwargs):
+ calls.append(
+ {
+ "q_shape": tuple(q_arg.shape),
+ "k_shape": tuple(k_arg.shape),
+ "v_shape": tuple(v_arg.shape),
+ "heads": heads_arg,
+ "kwargs": kwargs,
+ }
+ )
+ return q_arg + v_arg
+
+ monkeypatch.setattr(attention, "optimized_attention", fake_optimized_attention)
+
+ out = var_attention_optimized_split(
+ q,
+ k,
+ v,
+ heads,
+ cu,
+ cu,
+ skip_reshape=True,
+ skip_output_reshape=True,
+ )
+
+ assert tuple(out.shape) == (5, heads, head_dim)
+ assert len(calls) == 2
+ assert calls[0]["q_shape"] == (1, heads, 2, head_dim)
+ assert calls[1]["q_shape"] == (1, heads, 3, head_dim)
+ assert all(call["heads"] == heads for call in calls)
+ assert all(call["kwargs"]["skip_reshape"] is True for call in calls)
+ assert all(call["kwargs"]["skip_output_reshape"] is True for call in calls)
+ torch.testing.assert_close(out, q + v, rtol=0, atol=0)
+
diff --git a/tests-unit/comfy_test/test_seedvr2_model.py b/tests-unit/comfy_test/test_seedvr2_model.py
new file mode 100644
index 000000000..1d454aaf1
--- /dev/null
+++ b/tests-unit/comfy_test/test_seedvr2_model.py
@@ -0,0 +1,320 @@
+"""SeedVR2 model, latent-format, and VAE graph regression tests."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+import pytest
+import torch
+from torch import nn
+
+from comfy.cli_args import args
+
+if not torch.cuda.is_available():
+ args.cpu = True
+
+import comfy # noqa: E402
+import comfy.latent_formats # noqa: E402
+import comfy.ldm.seedvr.model as seedvr_model # noqa: E402
+import comfy.ldm.seedvr.vae as seedvr_vae_mod # noqa: E402
+import comfy.model_management # noqa: E402
+import comfy.ops as comfy_ops # noqa: E402
+import comfy.sample # noqa: E402
+import comfy.sd as sd_mod # noqa: E402
+import nodes as nodes_mod # noqa: E402
+from comfy.ldm.seedvr.model import NaDiT # noqa: E402
+
+
+_LATENT_CHANNELS = seedvr_vae_mod.SEEDVR2_LATENT_CHANNELS
+
+
+def _make_standin(positive_conditioning):
+ class _StandIn(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.register_buffer(
+ "positive_conditioning", positive_conditioning
+ )
+
+ _resolve_text_conditioning = NaDiT._resolve_text_conditioning
+
+ return _StandIn()
+
+
+class _StubModule(nn.Module):
+ def __init__(self, *args, **kwargs):
+ super().__init__()
+
+
+def _capture_last_layer_flags(monkeypatch, vid_dim: int, txt_in_dim: int) -> list[bool]:
+ flags = []
+
+ class _Block(_StubModule):
+ def __init__(self, *args, **kwargs):
+ flags.append(kwargs["is_last_layer"])
+ super().__init__()
+
+ monkeypatch.setattr(seedvr_model, "NaPatchIn", _StubModule)
+ monkeypatch.setattr(seedvr_model, "NaPatchOut", _StubModule)
+ monkeypatch.setattr(seedvr_model, "TimeEmbedding", _StubModule)
+ monkeypatch.setattr(seedvr_model, "NaMMSRTransformerBlock", _Block)
+
+ seedvr_model.NaDiT(
+ norm_eps=1e-5,
+ num_layers=4,
+ mlp_type="normal",
+ vid_dim=vid_dim,
+ txt_in_dim=txt_in_dim,
+ heads=24,
+ mm_layers=3,
+ operations=comfy_ops.disable_weight_init,
+ )
+
+ return flags
+
+
+class _Model:
+ def __init__(self, latent_format):
+ self._latent_format = latent_format
+
+ def get_model_object(self, name):
+ assert name == "latent_format"
+ return self._latent_format
+
+
+class _Patcher:
+ def get_free_memory(self, device):
+ return 1024 * 1024 * 1024
+
+
+class _EncodeWrapper(seedvr_vae_mod.VideoAutoencoderKLWrapper):
+ def __init__(self, encoded):
+ nn.Module.__init__(self)
+ self.encoded = encoded
+ self.spatial_downsample_factor = 8
+ self.temporal_downsample_factor = 4
+ self.seen = []
+
+ def encode(self, x):
+ self.seen.append(tuple(x.shape))
+ return self.encoded.to(device=x.device, dtype=x.dtype)
+
+
+class _DecodeWrapper(seedvr_vae_mod.VideoAutoencoderKLWrapper):
+ def __init__(self):
+ nn.Module.__init__(self)
+ self.spatial_downsample_factor = 8
+ self.temporal_downsample_factor = 4
+ self.calls = []
+
+ def decode(self, z, seedvr2_tiling=None):
+ self.calls.append({"shape": tuple(z.shape), "seedvr2_tiling": seedvr2_tiling})
+ if z.ndim == 4:
+ b, tc, h, w = z.shape
+ t = tc // _LATENT_CHANNELS
+ else:
+ b, _, t, h, w = z.shape
+ return torch.zeros(b, 3, t, h * 8, w * 8, dtype=z.dtype, device=z.device)
+
+
+def test_seedvr2_wrapper_public_encode_returns_tensor(monkeypatch):
+ raw_latent = torch.full((1, _LATENT_CHANNELS, 1, 4, 5), 2.0)
+ seen_shapes = []
+
+ def base_encode(self, x):
+ seen_shapes.append(tuple(x.shape))
+ return raw_latent.to(device=x.device, dtype=x.dtype)
+
+ monkeypatch.setattr(seedvr_vae_mod.VideoAutoencoderKL, "encode", base_encode)
+
+ vae = seedvr_vae_mod.VideoAutoencoderKLWrapper.__new__(seedvr_vae_mod.VideoAutoencoderKLWrapper)
+ nn.Module.__init__(vae)
+ vae._dummy = nn.Parameter(torch.zeros((), dtype=torch.float32))
+
+ latent = vae.encode(torch.zeros(1, 3, 32, 40))
+
+ assert type(latent) is torch.Tensor
+ assert tuple(latent.shape) == (1, _LATENT_CHANNELS, 4, 5)
+ assert seen_shapes == [(1, 3, 1, 32, 40)]
+
+
+def test_seedvr2_wrapper_private_encode_helper_keeps_raw_latent(monkeypatch):
+ raw_latent = torch.full((1, _LATENT_CHANNELS, 1, 4, 5), 3.0)
+
+ def base_encode(self, x):
+ return raw_latent.to(device=x.device, dtype=x.dtype)
+
+ monkeypatch.setattr(seedvr_vae_mod.VideoAutoencoderKL, "encode", base_encode)
+
+ vae = seedvr_vae_mod.VideoAutoencoderKLWrapper.__new__(seedvr_vae_mod.VideoAutoencoderKLWrapper)
+ nn.Module.__init__(vae)
+ vae._dummy = nn.Parameter(torch.zeros((), dtype=torch.float32))
+
+ latent, raw = vae._encode_with_raw_latent(torch.zeros(1, 3, 32, 40))
+
+ assert tuple(latent.shape) == (1, _LATENT_CHANNELS, 4, 5)
+ assert tuple(raw.shape) == (1, _LATENT_CHANNELS, 1, 4, 5)
+ assert torch.equal(raw, raw_latent)
+
+
+def _make_vae(wrapper):
+ vae = sd_mod.VAE.__new__(sd_mod.VAE)
+ vae.first_stage_model = wrapper
+ vae.device = torch.device("cpu")
+ vae.output_device = torch.device("cpu")
+ vae.vae_dtype = torch.float32
+ vae.latent_channels = _LATENT_CHANNELS
+ vae.latent_dim = 3
+ vae.downscale_ratio = (lambda a: max(0, (a + 3) // 4), 8, 8)
+ vae.upscale_ratio = (lambda a: max(0, a * 4 - 3), 8, 8)
+ vae.output_channels = 3
+ vae.disable_offload = True
+ vae.extra_1d_channel = None
+ vae.crop_input = False
+ vae.not_video = False
+ vae.handles_tiling = isinstance(wrapper, seedvr_vae_mod.VideoAutoencoderKLWrapper)
+ vae.format_encoded = wrapper.comfy_format_encoded
+ vae.patcher = _Patcher()
+ vae.process_input = lambda image: image
+ vae.process_output = lambda image: image.add(1.0).div(2.0).clamp(0.0, 1.0)
+ vae.vae_output_dtype = lambda: torch.float32
+ vae.memory_used_encode = lambda shape, dtype: 1
+ vae.memory_used_decode = lambda shape, dtype: 1
+ vae.throw_exception_if_invalid = lambda: None
+ vae.vae_encode_crop_pixels = lambda pixels: pixels
+ vae.spacial_compression_decode = lambda: 8
+ vae.temporal_compression_decode = lambda: 4
+ return vae
+
+
+def test_missing_context_falls_back_to_positive_buffer():
+ pos_buffer = torch.full((58, 5120), 7.0)
+ standin = _make_standin(pos_buffer)
+ txt, txt_shape = standin._resolve_text_conditioning(None)
+ assert txt.shape == (58, 5120)
+ assert (txt == 7.0).all(), (
+ "fallback path must use the positive_conditioning buffer "
+ "verbatim, not a zero tensor"
+ )
+ assert txt_shape.shape == (1, 1)
+ assert txt_shape[0, 0].item() == 58
+
+
+def test_seedvr2_7b_keeps_final_block_text_path(monkeypatch):
+ assert _capture_last_layer_flags(monkeypatch, vid_dim=3072, txt_in_dim=3072) == [
+ False,
+ False,
+ False,
+ False,
+ ]
+
+
+def test_seedvr2_7b_rope3d_matches_wrapper_oracle():
+ rope = seedvr_model.get_na_rope("rope3d", dim=64)
+ generator = torch.Generator(device="cpu").manual_seed(0)
+ q = torch.randn(4, 2, 128, generator=generator)
+ k = torch.randn(4, 2, 128, generator=generator)
+ shape = torch.tensor([[1, 2, 2]], dtype=torch.long)
+ freqs = rope.get_axial_freqs(1, 2, 2).reshape(4, -1)
+
+ expected_q = seedvr_model._apply_seedvr2_rotary_emb(
+ freqs,
+ q.permute(1, 0, 2).float(),
+ ).to(q.dtype).permute(1, 0, 2)
+ expected_k = seedvr_model._apply_seedvr2_rotary_emb(
+ freqs,
+ k.permute(1, 0, 2).float(),
+ ).to(k.dtype).permute(1, 0, 2)
+
+ actual_q, actual_k = rope(q.clone(), k.clone(), shape, seedvr_model.Cache(disable=True))
+
+ torch.testing.assert_close(actual_q, expected_q, rtol=0, atol=0)
+ torch.testing.assert_close(actual_k, expected_k, rtol=0, atol=0)
+
+
+def test_seedvr2_forward_requires_conditioning_latents():
+ model = NaDiT.__new__(NaDiT)
+ x = torch.zeros(1, _LATENT_CHANNELS, 1, 4, 5)
+
+ with pytest.raises(ValueError, match="requires conditioning latents"):
+ NaDiT.forward(model, x, timestep=torch.tensor([1.0]), context=None)
+
+
+def test_seedvr2_latent_format_uses_native_video_latent_shape():
+ latent_format = comfy.latent_formats.SeedVR2()
+ latent_image = torch.zeros(1, 1, 4, 5)
+
+ fixed = comfy.sample.fix_empty_latent_channels(_Model(latent_format), latent_image)
+
+ assert latent_format.latent_channels == _LATENT_CHANNELS
+ assert latent_format.latent_dimensions == 3
+ assert fixed.shape == (1, _LATENT_CHANNELS, 1, 4, 5)
+
+
+def test_seedvr2_model_requires_native_5d_latent():
+ latent = torch.zeros(1, _LATENT_CHANNELS, 2, 4, 5)
+ assert NaDiT._check_seedvr2_video_latent(latent, _LATENT_CHANNELS, "latent") is latent
+
+ with pytest.raises(ValueError, match="5-D native latent"):
+ NaDiT._check_seedvr2_video_latent(torch.zeros(1, _LATENT_CHANNELS * 2, 4, 5), _LATENT_CHANNELS, "latent")
+
+
+def test_seedvr2_encode_and_encode_tiled_preserve_native_latent_contract(monkeypatch):
+ monkeypatch.setattr(sd_mod.model_management, "load_models_gpu", lambda *a, **k: None)
+
+ encoded = torch.full((1, _LATENT_CHANNELS, 2, 4, 5), 2.0)
+ vae = _make_vae(_EncodeWrapper(encoded))
+ pixels = torch.zeros(1, 5, 32, 40, 3)
+
+ node_output = nodes_mod.VAEEncode().encode(vae, pixels)[0]
+ node_latent = node_output["samples"]
+ assert set(node_output) == {"samples"}
+ assert tuple(node_latent.shape) == (1, _LATENT_CHANNELS, 2, 4, 5)
+ assert node_latent.dtype == torch.float32
+ assert node_latent.stride()[-1] == 1
+ assert torch.equal(node_latent, torch.full_like(node_latent, 2.0 * seedvr_vae_mod.BYTEDANCE_VAE_SCALING_FACTOR))
+
+ tiled = torch.full((1, _LATENT_CHANNELS, 2, 4, 5), 3.0)
+ monkeypatch.setattr(seedvr_vae_mod, "tiled_vae", MagicMock(return_value=tiled))
+ tiled_output = nodes_mod.VAEEncodeTiled().encode(
+ vae,
+ pixels,
+ tile_size=512,
+ overlap=64,
+ temporal_size=16,
+ temporal_overlap=4,
+ )[0]
+ tiled_latent = tiled_output["samples"]
+ assert set(tiled_output) == {"samples"}
+ assert tuple(tiled_latent.shape) == (1, _LATENT_CHANNELS, 2, 4, 5)
+ assert tiled_latent.dtype == torch.float32
+ assert torch.equal(tiled_latent, torch.full_like(tiled_latent, 3.0 * seedvr_vae_mod.BYTEDANCE_VAE_SCALING_FACTOR))
+
+
+def test_vaedecode_tiled_spatial_applies_temporal_discarded(monkeypatch):
+ monkeypatch.setattr(sd_mod.model_management, "load_models_gpu", lambda *a, **k: None)
+ vae = _make_vae(_DecodeWrapper())
+
+ nodes_mod.VAEDecodeTiled().decode(
+ vae,
+ {"samples": torch.zeros(1, _LATENT_CHANNELS, 2, 4, 5)},
+ tile_size=512,
+ overlap=64,
+ temporal_size=16,
+ temporal_overlap=4,
+ )
+
+ # Spatial inputs flow through; temporal inputs are discarded as public tiling
+ # knobs, but SeedVR2's internal MemoryState causal slicing is left intact.
+ assert vae.first_stage_model.calls == [
+ {
+ "shape": (1, _LATENT_CHANNELS, 2, 4, 5),
+ "seedvr2_tiling": {
+ "enable_tiling": True,
+ "tile_size": (512, 512),
+ "tile_overlap": (64, 64),
+ "temporal_size": None,
+ "temporal_overlap": None,
+ },
+ }
+ ]
diff --git a/tests-unit/comfy_test/test_seedvr2_vae_decode.py b/tests-unit/comfy_test/test_seedvr2_vae_decode.py
new file mode 100644
index 000000000..c486b9195
--- /dev/null
+++ b/tests-unit/comfy_test/test_seedvr2_vae_decode.py
@@ -0,0 +1,94 @@
+from unittest.mock import patch
+
+import pytest
+import torch
+import torch.nn as nn
+
+from comfy.cli_args import args as cli_args
+
+if not torch.cuda.is_available():
+ cli_args.cpu = True
+
+import comfy.ldm.seedvr.vae as vae_mod # noqa: E402
+from comfy_extras import nodes_seedvr # noqa: E402
+
+
+_LATENT_CHANNELS = vae_mod.SEEDVR2_LATENT_CHANNELS
+
+
+def _make_wrapper() -> vae_mod.VideoAutoencoderKLWrapper:
+ wrapper = vae_mod.VideoAutoencoderKLWrapper.__new__(
+ vae_mod.VideoAutoencoderKLWrapper
+ )
+ nn.Module.__init__(wrapper)
+ return wrapper
+
+
+def _fingerprint_decode_(self, z, return_dict=True):
+ b = int(z.shape[0])
+ t = int(z.shape[2])
+ h = int(z.shape[3])
+ w = int(z.shape[4])
+ out = torch.empty(b, 3, t, h * 8, w * 8)
+ for batch_idx in range(b):
+ out[batch_idx].fill_(float(batch_idx + 1))
+ return out
+
+
+def _decode_with_patches(wrapper, z):
+ with patch.object(vae_mod.VideoAutoencoderKL, "decode_", _fingerprint_decode_):
+ return wrapper.decode(z)
+
+
+def test_decode_b2_t3_multi_frame_batch_unchanged():
+ wrapper = _make_wrapper()
+
+ out = _decode_with_patches(wrapper, torch.zeros(2, _LATENT_CHANNELS * 3, 2, 2))
+
+ assert tuple(out.shape) == (2, 3, 3, 16, 16)
+
+
+class _Wrapper(vae_mod.VideoAutoencoderKLWrapper):
+ def __init__(self):
+ nn.Module.__init__(self)
+ self.calls = []
+
+ def parameters(self):
+ return iter([torch.nn.Parameter(torch.zeros(()))])
+
+def _decode_stub(self, latent):
+ self.calls.append(tuple(latent.shape))
+ return torch.zeros(latent.shape[0], 3, latent.shape[2], latent.shape[3] * 8, latent.shape[4] * 8)
+
+
+def test_seedvr2_wrapper_decode_accepts_5d_channel_first_latents_without_preprocessor_state():
+ wrapper = _Wrapper()
+
+ with patch.object(vae_mod.VideoAutoencoderKL, "decode_", _decode_stub):
+ out = wrapper.decode(torch.zeros(1, _LATENT_CHANNELS, 2, 4, 5))
+
+ assert tuple(out.shape) == (1, 3, 2, 32, 40)
+ assert wrapper.calls == [(1, _LATENT_CHANNELS, 2, 4, 5)]
+
+
+def test_seedvr2_wrapper_decode_rejects_wrong_rank_latents():
+ wrapper = _Wrapper()
+
+ with pytest.raises(RuntimeError, match=r"latent input must be 4-D collapsed .* or 5-D"):
+ wrapper.decode(torch.zeros(1, _LATENT_CHANNELS, 4))
+
+
+def _t_padded(t_in: int) -> int:
+ if t_in == 1:
+ return 1
+ if t_in <= 4:
+ return 5
+ if (t_in - 1) % 4 == 0:
+ return t_in
+ return t_in + (4 - ((t_in - 1) % 4))
+
+
[email protected]("t_in", [1, 5, 9])
+def test_t_padded_matches_cut_videos(t_in):
+ dummy = torch.zeros(1, t_in, 1, 1, 1)
+ assert nodes_seedvr.cut_videos(dummy).shape[1] == _t_padded(t_in)
diff --git a/tests-unit/comfy_test/test_seedvr2_vae_tiled.py b/tests-unit/comfy_test/test_seedvr2_vae_tiled.py
new file mode 100644
index 000000000..a2866b609
--- /dev/null
+++ b/tests-unit/comfy_test/test_seedvr2_vae_tiled.py
@@ -0,0 +1,382 @@
+from contextlib import ExitStack
+from unittest.mock import MagicMock, patch
+
+import pytest
+import torch
+import torch.nn as nn
+
+from comfy.cli_args import args as cli_args
+
+if not torch.cuda.is_available():
+ cli_args.cpu = True
+
+import comfy.ldm.seedvr.vae as vae_mod # noqa: E402
+import comfy.ldm.seedvr.vae as seedvr_vae_mod # noqa: E402
+import comfy.sd as sd_mod # noqa: E402
+from comfy.ldm.seedvr.vae import MemoryState, tiled_vae # noqa: E402
+
+
+_LATENT_CHANNELS = seedvr_vae_mod.SEEDVR2_LATENT_CHANNELS
+
+
+def test_runtime_decode_zero_temporal_size_preserves_model_slicing():
+ class StubVAEModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.slicing_latent_min_size = 2
+ self.spatial_downsample_factor = 8
+ self.temporal_downsample_factor = 4
+ self.device = torch.device("cpu")
+ self.use_slicing = True
+ self._dummy = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32))
+ self.decode_min_sizes = []
+ self.memory_states = []
+
+ def decode_(self, t_chunk):
+ self.decode_min_sizes.append(self.slicing_latent_min_size)
+ return vae_mod.VideoAutoencoderKL.slicing_decode(self, t_chunk)
+
+ def _decode(self, z, memory_state=MemoryState.DISABLED, memory_cache=None):
+ self.memory_states.append(memory_state)
+ b, c, d, h, w = z.shape
+ return torch.zeros((b, 3, d, h * 8, w * 8), dtype=z.dtype)
+
+ vae = StubVAEModel()
+ z = torch.zeros((1, _LATENT_CHANNELS, 5, 8, 8), dtype=torch.float32)
+
+ tiled_vae(
+ z,
+ vae,
+ tile_size=(64, 64),
+ tile_overlap=(0, 0),
+ temporal_size=0,
+ temporal_overlap=0,
+ encode=False,
+ )
+
+ assert vae.decode_min_sizes == [2]
+ assert vae.memory_states == [MemoryState.INITIALIZING, MemoryState.ACTIVE]
+ assert vae.slicing_latent_min_size == 2
+
+
+def test_zero_temporal_size_preserves_min_size_when_encode_raises():
+ class RaisingVAEModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.slicing_sample_min_size = 4
+ self.spatial_downsample_factor = 8
+ self.temporal_downsample_factor = 4
+ self.device = torch.device("cpu")
+ self._dummy = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32))
+
+ def encode(self, t_chunk):
+ raise RuntimeError("simulated encode failure")
+
+ vae = RaisingVAEModel()
+ x = torch.zeros((1, 3, 12, 64, 64), dtype=torch.float32)
+
+ with pytest.raises(RuntimeError, match="simulated encode failure"):
+ tiled_vae(
+ x,
+ vae,
+ tile_size=(64, 64),
+ tile_overlap=(0, 0),
+ temporal_size=0,
+ temporal_overlap=0,
+ encode=True,
+ )
+
+ assert vae.slicing_sample_min_size == 4
+
+
+def test_tiled_vae_encode_uses_tensor_return_without_indexing():
+ class TensorEncodeVAEModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.slicing_sample_min_size = 4
+ self.spatial_downsample_factor = 8
+ self.temporal_downsample_factor = 4
+ self.device = torch.device("cpu")
+ self._dummy = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32))
+ self.calls = []
+
+ def encode(self, t_chunk):
+ self.calls.append(tuple(t_chunk.shape))
+ b, _, _, h, w = t_chunk.shape
+ return torch.ones((b, _LATENT_CHANNELS, 1, h // 8, w // 8), dtype=t_chunk.dtype)
+
+ vae = TensorEncodeVAEModel()
+ x = torch.zeros((2, 3, 1, 64, 64), dtype=torch.float32)
+
+ out = tiled_vae(
+ x,
+ vae,
+ tile_size=(64, 64),
+ tile_overlap=(0, 0),
+ temporal_size=0,
+ temporal_overlap=0,
+ encode=True,
+ )
+
+ assert vae.calls == [(2, 3, 1, 64, 64)]
+ assert tuple(out.shape) == (2, _LATENT_CHANNELS, 1, 8, 8)
+
+
+def test_tiled_vae_preserves_input_dtype_on_single_tile():
+ class FloatOutputVAEModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.slicing_sample_min_size = 4
+ self.spatial_downsample_factor = 8
+ self.temporal_downsample_factor = 4
+ self.device = torch.device("cpu")
+ self._dummy = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32))
+
+ def encode(self, t_chunk):
+ b, _, _, h, w = t_chunk.shape
+ return torch.ones((b, _LATENT_CHANNELS, 1, h // 8, w // 8), dtype=torch.float32)
+
+ out = tiled_vae(
+ torch.zeros((1, 3, 1, 64, 64), dtype=torch.float16),
+ FloatOutputVAEModel(),
+ tile_size=(64, 64),
+ tile_overlap=(0, 0),
+ temporal_size=0,
+ temporal_overlap=0,
+ encode=True,
+ )
+
+ assert out.dtype == torch.float16
+
+
+class _SlicingDecodeVAE(nn.Module):
+ def __init__(self, slicing_latent_min_size):
+ super().__init__()
+ self.slicing_latent_min_size = slicing_latent_min_size
+ self.spatial_downsample_factor = 8
+ self.temporal_downsample_factor = 4
+ self.device = torch.device("cpu")
+ self.use_slicing = True
+ self._dummy = nn.Parameter(torch.zeros(1, dtype=torch.float32))
+ self.decode_min_sizes = []
+ self.memory_states = []
+
+ def decode_(self, z):
+ self.decode_min_sizes.append(self.slicing_latent_min_size)
+ return vae_mod.VideoAutoencoderKL.slicing_decode(self, z)
+
+ def _decode(self, z, memory_state=MemoryState.DISABLED, memory_cache=None):
+ self.memory_states.append(memory_state)
+ x = z[:, :1].repeat(
+ 1,
+ 3,
+ 1,
+ self.spatial_downsample_factor,
+ self.spatial_downsample_factor,
+ )
+ return x
+
+
+def test_decode_tiled_vae_maps_temporal_args_to_latent_slicing_min_size():
+ vae = _SlicingDecodeVAE(slicing_latent_min_size=2)
+ z = torch.arange(
+ _LATENT_CHANNELS * 5 * 8 * 8,
+ dtype=torch.float32,
+ ).reshape(1, _LATENT_CHANNELS, 5, 8, 8)
+
+ tiled_vae(
+ z,
+ vae,
+ tile_size=(64, 64),
+ tile_overlap=(0, 0),
+ temporal_size=12,
+ temporal_overlap=4,
+ encode=False,
+ )
+
+ assert vae.decode_min_sizes == [2]
+ assert vae.memory_states == [MemoryState.INITIALIZING, MemoryState.ACTIVE]
+ assert vae.slicing_latent_min_size == 2
+
+ wrapper = vae_mod.VideoAutoencoderKLWrapper.__new__(
+ vae_mod.VideoAutoencoderKLWrapper
+ )
+ nn.Module.__init__(wrapper)
+ seedvr2_tiling = {
+ "enable_tiling": True,
+ "tile_size": (64, 64),
+ "tile_overlap": (0, 0),
+ "temporal_size": 8,
+ "temporal_overlap": 7,
+ }
+
+ captured = {}
+
+ def _fake_tiled_vae(latent, model, **kwargs):
+ captured.update(kwargs)
+ return torch.zeros(1, 3, 1, 16, 16)
+
+ with patch.object(vae_mod, "tiled_vae", side_effect=_fake_tiled_vae):
+ wrapper.decode(torch.zeros(1, _LATENT_CHANNELS, 2, 2), seedvr2_tiling=seedvr2_tiling)
+
+ assert captured["temporal_overlap"] == 7
+
+
+def _force_oom(*a, **k):
+ raise torch.cuda.OutOfMemoryError("forced OOM for dispatcher test")
+
+
+def _make_vae(first_stage_model, latent_channels, latent_dim):
+ vae = sd_mod.VAE.__new__(sd_mod.VAE)
+ vae.first_stage_model = first_stage_model
+ vae.patcher = MagicMock()
+ vae.patcher.get_free_memory = MagicMock(return_value=8 * 1024 * 1024 * 1024)
+ vae.device = vae.output_device = torch.device("cpu")
+ vae.vae_dtype = torch.float32
+ vae.disable_offload = True
+ vae.extra_1d_channel = None
+ vae.upscale_ratio = vae.downscale_ratio = 8
+ vae.upscale_index_formula = vae.downscale_index_formula = None
+ vae.output_channels = 3
+ vae.latent_channels = latent_channels
+ vae.latent_dim = latent_dim
+ vae.vae_output_dtype = lambda: torch.float32
+ vae.spacial_compression_decode = lambda: 8
+ vae.handles_tiling = isinstance(first_stage_model, seedvr_vae_mod.VideoAutoencoderKLWrapper)
+ vae.format_encoded = None
+ vae.process_input = lambda x: x
+ vae.process_output = lambda x: x
+ vae.throw_exception_if_invalid = lambda: None
+ vae.memory_used_decode = lambda *a, **k: 1
+ return vae
+
+
+def _dispatch(vae, samples, seedvr2_call, generic_call, patch_wrapper_decode):
+ mm = sd_mod.model_management
+ with ExitStack() as stack:
+ stack.enter_context(patch.object(mm, "raise_non_oom", lambda e: None))
+ stack.enter_context(patch.object(mm, "load_models_gpu", lambda *a, **k: None))
+ stack.enter_context(patch.object(mm, "soft_empty_cache", lambda: None))
+ stack.enter_context(patch.object(sd_mod.VAE, "_decode_tiled_owned", seedvr2_call))
+ stack.enter_context(patch.object(sd_mod.VAE, "decode_tiled_", generic_call))
+ if patch_wrapper_decode:
+ stack.enter_context(patch.object(
+ seedvr_vae_mod.VideoAutoencoderKLWrapper, "decode",
+ side_effect=_force_oom))
+ vae.decode(samples)
+
+
+def test_4d_seedvr2_latent_routes_to_owned_decode_tiled():
+ wrapper = seedvr_vae_mod.VideoAutoencoderKLWrapper.__new__(
+ seedvr_vae_mod.VideoAutoencoderKLWrapper)
+ vae = _make_vae(wrapper, latent_channels=_LATENT_CHANNELS, latent_dim=3)
+ seedvr2_call = MagicMock(return_value=torch.zeros(1, 3, 9, 64, 64))
+ generic_call = MagicMock(return_value=torch.zeros(1, 3, 64, 64))
+ _dispatch(vae, torch.zeros(1, _LATENT_CHANNELS * 3, 8, 8), seedvr2_call, generic_call, True)
+ assert seedvr2_call.call_count == 1
+ assert generic_call.call_count == 0
+
+
+def test_4d_non_seedvr2_latent_still_routes_to_generic_decode_tiled():
+ first_stage = MagicMock()
+ first_stage.decode = MagicMock(side_effect=_force_oom)
+ vae = _make_vae(first_stage, latent_channels=4, latent_dim=2)
+ seedvr2_call = MagicMock(return_value=torch.zeros(1, 3, 9, 64, 64))
+ generic_call = MagicMock(return_value=torch.zeros(1, 3, 64, 64))
+ _dispatch(vae, torch.zeros(1, 4, 8, 8), seedvr2_call, generic_call, False)
+ assert generic_call.call_count == 1
+ assert seedvr2_call.call_count == 0
+
+
+def _populate_common_vae_attrs_fallback(vae):
+ vae.patcher = MagicMock()
+ vae.patcher.get_free_memory = MagicMock(return_value=8 * 1024 * 1024 * 1024)
+ vae.device = torch.device("cpu")
+ vae.output_device = torch.device("cpu")
+ vae.vae_dtype = torch.float32
+ vae.disable_offload = True
+ vae.extra_1d_channel = None
+ vae.upscale_ratio = 8
+ vae.upscale_index_formula = None
+ vae.output_channels = 3
+ vae.latent_channels = _LATENT_CHANNELS
+ vae.latent_dim = 3
+ vae.downscale_ratio = 8
+ vae.downscale_index_formula = None
+ vae.not_video = False
+ vae.crop_input = False
+ vae.pad_channel_value = None
+ vae.handles_tiling = isinstance(vae.first_stage_model, seedvr_vae_mod.VideoAutoencoderKLWrapper)
+ vae.format_encoded = None
+
+ vae.vae_output_dtype = lambda: torch.float32
+ vae.spacial_compression_encode = lambda: 8
+ vae.process_input = lambda x: x
+ vae.process_output = lambda x: x
+ vae.throw_exception_if_invalid = lambda: None
+ vae.memory_used_encode = lambda *a, **k: 1
+
+
+def _make_seedvr2_vae_fallback():
+ vae = sd_mod.VAE.__new__(sd_mod.VAE)
+ wrapper = seedvr_vae_mod.VideoAutoencoderKLWrapper.__new__(
+ seedvr_vae_mod.VideoAutoencoderKLWrapper
+ )
+ vae.first_stage_model = wrapper
+ _populate_common_vae_attrs_fallback(vae)
+ return vae
+
+
+def _make_non_seedvr2_vae_fallback():
+ vae = sd_mod.VAE.__new__(sd_mod.VAE)
+ vae.first_stage_model = MagicMock()
+ _populate_common_vae_attrs_fallback(vae)
+ return vae
+
+
+def _force_regular_encode_oom(*args, **kwargs):
+ raise torch.cuda.OutOfMemoryError("forced OOM for dispatcher test")
+
+
+def test_seedvr2_3d_routes_to_owned_encode_tiled_on_oom():
+ vae = _make_seedvr2_vae_fallback()
+ pixel_samples = torch.zeros((1, 8, 64, 64, 3))
+
+ seedvr2_call = MagicMock(return_value=torch.zeros(1, _LATENT_CHANNELS, 2, 8, 8))
+ generic_call = MagicMock(return_value=torch.zeros(1, _LATENT_CHANNELS, 2, 8, 8))
+
+ with patch.object(sd_mod.model_management, "raise_non_oom",
+ lambda e: None), \
+ patch.object(sd_mod.model_management, "load_models_gpu",
+ lambda *a, **k: None), \
+ patch.object(sd_mod.model_management, "soft_empty_cache",
+ lambda: None), \
+ patch.object(seedvr_vae_mod.VideoAutoencoderKLWrapper, "encode",
+ side_effect=_force_regular_encode_oom), \
+ patch.object(sd_mod.VAE, "_encode_tiled_owned", seedvr2_call), \
+ patch.object(sd_mod.VAE, "encode_tiled_3d", generic_call):
+ vae.encode(pixel_samples)
+
+ assert seedvr2_call.call_count == 1, (
+ f"Expected _encode_tiled_owned to be called once for a SeedVR2 3D "
+ f"input under OOM fallback; got {seedvr2_call.call_count} calls."
+ )
+ assert generic_call.call_count == 0, (
+ f"encode_tiled_3d must NOT be called for a SeedVR2 input; got "
+ f"{generic_call.call_count} calls."
+ )
+
+
+def test_non_seedvr2_encode_tiled_3d_default_overlap_is_concrete():
+ vae = _make_non_seedvr2_vae_fallback()
+ vae.downscale_ratio = (lambda a: max(1, a // 4), 8, 8)
+ vae.upscale_ratio = (lambda a: a * 4, 8, 8)
+ generic_call = MagicMock(return_value=torch.zeros(1, _LATENT_CHANNELS, 2, 8, 8))
+ pixel_samples = torch.zeros((1, 8, 64, 64, 3))
+
+ with patch.object(sd_mod.model_management, "load_models_gpu",
+ lambda *a, **k: None), \
+ patch.object(sd_mod.VAE, "encode_tiled_3d", generic_call):
+ vae.encode_tiled(pixel_samples)
+
+ assert generic_call.call_args.kwargs["overlap"] == (1, 64, 64)