The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add SeedVR2 support (CORE-6) (#14110)
John Pollock7863cf0e53ca
diff --git a/comfy/latent_formats.py b/comfy/latent_formats.pyindex bbdfd4bc2..fcbd97c59 100644--- a/comfy/latent_formats.py+++ b/comfy/latent_formats.py@@ -4,6 +4,7 @@ class LatentFormat:scale_factor = 1.0latent_channels = 4latent_dimensions = 2+ preserve_empty_channel_multiples = Falselatent_rgb_factors = Nonelatent_rgb_factors_bias = Nonelatent_rgb_factors_reshape = None@@ -779,6 +780,10 @@ class ACEAudio(LatentFormat):latent_channels = 8latent_dimensions = 2+class SeedVR2(LatentFormat):+ latent_channels = 16+ preserve_empty_channel_multiples = True+class ACEAudio15(LatentFormat):latent_channels = 64latent_dimensions = 1diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.pyindex 55360535a..b78e764c7 100644--- a/comfy/ldm/modules/attention.py+++ b/comfy/ldm/modules/attention.py@@ -735,7 +735,86 @@ def attention_flash(q, k, v, heads, mask=None, attn_precision=None, skip_reshape)return out+def _var_attention_qkv(q, k, v, heads, skip_reshape):+ if skip_reshape:+ return q, k, v, q.shape[-1]+ total_tokens, embed_dim = q.shape+ head_dim = embed_dim // heads+ return (+ q.view(total_tokens, heads, head_dim),+ k.view(k.shape[0], heads, head_dim),+ v.view(v.shape[0], heads, head_dim),+ head_dim,+ )++def _var_attention_output(out, heads, head_dim, skip_output_reshape):+ if skip_output_reshape:+ return out+ return out.reshape(-1, heads * head_dim)+++def _use_blackwell_attention():+ device = model_management.get_torch_device()+ if device.type != "cuda":+ return False+ major, minor = torch.cuda.get_device_capability(device)+ return (major, minor) >= (12, 0)+++def _validate_split_cu_seqlens(name, cu_seqlens, token_count):+ if cu_seqlens.dtype not in (torch.int32, torch.int64):+ raise ValueError(f"{name} must use an integer dtype")+ if cu_seqlens.ndim != 1 or cu_seqlens.numel() < 2:+ raise ValueError(f"{name} must be a 1D tensor with at least two offsets")+ if cu_seqlens[0].item() != 0:+ raise ValueError(f"{name} must start at 0")+ if (cu_seqlens[1:] <= cu_seqlens[:-1]).any().item():+ raise ValueError(f"{name} must be strictly increasing")+ if cu_seqlens[-1].item() != token_count:+ raise ValueError(f"{name} does not match token count")+++def _split_indices(cu_seqlens):+ return cu_seqlens[1:-1].to(device="cpu", dtype=torch.long)+++def var_attention_optimized_split(q, k, v, heads, cu_seqlens_q, cu_seqlens_k, *args, skip_reshape=False, skip_output_reshape=False, **kwargs):+ q, k, v, head_dim = _var_attention_qkv(q, k, v, heads, skip_reshape)++ _validate_split_cu_seqlens("cu_seqlens_q", cu_seqlens_q, q.shape[0])+ _validate_split_cu_seqlens("cu_seqlens_k", cu_seqlens_k, k.shape[0])+ if cu_seqlens_k[-1].item() != v.shape[0]:+ raise ValueError("cu_seqlens_k does not match v token count")++ q_split_indices = _split_indices(cu_seqlens_q)+ k_split_indices = _split_indices(cu_seqlens_k)+ q_splits = torch.tensor_split(q, q_split_indices, dim=0)+ k_splits = torch.tensor_split(k, k_split_indices, dim=0)+ v_splits = torch.tensor_split(v, k_split_indices, dim=0)+ if len(q_splits) != len(k_splits) or len(q_splits) != len(v_splits):+ raise ValueError("cu_seqlens_q and cu_seqlens_k must describe the same sequence count")++ out = []+ for q_i, k_i, v_i in zip(q_splits, k_splits, v_splits):+ q_i = q_i.permute(1, 0, 2).unsqueeze(0)+ k_i = k_i.permute(1, 0, 2).unsqueeze(0)+ v_i = v_i.permute(1, 0, 2).unsqueeze(0)+ out_dtype = q_i.dtype+ if optimized_attention is attention_sage and q_i.dtype not in (torch.float16, torch.bfloat16):+ q_i = q_i.to(torch.bfloat16)+ k_i = k_i.to(torch.bfloat16)+ v_i = v_i.to(torch.bfloat16)+ out_i = optimized_attention(q_i, k_i, v_i, heads, skip_reshape=True, skip_output_reshape=True)+ if out_i.dtype != out_dtype:+ out_i = out_i.to(out_dtype)+ out.append(out_i.squeeze(0).permute(1, 0, 2))++ out = torch.cat(out, dim=0)+ return _var_attention_output(out, heads, head_dim, skip_output_reshape)+++optimized_var_attention = var_attention_optimized_splitoptimized_attention = attention_basicif model_management.sage_attention_enabled():@@ -758,6 +837,8 @@ else:logging.info("Using sub quadratic optimization for attention, if you have memory or speed issues try using: --use-split-cross-attention")optimized_attention = attention_sub_quad+logging.info("Using optimized_attention split-loop for variable-length attention")+optimized_attention_masked = optimized_attention@@ -773,6 +854,7 @@ if model_management.xformers_enabled():register_attention_function("pytorch", attention_pytorch)register_attention_function("sub_quad", attention_sub_quad)register_attention_function("split", attention_split)+register_attention_function("var_attention_optimized_split", var_attention_optimized_split)def optimized_attention_for_device(device, mask=False, small_input=False):@@ -1209,5 +1291,3 @@ class SpatialVideoTransformer(SpatialTransformer):x = self.proj_out(x)out = x + x_inreturn out--diff --git a/comfy/ldm/modules/diffusionmodules/model.py b/comfy/ldm/modules/diffusionmodules/model.pyindex fcbaa074f..235df0b83 100644--- a/comfy/ldm/modules/diffusionmodules/model.py+++ b/comfy/ldm/modules/diffusionmodules/model.py@@ -13,6 +13,7 @@ if model_management.xformers_enabled_vae():import xformersimport xformers.ops+def torch_cat_if_needed(xl, dim):xl = [x for x in xl if x is not None and x.shape[dim] > 0]if len(xl) > 1:@@ -22,7 +23,8 @@ def torch_cat_if_needed(xl, dim):else:return None-def get_timestep_embedding(timesteps, embedding_dim):++def get_timestep_embedding(timesteps, embedding_dim, flip_sin_to_cos=False, downscale_freq_shift=1):"""This matches the implementation in Denoising Diffusion Probabilistic Models:From Fairseq.@@ -33,11 +35,13 @@ def get_timestep_embedding(timesteps, embedding_dim):assert len(timesteps.shape) == 1half_dim = embedding_dim // 2- emb = math.log(10000) / (half_dim - 1)+ emb = math.log(10000) / (half_dim - downscale_freq_shift)emb = torch.exp(torch.arange(half_dim, dtype=torch.float32) * -emb)emb = emb.to(device=timesteps.device)emb = timesteps.float()[:, None] * emb[None, :]emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)+ if flip_sin_to_cos:+ emb = torch.cat([emb[:, half_dim:], emb[:, :half_dim]], dim=-1)if embedding_dim % 2 == 1: # zero pademb = torch.nn.functional.pad(emb, (0,1,0,0))return embdiff --git a/comfy/ldm/seedvr/color_fix.py b/comfy/ldm/seedvr/color_fix.pynew file mode 100644index 000000000..7ddfc03af--- /dev/null+++ b/comfy/ldm/seedvr/color_fix.py@@ -0,0 +1,340 @@+import torch+import torch.nn.functional as F+from torch import Tensor++from comfy.ldm.seedvr.model import safe_pad_operation+from comfy.ldm.seedvr.vae import safe_interpolate_operation+from comfy.ldm.seedvr.constants import (+ CIELAB_DELTA,+ CIELAB_KAPPA,+ D65_WHITE_X,+ D65_WHITE_Z,+ WAVELET_DECOMP_LEVELS,+)+++def wavelet_blur(image: Tensor, radius):+ max_safe_radius = max(1, min(image.shape[-2:]) // 8)+ if radius > max_safe_radius:+ radius = max_safe_radius++ num_channels = image.shape[1]++ kernel_vals = [+ [0.0625, 0.125, 0.0625],+ [0.125, 0.25, 0.125],+ [0.0625, 0.125, 0.0625],+ ]+ kernel = torch.tensor(kernel_vals, dtype=image.dtype, device=image.device)+ kernel = kernel[None, None].repeat(num_channels, 1, 1, 1)++ image = safe_pad_operation(image, (radius, radius, radius, radius), mode='replicate')+ output = F.conv2d(image, kernel, groups=num_channels, dilation=radius)++ return output++def wavelet_decomposition(image: Tensor, levels: int = WAVELET_DECOMP_LEVELS):+ high_freq = torch.zeros_like(image)++ for i in range(levels):+ radius = 2 ** i+ low_freq = wavelet_blur(image, radius)+ high_freq.add_(image).sub_(low_freq)+ image = low_freq++ return high_freq, low_freq++def wavelet_reconstruction(content_feat: Tensor, style_feat: Tensor) -> Tensor:++ if content_feat.shape != style_feat.shape:+ # Resize style to match content spatial dimensions+ if len(content_feat.shape) >= 3:+ # safe_interpolate_operation handles FP16 conversion automatically+ style_feat = safe_interpolate_operation(+ style_feat,+ size=content_feat.shape[-2:],+ mode='bilinear',+ align_corners=False+ )++ # Decompose both features into frequency components+ content_high_freq, content_low_freq = wavelet_decomposition(content_feat)+ del content_low_freq # Free memory immediately++ style_high_freq, style_low_freq = wavelet_decomposition(style_feat)+ del style_high_freq # Free memory immediately++ if content_high_freq.shape != style_low_freq.shape:+ style_low_freq = safe_interpolate_operation(+ style_low_freq,+ size=content_high_freq.shape[-2:],+ mode='bilinear',+ align_corners=False+ )++ content_high_freq.add_(style_low_freq)++ return content_high_freq.clamp_(-1.0, 1.0)++def _histogram_matching_channel(source: Tensor, reference: Tensor, device: torch.device) -> Tensor:+ original_shape = source.shape++ # Flatten+ source_flat = source.flatten()+ reference_flat = reference.flatten()++ # Sort both arrays+ source_sorted, source_indices = torch.sort(source_flat)+ reference_sorted, _ = torch.sort(reference_flat)+ del reference_flat++ # Quantile mapping+ n_source = len(source_sorted)+ n_reference = len(reference_sorted)++ if n_source == n_reference:+ matched_sorted = reference_sorted+ else:+ # Interpolate reference to match source quantiles+ source_quantiles = torch.linspace(0, 1, n_source, device=device)+ ref_indices = (source_quantiles * (n_reference - 1)).long()+ ref_indices.clamp_(0, n_reference - 1)+ matched_sorted = reference_sorted[ref_indices]+ del source_quantiles, ref_indices, reference_sorted++ del source_sorted, source_flat++ # Reconstruct using argsort (portable across CUDA/ROCm/MPS)+ inverse_indices = torch.argsort(source_indices)+ del source_indices+ matched_flat = matched_sorted[inverse_indices]+ del matched_sorted, inverse_indices++ return matched_flat.reshape(original_shape)++def _lab_to_rgb_batch(lab: Tensor, device: torch.device, matrix_inv: Tensor, epsilon: float, kappa: float) -> Tensor:+ """Convert batch of CIELAB images to RGB color space."""+ L, a, b = lab[:, 0], lab[:, 1], lab[:, 2]++ # LAB to XYZ+ fy = (L + 16.0) / 116.0+ fx = a.div(500.0).add_(fy)+ fz = fy - b / 200.0+ del L, a, b++ # XYZ transformation+ x = torch.where(+ fx > epsilon,+ torch.pow(fx, 3.0),+ fx.mul(116.0).sub_(16.0).div_(kappa)+ )+ y = torch.where(+ fy > epsilon,+ torch.pow(fy, 3.0),+ fy.mul(116.0).sub_(16.0).div_(kappa)+ )+ z = torch.where(+ fz > epsilon,+ torch.pow(fz, 3.0),+ fz.mul(116.0).sub_(16.0).div_(kappa)+ )+ del fx, fy, fz++ # Apply D65 white point (in-place)+ x.mul_(D65_WHITE_X)+ # y *= 1.00000 # (no-op, skip)+ z.mul_(D65_WHITE_Z)++ xyz = torch.stack([x, y, z], dim=1)+ del x, y, z++ # Matrix multiplication: XYZ -> RGB+ B, C, H, W = xyz.shape+ xyz_flat = xyz.permute(0, 2, 3, 1).reshape(-1, 3)+ del xyz++ # Ensure dtype consistency for matrix multiplication+ xyz_flat = xyz_flat.to(dtype=matrix_inv.dtype)+ rgb_linear_flat = torch.matmul(xyz_flat, matrix_inv.T)+ del xyz_flat++ rgb_linear = rgb_linear_flat.reshape(B, H, W, 3).permute(0, 3, 1, 2)+ del rgb_linear_flat++ # Apply inverse gamma correction (delinearize)+ mask = rgb_linear > 0.0031308+ rgb = torch.where(+ mask,+ torch.pow(torch.clamp(rgb_linear, min=0.0), 1.0 / 2.4).mul_(1.055).sub_(0.055),+ rgb_linear * 12.92+ )+ del mask, rgb_linear++ return torch.clamp(rgb, 0.0, 1.0)++def _rgb_to_lab_batch(rgb: Tensor, device: torch.device, matrix: Tensor, epsilon: float, kappa: float) -> Tensor:+ """Convert batch of RGB images to CIELAB color space using D65 illuminant."""+ # Apply sRGB gamma correction (linearize)+ mask = rgb > 0.04045+ rgb_linear = torch.where(+ mask,+ torch.pow((rgb + 0.055) / 1.055, 2.4),+ rgb / 12.92+ )+ del mask++ # Matrix multiplication: RGB -> XYZ+ B, C, H, W = rgb_linear.shape+ rgb_flat = rgb_linear.permute(0, 2, 3, 1).reshape(-1, 3)+ del rgb_linear++ # Ensure dtype consistency for matrix multiplication+ rgb_flat = rgb_flat.to(dtype=matrix.dtype)+ xyz_flat = torch.matmul(rgb_flat, matrix.T)+ del rgb_flat++ xyz = xyz_flat.reshape(B, H, W, 3).permute(0, 3, 1, 2)+ del xyz_flat++ # Normalize by D65 white point (in-place)+ xyz[:, 0].div_(D65_WHITE_X) # X+ # xyz[:, 1] /= 1.00000 # Y (no-op, skip)+ xyz[:, 2].div_(D65_WHITE_Z) # Z++ # XYZ to LAB transformation+ epsilon_cubed = epsilon ** 3+ mask = xyz > epsilon_cubed+ f_xyz = torch.where(+ mask,+ torch.pow(xyz, 1.0 / 3.0),+ xyz.mul(kappa).add_(16.0).div_(116.0)+ )+ del xyz, mask++ # Extract channels and compute LAB+ L = f_xyz[:, 1].mul(116.0).sub_(16.0) # Lightness [0, 100]+ a = (f_xyz[:, 0] - f_xyz[:, 1]).mul_(500.0) # Green-Red [-128, 127]+ b = (f_xyz[:, 1] - f_xyz[:, 2]).mul_(200.0) # Blue-Yellow [-128, 127]+ del f_xyz++ return torch.stack([L, a, b], dim=1)++def lab_color_transfer(+ content_feat: Tensor,+ style_feat: Tensor,+ luminance_weight: float = 0.8+) -> Tensor:+ content_feat = wavelet_reconstruction(content_feat, style_feat)++ if content_feat.shape != style_feat.shape:+ style_feat = safe_interpolate_operation(+ style_feat,+ size=content_feat.shape[-2:],+ mode='bilinear',+ align_corners=False+ )++ device = content_feat.device++ def ensure_float32_precision(c):+ orig_dtype = c.dtype+ c = c.float()+ return c, orig_dtype+ content_feat, original_dtype = ensure_float32_precision(content_feat)+ style_feat, _ = ensure_float32_precision(style_feat)++ rgb_to_xyz_matrix = torch.tensor([+ [0.4124564, 0.3575761, 0.1804375],+ [0.2126729, 0.7151522, 0.0721750],+ [0.0193339, 0.1191920, 0.9503041]+ ], dtype=torch.float32, device=device)++ xyz_to_rgb_matrix = torch.tensor([+ [ 3.2404542, -1.5371385, -0.4985314],+ [-0.9692660, 1.8760108, 0.0415560],+ [ 0.0556434, -0.2040259, 1.0572252]+ ], dtype=torch.float32, device=device)++ epsilon = CIELAB_DELTA+ kappa = CIELAB_KAPPA++ content_feat.add_(1.0).mul_(0.5).clamp_(0.0, 1.0)+ style_feat.add_(1.0).mul_(0.5).clamp_(0.0, 1.0)++ # Convert to LAB color space+ content_lab = _rgb_to_lab_batch(content_feat, device, rgb_to_xyz_matrix, epsilon, kappa)+ del content_feat++ style_lab = _rgb_to_lab_batch(style_feat, device, rgb_to_xyz_matrix, epsilon, kappa)+ del style_feat, rgb_to_xyz_matrix++ # Match chrominance channels (a*, b*) for accurate color transfer+ matched_a = _histogram_matching_channel(content_lab[:, 1], style_lab[:, 1], device)+ matched_b = _histogram_matching_channel(content_lab[:, 2], style_lab[:, 2], device)++ # Handle luminance with weighted blending+ if luminance_weight < 1.0:+ # Partially match luminance for better overall color accuracy+ matched_L = _histogram_matching_channel(content_lab[:, 0], style_lab[:, 0], device)+ # Blend: preserve some content L* for detail, adopt some style L* for color+ result_L = content_lab[:, 0].mul(luminance_weight).add_(matched_L.mul(1.0 - luminance_weight))+ del matched_L+ else:+ # Fully preserve content luminance+ result_L = content_lab[:, 0]++ del content_lab, style_lab++ # Reconstruct LAB with corrected channels+ result_lab = torch.stack([result_L, matched_a, matched_b], dim=1)+ del result_L, matched_a, matched_b++ # Convert back to RGB+ result_rgb = _lab_to_rgb_batch(result_lab, device, xyz_to_rgb_matrix, epsilon, kappa)+ del result_lab, xyz_to_rgb_matrix++ # Convert back to [-1, 1] range (in-place)+ result = result_rgb.mul_(2.0).sub_(1.0)+ del result_rgb++ result = result.to(original_dtype)++ return result+++def wavelet_color_transfer(content_feat: Tensor, style_feat: Tensor) -> Tensor:+ return wavelet_reconstruction(content_feat, style_feat)+++def adain_color_transfer(content_feat: Tensor, style_feat: Tensor, eps: float = 1e-5) -> Tensor:+ if content_feat.shape != style_feat.shape:+ style_feat = safe_interpolate_operation(+ style_feat,+ size=content_feat.shape[-2:],+ mode='bilinear',+ align_corners=False,+ )++ original_dtype = content_feat.dtype+ content_feat = content_feat.float()+ style_feat = style_feat.float()++ b, c = content_feat.shape[:2]+ content_flat = content_feat.reshape(b, c, -1)+ style_flat = style_feat.reshape(b, c, -1)++ content_mean = content_flat.mean(dim=2).reshape(b, c, 1, 1)+ content_std = (content_flat.var(dim=2, correction=0) + eps).sqrt().reshape(b, c, 1, 1)+ style_mean = style_flat.mean(dim=2).reshape(b, c, 1, 1)+ style_std = (style_flat.var(dim=2, correction=0) + eps).sqrt().reshape(b, c, 1, 1)+ del content_flat, style_flat++ normalized = (content_feat - content_mean) / content_std+ del content_mean, content_std+ result = normalized * style_std + style_mean+ del normalized, style_mean, style_std++ result = result.clamp_(-1.0, 1.0)+ if result.dtype != original_dtype:+ result = result.to(original_dtype)+ return resultdiff --git a/comfy/ldm/seedvr/constants.py b/comfy/ldm/seedvr/constants.pynew file mode 100644index 000000000..95838d1dd--- /dev/null+++ b/comfy/ldm/seedvr/constants.py@@ -0,0 +1,79 @@+"""Named constants for the SeedVR2 integration, grouped by provenance.++Provenance prefixes:+- ``SEEDVR2_*`` - introduced by this integration (no external origin); rationale inline.+- ``BYTEDANCE_*`` - ported from the official ByteDance-Seed/SeedVR release; each cites+ the upstream config/source path it was lifted from.+- unprefixed standards (``ROPE_THETA``, ``CIELAB_*``, ``D65_*``) - published literature /+ ISO / CIE values; cite the standard.+"""++# --------------------------------------------------------------------------------------+# A. Progressive-sampler chunk-size law (SEEDVR2 - this integration's VRAM experiment)+# n_max(frames/chunk) = SEEDVR2_CHUNK_FRAMES_PER_GB * (free_GB - SEEDVR2_CHUNK_GB_MARGIN)+# rounded to the 4n+1 grid. Fit on 22 blocked-5090 cells, validated on a real RTX 4070+# (3b and 7b). Resolution-independent (the VAE tiling sets the wall, not the DiT).+# --------------------------------------------------------------------------------------+SEEDVR2_CHUNK_GB_MARGIN = 3 # fixed VRAM overhead before chunks scale (GiB)+SEEDVR2_CHUNK_FRAMES_PER_GB = 4 # empirical slope: pixel frames admitted per free GiB++# --------------------------------------------------------------------------------------+# B. Fork heuristics (SEEDVR2 - this integration)+# --------------------------------------------------------------------------------------+SEEDVR2_7B_VID_DIM = 3072 # runtime 3b-vs-7b sentinel; tested against vid_dim.+ # (3072 is ByteDance's 7b vid_dim; the sentinel use is ours.)+SEEDVR2_OOM_BACKOFF_DIVISOR = 2 # auto-chunk OOM retry: halve the chunk and retry.+SEEDVR2_DTYPE_BYTES_FLOOR = 4 # per-element byte floor for memory math (fp32 worst case).+SEEDVR2_7B_MLP_CHUNK = 8192 # 7b MLP token-chunk to bound peak VRAM.+SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS = 4096 # partial-RoPE application token-chunk.+SEEDVR2_LATENT_CHANNELS = 16 # SeedVR2 latent channel count (== BYTEDANCE latent_channels).+SEEDVR2_COND_CHANNELS = 17 # conditioning channels = vid_in_channels(33) - latent(16).+SEEDVR2_DEFAULT_TEMPORAL_SIZE = 16 # default VAE temporal tile when unset.++# Color-correction memory model (fork tuning; per-frame VRAM estimate for chunk sizing)+SEEDVR2_COLOR_MEM_HEADROOM = 0.75 # fraction of free VRAM usable per color-correction chunk.+SEEDVR2_LAB_SCALE_MULTIPLIER = 13 # per-frame byte multiplier, LAB path.+SEEDVR2_WAVELET_SCALE_MULTIPLIER = 10 # per-frame byte multiplier, wavelet path.+SEEDVR2_ADAIN_SCALE_MULTIPLIER = 6 # per-frame byte multiplier, AdaIN path.++# --------------------------------------------------------------------------------------+# C. ByteDance config / source (BYTEDANCE - cite ByteDance-Seed/SeedVR)+# --------------------------------------------------------------------------------------+BYTEDANCE_VAE_SCALING_FACTOR = 0.9152 # configs_3b/main.yaml:57 (scaling_factor); latent denorm.+BYTEDANCE_VAE_SHIFTING_FACTOR = 0.0 # infer.py (shifting_factor default); latent denorm shift.+BYTEDANCE_VAE_CONV_MEM_GIB = 0.5 # configs_3b/main.yaml:54 (conv_max_mem).+BYTEDANCE_VAE_NORM_MEM_GIB = 0.5 # configs_3b/main.yaml:55 (norm_max_mem).+BYTEDANCE_LOGVAR_CLAMP_MIN = -30.0 # video_vae_v3/modules/types.py:28.+BYTEDANCE_LOGVAR_CLAMP_MAX = 20.0 # video_vae_v3/modules/types.py:28.+BYTEDANCE_GN_CHUNKS_FP16 = 4 # causal_inflation_lib.py:351 (GroupNorm chunk count, fp16).+BYTEDANCE_GN_CHUNKS_FP32 = 2 # causal_inflation_lib.py:351 (GroupNorm chunk count, fp32).+BYTEDANCE_CONTIGUOUS_BATCH_THRESHOLD = 64 # attn_video_vae.py:308 (force .contiguous() above this b*t).+BYTEDANCE_BLOCK_OUT_CHANNELS = (128, 256, 512, 512) # s8_c16_t4_inflation_sd3.yaml:7-11.+BYTEDANCE_SLICING_SAMPLE_MIN = 4 # s8_c16_t4_inflation_sd3.yaml:22 (slicing_sample_min_size).+BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE = 4 # infer.py:230 (temporal_downsample_factor); the 4n+1 factor.+BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE = 8 # infer.py:231 (spatial_downsample_factor).+BYTEDANCE_SCHEDULE_T = 1000.0 # configs_3b/main.yaml:65 (schedule.T); timestep range.+BYTEDANCE_SPATIAL_DIVISOR = 16 # inference_seedvr2_3b.py:241 (DivisibleCrop((16,16))).+BYTEDANCE_720P_REF_AREA = 45 * 80 # dit_v2/window.py:32 (720p reference area for window scaling).+BYTEDANCE_MAX_TEMPORAL_WINDOW = 30 # dit_v2/window.py:35 (max temporal window frames).+BYTEDANCE_ROPE_MAX_FREQ = 256 # dit_v2/rope.py:31 (pixel-RoPE max frequency).+BYTEDANCE_SINUSOIDAL_DIM = 256 # dit_3b/nadit.py:120 (timestep sinusoidal embed dim).+# Resolution-dependent timestep-shift linear fits: (x1, y1, x2, y2) for get_lin_function.+BYTEDANCE_IMG_SHIFT_FIT = (256 * 256, 1.0, 1024 * 1024, 3.2) # infer.py:242.+BYTEDANCE_VID_SHIFT_FIT = (256 * 256 * 37, 1.0, 1280 * 720 * 145, 5.0) # infer.py:243.++# --------------------------------------------------------------------------------------+# D. Published standards (cite the literature)+# --------------------------------------------------------------------------------------+ROPE_THETA = 10000 # RoPE base; Su et al., "RoFormer", arXiv:2104.09864.++# CIELAB f(t) piecewise constants and D65 white point (CIE 15 colorimetry; CIE D65).+CIELAB_DELTA = 6.0 / 29.0 # CIE 15 (delta).+CIELAB_KAPPA = (29.0 / 3.0) ** 3 # CIE 15 (kappa).+D65_WHITE_X = 0.95047 # CIE D65 standard illuminant Xn (Yn = 1).+D65_WHITE_Z = 1.08883 # CIE D65 standard illuminant Zn.+WAVELET_DECOMP_LEVELS = 5 # wavelet color-fix decomposition depth (GIMP/Krita; StableSR).++# NOTE: the sRGB<->XYZ D65 3x3 matrices (IEC 61966-2-1) remain inline in the color code and+# are named (SRGB_TO_XYZ_D65 / XYZ_TO_SRGB_D65) during the color-module extraction, where the+# exact existing coefficients move verbatim rather than being retyped here.diff --git a/comfy/ldm/seedvr/model.py b/comfy/ldm/seedvr/model.pynew file mode 100644index 000000000..3fa9fe07e--- /dev/null+++ b/comfy/ldm/seedvr/model.py@@ -0,0 +1,1665 @@+from dataclasses import dataclass+from typing import Optional, Tuple, Union, List, Dict, Any, Callable+import einops+from einops import rearrange+import torch.nn.functional as F+from math import ceil, pi+import torch+from itertools import chain+from comfy.ldm.modules.diffusionmodules.model import get_timestep_embedding+from comfy.ldm.modules.attention import optimized_var_attention+from torch.nn.modules.utils import _triple+from torch import nn+import math+from comfy.ldm.flux.math import apply_rope1+from comfy.ldm.seedvr.constants import (+ BYTEDANCE_720P_REF_AREA,+ BYTEDANCE_MAX_TEMPORAL_WINDOW,+ BYTEDANCE_ROPE_MAX_FREQ,+ BYTEDANCE_SINUSOIDAL_DIM,+ ROPE_THETA,+ SEEDVR2_7B_MLP_CHUNK,+ SEEDVR2_7B_VID_DIM,+ SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS,+)+import comfy.model_management+import numbers++def _torch_float8_types():+ return tuple(+ getattr(torch, name)+ for name in (+ "float8_e4m3fn",+ "float8_e4m3fnuz",+ "float8_e5m2",+ "float8_e5m2fnuz",+ "float8_e8m0fnu",+ )+ if hasattr(torch, name)+ )++class CustomRMSNorm(nn.Module):++ def __init__(self, normalized_shape, eps=1e-5, elementwise_affine=True, device=None, dtype=None):+ super(CustomRMSNorm, self).__init__()++ if isinstance(normalized_shape, numbers.Integral):+ normalized_shape = (normalized_shape,)+ self.normalized_shape = torch.Size(normalized_shape)+ self.eps = eps+ self.elementwise_affine = elementwise_affine++ if self.elementwise_affine:+ self.weight = nn.Parameter(torch.ones(*normalized_shape, device=device, dtype=dtype))+ else:+ self.register_parameter('weight', None)++ def forward(self, input):++ dims = tuple(range(-len(self.normalized_shape), 0))++ normalized = input.float()+ variance = normalized.pow(2).mean(dim=dims, keepdim=True)+ rms = torch.sqrt(variance + self.eps)++ normalized = normalized / rms++ if self.elementwise_affine:+ return normalized * self.weight.to(input.dtype)+ return normalized++class Cache:+ def __init__(self, disable=False, prefix="", cache=None):+ self.cache = cache if cache is not None else {}+ self.disable = disable+ self.prefix = prefix++ def __call__(self, key: str, fn: Callable):+ if self.disable:+ return fn()++ key = self.prefix + key+ try:+ result = self.cache[key]+ except KeyError:+ result = fn()+ self.cache[key] = result+ return result++ def namespace(self, namespace: str):+ return Cache(+ disable=self.disable,+ prefix=self.prefix + namespace + ".",+ cache=self.cache,+ )++ def get(self, key: str):+ key = self.prefix + key+ return self.cache[key]++def repeat_concat(+ vid: torch.FloatTensor, # (VL ... c)+ txt: torch.FloatTensor, # (TL ... c)+ vid_len: torch.LongTensor, # (n*b)+ txt_len: torch.LongTensor, # (b)+ txt_repeat: List, # (n)+) -> torch.FloatTensor: # (L ... c)+ vid = torch.split(vid, vid_len.tolist())+ txt = torch.split(txt, txt_len.tolist())+ txt = [[x] * n for x, n in zip(txt, txt_repeat)]+ txt = list(chain(*txt))+ return torch.cat(list(chain(*zip(vid, txt))))++def concat(+ vid: torch.FloatTensor, # (VL ... c)+ txt: torch.FloatTensor, # (TL ... c)+ vid_len: torch.LongTensor, # (b)+ txt_len: torch.LongTensor, # (b)+) -> torch.FloatTensor: # (L ... c)+ vid = torch.split(vid, vid_len.tolist())+ txt = torch.split(txt, txt_len.tolist())+ return torch.cat(list(chain(*zip(vid, txt))))++def concat_idx(+ vid_len: torch.LongTensor, # (b)+ txt_len: torch.LongTensor, # (b)+) -> Tuple[+ Callable,+ Callable,+]:+ device = vid_len.device+ vid_idx = torch.arange(vid_len.sum(), device=device)+ txt_idx = torch.arange(len(vid_idx), len(vid_idx) + txt_len.sum(), device=device)+ tgt_idx = concat(vid_idx, txt_idx, vid_len, txt_len)+ src_idx = torch.argsort(tgt_idx)+ return (+ lambda vid, txt: torch.index_select(torch.cat([vid, txt]), 0, tgt_idx),+ lambda all: torch.index_select(all, 0, src_idx).split([len(vid_idx), len(txt_idx)]),+ )+++def repeat_concat_idx(+ vid_len: torch.LongTensor, # (n*b)+ txt_len: torch.LongTensor, # (b)+ txt_repeat: torch.LongTensor, # (n)+) -> Tuple[+ Callable,+ Callable,+]:+ device = vid_len.device+ vid_idx = torch.arange(vid_len.sum(), device=device)+ txt_idx = torch.arange(len(vid_idx), len(vid_idx) + txt_len.sum(), device=device)+ txt_repeat_list = txt_repeat.tolist()+ tgt_idx = repeat_concat(vid_idx, txt_idx, vid_len, txt_len, txt_repeat)+ src_idx = torch.argsort(tgt_idx)+ txt_idx_len = len(tgt_idx) - len(vid_idx)+ repeat_txt_len = (txt_len * txt_repeat).tolist()++ def unconcat_coalesce(all):+ vid_out, txt_out = all[src_idx].split([len(vid_idx), txt_idx_len])+ txt_out_coalesced = []+ for txt, repeat_time in zip(txt_out.split(repeat_txt_len), txt_repeat_list):+ txt = txt.reshape(-1, repeat_time, *txt.shape[1:]).mean(1)+ txt_out_coalesced.append(txt)+ return vid_out, torch.cat(txt_out_coalesced)++ return (+ lambda vid, txt: torch.cat([vid, txt])[tgt_idx],+ lambda all: unconcat_coalesce(all),+ )+++@dataclass+class MMArg:+ vid: Any+ txt: Any++def safe_pad_operation(x, padding, mode='constant', value=0.0):+ """Safe padding operation that handles Half precision only for problematic modes"""+ # Modes qui nécessitent le fix Half precision+ problematic_modes = ['replicate', 'reflect', 'circular']++ if mode in problematic_modes:+ try:+ return F.pad(x, padding, mode=mode, value=value)+ except RuntimeError as e:+ if "not implemented for 'Half'" in str(e):+ original_dtype = x.dtype+ return F.pad(x.float(), padding, mode=mode, value=value).to(original_dtype)+ else:+ raise e+ else:+ # Pour 'constant' et autres modes compatibles, pas de fix nécessaire+ return F.pad(x, padding, mode=mode, value=value)+++def get_args(key: str, args: List[Any]) -> List[Any]:+ return [getattr(v, key) if isinstance(v, MMArg) else v for v in args]+++def get_kwargs(key: str, kwargs: Dict[str, Any]) -> Dict[str, Any]:+ return {k: getattr(v, key) if isinstance(v, MMArg) else v for k, v in kwargs.items()}+++def get_window_op(name: str):+ if name == "720pwin_by_size_bysize":+ return make_720Pwindows_bysize+ if name == "720pswin_by_size_bysize":+ return make_shifted_720Pwindows_bysize+ raise ValueError(f"Unknown windowing method: {name}")+++# -------------------------------- Windowing -------------------------------- #+def make_720Pwindows_bysize(size: Tuple[int, int, int], num_windows: Tuple[int, int, int]):+ t, h, w = size+ resized_nt, resized_nh, resized_nw = num_windows+ #cal windows under 720p+ scale = math.sqrt(BYTEDANCE_720P_REF_AREA / (h * w))+ resized_h, resized_w = round(h * scale), round(w * scale)+ wh, ww = ceil(resized_h / resized_nh), ceil(resized_w / resized_nw) # window size.+ wt = ceil(min(t, BYTEDANCE_MAX_TEMPORAL_WINDOW) / resized_nt) # window size.+ nt, nh, nw = ceil(t / wt), ceil(h / wh), ceil(w / ww) # window size.+ return [+ (+ slice(it * wt, min((it + 1) * wt, t)),+ slice(ih * wh, min((ih + 1) * wh, h)),+ slice(iw * ww, min((iw + 1) * ww, w)),+ )+ for iw in range(nw)+ if min((iw + 1) * ww, w) > iw * ww+ for ih in range(nh)+ if min((ih + 1) * wh, h) > ih * wh+ for it in range(nt)+ if min((it + 1) * wt, t) > it * wt+ ]++def make_shifted_720Pwindows_bysize(size: Tuple[int, int, int], num_windows: Tuple[int, int, int]):+ t, h, w = size+ resized_nt, resized_nh, resized_nw = num_windows+ #cal windows under 720p+ scale = math.sqrt(BYTEDANCE_720P_REF_AREA / (h * w))+ resized_h, resized_w = round(h * scale), round(w * scale)+ wh, ww = ceil(resized_h / resized_nh), ceil(resized_w / resized_nw) # window size.+ wt = ceil(min(t, BYTEDANCE_MAX_TEMPORAL_WINDOW) / resized_nt) # window size.++ st, sh, sw = ( # shift size.+ 0.5 if wt < t else 0,+ 0.5 if wh < h else 0,+ 0.5 if ww < w else 0,+ )+ nt, nh, nw = ceil((t - st) / wt), ceil((h - sh) / wh), ceil((w - sw) / ww) # window size.+ nt, nh, nw = ( # number of window.+ nt + 1 if st > 0 else 1,+ nh + 1 if sh > 0 else 1,+ nw + 1 if sw > 0 else 1,+ )+ return [+ (+ slice(max(int((it - st) * wt), 0), min(int((it - st + 1) * wt), t)),+ slice(max(int((ih - sh) * wh), 0), min(int((ih - sh + 1) * wh), h)),+ slice(max(int((iw - sw) * ww), 0), min(int((iw - sw + 1) * ww), w)),+ )+ for iw in range(nw)+ if min(int((iw - sw + 1) * ww), w) > max(int((iw - sw) * ww), 0)+ for ih in range(nh)+ if min(int((ih - sh + 1) * wh), h) > max(int((ih - sh) * wh), 0)+ for it in range(nt)+ if min(int((it - st + 1) * wt), t) > max(int((it - st) * wt), 0)+ ]++class RotaryEmbedding(nn.Module):+ def __init__(+ self,+ dim,+ custom_freqs = None,+ freqs_for = 'lang',+ theta = 10000,+ max_freq = 10,+ num_freqs = 1,+ learned_freq = False,+ use_xpos = False,+ xpos_scale_base = 512,+ interpolate_factor = 1.,+ theta_rescale_factor = 1.,+ seq_before_head_dim = False,+ cache_if_possible = True,+ cache_max_seq_len = 8192+ ):+ super().__init__()++ theta *= theta_rescale_factor ** (dim / (dim - 2))++ self.freqs_for = freqs_for++ if exists(custom_freqs):+ freqs = custom_freqs+ elif freqs_for == 'lang':+ freqs = 1. / (theta ** (torch.arange(0, dim, 2)[:(dim // 2)].float() / dim))+ elif freqs_for == 'pixel':+ freqs = torch.linspace(1., max_freq / 2, dim // 2) * pi+ elif freqs_for == 'constant':+ freqs = torch.ones(num_freqs).float()++ self.cache_if_possible = cache_if_possible+ self.cache_max_seq_len = cache_max_seq_len++ self.register_buffer('cached_freqs', torch.zeros(cache_max_seq_len, dim), persistent = False)+ self.cached_freqs_seq_len = 0++ self.freqs = nn.Parameter(freqs, requires_grad = learned_freq)++ self.learned_freq = learned_freq++ # dummy for device++ self.register_buffer('dummy', torch.tensor(0), persistent = False)++ # default sequence dimension++ self.seq_before_head_dim = seq_before_head_dim+ self.default_seq_dim = -3 if seq_before_head_dim else -2++ # interpolation factors++ assert interpolate_factor >= 1.+ self.interpolate_factor = interpolate_factor++ # xpos++ self.use_xpos = use_xpos++ if not use_xpos:+ return++ scale = (torch.arange(0, dim, 2) + 0.4 * dim) / (1.4 * dim)+ self.scale_base = xpos_scale_base++ self.register_buffer('scale', scale, persistent = False)+ self.register_buffer('cached_scales', torch.zeros(cache_max_seq_len, dim), persistent = False)+ self.cached_scales_seq_len = 0++ # add apply_rotary_emb as static method++ self.apply_rotary_emb = staticmethod(apply_rotary_emb)++ @property+ def device(self):+ return self.dummy.device++ def get_axial_freqs(+ self,+ *dims,+ offsets = None+ ):+ Colon = slice(None)+ all_freqs = []++ # handle offset++ if exists(offsets):+ assert len(offsets) == len(dims)++ for ind, dim in enumerate(dims):++ offset = 0+ if exists(offsets):+ offset = offsets[ind]++ if self.freqs_for == 'pixel':+ pos = torch.linspace(-1, 1, steps = dim, device = self.device)+ else:+ pos = torch.arange(dim, device = self.device)++ pos = pos + offset++ freqs = self.forward(pos, seq_len = dim)++ all_axis = [None] * len(dims)+ all_axis[ind] = Colon++ new_axis_slice = (Ellipsis, *all_axis, Colon)+ all_freqs.append(freqs[new_axis_slice])++ # concat all freqs++ all_freqs = torch.broadcast_tensors(*all_freqs)+ return torch.cat(all_freqs, dim = -1)++ def forward(+ self,+ t,+ seq_len: int | None = None,+ offset = 0+ ):+ should_cache = (+ self.cache_if_possible and+ not self.learned_freq and+ exists(seq_len) and+ self.freqs_for != 'pixel' and+ (offset + seq_len) <= self.cache_max_seq_len+ )++ if (+ should_cache and \+ exists(self.cached_freqs) and \+ (offset + seq_len) <= self.cached_freqs_seq_len+ ):+ return self.cached_freqs[offset:(offset + seq_len)].detach()++ freqs = self.freqs++ freqs = torch.einsum('..., f -> ... f', t.type(freqs.dtype), freqs)+ freqs = einops.repeat(freqs, '... n -> ... (n r)', r = 2)++ if should_cache and offset == 0:+ self.cached_freqs[:seq_len] = freqs.detach()+ self.cached_freqs_seq_len = seq_len++ return freqs++class RotaryEmbeddingBase(nn.Module):+ def __init__(self, dim: int, rope_dim: int):+ super().__init__()+ self.rope = RotaryEmbedding(+ dim=dim // rope_dim,+ freqs_for="pixel",+ max_freq=BYTEDANCE_ROPE_MAX_FREQ,+ )+ freqs = self.rope.freqs+ del self.rope.freqs+ self.rope.register_buffer("freqs", freqs.data)++ def get_axial_freqs(self, *dims):+ return self.rope.get_axial_freqs(*dims)+++class RotaryEmbedding3d(RotaryEmbeddingBase):+ def __init__(self, dim: int):+ super().__init__(dim, rope_dim=3)+ self.mm = False++ def forward(+ self,+ q: torch.FloatTensor, # b h l d+ k: torch.FloatTensor, # b h l d+ size: Tuple[int, int, int],+ ) -> Tuple[+ torch.FloatTensor,+ torch.FloatTensor,+ ]:+ T, H, W = size+ freqs = self.get_axial_freqs(T, H, W)+ q = rearrange(q, "b h (T H W) d -> b h T H W d", T=T, H=H, W=W)+ k = rearrange(k, "b h (T H W) d -> b h T H W d", T=T, H=H, W=W)+ q = apply_rotary_emb(freqs, q.float()).to(q.dtype)+ k = apply_rotary_emb(freqs, k.float()).to(k.dtype)+ q = rearrange(q, "b h T H W d -> b h (T H W) d")+ k = rearrange(k, "b h T H W d -> b h (T H W) d")+ return q, k+++class NaRotaryEmbedding3d(RotaryEmbedding3d):+ def forward(+ self,+ q: torch.FloatTensor,+ k: torch.FloatTensor,+ shape: torch.LongTensor,+ cache: Cache,+ ) -> Tuple[+ torch.FloatTensor,+ torch.FloatTensor,+ ]:+ freqs = cache("rope_freqs_3d", lambda: self.get_freqs(shape))+ freqs = freqs.to(device=q.device)+ q = rearrange(q, "L h d -> h L d")+ k = rearrange(k, "L h d -> h L d")+ q = _apply_seedvr2_rotary_emb(freqs, q.float()).to(q.dtype)+ k = _apply_seedvr2_rotary_emb(freqs, k.float()).to(k.dtype)+ q = rearrange(q, "h L d -> L h d")+ k = rearrange(k, "h L d -> L h d")+ return q, k++ @torch._dynamo.disable+ def get_freqs(+ self,+ shape: torch.LongTensor,+ ) -> torch.Tensor:+ # Primary provenance: ByteDance-Seed/SeedVR models/dit/rope.py builds+ # 7B pixel RoPE with the interleaved-angle convention, not Comfy's+ # Flux freqs_cis matrix.+ plain_rope = RotaryEmbedding(+ dim=self.rope.freqs.numel() * 2,+ freqs_for="pixel",+ max_freq=BYTEDANCE_ROPE_MAX_FREQ,+ )+ plain_rope = plain_rope.to(self.rope.dummy.device)+ freq_list = []+ for f, h, w in shape.tolist():+ freqs = plain_rope.get_axial_freqs(f, h, w)+ freq_list.append(freqs.view(-1, freqs.size(-1)))+ return torch.cat(freq_list, dim=0)+++class MMRotaryEmbeddingBase(RotaryEmbeddingBase):+ def __init__(self, dim: int, rope_dim: int):+ super().__init__(dim, rope_dim)+ self.rope = RotaryEmbedding(+ dim=dim // rope_dim,+ freqs_for="lang",+ theta=ROPE_THETA,+ cache_if_possible=False,+ )+ freqs = self.rope.freqs+ del self.rope.freqs+ self.rope.register_buffer("freqs", freqs.data)+ self.mm = True++def slice_at_dim(t, dim_slice: slice, *, dim):+ dim += (t.ndim if dim < 0 else 0)+ colons = [slice(None)] * t.ndim+ colons[dim] = dim_slice+ return t[tuple(colons)]++# rotary embedding helper functions++def rotate_half(x):+ x = rearrange(x, '... (d r) -> ... d r', r = 2)+ x1, x2 = x.unbind(dim = -1)+ x = torch.stack((-x2, x1), dim = -1)+ return rearrange(x, '... d r -> ... (d r)')+def exists(val):+ return val is not None++def apply_rotary_emb(+ freqs,+ t,+ start_index = 0,+ scale = 1.,+ seq_dim = -2,+ freqs_seq_dim = None+):+ dtype = t.dtype+ if not exists(freqs_seq_dim):+ if freqs.ndim == 2 or t.ndim == 3:+ freqs_seq_dim = 0++ if t.ndim == 3 or exists(freqs_seq_dim):+ seq_len = t.shape[seq_dim]+ freqs = slice_at_dim(freqs, slice(-seq_len, None), dim = freqs_seq_dim)++ rot_feats = freqs.shape[-1]+ end_index = start_index + rot_feats++ t_left = t[..., :start_index]+ t_middle = t[..., start_index:end_index]+ t_right = t[..., end_index:]++ angles = freqs.to(t_middle.device)[..., ::2]+ cos = torch.cos(angles) * scale+ sin = torch.sin(angles) * scale++ col0 = torch.stack([cos, sin], dim=-1)+ col1 = torch.stack([-sin, cos], dim=-1)+ freqs_mat = torch.stack([col0, col1], dim=-1)++ t_middle_out = apply_rope1(t_middle, freqs_mat)+ out = torch.cat((t_left, t_middle_out, t_right), dim=-1)+ return out.type(dtype)+++def _apply_seedvr2_rotary_emb(+ freqs: torch.Tensor,+ t: torch.Tensor,+ start_index: int = 0,+ scale: float = 1.0,+ seq_dim: int = -2,+ freqs_seq_dim: int | None = None,+) -> torch.Tensor:+ dtype = t.dtype+ if freqs_seq_dim is None and (freqs.ndim == 2 or t.ndim == 3):+ freqs_seq_dim = 0++ if t.ndim == 3 or freqs_seq_dim is not None:+ seq_len = t.shape[seq_dim]+ freqs = slice_at_dim(freqs, slice(-seq_len, None), dim=freqs_seq_dim)++ rot_feats = freqs.shape[-1]+ end_index = start_index + rot_feats++ t_left = t[..., :start_index]+ t_middle = t[..., start_index:end_index]+ t_right = t[..., end_index:]++ freqs = freqs.to(device=t_middle.device, dtype=t_middle.dtype)+ cos = freqs.cos() * scale+ sin = freqs.sin() * scale+ t_middle = (t_middle * cos) + (rotate_half(t_middle) * sin)+ return torch.cat((t_left, t_middle, t_right), dim=-1).to(dtype)++def _to_flux_freqs_cis(freqs_interleaved: torch.Tensor) -> torch.Tensor:+ """Convert lucidrains-interleaved freqs to flux-canonical fp32 freqs_cis `[..., d/2, 2, 2]` (cos/-sin/sin/cos), per `comfy/ldm/flux/math.py:rope`."""+ angles = freqs_interleaved[..., ::2].float()+ cos = torch.cos(angles)+ sin = torch.sin(angles)+ out = torch.stack([cos, -sin, sin, cos], dim=-1)+ return rearrange(out, "... d (i j) -> ... d i j", i=2, j=2)+++def _apply_rope1_partial(t: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor:+ """Rotate the leading ``rot_d = 2 * freqs_cis.shape[-3]`` dims of ``t`` and pass the rest+ through; in-place for inference, cloned for training (autograd). Mirrors the legacy+ ``apply_rotary_emb`` ``t_left``/``t_middle``/``t_right`` split: 3B ``rope_dim=128`` gives+ ``42*3 = 126`` rotated of head_dim 128 (trailing 2 unrotated). Fast path skips the cat when+ ``rot_d == t.shape[-1]``.+ """+ out = t.clone() if t.requires_grad or comfy.model_management.in_training else t+ rot_d = 2 * freqs_cis.shape[-3]+ seq_len = out.shape[-2]+ for start in range(0, seq_len, SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS):+ end = min(start + SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS, seq_len)+ freqs_chunk = freqs_cis[start:end]+ if rot_d == out.shape[-1]:+ out[..., start:end, :] = apply_rope1(out[..., start:end, :], freqs_chunk).to(out.dtype)+ else:+ out[..., start:end, :rot_d] = apply_rope1(out[..., start:end, :rot_d], freqs_chunk).to(out.dtype)+ return out+++class NaMMRotaryEmbedding3d(MMRotaryEmbeddingBase):+ def __init__(self, dim: int):+ super().__init__(dim, rope_dim=3)++ def forward(+ self,+ vid_q: torch.FloatTensor, # L h d+ vid_k: torch.FloatTensor, # L h d+ vid_shape: torch.LongTensor, # B 3+ txt_q: torch.FloatTensor, # L h d+ txt_k: torch.FloatTensor, # L h d+ txt_shape: torch.LongTensor, # B 1+ cache: Cache,+ ) -> Tuple[+ torch.FloatTensor,+ torch.FloatTensor,+ torch.FloatTensor,+ torch.FloatTensor,+ ]:+ vid_freqs, txt_freqs = cache(+ "mmrope_freqs_3d",+ lambda: self.get_freqs(vid_shape, txt_shape),+ )+ target_device = vid_q.device+ if vid_freqs.device != target_device:+ vid_freqs = vid_freqs.to(target_device)+ if txt_freqs.device != target_device:+ txt_freqs = txt_freqs.to(target_device)+ vid_q = rearrange(vid_q, "L h d -> h L d")+ vid_k = rearrange(vid_k, "L h d -> h L d")+ vid_q = _apply_rope1_partial(vid_q, vid_freqs)+ vid_k = _apply_rope1_partial(vid_k, vid_freqs)+ vid_q = rearrange(vid_q, "h L d -> L h d")+ vid_k = rearrange(vid_k, "h L d -> L h d")++ txt_q = rearrange(txt_q, "L h d -> h L d")+ txt_k = rearrange(txt_k, "L h d -> h L d")+ txt_q = _apply_rope1_partial(txt_q, txt_freqs)+ txt_k = _apply_rope1_partial(txt_k, txt_freqs)+ txt_q = rearrange(txt_q, "h L d -> L h d")+ txt_k = rearrange(txt_k, "h L d -> L h d")+ return vid_q, vid_k, txt_q, txt_k++ @torch._dynamo.disable # Disable compilation: .tolist() is data-dependent and causes graph breaks+ def get_freqs(+ self,+ vid_shape: torch.LongTensor,+ txt_shape: torch.LongTensor,+ ) -> Tuple[+ torch.Tensor,+ torch.Tensor,+ ]:++ # Calculate actual max dimensions needed for this batch+ max_temporal = 0+ max_height = 0+ max_width = 0+ max_txt_len = 0++ for (f, h, w), l in zip(vid_shape.tolist(), txt_shape[:, 0].tolist()):+ max_temporal = max(max_temporal, l + f) # Need up to l+f for temporal+ max_height = max(max_height, h)+ max_width = max(max_width, w)+ max_txt_len = max(max_txt_len, l)++ autocast_device = "cuda" if torch.cuda.is_available() else "cpu"+ with torch.amp.autocast(autocast_device, enabled=False):+ vid_freqs = self.get_axial_freqs(+ max_temporal + 16,+ max_height + 4,+ max_width + 4,+ ).float()+ txt_freqs = self.get_axial_freqs(max_txt_len + 16)++ # Now slice as before+ vid_freq_list, txt_freq_list = [], []+ for (f, h, w), l in zip(vid_shape.tolist(), txt_shape[:, 0].tolist()):+ vid_freq = vid_freqs[l : l + f, :h, :w].reshape(-1, vid_freqs.size(-1))+ txt_freq = txt_freqs[:l].repeat(1, 3).reshape(-1, vid_freqs.size(-1))+ vid_freq_list.append(vid_freq)+ txt_freq_list.append(txt_freq)+ vid_freqs_interleaved = torch.cat(vid_freq_list, dim=0)+ txt_freqs_interleaved = torch.cat(txt_freq_list, dim=0)++ # Convert from lucidrains-interleaved layout `[θ0, θ0, θ1, θ1, ...]`+ # (produced by `repeat(freqs, '... n -> ... (n r)', r=2)` in the+ # upstream `RotaryEmbedding.forward`) to flux-canonical `freqs_cis`+ # in shape `[..., d/2, 2, 2]` with `cos/-sin/sin/cos` baked in.+ # Mirrors `comfy/ldm/flux/math.py:rope` (line 27) so the trailing+ # 2x2 is the per-frequency rotation matrix that+ # `comfy.ldm.flux.math.apply_rope1` expects.+ return _to_flux_freqs_cis(vid_freqs_interleaved), _to_flux_freqs_cis(txt_freqs_interleaved)++class MMModule(nn.Module):+ def __init__(+ self,+ module: Callable[..., nn.Module],+ *args,+ shared_weights: bool = False,+ vid_only: bool = False,+ **kwargs,+ ):+ super().__init__()+ self.shared_weights = shared_weights+ self.vid_only = vid_only+ if self.shared_weights:+ assert get_args("vid", args) == get_args("txt", args)+ assert get_kwargs("vid", kwargs) == get_kwargs("txt", kwargs)+ self.all = module(*get_args("vid", args), **get_kwargs("vid", kwargs))+ else:+ self.vid = module(*get_args("vid", args), **get_kwargs("vid", kwargs))+ self.txt = (+ module(*get_args("txt", args), **get_kwargs("txt", kwargs))+ if not vid_only+ else None+ )++ def forward(+ self,+ vid: torch.FloatTensor,+ txt: torch.FloatTensor,+ *args,+ **kwargs,+ ) -> Tuple[+ torch.FloatTensor,+ torch.FloatTensor,+ ]:+ vid_module = self.vid if not self.shared_weights else self.all+ vid = vid_module(vid, *get_args("vid", args), **get_kwargs("vid", kwargs))+ if not self.vid_only:+ txt_module = self.txt if not self.shared_weights else self.all+ txt = txt.to(device=vid.device, dtype=vid.dtype)+ txt = txt_module(txt, *get_args("txt", args), **get_kwargs("txt", kwargs))+ return vid, txt++def get_na_rope(rope_type: Optional[str], dim: int):+ if rope_type is None:+ return None+ if rope_type == "rope3d":+ return NaRotaryEmbedding3d(dim=dim)+ if rope_type == "mmrope3d":+ return NaMMRotaryEmbedding3d(dim=dim)++class NaMMAttention(nn.Module):+ def __init__(+ self,+ vid_dim: int,+ txt_dim: int,+ heads: int,+ head_dim: int,+ qk_bias: bool,+ qk_norm,+ qk_norm_eps: float,+ rope_type: Optional[str],+ rope_dim: int,+ shared_weights: bool,+ device, dtype, operations,+ **kwargs,+ ):+ super().__init__()+ dim = MMArg(vid_dim, txt_dim)+ self.heads = heads+ inner_dim = heads * head_dim+ qkv_dim = inner_dim * 3+ self.head_dim = head_dim+ self.proj_qkv = MMModule(+ operations.Linear, dim, qkv_dim, bias=qk_bias, shared_weights=shared_weights, device=device, dtype=dtype+ )+ self.proj_out = MMModule(operations.Linear, inner_dim, dim, shared_weights=shared_weights, device=device, dtype=dtype)+ self.norm_q = MMModule(+ qk_norm,+ normalized_shape=head_dim,+ eps=qk_norm_eps,+ elementwise_affine=True,+ shared_weights=shared_weights,+ device=device, dtype=dtype+ )+ self.norm_k = MMModule(+ qk_norm,+ normalized_shape=head_dim,+ eps=qk_norm_eps,+ elementwise_affine=True,+ shared_weights=shared_weights,+ device=device, dtype=dtype+ )+++ self.rope = get_na_rope(rope_type=rope_type, dim=rope_dim)++ def forward(self):+ pass++def window(+ hid: torch.FloatTensor, # (L c)+ hid_shape: torch.LongTensor, # (b n)+ window_fn: Callable[[torch.Tensor], List[torch.Tensor]],+):+ hid = unflatten(hid, hid_shape)+ hid = list(map(window_fn, hid))+ hid_windows = torch.tensor(list(map(len, hid)), device=hid_shape.device)+ hid, hid_shape = flatten(list(chain(*hid)))+ return hid, hid_shape, hid_windows++def window_idx(+ hid_shape: torch.LongTensor, # (b n)+ window_fn: Callable[[torch.Tensor], List[torch.Tensor]],+):+ hid_idx = torch.arange(hid_shape.prod(-1).sum(), device=hid_shape.device).unsqueeze(-1)+ tgt_idx, tgt_shape, tgt_windows = window(hid_idx, hid_shape, window_fn)+ tgt_idx = tgt_idx.squeeze(-1)+ src_idx = torch.argsort(tgt_idx)+ return (+ lambda hid: torch.index_select(hid, 0, tgt_idx),+ lambda hid: torch.index_select(hid, 0, src_idx),+ tgt_shape,+ tgt_windows,+ )++class NaSwinAttention(NaMMAttention):+ def __init__(+ self,+ *args,+ window: Union[int, Tuple[int, int, int]],+ window_method: bool, # shifted or not+ **kwargs,+ ):+ super().__init__(*args, **kwargs)+ self.version_7b = kwargs.get("version", False)+ self.window = _triple(window)+ self.window_method = window_method+ assert all(map(lambda v: isinstance(v, int) and v >= 0, self.window))++ self.window_op = get_window_op(window_method)++ def forward(+ self,+ vid: torch.FloatTensor, # l c+ txt: torch.FloatTensor, # l c+ vid_shape: torch.LongTensor, # b 3+ txt_shape: torch.LongTensor, # b 1+ cache: Cache,+ ) -> Tuple[+ torch.FloatTensor,+ torch.FloatTensor,+ ]:++ vid_qkv, txt_qkv = self.proj_qkv(vid, txt)++ # re-org the input seq for window attn+ cache_win = cache.namespace(f"{self.window_method}_{self.window}_sd3")++ def make_window(x: torch.Tensor):+ t, h, w, _ = x.shape+ window_slices = self.window_op((t, h, w), self.window)+ return [x[st, sh, sw] for (st, sh, sw) in window_slices]++ window_partition, window_reverse, window_shape, window_count = cache_win(+ "win_transform",+ lambda: window_idx(vid_shape, make_window),+ )+ vid_qkv_win = window_partition(vid_qkv)++ vid_qkv_win = rearrange(vid_qkv_win, "l (o h d) -> l o h d", o=3, d=self.head_dim)+ txt_qkv = rearrange(txt_qkv, "l (o h d) -> l o h d", o=3, d=self.head_dim)++ vid_q, vid_k, vid_v = vid_qkv_win.unbind(1)+ txt_q, txt_k, txt_v = txt_qkv.unbind(1)++ vid_q, txt_q = self.norm_q(vid_q, txt_q)+ vid_k, txt_k = self.norm_k(vid_k, txt_k)++ txt_len = cache("txt_len", lambda: txt_shape.prod(-1))++ vid_len_win = cache_win("vid_len", lambda: window_shape.prod(-1))+ txt_len = txt_len.to(window_count.device)++ # window rope+ if self.rope:+ if self.version_7b:+ vid_q, vid_k = self.rope(vid_q, vid_k, window_shape, cache_win)+ elif self.rope.mm:+ # repeat text q and k for window mmrope+ _, num_h, _ = txt_q.shape+ txt_q_repeat = rearrange(txt_q, "l h d -> l (h d)")+ txt_q_repeat = unflatten(txt_q_repeat, txt_shape)+ txt_q_repeat = [[x] * n for x, n in zip(txt_q_repeat, window_count)]+ txt_q_repeat = list(chain(*txt_q_repeat))+ txt_q_repeat, txt_shape_repeat = flatten(txt_q_repeat)+ txt_q_repeat = rearrange(txt_q_repeat, "l (h d) -> l h d", h=num_h)++ txt_k_repeat = rearrange(txt_k, "l h d -> l (h d)")+ txt_k_repeat = unflatten(txt_k_repeat, txt_shape)+ txt_k_repeat = [[x] * n for x, n in zip(txt_k_repeat, window_count)]+ txt_k_repeat = list(chain(*txt_k_repeat))+ txt_k_repeat, _ = flatten(txt_k_repeat)+ txt_k_repeat = rearrange(txt_k_repeat, "l (h d) -> l h d", h=num_h)++ vid_q, vid_k, txt_q, txt_k = self.rope(+ vid_q, vid_k, window_shape, txt_q_repeat, txt_k_repeat, txt_shape_repeat, cache_win+ )+ else:+ vid_q, vid_k = self.rope(vid_q, vid_k, window_shape, cache_win)++ txt_len_win = cache_win("txt_len", lambda: txt_len.repeat_interleave(window_count))+ all_len_win = cache_win("all_len", lambda: vid_len_win + txt_len_win)+ concat_win, unconcat_win = cache_win(+ "mm_pnp", lambda: repeat_concat_idx(vid_len_win, txt_len, window_count)+ )+ out = optimized_var_attention(+ q=concat_win(vid_q, txt_q),+ k=concat_win(vid_k, txt_k),+ v=concat_win(vid_v, txt_v),+ heads=self.heads, skip_reshape=True, skip_output_reshape=True,+ cu_seqlens_q=cache_win(+ "vid_seqlens_q", lambda: safe_pad_operation(all_len_win.cumsum(0), (1, 0)).int()+ ),+ cu_seqlens_k=cache_win(+ "vid_seqlens_k", lambda: safe_pad_operation(all_len_win.cumsum(0), (1, 0)).int()+ ),+ )+ vid_out, txt_out = unconcat_win(out)++ vid_out = rearrange(vid_out, "l h d -> l (h d)")+ txt_out = rearrange(txt_out, "l h d -> l (h d)")+ vid_out = window_reverse(vid_out)++ vid_out, txt_out = self.proj_out(vid_out, txt_out)++ return vid_out, txt_out++class MLP(nn.Module):+ def __init__(+ self,+ dim: int,+ expand_ratio: int,+ device, dtype, operations+ ):+ super().__init__()+ self.proj_in = operations.Linear(dim, dim * expand_ratio, device=device, dtype=dtype)+ self.act = nn.GELU("tanh")+ self.proj_out = operations.Linear(dim * expand_ratio, dim, device=device, dtype=dtype)++ def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:+ x = self.proj_in(x)+ x = self.act(x)+ x = self.proj_out(x)+ return x+++class SwiGLUMLP(nn.Module):+ def __init__(+ self,+ dim: int,+ expand_ratio: int,+ multiple_of: int = 256,+ device=None, dtype=None, operations=None+ ):+ super().__init__()+ hidden_dim = int(2 * dim * expand_ratio / 3)+ hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)+ self.proj_in_gate = operations.Linear(dim, hidden_dim, bias=False, device=device, dtype=dtype)+ self.proj_out = operations.Linear(hidden_dim, dim, bias=False, device=device, dtype=dtype)+ self.proj_in = operations.Linear(dim, hidden_dim, bias=False, device=device, dtype=dtype)++ def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:+ return self.proj_out(F.silu(self.proj_in_gate(x)) * self.proj_in(x))++def get_mlp(mlp_type: Optional[str] = "normal"):+ # 3b and 7b uses different mlp types+ if mlp_type == "normal":+ return MLP+ elif mlp_type == "swiglu":+ return SwiGLUMLP++class NaMMSRTransformerBlock(nn.Module):+ def __init__(+ self,+ *,+ vid_dim: int,+ txt_dim: int,+ emb_dim: int,+ heads: int,+ head_dim: int,+ expand_ratio: int,+ norm,+ norm_eps: float,+ ada,+ qk_bias: bool,+ qk_norm,+ mlp_type: str,+ shared_weights: bool,+ rope_type: str,+ rope_dim: int,+ is_last_layer: bool,+ device, dtype, operations,+ **kwargs,+ ):+ super().__init__()+ version = kwargs.get("version", False)+ dim = MMArg(vid_dim, txt_dim)+ self.attn_norm = MMModule(norm, normalized_shape=dim, eps=norm_eps, elementwise_affine=False, shared_weights=shared_weights, device=device, dtype=dtype)++ self.attn = NaSwinAttention(+ vid_dim=vid_dim,+ txt_dim=txt_dim,+ heads=heads,+ head_dim=head_dim,+ qk_bias=qk_bias,+ qk_norm=qk_norm,+ qk_norm_eps=norm_eps,+ rope_type=rope_type,+ rope_dim=rope_dim,+ shared_weights=shared_weights,+ window=kwargs.pop("window", None),+ window_method=kwargs.pop("window_method", None),+ version=version,+ device=device, dtype=dtype, operations=operations+ )++ self.mlp_norm = MMModule(norm, normalized_shape=dim, eps=norm_eps, elementwise_affine=False, shared_weights=shared_weights, vid_only=is_last_layer, device=device, dtype=dtype)+ self.mlp = MMModule(+ get_mlp(mlp_type),+ dim=dim,+ expand_ratio=expand_ratio,+ shared_weights=shared_weights,+ vid_only=is_last_layer,+ device=device, dtype=dtype, operations=operations+ )+ self.ada = MMModule(ada, dim=dim, emb_dim=emb_dim, layers=["attn", "mlp"], shared_weights=shared_weights, vid_only=is_last_layer, device=device, dtype=dtype)+ self.is_last_layer = is_last_layer+ self.version = version++ def _seedvr2_7b_mlp(+ self,+ vid: torch.FloatTensor,+ txt: torch.FloatTensor,+ ) -> Tuple[+ torch.FloatTensor,+ torch.FloatTensor,+ ]:+ vid_module = self.mlp.vid if not self.mlp.shared_weights else self.mlp.all+ if comfy.model_management.in_training or vid.requires_grad:+ vid = torch.cat([vid_module(chunk) for chunk in vid.split(SEEDVR2_7B_MLP_CHUNK, dim=0)], dim=0)+ else:+ vid_out = None+ offset = 0+ for chunk in vid.split(SEEDVR2_7B_MLP_CHUNK, dim=0):+ chunk_out = vid_module(chunk)+ if vid_out is None:+ vid_out = chunk_out.new_empty((vid.shape[0], *chunk_out.shape[1:]))+ vid_out[offset:offset + chunk_out.shape[0]] = chunk_out+ offset += chunk_out.shape[0]+ vid = vid_out+ if not self.mlp.vid_only:+ txt_module = self.mlp.txt if not self.mlp.shared_weights else self.mlp.all+ txt = txt.to(device=vid.device, dtype=vid.dtype)+ txt = txt_module(txt)+ return vid, txt++ def forward(+ self,+ vid: torch.FloatTensor, # l c+ txt: torch.FloatTensor, # l c+ vid_shape: torch.LongTensor, # b 3+ txt_shape: torch.LongTensor, # b 1+ emb: torch.FloatTensor,+ cache: Cache,+ ) -> Tuple[+ torch.FloatTensor,+ torch.FloatTensor,+ torch.LongTensor,+ torch.LongTensor,+ ]:+ hid_len = MMArg(+ cache("vid_len", lambda: vid_shape.prod(-1)),+ cache("txt_len", lambda: txt_shape.prod(-1)),+ )+ ada_kwargs = {+ "emb": emb,+ "hid_len": hid_len,+ "cache": cache,+ "branch_tag": MMArg("vid", "txt"),+ }++ vid_attn, txt_attn = self.attn_norm(vid, txt)+ vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="in", **ada_kwargs)+ vid_attn, txt_attn = self.attn(vid_attn, txt_attn, vid_shape, txt_shape, cache)+ vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="out", **ada_kwargs)+ vid_attn, txt_attn = (vid_attn + vid), (txt_attn + txt)++ vid_mlp, txt_mlp = self.mlp_norm(vid_attn, txt_attn)+ vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, layer="mlp", mode="in", **ada_kwargs)+ if self.version:+ vid_mlp, txt_mlp = self._seedvr2_7b_mlp(vid_mlp, txt_mlp)+ else:+ vid_mlp, txt_mlp = self.mlp(vid_mlp, txt_mlp)+ vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, layer="mlp", mode="out", **ada_kwargs)+ vid_mlp, txt_mlp = (vid_mlp + vid_attn), (txt_mlp + txt_attn)++ return vid_mlp, txt_mlp, vid_shape, txt_shape++class PatchOut(nn.Module):+ def __init__(+ self,+ out_channels: int,+ patch_size: Union[int, Tuple[int, int, int]],+ dim: int,+ device, dtype, operations+ ):+ super().__init__()+ t, h, w = _triple(patch_size)+ self.patch_size = t, h, w+ self.proj = operations.Linear(dim, out_channels * t * h * w, device=device, dtype=dtype)++ def forward(+ self,+ vid: torch.Tensor,+ ) -> torch.Tensor:+ t, h, w = self.patch_size+ vid = self.proj(vid)+ vid = rearrange(vid, "b T H W (t h w c) -> b c (T t) (H h) (W w)", t=t, h=h, w=w)+ if t > 1:+ vid = vid[:, :, (t - 1) :]+ return vid++class NaPatchOut(PatchOut):+ def forward(+ self,+ vid: torch.FloatTensor, # l c+ vid_shape: torch.LongTensor,+ cache: Cache = Cache(disable=True), # for test+ vid_shape_before_patchify = None+ ) -> Tuple[+ torch.FloatTensor,+ torch.LongTensor,+ ]:++ t, h, w = self.patch_size+ vid = self.proj(vid)++ if not (t == h == w == 1):+ vid = unflatten(vid, vid_shape)+ for i in range(len(vid)):+ vid[i] = rearrange(vid[i], "T H W (t h w c) -> (T t) (H h) (W w) c", t=t, h=h, w=w)+ if t > 1 and vid_shape_before_patchify[i, 0] % t != 0:+ vid[i] = vid[i][(t - vid_shape_before_patchify[i, 0] % t) :]+ vid, vid_shape = flatten(vid)++ return vid, vid_shape++class PatchIn(nn.Module):+ def __init__(+ self,+ in_channels: int,+ patch_size: Union[int, Tuple[int, int, int]],+ dim: int,+ device, dtype, operations+ ):+ super().__init__()+ t, h, w = _triple(patch_size)+ self.patch_size = t, h, w+ self.proj = operations.Linear(in_channels * t * h * w, dim, device=device, dtype=dtype)++ def forward(+ self,+ vid: torch.Tensor,+ ) -> torch.Tensor:+ t, h, w = self.patch_size+ if t > 1:+ assert vid.size(2) % t == 1+ vid = torch.cat([vid[:, :, :1]] * (t - 1) + [vid], dim=2)+ vid = rearrange(vid, "b c (T t) (H h) (W w) -> b T H W (t h w c)", t=t, h=h, w=w)+ vid = self.proj(vid)+ return vid++class NaPatchIn(PatchIn):+ def forward(+ self,+ vid: torch.Tensor, # l c+ vid_shape: torch.LongTensor,+ cache: Cache = Cache(disable=True), # for test+ ) -> torch.Tensor:+ cache = cache.namespace("patch")+ vid_shape_before_patchify = cache("vid_shape_before_patchify", lambda: vid_shape)+ t, h, w = self.patch_size+ if not (t == h == w == 1):+ vid = unflatten(vid, vid_shape)+ for i in range(len(vid)):+ if t > 1 and vid_shape_before_patchify[i, 0] % t != 0:+ vid[i] = torch.cat([vid[i][:1]] * (t - vid[i].size(0) % t) + [vid[i]], dim=0)+ vid[i] = rearrange(vid[i], "(T t) (H h) (W w) c -> T H W (t h w c)", t=t, h=h, w=w)+ vid, vid_shape = flatten(vid)++ vid = self.proj(vid)+ return vid, vid_shape++def expand_dims(x: torch.Tensor, dim: int, ndim: int):+ shape = x.shape+ shape = shape[:dim] + (1,) * (ndim - len(shape)) + shape[dim:]+ return x.reshape(shape)+++class AdaSingle(nn.Module):+ def __init__(+ self,+ dim: int,+ emb_dim: int,+ layers: List[str],+ modes: List[str] = ["in", "out"],+ device = None, dtype = None,+ ):+ assert emb_dim == 6 * dim, "AdaSingle requires emb_dim == 6 * dim"+ super().__init__()+ self.dim = dim+ self.emb_dim = emb_dim+ self.layers = layers++ randn_kwargs = {"device": device}+ fp8_types = _torch_float8_types()+ if dtype is not None and dtype not in fp8_types:+ randn_kwargs["dtype"] = dtype++ for l in layers:+ if "in" in modes:+ # Passing fp8 ``dtype=`` here would break CPU weight+ # loads: CPU has no ``normal_kernel_cpu`` for fp8.+ self.register_parameter(f"{l}_shift", nn.Parameter(torch.randn(dim, **randn_kwargs) / dim**0.5))+ self.register_parameter(+ f"{l}_scale", nn.Parameter(torch.randn(dim, **randn_kwargs) / dim**0.5 + 1)+ )+ if "out" in modes:+ self.register_parameter(f"{l}_gate", nn.Parameter(torch.randn(dim, **randn_kwargs) / dim**0.5))++ def forward(+ self,+ hid: torch.FloatTensor, # b ... c+ emb: torch.FloatTensor, # b d+ layer: str,+ mode: str,+ cache: Cache = Cache(disable=True),+ branch_tag: str = "",+ hid_len: Optional[torch.LongTensor] = None, # b+ ) -> torch.FloatTensor:+ idx = self.layers.index(layer)+ emb = rearrange(emb, "b (d l g) -> b d l g", l=len(self.layers), g=3)[..., idx, :]+ emb = expand_dims(emb, 1, hid.ndim + 1)++ if hid_len is not None:+ slice_inputs = lambda x, dim: x+ emb = cache(+ f"emb_repeat_{idx}_{branch_tag}",+ lambda: slice_inputs(+ torch.repeat_interleave(emb, hid_len, dim=0),+ dim=0,+ ),+ )++ shiftA, scaleA, gateA = emb.unbind(-1)+ shiftB, scaleB, gateB = (+ getattr(self, f"{layer}_shift", None),+ getattr(self, f"{layer}_scale", None),+ getattr(self, f"{layer}_gate", None),+ )++ fp8_types = _torch_float8_types()+ if fp8_types:+ target_dtype = hid.dtype++ if shiftB is not None and shiftB.dtype in fp8_types:+ shiftB = shiftB.to(target_dtype)+ if scaleB is not None and scaleB.dtype in fp8_types:+ scaleB = scaleB.to(target_dtype)+ if gateB is not None and gateB.dtype in fp8_types:+ gateB = gateB.to(target_dtype)++ if mode == "in":+ return hid.mul_(scaleA + scaleB).add_(shiftA + shiftB)+ if mode == "out":+ if gateB is not None:+ return hid.mul_(gateA + gateB)+ else:+ return hid.mul_(gateA)++ raise NotImplementedError+++def emb_add(emb1: torch.Tensor, emb2: Optional[torch.Tensor]):+ return emb1 if emb2 is None else emb1 + emb2+++class TimeEmbedding(nn.Module):+ def __init__(+ self,+ sinusoidal_dim: int,+ hidden_dim: int,+ output_dim: int,+ device, dtype, operations+ ):+ super().__init__()+ self.sinusoidal_dim = sinusoidal_dim+ self.proj_in = operations.Linear(sinusoidal_dim, hidden_dim, device=device, dtype=dtype)+ self.proj_hid = operations.Linear(hidden_dim, hidden_dim, device=device, dtype=dtype)+ self.proj_out = operations.Linear(hidden_dim, output_dim, device=device, dtype=dtype)+ self.act = nn.SiLU()++ def forward(+ self,+ timestep: Union[int, float, torch.IntTensor, torch.FloatTensor],+ device: torch.device,+ dtype: torch.dtype,+ ) -> torch.FloatTensor:+ if not torch.is_tensor(timestep):+ timestep = torch.tensor([timestep], device=device, dtype=dtype)+ if timestep.ndim == 0:+ timestep = timestep[None]++ emb = get_timestep_embedding(+ timesteps=timestep,+ embedding_dim=self.sinusoidal_dim,+ flip_sin_to_cos=False,+ downscale_freq_shift=0,+ ).to(dtype)+ emb = self.proj_in(emb)+ emb = self.act(emb)+ emb = self.proj_hid(emb)+ emb = self.act(emb)+ emb = self.proj_out(emb)+ return emb++def flatten(+ hid: List[torch.FloatTensor], # List of (*** c)+) -> Tuple[+ torch.FloatTensor, # (L c)+ torch.LongTensor, # (b n)+]:+ assert len(hid) > 0+ shape = torch.stack([torch.tensor(x.shape[:-1], device=hid[0].device) for x in hid])+ hid = torch.cat([x.flatten(0, -2) for x in hid])+ return hid, shape+++def unflatten(+ hid: torch.FloatTensor, # (L c) or (L ... c)+ hid_shape: torch.LongTensor, # (b n)+) -> List[torch.Tensor]: # List of (*** c) or (*** ... c)+ hid_len = hid_shape.prod(-1)+ hid = hid.split(hid_len.tolist())+ hid = [x.unflatten(0, s.tolist()) for x, s in zip(hid, hid_shape)]+ return hid++def repeat(+ hid: torch.FloatTensor, # (L c)+ hid_shape: torch.LongTensor, # (b n)+ pattern: str,+ **kwargs: Dict[str, torch.LongTensor], # (b)+) -> Tuple[+ torch.FloatTensor,+ torch.LongTensor,+]:+ hid = unflatten(hid, hid_shape)+ kwargs = [{k: v[i].item() for k, v in kwargs.items()} for i in range(len(hid))]+ return flatten([einops.repeat(h, pattern, **a) for h, a in zip(hid, kwargs)])++class NaDiT(nn.Module):++ def __init__(+ self,+ norm_eps,+ qk_rope,+ num_layers,+ mlp_type,+ vid_in_channels = 33,+ vid_out_channels = 16,+ vid_dim = 2560,+ txt_in_dim = 5120,+ heads = 20,+ head_dim = 128,+ mm_layers = 10,+ expand_ratio = 4,+ qk_bias = False,+ patch_size = [ 1,2,2 ],+ shared_qkv: bool = False,+ shared_mlp: bool = False,+ window_method: Optional[Tuple[str]] = None,+ temporal_window_size: int = None,+ temporal_shifted: bool = False,+ rope_dim = 128,+ rope_type = "mmrope3d",+ vid_out_norm: Optional[str] = None,+ device = None,+ dtype = None,+ operations = None,+ **kwargs,+ ):+ self._7b_version = vid_dim == SEEDVR2_7B_VID_DIM+ if self._7b_version:+ rope_type = "rope3d"+ self.dtype = dtype+ factory_kwargs = {"device": device, "dtype": dtype}+ window_method = num_layers // 2 * ["720pwin_by_size_bysize","720pswin_by_size_bysize"]+ txt_dim = vid_dim+ emb_dim = vid_dim * 6+ block_type = ["mmdit_sr"] * num_layers+ window = num_layers * [(4,3,3)]+ ada = AdaSingle+ norm = CustomRMSNorm+ qk_norm = CustomRMSNorm+ if isinstance(block_type, str):+ block_type = [block_type] * num_layers+ elif len(block_type) != num_layers:+ raise ValueError("The ``block_type`` list should equal to ``num_layers``.")+ super().__init__()+ # ``torch.empty`` returns uninitialized memory, not zeros. The+ # SeedVR2Conditioning fail-loud guard at+ # ``comfy_extras/nodes_seedvr.py`` distinguishes "buffer was loaded"+ # from "buffer was never populated by the file" by checking+ # ``positive_conditioning.abs().sum() == 0``. That sentinel is only+ # reliable if the post-construction buffer state is deterministically+ # zero, so explicitly zero-fill here rather than relying on the+ # allocator's zero-on-alloc behavior (allocator-dependent and not+ # contractual). When ``load_state_dict`` populates these buffers+ # from a properly-baked SeedVR2 .safetensors, the in-place copy+ # overwrites the zeros with the universal SeedVR2 conditioning+ # tensors (shape (58, 5120) and (64, 5120) bf16).+ self.register_buffer("positive_conditioning", torch.zeros((58, 5120), device=device, dtype=dtype))+ self.register_buffer("negative_conditioning", torch.zeros((64, 5120), device=device, dtype=dtype))+ self.vid_in = NaPatchIn(+ in_channels=vid_in_channels,+ patch_size=patch_size,+ dim=vid_dim,+ device=device, dtype=dtype, operations=operations+ )+ self.txt_in = (+ operations.Linear(txt_in_dim, txt_dim, **factory_kwargs)+ if txt_in_dim and txt_in_dim != txt_dim+ else nn.Identity()+ )+ self.emb_in = TimeEmbedding(+ sinusoidal_dim=BYTEDANCE_SINUSOIDAL_DIM,+ hidden_dim=max(vid_dim, txt_dim),+ output_dim=emb_dim,+ device=device, dtype=dtype, operations=operations+ )++ if window is None or isinstance(window[0], int):+ window = [window] * num_layers+ if window_method is None or isinstance(window_method, str):+ window_method = [window_method] * num_layers+ if temporal_window_size is None or isinstance(temporal_window_size, int):+ temporal_window_size = [temporal_window_size] * num_layers+ if temporal_shifted is None or isinstance(temporal_shifted, bool):+ temporal_shifted = [temporal_shifted] * num_layers++ rope_dim = rope_dim if rope_dim is not None else head_dim // 2+ self.blocks = nn.ModuleList(+ [+ NaMMSRTransformerBlock(+ vid_dim=vid_dim,+ txt_dim=txt_dim,+ emb_dim=emb_dim,+ heads=heads,+ head_dim=head_dim,+ expand_ratio=expand_ratio,+ norm=norm,+ norm_eps=norm_eps,+ ada=ada,+ qk_bias=qk_bias,+ qk_rope=qk_rope,+ qk_norm=qk_norm,+ shared_qkv=shared_qkv,+ shared_mlp=shared_mlp,+ mlp_type=mlp_type,+ rope_dim = rope_dim,+ window=window[i],+ window_method=window_method[i],+ temporal_window_size=temporal_window_size[i],+ temporal_shifted=temporal_shifted[i],+ is_last_layer=(i == num_layers - 1) and not self._7b_version,+ rope_type = rope_type,+ shared_weights=not (+ (i < mm_layers) if isinstance(mm_layers, int) else mm_layers[i]+ ),+ version = self._7b_version,+ operations = operations,+ **kwargs,+ **factory_kwargs+ )+ for i in range(num_layers)+ ]+ )+ self.vid_out = NaPatchOut(+ out_channels=vid_out_channels,+ patch_size=patch_size,+ dim=vid_dim,+ device=device, dtype=dtype, operations=operations+ )++ self.need_txt_repeat = block_type[0] in [+ "mmdit_stwin",+ "mmdit_stwin_spatial",+ "mmdit_stwin_3d_spatial",+ ]++ self.vid_out_norm = None+ if vid_out_norm is not None:+ self.vid_out_norm = CustomRMSNorm(+ normalized_shape=vid_dim,+ eps=norm_eps,+ elementwise_affine=True,+ device=device, dtype=dtype+ )+ self.vid_out_ada = ada(+ dim=vid_dim,+ emb_dim=emb_dim,+ layers=["out"],+ modes=["in"],+ device=device, dtype=dtype+ )++ def _resolve_text_conditioning(self, context, cond_or_uncond=None):+ if context is None or getattr(context, "numel", lambda: None)() == 0:+ context = self.positive_conditioning+ return flatten([context])+ if NaDiT._seedvr2_is_single_conditioning_branch(cond_or_uncond):+ if context.shape[0] == 1:+ context = context.squeeze(0)+ return flatten([context])+ return flatten(context.unbind(0))+ if context.shape[0] % 2 != 0:+ raise ValueError(f"SeedVR2 expected an even text-conditioning batch, got shape {tuple(context.shape)}")+ neg_cond, pos_cond = context.chunk(2, dim=0)+ if pos_cond.shape[0] == 1:+ pos_cond, neg_cond = pos_cond.squeeze(0), neg_cond.squeeze(0)+ return flatten([pos_cond, neg_cond])+ return flatten((*pos_cond.unbind(0), *neg_cond.unbind(0)))++ @staticmethod+ def _seedvr2_is_single_conditioning_branch(cond_or_uncond):+ if cond_or_uncond is None or len(cond_or_uncond) == 0:+ return False+ first = cond_or_uncond[0]+ return all(entry == first for entry in cond_or_uncond)++ def _swap_pos_neg_halves(self, out, cond_or_uncond=None):+ if NaDiT._seedvr2_is_single_conditioning_branch(cond_or_uncond):+ return out+ # ``dim=0`` is explicit on both calls. The contract is "split+ # the batch axis into two halves and swap them"; making the+ # axis load-bearing in source guards against silent drift if a+ # future refactor reorders tensor axes.+ pos, neg = out.chunk(2, dim=0)+ return torch.cat([neg, pos], dim=0)++ def forward(+ self,+ x,+ timestep,+ context, # l c+ disable_cache: bool = False, # for test # TODO ? // gives an error when set to True+ **kwargs+ ):+ transformer_options = kwargs.get("transformer_options", {})+ patches_replace = transformer_options.get("patches_replace", {})+ blocks_replace = patches_replace.get("dit", {})+ conditions = kwargs.get("condition")+ b, tc, h, w = x.shape+ x = x.view(b, 16, -1, h, w)+ conditions = conditions.view(b, 17, -1, h, w)+ x = x.movedim(1, -1)+ conditions = conditions.movedim(1, -1)+ cache = Cache(disable=disable_cache)++ txt, txt_shape = self._resolve_text_conditioning(context, transformer_options.get("cond_or_uncond"))++ vid, vid_shape = flatten(x)+ cond_latent, _ = flatten(conditions)++ vid = torch.cat([vid, cond_latent], dim=-1)+ if txt_shape.size(-1) == 1 and self.need_txt_repeat:+ txt, txt_shape = repeat(txt, txt_shape, "l c -> t l c", t=vid_shape[:, 0])++ txt = self.txt_in(txt)++ vid_shape_before_patchify = vid_shape+ vid, vid_shape = self.vid_in(vid, vid_shape, cache=cache)++ emb = self.emb_in(timestep, device=vid.device, dtype=vid.dtype)++ for i, block in enumerate(self.blocks):+ if ("block", i) in blocks_replace:+ def block_wrap(args):+ out = {}+ out["vid"], out["txt"], out["vid_shape"], out["txt_shape"] = block(+ vid=args["vid"],+ txt=args["txt"],+ vid_shape=args["vid_shape"],+ txt_shape=args["txt_shape"],+ emb=args["emb"],+ cache=args["cache"],+ )+ return out+ out = blocks_replace[("block", i)]({+ "vid":vid,+ "txt":txt,+ "vid_shape":vid_shape,+ "txt_shape":txt_shape,+ "emb":emb,+ "cache":cache,+ }, {"original_block": block_wrap})+ vid, txt, vid_shape, txt_shape = out["vid"], out["txt"], out["vid_shape"], out["txt_shape"]+ else:+ vid, txt, vid_shape, txt_shape = block(+ vid=vid,+ txt=txt,+ vid_shape=vid_shape,+ txt_shape=txt_shape,+ emb=emb,+ cache=cache,+ )++ if self.vid_out_norm:+ vid = self.vid_out_norm(vid)+ vid = self.vid_out_ada(+ vid,+ emb=emb,+ layer="out",+ mode="in",+ hid_len=cache("vid_len", lambda: vid_shape.prod(-1)),+ cache=cache,+ branch_tag="vid",+ )++ vid, vid_shape = self.vid_out(vid, vid_shape, cache, vid_shape_before_patchify = vid_shape_before_patchify)+ vid = unflatten(vid, vid_shape)+ out = torch.stack(vid)+ out = out.movedim(-1, 1)+ out = rearrange(out, "b c t h w -> b (c t) h w")+ return self._swap_pos_neg_halves(out, transformer_options.get("cond_or_uncond"))diff --git a/comfy/ldm/seedvr/vae.py b/comfy/ldm/seedvr/vae.pynew file mode 100644index 000000000..68b11c0ff--- /dev/null+++ b/comfy/ldm/seedvr/vae.py@@ -0,0 +1,2110 @@+from contextlib import nullcontext+from typing import Literal, Optional, Tuple+import gc+import torch+import torch.nn as nn+import torch.nn.functional as F+from einops import rearrange+from torch import Tensor+from contextlib import contextmanager+from comfy.utils import ProgressBar++from comfy.ldm.seedvr.model import safe_pad_operation+from comfy.ldm.seedvr.constants import (+ BYTEDANCE_BLOCK_OUT_CHANNELS,+ BYTEDANCE_CONTIGUOUS_BATCH_THRESHOLD,+ BYTEDANCE_GN_CHUNKS_FP16,+ BYTEDANCE_GN_CHUNKS_FP32,+ BYTEDANCE_LOGVAR_CLAMP_MAX,+ BYTEDANCE_LOGVAR_CLAMP_MIN,+ BYTEDANCE_SLICING_SAMPLE_MIN,+ BYTEDANCE_VAE_CONV_MEM_GIB,+ BYTEDANCE_VAE_NORM_MEM_GIB,+ BYTEDANCE_VAE_SCALING_FACTOR,+ BYTEDANCE_VAE_SHIFTING_FACTOR,+ BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE,+ BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE,+ SEEDVR2_LATENT_CHANNELS,+)+from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.diffusionmodules.model import vae_attention++import math+from enum import Enum+from comfy.ops import NVIDIA_MEMORY_CONV_BUG_WORKAROUND++import logging+import comfy.model_management+import comfy.ops+ops = comfy.ops.disable_weight_init+++def _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap, temporal_scale=1):+ if temporal_size is None:+ return None++ temporal_size = int(temporal_size)+ if temporal_size <= 0:+ return 0++ temporal_overlap = max(0, int(temporal_overlap or 0))+ temporal_overlap = min(temporal_overlap, temporal_size - 1)+ temporal_step = temporal_size - temporal_overlap+ temporal_scale = max(1, int(temporal_scale))+ return max(1, math.ceil(temporal_step / temporal_scale))+++def _seedvr2_clamped_spatial_overlap(overlap, tile_size):+ overlap = max(0, int(overlap))+ tile_size = max(1, int(tile_size))+ return min(overlap, tile_size - 1)+++def _seedvr2_clear_temporal_memory(model):+ for module in model.modules():+ if hasattr(module, "memory"):+ module.memory = None++[email protected]_mode()+def tiled_vae(+ x,+ vae_model,+ tile_size=(512, 512),+ tile_overlap=(64, 64),+ temporal_size=16,+ temporal_overlap=0,+ encode=True,+ **kwargs,+):+ gc.collect()+ comfy.model_management.soft_empty_cache()++ x = x.to(next(vae_model.parameters()).dtype)+ if x.ndim != 5:+ x = x.unsqueeze(2)++ _, _, d, h, w = x.shape++ sf_s = getattr(vae_model, "spatial_downsample_factor", BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE)+ sf_t = getattr(vae_model, "temporal_downsample_factor", BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE)+ if encode:+ slicing_attr = "slicing_sample_min_size"+ slicing_min_size = _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap)+ else:+ slicing_attr = "slicing_latent_min_size"+ slicing_min_size = _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap, sf_t)+ if encode:+ ti_h, ti_w = tile_size+ ov_h = _seedvr2_clamped_spatial_overlap(tile_overlap[0], ti_h)+ ov_w = _seedvr2_clamped_spatial_overlap(tile_overlap[1], ti_w)+ blend_ov_h = max(0, ov_h // sf_s)+ blend_ov_w = max(0, ov_w // sf_s)+ target_d = (d + sf_t - 1) // sf_t+ target_h = (h + sf_s - 1) // sf_s+ target_w = (w + sf_s - 1) // sf_s+ else:+ ti_h = max(1, tile_size[0] // sf_s)+ ti_w = max(1, tile_size[1] // sf_s)+ ov_h = _seedvr2_clamped_spatial_overlap(tile_overlap[0] // sf_s, ti_h)+ ov_w = _seedvr2_clamped_spatial_overlap(tile_overlap[1] // sf_s, ti_w)+ blend_ov_h = ov_h * sf_s+ blend_ov_w = ov_w * sf_s++ target_d = max(1, d * sf_t - (sf_t - 1))+ target_h = h * sf_s+ target_w = w * sf_s++ stride_h = max(1, ti_h - ov_h)+ stride_w = max(1, ti_w - ov_w)++ storage_device = vae_model.device+ result = None+ count = None+ def run_temporal_chunks(spatial_tile, model=vae_model, device=storage_device):+ device = torch.device(device)+ _seedvr2_clear_temporal_memory(model)+ t_chunk = spatial_tile.to(device=device, dtype=next(model.parameters()).dtype, non_blocking=True).contiguous()+ old_device = getattr(model, "device", None)+ model.device = device+ old_slicing_min_size = getattr(model, slicing_attr, None)+ if old_slicing_min_size is not None and slicing_min_size is not None:+ if slicing_min_size <= 0:+ setattr(model, slicing_attr, t_chunk.shape[2])+ else:+ setattr(model, slicing_attr, slicing_min_size)+ try:+ if encode:+ out = model.encode(t_chunk)[0]+ else:+ out = model.decode_(t_chunk)+ finally:+ if old_slicing_min_size is not None and slicing_min_size is not None:+ setattr(model, slicing_attr, old_slicing_min_size)+ if old_device is not None:+ model.device = old_device+ if isinstance(out, (tuple, list)):+ out = out[0]+ if out.ndim == 4:+ out = out.unsqueeze(2)+ return out.to(storage_device)++ ramp_cache = {}+ def get_ramp(steps):+ if steps not in ramp_cache:+ t = torch.linspace(0, 1, steps=steps, device=storage_device, dtype=torch.float32)+ ramp_cache[steps] = 0.5 - 0.5 * torch.cos(t * torch.pi)+ return ramp_cache[steps]++ tile_ranges = []+ for y_idx in range(0, h, stride_h):+ y_end = min(y_idx + ti_h, h)+ if y_idx > 0 and (y_end - y_idx) <= ov_h:+ continue+ for x_idx in range(0, w, stride_w):+ x_end = min(x_idx + ti_w, w)+ if x_idx > 0 and (x_end - x_idx) <= ov_w:+ continue+ tile_ranges.append((y_idx, y_end, x_idx, x_end))++ total_tiles = len(tile_ranges)+ bar = ProgressBar(total_tiles)+ single_spatial_tile = h <= ti_h and w <= ti_w++ _seedvr2_clear_temporal_memory(vae_model)++ def run_tile(tile_index, tile_range):+ y_idx, y_end, x_idx, x_end = tile_range+ tile_x = x[:, :, :, y_idx:y_end, x_idx:x_end]+ tile_out = run_temporal_chunks(tile_x)+ return tile_index, y_idx, y_end, x_idx, x_end, tile_out++ ordered_tile_outputs = (+ run_tile(tile_index, tile_range)+ for tile_index, tile_range in enumerate(tile_ranges)+ )++ for _, y_idx, y_end, x_idx, x_end, tile_out in ordered_tile_outputs:++ if single_spatial_tile:+ result = tile_out[:, :, :target_d, :target_h, :target_w]+ if result.device != x.device:+ result = result.to(x.device).to(x.dtype)+ if x.shape[2] == 1 and sf_t == 1:+ result = result.squeeze(2)+ bar.update(1)+ return result++ if result is None:+ b_out, c_out = tile_out.shape[0], tile_out.shape[1]+ result = torch.zeros((b_out, c_out, target_d, target_h, target_w), device=storage_device, dtype=torch.float32)+ count = torch.zeros((1, 1, 1, target_h, target_w), device=storage_device, dtype=torch.float32)++ if encode:+ ys, ye = y_idx // sf_s, (y_idx // sf_s) + tile_out.shape[3]+ xs, xe = x_idx // sf_s, (x_idx // sf_s) + tile_out.shape[4]+ cur_ov_h = max(0, min(blend_ov_h, tile_out.shape[3] // 2))+ cur_ov_w = max(0, min(blend_ov_w, tile_out.shape[4] // 2))+ else:+ ys, ye = y_idx * sf_s, (y_idx * sf_s) + tile_out.shape[3]+ xs, xe = x_idx * sf_s, (x_idx * sf_s) + tile_out.shape[4]+ cur_ov_h = max(0, min(blend_ov_h, tile_out.shape[3] // 2))+ cur_ov_w = max(0, min(blend_ov_w, tile_out.shape[4] // 2))++ w_h = torch.ones((tile_out.shape[3],), device=storage_device)+ w_w = torch.ones((tile_out.shape[4],), device=storage_device)++ if cur_ov_h > 0:+ r = get_ramp(cur_ov_h)+ if y_idx > 0:+ w_h[:cur_ov_h] = r+ if y_end < h:+ w_h[-cur_ov_h:] = 1.0 - r++ if cur_ov_w > 0:+ r = get_ramp(cur_ov_w)+ if x_idx > 0:+ w_w[:cur_ov_w] = r+ if x_end < w:+ w_w[-cur_ov_w:] = 1.0 - r++ final_weight = w_h.view(1,1,1,-1,1) * w_w.view(1,1,1,1,-1)++ valid_d = min(tile_out.shape[2], result.shape[2])+ tile_out = tile_out[:, :, :valid_d, :, :]++ tile_out.mul_(final_weight)++ result[:, :, :valid_d, ys:ye, xs:xe] += tile_out+ count[:, :, :, ys:ye, xs:xe] += final_weight++ del tile_out, final_weight, w_h, w_w+ bar.update(1)++ result.div_(count.clamp(min=1e-6))+ _seedvr2_clear_temporal_memory(vae_model)++ if result.device != x.device:+ result = result.to(x.device).to(x.dtype)++ if x.shape[2] == 1 and sf_t == 1:+ result = result.squeeze(2)++ return result++_NORM_LIMIT = float("inf")+def get_norm_limit():+ return _NORM_LIMIT+++def set_norm_limit(value: Optional[float] = None):+ global _NORM_LIMIT+ if value is None:+ value = float("inf")+ _NORM_LIMIT = value++@contextmanager+def ignore_padding(model):+ orig_padding = model.padding+ model.padding = (0, 0, 0)+ try:+ yield+ finally:+ model.padding = orig_padding++class MemoryState(Enum):+ DISABLED = 0+ INITIALIZING = 1+ ACTIVE = 2+ UNSET = 3++def get_cache_size(conv_module, input_len, pad_len, dim=0):+ dilated_kernerl_size = conv_module.dilation[dim] * (conv_module.kernel_size[dim] - 1) + 1+ output_len = (input_len + pad_len - dilated_kernerl_size) // conv_module.stride[dim] + 1+ remain_len = (+ input_len + pad_len - ((output_len - 1) * conv_module.stride[dim] + dilated_kernerl_size)+ )+ overlap_len = dilated_kernerl_size - conv_module.stride[dim]+ cache_len = overlap_len + remain_len # >= 0++ assert output_len > 0+ return cache_len++class DiagonalGaussianDistribution(object):+ def __init__(self, parameters: torch.Tensor, deterministic: bool = False):+ self.parameters = parameters+ self.mean, self.logvar = torch.chunk(parameters, 2, dim=1)+ self.logvar = torch.clamp(self.logvar, BYTEDANCE_LOGVAR_CLAMP_MIN, BYTEDANCE_LOGVAR_CLAMP_MAX)+ self.deterministic = deterministic+ self.std = torch.exp(0.5 * self.logvar)+ self.var = torch.exp(self.logvar)+ if self.deterministic:+ self.var = self.std = torch.zeros_like(+ self.mean, device=self.parameters.device, dtype=self.parameters.dtype+ )++ def sample(self, generator: Optional[torch.Generator] = None) -> torch.Tensor:+ sample = torch.randn(+ self.mean.shape,+ generator=generator,+ device=self.parameters.device,+ dtype=self.parameters.dtype,+ )+ x = self.mean + self.std * sample+ return x++ def mode(self):+ return self.mean++class SpatialNorm(nn.Module):+ def __init__(+ self,+ f_channels: int,+ zq_channels: int,+ ):+ super().__init__()+ self.norm_layer = ops.GroupNorm(num_channels=f_channels, num_groups=32, eps=1e-6, affine=True)+ self.conv_y = ops.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0)+ self.conv_b = ops.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0)++ def forward(self, f: torch.Tensor, zq: torch.Tensor) -> torch.Tensor:+ f_size = f.shape[-2:]+ zq = F.interpolate(zq, size=f_size, mode="nearest")+ norm_f = self.norm_layer(f)+ new_f = norm_f * self.conv_y(zq) + self.conv_b(zq)+ return new_f++# partial implementation of diffusers's Attention for comfyui+class Attention(nn.Module):+ def __init__(+ self,+ query_dim: int,+ cross_attention_dim: Optional[int] = None,+ heads: int = 8,+ kv_heads: Optional[int] = None,+ dim_head: int = 64,+ dropout: float = 0.0,+ bias: bool = False,+ upcast_attention: bool = False,+ upcast_softmax: bool = False,+ added_kv_proj_dim: Optional[int] = None,+ added_proj_bias: Optional[bool] = True,+ norm_num_groups: Optional[int] = None,+ spatial_norm_dim: Optional[int] = None,+ out_bias: bool = True,+ scale_qk: bool = True,+ only_cross_attention: bool = False,+ eps: float = 1e-5,+ rescale_output_factor: float = 1.0,+ residual_connection: bool = False,+ _from_deprecated_attn_block: bool = False,+ out_dim: int = None,+ out_context_dim: int = None,+ context_pre_only=None,+ pre_only=False,+ is_causal: bool = False,+ ):+ super().__init__()++ self.inner_dim = out_dim if out_dim is not None else dim_head * heads+ self.inner_kv_dim = self.inner_dim if kv_heads is None else dim_head * kv_heads+ self.query_dim = query_dim+ self.use_bias = bias+ self.is_cross_attention = cross_attention_dim is not None+ self.cross_attention_dim = cross_attention_dim if cross_attention_dim is not None else query_dim+ self.upcast_attention = upcast_attention+ self.upcast_softmax = upcast_softmax+ self.rescale_output_factor = rescale_output_factor+ self.residual_connection = residual_connection+ self.dropout = dropout+ self.fused_projections = False+ self.out_dim = out_dim if out_dim is not None else query_dim+ self.out_context_dim = out_context_dim if out_context_dim is not None else query_dim+ self.context_pre_only = context_pre_only+ self.pre_only = pre_only+ self.is_causal = is_causal++ # we make use of this private variable to know whether this class is loaded+ # with an deprecated state dict so that we can convert it on the fly+ self._from_deprecated_attn_block = _from_deprecated_attn_block++ self.scale_qk = scale_qk+ self.scale = dim_head**-0.5 if self.scale_qk else 1.0++ self.heads = out_dim // dim_head if out_dim is not None else heads+ self.sliceable_head_dim = heads++ self.added_kv_proj_dim = added_kv_proj_dim+ self.only_cross_attention = only_cross_attention++ if norm_num_groups is not None:+ self.group_norm = ops.GroupNorm(num_channels=query_dim, num_groups=norm_num_groups, eps=eps, affine=True)+ else:+ self.group_norm = None++ if spatial_norm_dim is not None:+ self.spatial_norm = SpatialNorm(f_channels=query_dim, zq_channels=spatial_norm_dim)+ else:+ self.spatial_norm = None++ self.norm_q = None+ self.norm_k = None++ self.norm_cross = None+ self.to_q = ops.Linear(query_dim, self.inner_dim, bias=bias)++ if not self.only_cross_attention:+ # only relevant for the `AddedKVProcessor` classes+ self.to_k = ops.Linear(self.cross_attention_dim, self.inner_kv_dim, bias=bias)+ self.to_v = ops.Linear(self.cross_attention_dim, self.inner_kv_dim, bias=bias)+ else:+ self.to_k = None+ self.to_v = None++ self.added_proj_bias = added_proj_bias+ if self.added_kv_proj_dim is not None:+ self.add_k_proj = ops.Linear(added_kv_proj_dim, self.inner_kv_dim, bias=added_proj_bias)+ self.add_v_proj = ops.Linear(added_kv_proj_dim, self.inner_kv_dim, bias=added_proj_bias)+ if self.context_pre_only is not None:+ self.add_q_proj = ops.Linear(added_kv_proj_dim, self.inner_dim, bias=added_proj_bias)+ else:+ self.add_q_proj = None+ self.add_k_proj = None+ self.add_v_proj = None++ if not self.pre_only:+ self.to_out = nn.ModuleList([])+ self.to_out.append(ops.Linear(self.inner_dim, self.out_dim, bias=out_bias))+ self.to_out.append(nn.Dropout(dropout))+ else:+ self.to_out = None++ if self.context_pre_only is not None and not self.context_pre_only:+ self.to_add_out = ops.Linear(self.inner_dim, self.out_context_dim, bias=out_bias)+ else:+ self.to_add_out = None++ self.norm_added_q = None+ self.norm_added_k = None+ self.optimized_vae_attention = vae_attention()++ def __call__(+ self,+ hidden_states: torch.Tensor,+ encoder_hidden_states: Optional[torch.Tensor] = None,+ attention_mask: Optional[torch.Tensor] = None,+ temb: Optional[torch.Tensor] = None,+ *args,+ **kwargs,+ ) -> torch.Tensor:++ residual = hidden_states+ if self.spatial_norm is not None:+ hidden_states = self.spatial_norm(hidden_states, temb)++ input_ndim = hidden_states.ndim++ if input_ndim == 4:+ batch_size, channel, height, width = hidden_states.shape+ hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)++ batch_size, sequence_length, _ = (+ hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape+ )++ if attention_mask is not None:+ attention_mask = self.prepare_attention_mask(attention_mask, sequence_length, batch_size)+ attention_mask = attention_mask.view(batch_size, self.heads, -1, attention_mask.shape[-1])++ if self.group_norm is not None:+ hidden_states = self.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)++ query = self.to_q(hidden_states)++ if encoder_hidden_states is None:+ encoder_hidden_states = hidden_states+ elif self.norm_cross:+ encoder_hidden_states = self.norm_encoder_hidden_states(encoder_hidden_states)++ key = self.to_k(encoder_hidden_states)+ value = self.to_v(encoder_hidden_states)++ inner_dim = key.shape[-1]+ head_dim = inner_dim // self.heads++ query = query.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)++ key = key.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)+ value = value.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)++ if self.norm_q is not None:+ query = self.norm_q(query)+ if self.norm_k is not None:+ key = self.norm_k(key)++ if input_ndim == 4 and encoder_hidden_states is hidden_states and attention_mask is None and self.heads == 1:+ query = query.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width)+ key = key.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width)+ value = value.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width)+ hidden_states = self.optimized_vae_attention(query, key, value).reshape(batch_size, self.heads, head_dim, height * width).transpose(2, 3)+ else:+ hidden_states = optimized_attention(query, key, value, heads = self.heads, mask = attention_mask, skip_reshape=True, skip_output_reshape=True)++ hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, self.heads * head_dim)+ hidden_states = hidden_states.to(query.dtype)++ # linear proj+ hidden_states = self.to_out[0](hidden_states)+ # dropout+ hidden_states = self.to_out[1](hidden_states)++ if input_ndim == 4:+ hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)++ if self.residual_connection:+ hidden_states = hidden_states + residual++ hidden_states = hidden_states / self.rescale_output_factor++ return hidden_states+++def inflate_weight(weight_2d: torch.Tensor, weight_3d: torch.Tensor):+ with torch.no_grad():+ depth = weight_3d.size(2)+ weight_3d.copy_(weight_2d.unsqueeze(2).repeat(1, 1, depth, 1, 1) / depth)+ return weight_3d++def inflate_bias(bias_2d: torch.Tensor, bias_3d: torch.Tensor):+ with torch.no_grad():+ bias_3d.copy_(bias_2d)+ return bias_3d+++def modify_state_dict(layer, state_dict, prefix, inflate_weight_fn, inflate_bias_fn):+ weight_name = prefix + "weight"+ bias_name = prefix + "bias"+ if weight_name in state_dict:+ weight_2d = state_dict[weight_name]+ if weight_2d.dim() == 4:+ weight_3d = inflate_weight_fn(+ weight_2d=weight_2d,+ weight_3d=layer.weight,+ )+ state_dict[weight_name] = weight_3d+ else:+ return state_dict+ if bias_name in state_dict:+ bias_2d = state_dict[bias_name]+ if bias_2d.dim() == 1:+ bias_3d = inflate_bias_fn(+ bias_2d=bias_2d,+ bias_3d=layer.bias,+ )+ state_dict[bias_name] = bias_3d+ return state_dict++def causal_norm_wrapper(norm_layer: nn.Module, x: torch.Tensor) -> torch.Tensor:+ input_dtype = x.dtype+ if isinstance(norm_layer, (ops.LayerNorm, ops.RMSNorm)):+ if x.ndim == 4:+ x = rearrange(x, "b c h w -> b h w c")+ x = norm_layer(x)+ x = rearrange(x, "b h w c -> b c h w")+ return x.to(input_dtype)+ if x.ndim == 5:+ x = rearrange(x, "b c t h w -> b t h w c")+ x = norm_layer(x)+ x = rearrange(x, "b t h w c -> b c t h w")+ return x.to(input_dtype)+ if isinstance(norm_layer, (ops.GroupNorm, nn.BatchNorm2d, nn.SyncBatchNorm)):+ if x.ndim <= 4:+ return norm_layer(x).to(input_dtype)+ if x.ndim == 5:+ t = x.size(2)+ x = rearrange(x, "b c t h w -> (b t) c h w")+ memory_occupy = x.numel() * x.element_size() / 1024**3+ if isinstance(norm_layer, ops.GroupNorm) and memory_occupy > get_norm_limit():+ num_chunks = min(BYTEDANCE_GN_CHUNKS_FP16 if x.element_size() == 2 else BYTEDANCE_GN_CHUNKS_FP32, norm_layer.num_groups)+ assert norm_layer.num_groups % num_chunks == 0+ num_groups_per_chunk = norm_layer.num_groups // num_chunks++ x = list(x.chunk(num_chunks, dim=1))+ weights = norm_layer.weight.chunk(num_chunks, dim=0)+ biases = norm_layer.bias.chunk(num_chunks, dim=0)+ for i, (w, b) in enumerate(zip(weights, biases)):+ x[i] = F.group_norm(x[i], num_groups_per_chunk, w, b, norm_layer.eps)+ x[i] = x[i].to(input_dtype)+ x = torch.cat(x, dim=1)+ else:+ x = norm_layer(x)+ x = rearrange(x, "(b t) c h w -> b c t h w", t=t)+ return x.to(input_dtype)+ raise NotImplementedError++def safe_interpolate_operation(x, size=None, scale_factor=None, mode='nearest', align_corners=None, recompute_scale_factor=None):+ problematic_modes = ['bilinear', 'bicubic', 'trilinear']++ if mode in problematic_modes:+ try:+ return F.interpolate(+ x,+ size=size,+ scale_factor=scale_factor,+ mode=mode,+ align_corners=align_corners,+ recompute_scale_factor=recompute_scale_factor+ )+ except RuntimeError as e:+ if ("not implemented for 'Half'" in str(e) or+ "compute_indices_weights" in str(e)):+ original_dtype = x.dtype+ return F.interpolate(+ x.float(),+ size=size,+ scale_factor=scale_factor,+ mode=mode,+ align_corners=align_corners,+ recompute_scale_factor=recompute_scale_factor+ ).to(original_dtype)+ else:+ raise e+ else:+ # Pour 'nearest' et autres modes compatibles, pas de fix nécessaire+ return F.interpolate(+ x,+ size=size,+ scale_factor=scale_factor,+ mode=mode,+ align_corners=align_corners,+ recompute_scale_factor=recompute_scale_factor+ )++_receptive_field_t = Literal["half", "full"]++def extend_head(tensor, times: int = 2, memory = None):+ if memory is not None:+ return torch.cat((memory.to(tensor), tensor), dim=2)+ assert times >= 0, "Invalid input for function 'extend_head'!"+ if times == 0:+ return tensor+ else:+ tile_repeat = [1] * tensor.ndim+ tile_repeat[2] = times+ return torch.cat(tensors=(torch.tile(tensor[:, :, :1], tile_repeat), tensor), dim=2)++def cache_send_recv(tensor, cache_size, times, memory=None):+ recv_buffer = None++ if memory is not None:+ recv_buffer = memory.to(tensor[0])+ elif times > 0:+ tile_repeat = [1] * tensor[0].ndim+ tile_repeat[2] = times+ recv_buffer = torch.tile(tensor[0][:, :, :1], tile_repeat)++ return recv_buffer++class InflatedCausalConv3d(ops.Conv3d):+ def __init__(+ self,+ *args,+ inflation_mode,+ memory_device = "same",+ **kwargs,+ ):+ self.inflation_mode = inflation_mode+ self.memory = None+ super().__init__(*args, **kwargs)+ self.temporal_padding = self.padding[0]+ self.memory_device = memory_device+ self.padding = (0, *self.padding[1:])+ self.memory_limit = float("inf")+ self.logged_once = False++ def set_memory_limit(self, value: float):+ self.memory_limit = value++ def set_memory_device(self, memory_device):+ self.memory_device = memory_device++ def _conv_forward(self, input, weight, bias, *args, **kwargs):+ if (NVIDIA_MEMORY_CONV_BUG_WORKAROUND and+ weight.dtype in (torch.float16, torch.bfloat16) and+ hasattr(torch.backends.cudnn, 'is_available') and+ torch.backends.cudnn.is_available() and+ getattr(torch.backends.cudnn, 'enabled', True)):+ try:+ out = torch.cudnn_convolution(+ input, weight, self.padding, self.stride, self.dilation, self.groups,+ benchmark=False, deterministic=False, allow_tf32=True+ )+ if bias is not None:+ out += bias.reshape((1, -1) + (1,) * (out.ndim - 2))+ return out+ except RuntimeError:+ pass+ except NotImplementedError:+ pass+ try:+ return super()._conv_forward(input, weight, bias, *args, **kwargs)+ except NotImplementedError:+ # for: Could not run 'aten::cudnn_convolution' with arguments from the 'CPU' backend+ if not self.logged_once:+ logging.warning("VAE is on CPU for decoding. This is most likely due to not enough memory")+ self.logged_once = True+ return F.conv3d(input, weight, bias, *args, **kwargs)++ def memory_limit_conv(+ self,+ x,+ *,+ split_dim=3,+ padding=(0, 0, 0, 0, 0, 0),+ prev_cache=None,+ ):+ # Compatible with no limit.+ if math.isinf(self.memory_limit):+ if prev_cache is not None:+ x = torch.cat([prev_cache, x], dim=split_dim - 1)+ return super().forward(x)++ # Compute tensor shape after concat & padding.+ shape = torch.tensor(x.size())+ if prev_cache is not None:+ shape[split_dim - 1] += prev_cache.size(split_dim - 1)+ shape[-3:] += torch.tensor(padding).view(3, 2).sum(-1).flip(0)+ memory_occupy = shape.prod() * x.element_size() / 1024**3 # GiB+ if memory_occupy < self.memory_limit or split_dim == x.ndim:+ x_concat = x+ if prev_cache is not None:+ x_concat = torch.cat([prev_cache, x], dim=split_dim - 1)++ def pad_and_forward():+ padded = safe_pad_operation(x_concat, padding, mode='constant', value=0.0)+ if not padded.is_contiguous():+ padded = padded.contiguous()+ with ignore_padding(self):+ return torch.nn.Conv3d.forward(self, padded)++ return pad_and_forward()++ num_splits = math.ceil(memory_occupy / self.memory_limit)+ size_per_split = x.size(split_dim) // num_splits+ split_sizes = [size_per_split] * (num_splits - 1)+ split_sizes += [x.size(split_dim) - sum(split_sizes)]++ x = list(x.split(split_sizes, dim=split_dim))+ if prev_cache is not None:+ prev_cache = list(prev_cache.split(split_sizes, dim=split_dim))+ cache = None+ for idx in range(len(x)):+ if prev_cache is not None:+ x[idx] = torch.cat([prev_cache[idx], x[idx]], dim=split_dim - 1)++ lpad_dim = (x[idx].ndim - split_dim - 1) * 2+ rpad_dim = lpad_dim + 1+ padding = list(padding)+ padding[lpad_dim] = self.padding[split_dim - 2] if idx == 0 else 0+ padding[rpad_dim] = self.padding[split_dim - 2] if idx == len(x) - 1 else 0+ pad_len = padding[lpad_dim] + padding[rpad_dim]+ padding = tuple(padding)++ next_cache = None+ cache_len = cache.size(split_dim) if cache is not None else 0+ next_catch_size = get_cache_size(+ conv_module=self,+ input_len=x[idx].size(split_dim) + cache_len,+ pad_len=pad_len,+ dim=split_dim - 2,+ )+ if next_catch_size != 0:+ assert next_catch_size <= x[idx].size(split_dim)+ next_cache = (+ x[idx].transpose(0, split_dim)[-next_catch_size:].transpose(0, split_dim)+ )++ x[idx] = self.memory_limit_conv(+ x[idx],+ split_dim=split_dim + 1,+ padding=padding,+ prev_cache=cache+ )++ cache = next_cache++ output = torch.cat(x, dim=split_dim)+ return output++ def forward(+ self,+ input,+ memory_state: MemoryState = MemoryState.UNSET+ ) -> Tensor:+ assert memory_state != MemoryState.UNSET+ if memory_state != MemoryState.ACTIVE:+ self.memory = None+ if (+ math.isinf(self.memory_limit)+ and torch.is_tensor(input)+ ):+ return self.basic_forward(input, memory_state)+ return self.slicing_forward(input, memory_state)++ def basic_forward(self, input: Tensor, memory_state: MemoryState = MemoryState.UNSET):+ mem_size = self.stride[0] - self.kernel_size[0]+ if (self.memory is not None) and (memory_state == MemoryState.ACTIVE):+ input = extend_head(input, memory=self.memory, times=-1)+ else:+ input = extend_head(input, times=self.temporal_padding * 2)+ memory = (+ input[:, :, mem_size:].detach()+ if (mem_size != 0 and memory_state != MemoryState.DISABLED)+ else None+ )+ if (+ memory_state != MemoryState.DISABLED+ and not self.training+ and (self.memory_device is not None)+ ):+ self.memory = memory+ if self.memory_device == "cpu" and self.memory is not None:+ self.memory = self.memory.to("cpu")+ return super().forward(input)++ def slicing_forward(+ self,+ input,+ memory_state: MemoryState = MemoryState.UNSET,+ ) -> Tensor:+ squeeze_out = False+ if torch.is_tensor(input):+ input = [input]+ squeeze_out = True++ cache_size = self.kernel_size[0] - self.stride[0]+ cache = cache_send_recv(+ input, cache_size=cache_size, memory=self.memory, times=self.temporal_padding * 2+ )++ # Single GPU inference - simplified memory management+ if (+ memory_state in [MemoryState.INITIALIZING, MemoryState.ACTIVE] # use_slicing+ and not self.training+ and (self.memory_device is not None)+ and cache_size != 0+ ):+ if cache_size > input[-1].size(2) and cache is not None and len(input) == 1:+ input[0] = torch.cat([cache, input[0]], dim=2)+ cache = None+ if cache_size <= input[-1].size(2):+ self.memory = input[-1][:, :, -cache_size:].detach().contiguous()+ if self.memory_device == "cpu" and self.memory is not None:+ self.memory = self.memory.to("cpu")++ padding = tuple(x for x in reversed(self.padding) for _ in range(2))+ for i in range(len(input)):+ # Prepare cache for next input slice.+ next_cache = None+ cache_size = 0+ if i < len(input) - 1:+ cache_len = cache.size(2) if cache is not None else 0+ cache_size = get_cache_size(self, input[i].size(2) + cache_len, pad_len=0)+ if cache_size != 0:+ if cache_size > input[i].size(2) and cache is not None:+ input[i] = torch.cat([cache, input[i]], dim=2)+ cache = None+ assert cache_size <= input[i].size(2), f"{cache_size} > {input[i].size(2)}"+ next_cache = input[i][:, :, -cache_size:]++ # Conv forward for this input slice.+ input[i] = self.memory_limit_conv(+ input[i],+ padding=padding,+ prev_cache=cache+ )++ # Update cache.+ cache = next_cache++ return input[0] if squeeze_out else input++def remove_head(tensor: Tensor, times: int = 1) -> Tensor:+ if times == 0:+ return tensor+ return torch.cat(tensors=(tensor[:, :, :1], tensor[:, :, times + 1 :]), dim=2)++class Upsample3D(nn.Module):++ def __init__(+ self,+ channels,+ out_channels = None,+ inflation_mode = "tail",+ temporal_up: bool = False,+ spatial_up: bool = True,+ slicing: bool = False,+ interpolate = True,+ name: str = "conv",+ use_conv_transpose = False,+ use_conv: bool = False,+ padding = 1,+ bias = True,+ kernel_size = None,+ **kwargs,+ ):+ super().__init__()+ self.interpolate = interpolate+ self.channels = channels+ self.out_channels = out_channels or channels+ self.use_conv_transpose = use_conv_transpose+ self.use_conv = use_conv+ self.name = name++ self.conv = None+ if use_conv_transpose:+ if kernel_size is None:+ kernel_size = 4+ self.conv = ops.ConvTranspose2d(+ channels, self.out_channels, kernel_size=kernel_size, stride=2, padding=padding, bias=bias+ )+ elif use_conv:+ if kernel_size is None:+ kernel_size = 3+ self.conv = ops.Conv2d(self.channels, self.out_channels, kernel_size=kernel_size, padding=padding, bias=bias)++ conv = self.conv if self.name == "conv" else self.Conv2d_0++ # Note: lora_layer is not passed into constructor in the original implementation.+ # So we make a simplification.+ conv = InflatedCausalConv3d(+ self.channels,+ self.out_channels,+ 3,+ padding=1,+ inflation_mode=inflation_mode,+ )++ self.temporal_up = temporal_up+ self.spatial_up = spatial_up+ self.temporal_ratio = 2 if temporal_up else 1+ self.spatial_ratio = 2 if spatial_up else 1+ self.slicing = slicing++ assert not self.interpolate+ # [Override] MAGViT v2 implementation+ if not self.interpolate:+ upscale_ratio = (self.spatial_ratio**2) * self.temporal_ratio+ self.upscale_conv = ops.Conv3d(+ self.channels, self.channels * upscale_ratio, kernel_size=1, padding=0+ )+ identity = (+ torch.eye(self.channels)+ .repeat(upscale_ratio, 1)+ .reshape_as(self.upscale_conv.weight)+ )+ self.upscale_conv.weight.data.copy_(identity)++ if self.name == "conv":+ self.conv = conv+ else:+ self.Conv2d_0 = conv++ self.norm = None++ def forward(+ self,+ hidden_states: torch.FloatTensor,+ memory_state=None,+ **kwargs,+ ) -> torch.FloatTensor:+ assert hidden_states.shape[1] == self.channels++ if hasattr(self, "norm") and self.norm is not None:+ # [Overridden] change to causal norm.+ hidden_states = causal_norm_wrapper(self.norm, hidden_states)++ if self.use_conv_transpose:+ return self.conv(hidden_states)++ if self.slicing:+ split_size = hidden_states.size(2) // 2+ hidden_states = list(+ hidden_states.split([split_size, hidden_states.size(2) - split_size], dim=2)+ )+ else:+ hidden_states = [hidden_states]++ for i in range(len(hidden_states)):+ hidden_states[i] = self.upscale_conv(hidden_states[i])+ hidden_states[i] = rearrange(+ hidden_states[i],+ "b (x y z c) f h w -> b c (f z) (h x) (w y)",+ x=self.spatial_ratio,+ y=self.spatial_ratio,+ z=self.temporal_ratio,+ )++ if self.temporal_up and memory_state != MemoryState.ACTIVE:+ hidden_states[0] = remove_head(hidden_states[0])++ if not self.slicing:+ hidden_states = hidden_states[0]++ if self.use_conv:+ if self.name == "conv":+ hidden_states = self.conv(hidden_states, memory_state=memory_state)+ else:+ hidden_states = self.Conv2d_0(hidden_states, memory_state=memory_state)++ if not self.slicing:+ return hidden_states+ else:+ return torch.cat(hidden_states, dim=2)+++class Downsample3D(nn.Module):+ """A 3D downsampling layer with an optional convolution."""++ def __init__(+ self,+ channels,+ out_channels = None,+ inflation_mode = "tail",+ spatial_down: bool = False,+ temporal_down: bool = False,+ name: str = "conv",+ kernel_size=3,+ use_conv: bool = False,+ padding = 1,+ bias=True,+ **kwargs,+ ):+ super().__init__()+ self.padding = padding+ self.name = name+ self.channels = channels+ self.out_channels = out_channels or channels+ self.temporal_down = temporal_down+ self.spatial_down = spatial_down+ self.use_conv = use_conv+ self.padding = padding++ self.temporal_ratio = 2 if temporal_down else 1+ self.spatial_ratio = 2 if spatial_down else 1++ self.temporal_kernel = 3 if temporal_down else 1+ self.spatial_kernel = 3 if spatial_down else 1++ if use_conv:+ conv = InflatedCausalConv3d(+ self.channels,+ self.out_channels,+ kernel_size=(self.temporal_kernel, self.spatial_kernel, self.spatial_kernel),+ stride=(self.temporal_ratio, self.spatial_ratio, self.spatial_ratio),+ padding=(+ 1 if self.temporal_down else 0,+ self.padding if self.spatial_down else 0,+ self.padding if self.spatial_down else 0,+ ),+ inflation_mode=inflation_mode,+ )+ else:+ assert self.channels == self.out_channels+ conv = nn.AvgPool3d(+ kernel_size=(self.temporal_ratio, self.spatial_ratio, self.spatial_ratio),+ stride=(self.temporal_ratio, self.spatial_ratio, self.spatial_ratio),+ )++ self.conv = conv+++ def forward(+ self,+ hidden_states: torch.FloatTensor,+ memory_state = None,+ **kwargs,+ ) -> torch.FloatTensor:++ assert hidden_states.shape[1] == self.channels++ if hasattr(self, "norm") and self.norm is not None:+ # [Overridden] change to causal norm.+ hidden_states = causal_norm_wrapper(self.norm, hidden_states)++ if self.use_conv and self.padding == 0 and self.spatial_down:+ pad = (0, 1, 0, 1)+ hidden_states = safe_pad_operation(hidden_states, pad, mode="constant", value=0)++ assert hidden_states.shape[1] == self.channels++ hidden_states = self.conv(hidden_states, memory_state=memory_state)++ return hidden_states+++class ResnetBlock3D(nn.Module):+ def __init__(+ self,+ in_channels: int,+ out_channels: Optional[int] = None,+ conv_shortcut: bool = False,+ dropout: float = 0.0,+ temb_channels: int = 512,+ groups: int = 32,+ groups_out: Optional[int] = None,+ eps: float = 1e-6,+ non_linearity: str = "swish",+ time_embedding_norm: str = "default",+ output_scale_factor: float = 1.0,+ skip_time_act: bool = False,+ use_in_shortcut: Optional[bool] = None,+ up: bool = False,+ down: bool = False,+ conv_shortcut_bias: bool = True,+ conv_2d_out_channels: Optional[int] = None,+ inflation_mode = "tail",+ time_receptive_field: _receptive_field_t = "half",+ slicing: bool = False,+ **kwargs,+ ):+ super().__init__()+ self.up = up+ self.down = down+ self.in_channels = in_channels+ self.out_channels = in_channels if out_channels is None else out_channels+ conv_2d_out_channels = conv_2d_out_channels or out_channels+ self.use_in_shortcut = use_in_shortcut+ self.output_scale_factor = output_scale_factor+ self.skip_time_act = skip_time_act+ self.nonlinearity = nn.SiLU()+ if temb_channels is not None:+ self.time_emb_proj = ops.Linear(temb_channels, out_channels)+ else:+ self.time_emb_proj = None+ self.norm1 = ops.GroupNorm(num_groups=groups, num_channels=in_channels, eps=eps, affine=True)+ if groups_out is None:+ groups_out = groups+ self.norm2 = ops.GroupNorm(num_groups=groups_out, num_channels=out_channels, eps=eps, affine=True)+ self.use_in_shortcut = self.in_channels != out_channels+ self.dropout = torch.nn.Dropout(dropout)+ self.conv1 = InflatedCausalConv3d(+ self.in_channels,+ self.out_channels,+ kernel_size=(1, 3, 3) if time_receptive_field == "half" else (3, 3, 3),+ stride=1,+ padding=(0, 1, 1) if time_receptive_field == "half" else (1, 1, 1),+ inflation_mode=inflation_mode,+ )++ self.conv2 = InflatedCausalConv3d(+ self.out_channels,+ conv_2d_out_channels,+ kernel_size=3,+ stride=1,+ padding=1,+ inflation_mode=inflation_mode,+ )++ self.upsample = self.downsample = None+ if self.up:+ self.upsample = Upsample3D(+ self.in_channels,+ use_conv=False,+ inflation_mode=inflation_mode,+ slicing=slicing,+ )+ elif self.down:+ self.downsample = Downsample3D(+ self.in_channels,+ use_conv=False,+ padding=1,+ name="op",+ inflation_mode=inflation_mode,+ )++ self.conv_shortcut = None+ if self.use_in_shortcut:+ self.conv_shortcut = InflatedCausalConv3d(+ self.in_channels,+ conv_2d_out_channels,+ kernel_size=1,+ stride=1,+ padding=0,+ bias=True,+ inflation_mode=inflation_mode,+ )++ def forward(+ self, input_tensor, temb, memory_state = None, **kwargs+ ):+ hidden_states = input_tensor++ hidden_states = causal_norm_wrapper(self.norm1, hidden_states)++ hidden_states = self.nonlinearity(hidden_states)++ if self.upsample is not None:+ if hidden_states.shape[0] >= BYTEDANCE_CONTIGUOUS_BATCH_THRESHOLD:+ input_tensor = input_tensor.contiguous()+ hidden_states = hidden_states.contiguous()+ input_tensor = self.upsample(input_tensor, memory_state=memory_state)+ hidden_states = self.upsample(hidden_states, memory_state=memory_state)+ elif self.downsample is not None:+ input_tensor = self.downsample(input_tensor, memory_state=memory_state)+ hidden_states = self.downsample(hidden_states, memory_state=memory_state)++ hidden_states = self.conv1(hidden_states, memory_state=memory_state)++ if self.time_emb_proj is not None:+ if not self.skip_time_act:+ temb = self.nonlinearity(temb)+ temb = self.time_emb_proj(temb)[:, :, None, None]++ if temb is not None:+ hidden_states = hidden_states + temb++ hidden_states = causal_norm_wrapper(self.norm2, hidden_states)++ hidden_states = self.nonlinearity(hidden_states)++ hidden_states = self.dropout(hidden_states)+ hidden_states = self.conv2(hidden_states, memory_state=memory_state)++ if self.conv_shortcut is not None:+ input_tensor = self.conv_shortcut(input_tensor, memory_state=memory_state)++ output_tensor = (input_tensor + hidden_states) / self.output_scale_factor++ return output_tensor+++class DownEncoderBlock3D(nn.Module):+ def __init__(+ self,+ in_channels: int,+ out_channels: int,+ dropout: float = 0.0,+ num_layers: int = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default",+ resnet_act_fn: str = "swish",+ resnet_groups: int = 32,+ resnet_pre_norm: bool = True,+ output_scale_factor: float = 1.0,+ add_downsample: bool = True,+ downsample_padding: int = 1,+ inflation_mode = "tail",+ time_receptive_field: _receptive_field_t = "half",+ temporal_down: bool = True,+ spatial_down: bool = True,+ ):+ super().__init__()+ resnets = []+ temporal_modules = []++ for i in range(num_layers):+ in_channels = in_channels if i == 0 else out_channels+ resnets.append(+ # [Override] Replace module.+ ResnetBlock3D(+ in_channels=in_channels,+ out_channels=out_channels,+ temb_channels=None,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )+ )+ temporal_modules.append(nn.Identity())++ self.resnets = nn.ModuleList(resnets)+ self.temporal_modules = nn.ModuleList(temporal_modules)++ if add_downsample:+ self.downsamplers = nn.ModuleList(+ [+ Downsample3D(+ out_channels,+ use_conv=True,+ out_channels=out_channels,+ padding=downsample_padding,+ name="op",+ temporal_down=temporal_down,+ spatial_down=spatial_down,+ inflation_mode=inflation_mode,+ )+ ]+ )+ else:+ self.downsamplers = None++ def forward(+ self,+ hidden_states: torch.FloatTensor,+ memory_state = None,+ **kwargs,+ ) -> torch.FloatTensor:+ for resnet, temporal in zip(self.resnets, self.temporal_modules):+ hidden_states = resnet(hidden_states, temb=None, memory_state=memory_state)+ hidden_states = temporal(hidden_states)++ if self.downsamplers is not None:+ for downsampler in self.downsamplers:+ hidden_states = downsampler(hidden_states, memory_state=memory_state)++ return hidden_states+++class UpDecoderBlock3D(nn.Module):+ def __init__(+ self,+ in_channels: int,+ out_channels: int,+ dropout: float = 0.0,+ num_layers: int = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default", # default, spatial+ resnet_act_fn: str = "swish",+ resnet_groups: int = 32,+ resnet_pre_norm: bool = True,+ output_scale_factor: float = 1.0,+ add_upsample: bool = True,+ temb_channels: Optional[int] = None,+ inflation_mode = "tail",+ time_receptive_field: _receptive_field_t = "half",+ temporal_up: bool = True,+ spatial_up: bool = True,+ slicing: bool = False,+ ):+ super().__init__()+ resnets = []+ temporal_modules = []++ for i in range(num_layers):+ input_channels = in_channels if i == 0 else out_channels++ resnets.append(+ # [Override] Replace module.+ ResnetBlock3D(+ in_channels=input_channels,+ out_channels=out_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ slicing=slicing,+ )+ )++ temporal_modules.append(nn.Identity())++ self.resnets = nn.ModuleList(resnets)+ self.temporal_modules = nn.ModuleList(temporal_modules)++ if add_upsample:+ # [Override] Replace module & use learnable upsample+ self.upsamplers = nn.ModuleList(+ [+ Upsample3D(+ out_channels,+ use_conv=True,+ out_channels=out_channels,+ temporal_up=temporal_up,+ spatial_up=spatial_up,+ interpolate=False,+ inflation_mode=inflation_mode,+ slicing=slicing,+ )+ ]+ )+ else:+ self.upsamplers = None++ def forward(+ self,+ hidden_states: torch.FloatTensor,+ temb: Optional[torch.FloatTensor] = None,+ memory_state=None+ ) -> torch.FloatTensor:+ for resnet, temporal in zip(self.resnets, self.temporal_modules):+ hidden_states = resnet(hidden_states, temb=None, memory_state=memory_state)+ hidden_states = temporal(hidden_states)++ if self.upsamplers is not None:+ for upsampler in self.upsamplers:+ hidden_states = upsampler(hidden_states, memory_state=memory_state)++ return hidden_states+++class UNetMidBlock3D(nn.Module):+ def __init__(+ self,+ in_channels: int,+ temb_channels: int,+ dropout: float = 0.0,+ num_layers: int = 1,+ resnet_eps: float = 1e-6,+ resnet_time_scale_shift: str = "default", # default, spatial+ resnet_act_fn: str = "swish",+ resnet_groups: int = 32,+ resnet_pre_norm: bool = True,+ add_attention: bool = True,+ attention_head_dim: int = 1,+ output_scale_factor: float = 1.0,+ inflation_mode = "tail",+ time_receptive_field: _receptive_field_t = "half",+ ):+ super().__init__()+ resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)+ self.add_attention = add_attention++ # there is always at least one resnet+ resnets = [+ # [Override] Replace module.+ ResnetBlock3D(+ in_channels=in_channels,+ out_channels=in_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )+ ]+ attentions = []++ if attention_head_dim is None:+ attention_head_dim = in_channels++ for _ in range(num_layers):+ if self.add_attention:+ attentions.append(+ Attention(+ in_channels,+ heads=in_channels // attention_head_dim,+ dim_head=attention_head_dim,+ rescale_output_factor=output_scale_factor,+ eps=resnet_eps,+ norm_num_groups=(+ resnet_groups if resnet_time_scale_shift == "default" else None+ ),+ spatial_norm_dim=(+ temb_channels if resnet_time_scale_shift == "spatial" else None+ ),+ residual_connection=True,+ bias=True,+ upcast_softmax=True,+ _from_deprecated_attn_block=True,+ )+ )+ else:+ attentions.append(None)++ resnets.append(+ ResnetBlock3D(+ in_channels=in_channels,+ out_channels=in_channels,+ temb_channels=temb_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ time_embedding_norm=resnet_time_scale_shift,+ non_linearity=resnet_act_fn,+ output_scale_factor=output_scale_factor,+ pre_norm=resnet_pre_norm,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )+ )++ self.attentions = nn.ModuleList(attentions)+ self.resnets = nn.ModuleList(resnets)++ def forward(self, hidden_states, temb=None, memory_state=None):+ video_length, frame_height, frame_width = hidden_states.size()[-3:]+ hidden_states = self.resnets[0](hidden_states, temb, memory_state=memory_state)+ for attn, resnet in zip(self.attentions, self.resnets[1:]):+ if attn is not None:+ hidden_states = rearrange(hidden_states, "b c f h w -> (b f) c h w")+ hidden_states = attn(hidden_states, temb=temb)+ hidden_states = rearrange(+ hidden_states, "(b f) c h w -> b c f h w", f=video_length+ )+ hidden_states = resnet(hidden_states, temb, memory_state=memory_state)++ return hidden_states+++class Encoder3D(nn.Module):+ def __init__(+ self,+ in_channels: int = 3,+ out_channels: int = 3,+ down_block_types: Tuple[str, ...] = ("DownEncoderBlock3D",),+ block_out_channels: Tuple[int, ...] = (64,),+ layers_per_block: int = 2,+ norm_num_groups: int = 32,+ act_fn: str = "silu",+ double_z: bool = True,+ mid_block_add_attention=True,+ # [Override] add extra_cond_dim, temporal down num+ temporal_down_num: int = 2,+ extra_cond_dim: int = None,+ gradient_checkpoint: bool = False,+ inflation_mode = "tail",+ time_receptive_field: _receptive_field_t = "half",+ ):+ super().__init__()+ self.layers_per_block = layers_per_block+ self.temporal_down_num = temporal_down_num++ self.conv_in = InflatedCausalConv3d(+ in_channels,+ block_out_channels[0],+ kernel_size=3,+ stride=1,+ padding=1,+ inflation_mode=inflation_mode,+ )++ self.mid_block = None+ self.down_blocks = nn.ModuleList([])+ self.extra_cond_dim = extra_cond_dim++ self.conv_extra_cond = nn.ModuleList([])++ # down+ output_channel = block_out_channels[0]+ for i, down_block_type in enumerate(down_block_types):+ input_channel = output_channel+ output_channel = block_out_channels[i]+ is_final_block = i == len(block_out_channels) - 1+ # [Override] to support temporal down block design+ is_temporal_down_block = i >= len(block_out_channels) - self.temporal_down_num - 1+ # Note: take the last ones++ assert down_block_type == "DownEncoderBlock3D"++ down_block = DownEncoderBlock3D(+ num_layers=self.layers_per_block,+ in_channels=input_channel,+ out_channels=output_channel,+ add_downsample=not is_final_block,+ resnet_eps=1e-6,+ downsample_padding=0,+ # Note: Don't know why set it as 0+ resnet_act_fn=act_fn,+ resnet_groups=norm_num_groups,+ temporal_down=is_temporal_down_block,+ spatial_down=True,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )+ self.down_blocks.append(down_block)++ def zero_module(module):+ # Zero out the parameters of a module and return it.+ for p in module.parameters():+ p.detach().zero_()+ return module++ self.conv_extra_cond.append(+ zero_module(+ ops.Conv3d(extra_cond_dim, output_channel, kernel_size=1, stride=1, padding=0)+ )+ if self.extra_cond_dim is not None and self.extra_cond_dim > 0+ else None+ )++ # mid+ self.mid_block = UNetMidBlock3D(+ in_channels=block_out_channels[-1],+ resnet_eps=1e-6,+ resnet_act_fn=act_fn,+ output_scale_factor=1,+ resnet_time_scale_shift="default",+ attention_head_dim=block_out_channels[-1],+ resnet_groups=norm_num_groups,+ temb_channels=None,+ add_attention=mid_block_add_attention,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )++ # out+ self.conv_norm_out = ops.GroupNorm(+ num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6+ )+ self.conv_act = nn.SiLU()++ conv_out_channels = 2 * out_channels if double_z else out_channels+ self.conv_out = InflatedCausalConv3d(+ block_out_channels[-1], conv_out_channels, 3, padding=1, inflation_mode=inflation_mode+ )++ self.gradient_checkpointing = gradient_checkpoint++ def forward(+ self,+ sample: torch.FloatTensor,+ extra_cond=None,+ memory_state = None+ ) -> torch.FloatTensor:+ r"""The forward method of the `Encoder` class."""+ sample = sample.to(next(self.parameters()).device)+ sample = self.conv_in(sample, memory_state = memory_state)+ if self.training and self.gradient_checkpointing:++ def create_custom_forward(module):+ def custom_forward(*inputs):+ return module(*inputs)++ return custom_forward++ # down+ # [Override] add extra block and extra cond+ for down_block, extra_block in zip(self.down_blocks, self.conv_extra_cond):+ sample = torch.utils.checkpoint.checkpoint(+ create_custom_forward(down_block), sample, use_reentrant=False+ )+ if extra_block is not None:+ sample = sample + safe_interpolate_operation(extra_block(extra_cond), size=sample.shape[2:])++ # middle+ sample = self.mid_block(sample)++ else:+ # down+ # [Override] add extra block and extra cond+ for down_block, extra_block in zip(self.down_blocks, self.conv_extra_cond):+ sample = down_block(sample, memory_state=memory_state)+ if extra_block is not None:+ sample = sample + safe_interpolate_operation(extra_block(extra_cond), size=sample.shape[2:])++ # middle+ sample = self.mid_block(sample, memory_state=memory_state)++ # post-process+ sample = causal_norm_wrapper(self.conv_norm_out, sample)+ sample = self.conv_act(sample)+ sample = self.conv_out(sample, memory_state = memory_state)++ return sample+++class Decoder3D(nn.Module):++ def __init__(+ self,+ in_channels: int = 3,+ out_channels: int = 3,+ up_block_types: Tuple[str, ...] = ("UpDecoderBlock3D",),+ block_out_channels: Tuple[int, ...] = (64,),+ layers_per_block: int = 2,+ norm_num_groups: int = 32,+ act_fn: str = "silu",+ norm_type: str = "group", # group, spatial+ mid_block_add_attention=True,+ # [Override] add temporal up block+ inflation_mode = "tail",+ time_receptive_field: _receptive_field_t = "half",+ temporal_up_num: int = 2,+ slicing_up_num: int = 0,+ gradient_checkpoint: bool = False,+ ):+ super().__init__()+ self.layers_per_block = layers_per_block+ self.temporal_up_num = temporal_up_num++ self.conv_in = InflatedCausalConv3d(+ in_channels,+ block_out_channels[-1],+ kernel_size=3,+ stride=1,+ padding=1,+ inflation_mode=inflation_mode,+ )++ self.mid_block = None+ self.up_blocks = nn.ModuleList([])++ temb_channels = in_channels if norm_type == "spatial" else None++ # mid+ self.mid_block = UNetMidBlock3D(+ in_channels=block_out_channels[-1],+ resnet_eps=1e-6,+ resnet_act_fn=act_fn,+ output_scale_factor=1,+ resnet_time_scale_shift="default" if norm_type == "group" else norm_type,+ attention_head_dim=block_out_channels[-1],+ resnet_groups=norm_num_groups,+ temb_channels=temb_channels,+ add_attention=mid_block_add_attention,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )++ # up+ reversed_block_out_channels = list(reversed(block_out_channels))+ output_channel = reversed_block_out_channels[0]+ for i, up_block_type in enumerate(up_block_types):+ prev_output_channel = output_channel+ output_channel = reversed_block_out_channels[i]++ is_final_block = i == len(block_out_channels) - 1+ is_temporal_up_block = i < self.temporal_up_num+ is_slicing_up_block = i >= len(block_out_channels) - slicing_up_num+ # Note: Keep symmetric++ assert up_block_type == "UpDecoderBlock3D"+ up_block = UpDecoderBlock3D(+ num_layers=self.layers_per_block + 1,+ in_channels=prev_output_channel,+ out_channels=output_channel,+ add_upsample=not is_final_block,+ resnet_eps=1e-6,+ resnet_act_fn=act_fn,+ resnet_groups=norm_num_groups,+ resnet_time_scale_shift=norm_type,+ temb_channels=temb_channels,+ temporal_up=is_temporal_up_block,+ slicing=is_slicing_up_block,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )+ self.up_blocks.append(up_block)+ prev_output_channel = output_channel++ # out+ if norm_type == "spatial":+ self.conv_norm_out = SpatialNorm(block_out_channels[0], temb_channels)+ else:+ self.conv_norm_out = ops.GroupNorm(+ num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6+ )+ self.conv_act = nn.SiLU()+ self.conv_out = InflatedCausalConv3d(+ block_out_channels[0], out_channels, 3, padding=1, inflation_mode=inflation_mode+ )++ self.gradient_checkpointing = gradient_checkpoint++ # Note: Just copy from Decoder.+ def forward(+ self,+ sample: torch.FloatTensor,+ latent_embeds: Optional[torch.FloatTensor] = None,+ memory_state = None,+ ) -> torch.FloatTensor:++ sample = sample.to(next(self.parameters()).device)+ sample = self.conv_in(sample, memory_state=memory_state)++ upscale_dtype = next(iter(self.up_blocks.parameters())).dtype+ # middle+ sample = self.mid_block(sample, latent_embeds, memory_state=memory_state)+ sample = sample.to(upscale_dtype)++ # up+ for up_block in self.up_blocks:+ sample = up_block(sample, latent_embeds, memory_state=memory_state)++ # post-process+ sample = causal_norm_wrapper(self.conv_norm_out, sample)+ sample = self.conv_act(sample)+ sample = self.conv_out(sample, memory_state=memory_state)++ return sample++class VideoAutoencoderKL(nn.Module):+ def __init__(+ self,+ in_channels: int = 3,+ out_channels: int = 3,+ layers_per_block: int = 2,+ act_fn: str = "silu",+ latent_channels: int = SEEDVR2_LATENT_CHANNELS,+ norm_num_groups: int = 32,+ attention: bool = True,+ temporal_scale_num: int = 2,+ slicing_up_num: int = 0,+ gradient_checkpoint: bool = False,+ inflation_mode = "pad",+ time_receptive_field: _receptive_field_t = "full",+ use_quant_conv: bool = False,+ use_post_quant_conv: bool = False,+ slicing_sample_min_size = BYTEDANCE_SLICING_SAMPLE_MIN,+ *args,+ **kwargs,+ ):+ self.slicing_sample_min_size = slicing_sample_min_size+ self.slicing_latent_min_size = slicing_sample_min_size // (2**temporal_scale_num)+ extra_cond_dim = kwargs.pop("extra_cond_dim") if "extra_cond_dim" in kwargs else None+ block_out_channels = BYTEDANCE_BLOCK_OUT_CHANNELS+ down_block_types = ("DownEncoderBlock3D",) * 4+ up_block_types = ("UpDecoderBlock3D",) * 4+ super().__init__()++ # pass init params to Encoder+ self.encoder = Encoder3D(+ in_channels=in_channels,+ out_channels=latent_channels,+ down_block_types=down_block_types,+ block_out_channels=block_out_channels,+ layers_per_block=layers_per_block,+ act_fn=act_fn,+ norm_num_groups=norm_num_groups,+ double_z=True,+ extra_cond_dim=extra_cond_dim,+ # [Override] add temporal_down_num parameter+ temporal_down_num=temporal_scale_num,+ gradient_checkpoint=gradient_checkpoint,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )++ # pass init params to Decoder+ self.decoder = Decoder3D(+ in_channels=latent_channels,+ out_channels=out_channels,+ up_block_types=up_block_types,+ block_out_channels=block_out_channels,+ layers_per_block=layers_per_block,+ norm_num_groups=norm_num_groups,+ act_fn=act_fn,+ # [Override] add temporal_up_num parameter+ temporal_up_num=temporal_scale_num,+ slicing_up_num=slicing_up_num,+ gradient_checkpoint=gradient_checkpoint,+ inflation_mode=inflation_mode,+ time_receptive_field=time_receptive_field,+ )++ self.quant_conv = (+ InflatedCausalConv3d(+ in_channels=2 * latent_channels,+ out_channels=2 * latent_channels,+ kernel_size=1,+ inflation_mode=inflation_mode,+ )+ if use_quant_conv+ else None+ )+ self.post_quant_conv = (+ InflatedCausalConv3d(+ in_channels=latent_channels,+ out_channels=latent_channels,+ kernel_size=1,+ inflation_mode=inflation_mode,+ )+ if use_post_quant_conv+ else None+ )++ # A hacky way to remove attention.+ if not attention:+ self.encoder.mid_block.attentions = torch.nn.ModuleList([None])+ self.decoder.mid_block.attentions = torch.nn.ModuleList([None])++ self.use_slicing = True++ def encode(self, x: torch.FloatTensor, return_dict: bool = True):+ h = self.slicing_encode(x)+ posterior = DiagonalGaussianDistribution(h).mode()++ if not return_dict:+ return (posterior,)++ return posterior++ def decode_(+ self, z: torch.Tensor, return_dict: bool = True+ ):+ decoded = self.slicing_decode(z)++ if not return_dict:+ return (decoded,)++ return decoded++ def _encode(+ self, x, memory_state = MemoryState.DISABLED+ ) -> torch.Tensor:+ _x = x.to(self.device)+ h = self.encoder(_x, memory_state=memory_state)+ if self.quant_conv is not None:+ output = self.quant_conv(h, memory_state=memory_state)+ else:+ output = h+ return output.to(x.device)++ def _decode(+ self, z, memory_state = MemoryState.DISABLED+ ) -> torch.Tensor:+ _z = z.to(self.device)++ if self.post_quant_conv is not None:+ _z = self.post_quant_conv(_z, memory_state=memory_state)++ output = self.decoder(_z, memory_state=memory_state)+ return output.to(z.device)++ def slicing_encode(self, x: torch.Tensor) -> torch.Tensor:+ sp_size =1+ if self.use_slicing and (x.shape[2] - 1) > self.slicing_sample_min_size * sp_size:+ split_size = max(+ self.slicing_sample_min_size * sp_size,+ getattr(self, "temporal_downsample_factor", 1),+ )+ x_slices = list(x[:, :, 1:].split(split_size=split_size, dim=2))+ min_active_len = getattr(self, "temporal_downsample_factor", 1)+ if len(x_slices) > 1 and x_slices[-1].shape[2] < min_active_len:+ x_slices[-2] = torch.cat((x_slices[-2], x_slices[-1]), dim=2)+ x_slices.pop()+ encoded_slices = [+ self._encode(+ torch.cat((x[:, :, :1], x_slices[0]), dim=2),+ memory_state=MemoryState.INITIALIZING,+ )+ ]+ for x_idx in range(1, len(x_slices)):+ encoded_slices.append(+ self._encode(x_slices[x_idx], memory_state=MemoryState.ACTIVE)+ )+ out = torch.cat(encoded_slices, dim=2)+ modules_with_memory = [m for m in self.modules()+ if isinstance(m, InflatedCausalConv3d) and m.memory is not None]+ for m in modules_with_memory:+ m.memory = None+ return out+ else:+ return self._encode(x)++ def slicing_decode(self, z: torch.Tensor) -> torch.Tensor:+ sp_size = 1+ if self.use_slicing and (z.shape[2] - 1) > self.slicing_latent_min_size * sp_size:+ z_slices = z[:, :, 1:].split(split_size=self.slicing_latent_min_size * sp_size, dim=2)+ decoded_slices = [+ self._decode(+ torch.cat((z[:, :, :1], z_slices[0]), dim=2),+ memory_state=MemoryState.INITIALIZING+ )+ ]+ for z_idx in range(1, len(z_slices)):+ decoded_slices.append(+ self._decode(z_slices[z_idx], memory_state=MemoryState.ACTIVE)+ )+ out = torch.cat(decoded_slices, dim=2)+ modules_with_memory = [m for m in self.modules()+ if isinstance(m, InflatedCausalConv3d) and m.memory is not None]+ for m in modules_with_memory:+ m.memory = None+ return out+ else:+ return self._decode(z)++ def tiled_encode(self, x: torch.Tensor, **kwargs) -> torch.Tensor:+ raise NotImplementedError++ def tiled_decode(self, z: torch.Tensor, **kwargs) -> torch.Tensor:+ raise NotImplementedError++ def forward(+ self, x: torch.FloatTensor, mode: Literal["encode", "decode", "all"] = "all", **kwargs+ ):+ # x: [b c t h w]+ def _unwrap(value):+ return value[0] if isinstance(value, tuple) else value++ if mode == "encode":+ return _unwrap(self.encode(x))+ elif mode == "decode":+ return _unwrap(self.decode_(x))+ else:+ latent = _unwrap(self.encode(x))+ return _unwrap(self.decode_(latent))++class VideoAutoencoderKLWrapper(VideoAutoencoderKL):+ def __init__(+ self,+ *args,+ spatial_downsample_factor = 8,+ temporal_downsample_factor = 4,+ freeze_encoder = True,+ **kwargs,+ ):+ self.spatial_downsample_factor = spatial_downsample_factor+ self.temporal_downsample_factor = temporal_downsample_factor+ self.freeze_encoder = freeze_encoder+ self.enable_tiling = False+ super().__init__(*args, **kwargs)+ self.set_memory_limit(BYTEDANCE_VAE_CONV_MEM_GIB, BYTEDANCE_VAE_NORM_MEM_GIB)++ def forward(self, x: torch.FloatTensor):+ with torch.no_grad() if self.freeze_encoder else nullcontext():+ z, p = self.encode(x)+ x = self.decode(z)+ return x, z, p++ def encode(self, x, orig_dims=None):+ if x.ndim == 4:+ x = x.unsqueeze(2)+ x = x.to(dtype=next(self.parameters()).dtype)+ self.device = x.device+ p = super().encode(x)+ z = p.squeeze(2)+ return z, p++ def decode(self, z, seedvr2_tiling=None):+ seedvr2_tiling = {} if seedvr2_tiling is None else seedvr2_tiling+ if not isinstance(seedvr2_tiling, dict):+ raise RuntimeError(+ "SeedVR2 VideoAutoencoderKLWrapper.decode: `seedvr2_tiling` must be a dict; "+ f"got {type(seedvr2_tiling).__name__} with value {seedvr2_tiling!r}."+ )++ if z.ndim == 5:+ b, c, t_latent, h, w = z.shape+ if c != 16:+ raise RuntimeError(+ "SeedVR2 VideoAutoencoderKLWrapper.decode: 5-D latent input must "+ f"have 16 channels; got shape {tuple(z.shape)}."+ )+ latent = z+ elif z.ndim == 4:+ b, tc, h, w = z.shape+ if tc % 16 != 0:+ raise RuntimeError(+ "SeedVR2 VideoAutoencoderKLWrapper.decode: 4-D latent input must "+ "use collapsed channel layout (B, 16*T, H, W); "+ f"got shape {tuple(z.shape)}."+ )+ latent = z.reshape(b, 16, -1, h, w)+ else:+ raise RuntimeError(+ "SeedVR2 VideoAutoencoderKLWrapper.decode: latent input must be "+ "4-D collapsed (B, 16*T, H, W) or 5-D (B, 16, T, H, W); "+ f"got shape {tuple(z.shape)}."+ )+ scale = BYTEDANCE_VAE_SCALING_FACTOR+ shift = BYTEDANCE_VAE_SHIFTING_FACTOR+ latent = latent / scale + shift++ self.device = latent.device+ self.enable_tiling = seedvr2_tiling.get("enable_tiling", False)++ if self.enable_tiling:+ decode_seedvr2_args = dict(seedvr2_tiling)+ tile_h, tile_w = decode_seedvr2_args.get("tile_size", (512, 512))+ ov_h, ov_w = decode_seedvr2_args.get("tile_overlap", (64, 64))+ decode_seedvr2_args["tile_overlap"] = (+ min(ov_h, max(0, tile_h - 8)),+ min(ov_w, max(0, tile_w - 8)),+ )+ x = tiled_vae(latent, self, **decode_seedvr2_args, encode=False)+ if x.ndim == 4:+ # tiled_vae squeezes the temporal axis when+ # temporal_downsample_factor == 1 AND latent T == 1+ # (see tiled_vae line 179-180); re-add it so the post-decode+ # pipeline can keep batch and time distinct on the tiled path.+ x = x.unsqueeze(2)+ else:+ x = super().decode_(latent)++ # ensure even dims for save video+ h, w = x.shape[-2:]+ w2 = w - (w % 2)+ h2 = h - (h % 2)+ x = x[..., :h2, :w2]++ return x++ def set_memory_limit(self, conv_max_mem: Optional[float], norm_max_mem: Optional[float], memory_device = "same"):+ set_norm_limit(norm_max_mem)+ for m in self.modules():+ if isinstance(m, InflatedCausalConv3d):+ m.set_memory_limit(conv_max_mem if conv_max_mem is not None else float("inf"))++ for module in self.modules():+ if isinstance(module, InflatedCausalConv3d):+ module.set_memory_device(memory_device)diff --git a/comfy/model_base.py b/comfy/model_base.pyindex 042804771..c084e23bb 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -54,6 +54,8 @@ import comfy.ldm.pixeldit.modelimport comfy.ldm.pixeldit.pidimport comfy.ldm.ace.modelimport comfy.ldm.omnigen.omnigen2+import comfy.ldm.seedvr.model+import comfy.ldm.qwen_image.modelimport comfy.ldm.ideogram4.modelimport comfy.ldm.kandinsky5.model@@ -928,6 +930,16 @@ class HunyuanDiT(BaseModel):out['image_meta_size'] = comfy.conds.CONDRegular(torch.FloatTensor([[height, width, target_height, target_width, 0, 0]]))return out+class SeedVR2(BaseModel):+ def __init__(self, model_config, model_type=ModelType.FLOW, device=None):+ super().__init__(model_config, model_type, device, comfy.ldm.seedvr.model.NaDiT)+ def extra_conds(self, **kwargs):+ out = super().extra_conds(**kwargs)+ condition = kwargs.get("condition", None)+ if condition is not None:+ out["condition"] = comfy.conds.CONDRegular(condition)+ return out+class PixArt(BaseModel):def __init__(self, model_config, model_type=ModelType.EPS, device=None):super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.pixart.pixartms.PixArtMS)diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex 74c838d13..955581006 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -598,6 +598,56 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):return dit_config+ if "{}blocks.35.mlp.vid.proj_in.weight".format(key_prefix) in state_dict_keys and state_dict["{}blocks.35.mlp.vid.proj_in.weight".format(key_prefix)].shape[1] == 3072: # seedvr2 7b+ dit_config = {}+ dit_config["image_model"] = "seedvr2"+ dit_config["vid_dim"] = 3072+ dit_config["heads"] = 24+ dit_config["num_layers"] = 36+ # 7B uses non-shared MMModule layout (separate ``vid.`` / ``txt.``+ # submodules) at EVERY block — verified by inspecting the 7B+ # state_dict at ``blocks.31.ada.txt.attn_gate`` (txt. prefix means+ # ``MMModule.shared_weights=False``). Native NaDiT computes+ # per-block ``shared_weights = not (i < mm_layers)``, so to keep+ # every block non-shared we set ``mm_layers = num_layers``.+ # Without this, blocks at index >= mm_layers (default 10) try to+ # load ``blocks.N.*.all.*`` keys that don't exist in the file,+ # silently miss-load → all-black output.+ dit_config["mm_layers"] = 36+ dit_config["norm_eps"] = 1e-5+ dit_config["qk_rope"] = True+ dit_config["rope_type"] = "rope3d"+ dit_config["rope_dim"] = 64+ dit_config["mlp_type"] = "normal"+ return dit_config+ elif "{}blocks.35.mlp.all.proj_in_gate.weight".format(key_prefix) in state_dict_keys: # seedvr2 7b+ dit_config = {}+ dit_config["image_model"] = "seedvr2"+ dit_config["vid_dim"] = 3072+ dit_config["heads"] = 24+ dit_config["num_layers"] = 36+ # This checkpoint layout carries shared ``all.`` MMModule keys.+ # Preserve the historical split: the initial blocks use separate+ # vid/txt modules, later blocks use shared modules.+ dit_config["mm_layers"] = 10+ dit_config["norm_eps"] = 1e-5+ dit_config["qk_rope"] = True+ dit_config["rope_type"] = "rope3d"+ dit_config["rope_dim"] = 64+ dit_config["mlp_type"] = "swiglu"+ return dit_config+ elif "{}blocks.31.mlp.all.proj_in_gate.weight".format(key_prefix) in state_dict_keys: # seedvr2 3b+ dit_config = {}+ dit_config["image_model"] = "seedvr2"+ dit_config["vid_dim"] = 2560+ dit_config["heads"] = 20+ dit_config["num_layers"] = 32+ dit_config["norm_eps"] = 1.0e-05+ dit_config["qk_rope"] = None+ dit_config["mlp_type"] = "swiglu"+ dit_config["vid_out_norm"] = True+ return dit_config+if '{}head.modulation'.format(key_prefix) in state_dict_keys: # Wan 2.1dit_config = {}dit_config["image_model"] = "wan2.1"diff --git a/comfy/sample.py b/comfy/sample.pyindex 2be0cae5f..de71596b3 100644--- a/comfy/sample.py+++ b/comfy/sample.py@@ -44,7 +44,13 @@ def fix_empty_latent_channels(model, latent_image, downscale_ratio_spacial=None,is_empty = torch.count_nonzero(latent_image) == 0if is_empty:if latent_format.latent_channels != latent_image.shape[1]:- latent_image = comfy.utils.repeat_to_batch_size(latent_image, latent_format.latent_channels, dim=1)+ preserves_collapsed_channels = (+ getattr(latent_format, "preserve_empty_channel_multiples", False)+ and latent_image.ndim == 4+ and latent_image.shape[1] % latent_format.latent_channels == 0+ )+ if not preserves_collapsed_channels:+ latent_image = comfy.utils.repeat_to_batch_size(latent_image, latent_format.latent_channels, dim=1)if downscale_ratio_spacial is not None:if downscale_ratio_spacial != latent_format.spacial_downscale_ratio:ratio = downscale_ratio_spacial / latent_format.spacial_downscale_ratiodiff --git a/comfy/sd.py b/comfy/sd.pyindex a66ba1bfb..8ac08ac42 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -1,3 +1,4 @@+import inspectimport jsonimport torchfrom enum import Enum@@ -16,6 +17,7 @@ import comfy.ldm.cosmos.vaeimport comfy.ldm.wan.vaeimport comfy.ldm.wan.vae2_2import comfy.ldm.hunyuan3d.vae+import comfy.ldm.seedvr.vaeimport comfy.ldm.triposplat.vaeimport comfy.ldm.ace.vae.music_dcae_pipelineimport comfy.ldm.cogvideo.vae@@ -84,6 +86,36 @@ import comfy.latent_formatsimport comfy.ldm.flux.redux+SEEDVR2_VAE_DECODE_BYTES_PER_OUTPUT_PIXEL = 160+++def _seedvr2_vae_decode_output_pixels(latent_t, latent_h, latent_w):+ output_t = max(1, (latent_t - 1) * 4 + 1)+ return output_t * latent_h * 8 * latent_w * 8+++def _seedvr2_vae_decode_memory_used(shape):+ if len(shape) == 5:+ candidates = []+ if shape[1] == 16:+ candidates.append((shape[2], shape[3], shape[4]))+ if shape[-1] == 16:+ candidates.append((shape[1], shape[2], shape[3]))+ if len(candidates) == 0:+ candidates.append((shape[2], shape[3], shape[4]))+ output_pixels = max(_seedvr2_vae_decode_output_pixels(*candidate) for candidate in candidates)+ elif len(shape) == 4:+ latent_t = max(1, (shape[1] + 15) // 16)+ latent_h, latent_w = shape[2], shape[3]+ output_pixels = _seedvr2_vae_decode_output_pixels(latent_t, latent_h, latent_w)+ else:+ latent_t, latent_h, latent_w = 1, shape[-2], shape[-1]+ output_pixels = _seedvr2_vae_decode_output_pixels(latent_t, latent_h, latent_w)+ # SeedVR2 decode performs full-frame LAB histogram matching: fp32 channels+ # plus int64 sort indices dominate peak memory, not the VAE weight dtype.+ return output_pixels * SEEDVR2_VAE_DECODE_BYTES_PER_OUTPUT_PIXEL++def load_lora_for_models(model, clip, lora, strength_model, strength_clip, lora_metadata=None):key_map = {}if model is not None:@@ -467,8 +499,10 @@ class CLIP:class VAE:def __init__(self, sd=None, device=None, config=None, dtype=None, metadata=None):- if 'decoder.up_blocks.0.resnets.0.norm1.weight' in sd.keys(): #diffusers format- sd = diffusers_convert.convert_vae_state_dict(sd)+ is_seedvr2_vae = "decoder.up_blocks.2.upsamplers.0.upscale_conv.weight" in sd+ if not is_seedvr2_vae and 'decoder.up_blocks.0.resnets.0.norm1.weight' in sd.keys(): #diffusers format+ if metadata is None or metadata.get("keep_diffusers_format") != "true":+ sd = diffusers_convert.convert_vae_state_dict(sd)if model_management.is_amd():VAE_KL_MEM_RATIO = 2.73@@ -540,6 +574,20 @@ class VAE:self.first_stage_model = StageC_coder()self.downscale_ratio = 32self.latent_channels = 16+ elif "decoder.up_blocks.2.upsamplers.0.upscale_conv.weight" in sd: # seedvr2+ self.first_stage_model = comfy.ldm.seedvr.vae.VideoAutoencoderKLWrapper()+ self.latent_channels = 16+ self.latent_dim = 3+ self.disable_offload = True+ self.memory_used_decode = lambda shape, dtype: _seedvr2_vae_decode_memory_used(shape)+ self.memory_used_encode = lambda shape, dtype: (max(shape[2], 5) * shape[3] * shape[4] * 64) * model_management.dtype_size(dtype)+ self.working_dtypes = [torch.float16, torch.bfloat16, torch.float32]+ self.downscale_ratio = (lambda a: max(0, math.floor((a + 3) / 4)), 8, 8)+ self.downscale_index_formula = (4, 8, 8)+ self.upscale_ratio = (lambda a: max(0, a * 4 - 3), 8, 8)+ self.upscale_index_formula = (4, 8, 8)+ self.process_input = lambda image: image * 2.0 - 1.0+ self.crop_input = Falseelif "decoder.conv_in.weight" in sd:if sd['decoder.conv_in.weight'].shape[1] == 64:ddconfig = {"block_out_channels": [128, 256, 512, 512, 1024, 1024], "in_channels": 3, "out_channels": 3, "num_res_blocks": 2, "ffactor_spatial": 32, "downsample_match_channel": True, "upsample_match_channel": True}@@ -667,6 +715,7 @@ class VAE:self.downscale_ratio = (lambda a: max(0, math.floor((a + 7) / 8)), 32, 32)self.downscale_index_formula = (8, 32, 32)self.working_dtypes = [torch.bfloat16, torch.float32]+elif "decoder.conv_in.conv.weight" in sd and sd['decoder.conv_in.conv.weight'].shape[1] == 32:ddconfig = {"block_out_channels": [128, 256, 512, 1024, 1024], "in_channels": 3, "out_channels": 3, "num_res_blocks": 2, "ffactor_spatial": 16, "ffactor_temporal": 4, "downsample_match_channel": True, "upsample_match_channel": True}ddconfig['z_channels'] = sd["decoder.conv_in.conv.weight"].shape[1]@@ -1006,6 +1055,40 @@ class VAE:decode_fn = lambda a: self.first_stage_model.decode(a.to(self.vae_dtype).to(self.device)).to(dtype=self.vae_output_dtype())return self.process_output(comfy.utils.tiled_scale_multidim(samples, decode_fn, tile=(tile_t, tile_x, tile_y), overlap=overlap, upscale_amount=self.upscale_ratio, out_channels=self.output_channels, index_formulas=self.upscale_index_formula, output_device=self.output_device))+ def decode_tiled_seedvr2(self, samples, tile_x=32, tile_y=32, overlap=8, tile_t=16, overlap_t=4):+ sf_s = getattr(self.first_stage_model, "spatial_downsample_factor", 8)+ sf_t = getattr(self.first_stage_model, "temporal_downsample_factor", 4)+ if tile_t is None:+ tile_t = 16+ if overlap_t is None:+ overlap_t = 4+ if tile_t > 0:+ temporal_size = tile_t * sf_t+ temporal_overlap = max(0, overlap_t) * sf_t+ else:+ temporal_size = 0+ temporal_overlap = 0+ args = {+ "enable_tiling": True,+ "tile_size": (tile_y * sf_s, tile_x * sf_s),+ "tile_overlap": (overlap * sf_s, overlap * sf_s),+ "temporal_size": temporal_size,+ "temporal_overlap": temporal_overlap,+ }+ output = self.first_stage_model.decode(+ samples.to(self.vae_dtype).to(self.device),+ seedvr2_tiling=args,+ )+ return self.process_output(output.to(device=self.output_device, dtype=self.vae_output_dtype(), copy=True))++ def _format_seedvr2_encoded_samples(self, samples):+ if isinstance(self.first_stage_model, comfy.ldm.seedvr.vae.VideoAutoencoderKLWrapper):+ if samples.ndim == 4:+ samples = samples.unsqueeze(2)+ samples = samples.contiguous()+ samples = samples * 0.9152+ return samples+def encode_tiled_(self, pixel_samples, tile_x=512, tile_y=512, overlap = 64):steps = pixel_samples.shape[0] * comfy.utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x, tile_y, overlap)steps += pixel_samples.shape[0] * comfy.utils.get_tiled_scale_steps(pixel_samples.shape[3], pixel_samples.shape[2], tile_x // 2, tile_y * 2, overlap)@@ -1042,6 +1125,36 @@ class VAE:encode_fn = lambda a: self.first_stage_model.encode((self.process_input(a)).to(self.vae_dtype).to(self.device)).to(dtype=self.vae_output_dtype())return comfy.utils.tiled_scale_multidim(samples, encode_fn, tile=(tile_t, tile_x, tile_y), overlap=overlap, upscale_amount=self.downscale_ratio, out_channels=self.latent_channels, downscale=True, index_formulas=self.downscale_index_formula, output_device=self.output_device)+ def encode_tiled_seedvr2(self, pixel_samples, tile_x=None, tile_y=None, overlap=None, tile_t=None, overlap_t=None):+ if tile_y is None:+ tile_y = 512+ if tile_x is None:+ tile_x = 512+ if overlap is None:+ overlap_y = 64+ overlap_x = 64+ else:+ overlap_y = overlap+ overlap_x = overlap+ if tile_t is None:+ tile_t = 9999+ if overlap_t is None:+ overlap_t = 0+ overlap_y = min(overlap_y, max(0, tile_y - 8))+ overlap_x = min(overlap_x, max(0, tile_x - 8))+ self.first_stage_model.device = self.device+ x = self.process_input(pixel_samples).to(self.vae_dtype).to(self.device)+ output = comfy.ldm.seedvr.vae.tiled_vae(+ x,+ self.first_stage_model,+ tile_size=(tile_y, tile_x),+ tile_overlap=(overlap_y, overlap_x),+ temporal_size=tile_t,+ temporal_overlap=overlap_t,+ encode=True,+ )+ return output.to(device=self.output_device, dtype=self.vae_output_dtype())+def decode(self, samples_in, vae_options={}):self.throw_exception_if_invalid()pixel_samples = None@@ -1089,16 +1202,40 @@ class VAE:if dims == 1 or self.extra_1d_channel is not None:pixel_samples = self.decode_tiled_1d(samples_in)elif dims == 2:- pixel_samples = self.decode_tiled_(samples_in)+ # SeedVR2 latents arrive in 4D collapsed form ``(B, 16*T, H, W)``+ # downstream of ``SeedVR2Conditioning`` (which performs the+ # ``rearrange(b c t h w -> b (c t) h w)`` collapse). The+ # generic ``decode_tiled_`` would treat the channel dim as+ # spatial-only and crash on the collapsed (16, T) layout+ # under ``tiled_scale``'s mask broadcast; route SeedVR2 4D+ # latents to ``decode_tiled_seedvr2`` instead, whose wrapper+ # dispatch handles both 4D and 5D inputs.+ if isinstance(self.first_stage_model, comfy.ldm.seedvr.vae.VideoAutoencoderKLWrapper):+ tile = 256 // self.spacial_compression_decode()+ overlap = tile // 4+ pixel_samples = self.decode_tiled_seedvr2(samples_in, tile_x=tile, tile_y=tile, overlap=overlap)+ else:+ pixel_samples = self.decode_tiled_(samples_in)elif dims == 3:tile = 256 // self.spacial_compression_decode()overlap = tile // 4- pixel_samples = self.decode_tiled_3d(samples_in, tile_x=tile, tile_y=tile, overlap=(1, overlap, overlap))+ if isinstance(self.first_stage_model, comfy.ldm.seedvr.vae.VideoAutoencoderKLWrapper):+ pixel_samples = self.decode_tiled_seedvr2(samples_in, tile_x=tile, tile_y=tile, overlap=overlap)+ else:+ pixel_samples = self.decode_tiled_3d(samples_in, tile_x=tile, tile_y=tile, overlap=(1, overlap, overlap))pixel_samples = pixel_samples.to(self.output_device).movedim(1,-1)return pixel_samples- def decode_tiled(self, samples, tile_x=None, tile_y=None, overlap=None, tile_t=None, overlap_t=None):+ def decode_tiled(+ self,+ samples,+ tile_x=None,+ tile_y=None,+ overlap=None,+ tile_t=None,+ overlap_t=None,+ ):self.throw_exception_if_invalid()memory_used = self.memory_used_decode(samples.shape, self.vae_dtype) #TODO: calculate mem required for tilemodel_management.load_models_gpu([self.patcher], memory_required=memory_used, force_full_load=self.disable_offload)@@ -1112,7 +1249,20 @@ class VAE:args["overlap"] = overlapwith model_management.cuda_device_context(self.device):- if dims == 1 or self.extra_1d_channel is not None:+ if isinstance(self.first_stage_model, comfy.ldm.seedvr.vae.VideoAutoencoderKLWrapper) and dims in (2, 3):+ seedvr2_args = {}+ if tile_x is not None:+ seedvr2_args["tile_x"] = tile_x+ if tile_y is not None:+ seedvr2_args["tile_y"] = tile_y+ if overlap is not None:+ seedvr2_args["overlap"] = overlap+ if tile_t is not None:+ seedvr2_args["tile_t"] = tile_t+ if overlap_t is not None:+ seedvr2_args["overlap_t"] = overlap_t+ output = self.decode_tiled_seedvr2(samples, **seedvr2_args)+ elif dims == 1 or self.extra_1d_channel is not None:args.pop("tile_y")output = self.decode_tiled_1d(samples, **args)elif dims == 2:@@ -1154,6 +1304,8 @@ class VAE:else:pixels_in = pixels_in.to(self.device)out = self.first_stage_model.encode(pixels_in)+ if isinstance(out, tuple):+ out = out[0]out = out.to(self.output_device).to(dtype=self.vae_output_dtype())if samples is None:samples = torch.empty((pixel_samples.shape[0],) + tuple(out.shape[1:]), device=self.output_device, dtype=self.vae_output_dtype())@@ -1173,20 +1325,23 @@ class VAE:if self.latent_dim == 3:tile = 256overlap = tile // 4- samples = self.encode_tiled_3d(pixel_samples, tile_x=tile, tile_y=tile, overlap=(1, overlap, overlap))+ if isinstance(self.first_stage_model, comfy.ldm.seedvr.vae.VideoAutoencoderKLWrapper):+ samples = self.encode_tiled_seedvr2(pixel_samples, tile_x=tile, tile_y=tile, overlap=overlap)+ else:+ samples = self.encode_tiled_3d(pixel_samples, tile_x=tile, tile_y=tile, overlap=(1, overlap, overlap))elif self.latent_dim == 1 or self.extra_1d_channel is not None:samples = self.encode_tiled_1d(pixel_samples)else:samples = self.encode_tiled_(pixel_samples)- return samples+ return self._format_seedvr2_encoded_samples(samples)def encode_tiled(self, pixel_samples, tile_x=None, tile_y=None, overlap=None, tile_t=None, overlap_t=None):self.throw_exception_if_invalid()pixel_samples = self.vae_encode_crop_pixels(pixel_samples)dims = self.latent_dimpixel_samples = pixel_samples.movedim(-1, 1)- if dims == 3:+ if dims == 3 and pixel_samples.ndim < 5:if not self.not_video:pixel_samples = pixel_samples.movedim(1, 0).unsqueeze(0)else:@@ -1210,22 +1365,47 @@ class VAE:elif dims == 2:samples = self.encode_tiled_(pixel_samples, **args)elif dims == 3:- if tile_t is not None:- tile_t_latent = max(2, self.downscale_ratio[0](tile_t))+ if isinstance(self.first_stage_model, comfy.ldm.seedvr.vae.VideoAutoencoderKLWrapper):+ seedvr2_args = {}+ if tile_x is not None:+ seedvr2_args["tile_x"] = tile_x+ else:+ seedvr2_args["tile_x"] = 512+ if tile_y is not None:+ seedvr2_args["tile_y"] = tile_y+ else:+ seedvr2_args["tile_y"] = 512+ if overlap is not None:+ seedvr2_args["overlap"] = overlap+ else:+ seedvr2_args["overlap"] = 64+ if tile_t is not None:+ seedvr2_args["tile_t"] = tile_t+ else:+ seedvr2_args["tile_t"] = 9999+ if overlap_t is not None:+ seedvr2_args["overlap_t"] = overlap_t+ else:+ seedvr2_args["overlap_t"] = 0+ samples = self.encode_tiled_seedvr2(pixel_samples, **seedvr2_args)else:- tile_t_latent = 9999- args["tile_t"] = self.upscale_ratio[0](tile_t_latent)+ if tile_t is not None:+ tile_t_latent = max(2, self.downscale_ratio[0](tile_t))+ else:+ tile_t_latent = 9999+ args["tile_t"] = self.upscale_ratio[0](tile_t_latent)- if overlap_t is None:- args["overlap"] = (1, overlap, overlap)- else:- args["overlap"] = (self.upscale_ratio[0](max(1, min(tile_t_latent // 2, self.downscale_ratio[0](overlap_t)))), overlap, overlap)- maximum = pixel_samples.shape[2]- maximum = self.upscale_ratio[0](self.downscale_ratio[0](maximum))+ spatial_overlap = overlap if overlap is not None else 64+ if overlap_t is None:+ args["overlap"] = (1, spatial_overlap, spatial_overlap)+ else:+ args["overlap"] = (self.upscale_ratio[0](max(1, min(tile_t_latent // 2, self.downscale_ratio[0](overlap_t)))), spatial_overlap, spatial_overlap)+ maximum = pixel_samples.shape[2]+ maximum = self.upscale_ratio[0](self.downscale_ratio[0](maximum))- samples = self.encode_tiled_3d(pixel_samples[:,:,:maximum], **args)+ samples = self.encode_tiled_3d(pixel_samples[:,:,:maximum], **args)- return samples+ return self._format_seedvr2_encoded_samples(samples)def get_sd(self):return self.first_stage_model.state_dict()@@ -1752,6 +1932,17 @@ def load_checkpoint(config_path=None, ckpt_path=None, output_vae=True, output_clreturn (model, clip, vae)++def _set_model_config_inference_dtype(model_config, dtype, manual_cast_dtype, device):+ set_dtype = model_config.set_inference_dtype+ parameters = inspect.signature(set_dtype).parameters+ supports_device = "device" in parameters or any(p.kind == inspect.Parameter.VAR_KEYWORD for p in parameters.values())+ if supports_device:+ set_dtype(dtype, manual_cast_dtype, device=device)+ else:+ set_dtype(dtype, manual_cast_dtype)++def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, output_clipvision=False, embedding_directory=None, output_model=True, model_options={}, te_model_options={}, disable_dynamic=False):sd, metadata = comfy.utils.load_torch_file(ckpt_path, return_metadata=True)out = load_state_dict_guess_config(sd, output_vae, output_clip, output_clipvision, embedding_directory, output_model, model_options, te_model_options=te_model_options, metadata=metadata, disable_dynamic=disable_dynamic)@@ -1859,7 +2050,7 @@ def load_state_dict_guess_config(sd, output_vae=True, output_clip=True, output_cmanual_cast_dtype = model_management.unet_manual_cast(None, load_device, model_config.supported_inference_dtypes)else:manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device, model_config.supported_inference_dtypes)- model_config.set_inference_dtype(unet_dtype, manual_cast_dtype)+ _set_model_config_inference_dtype(model_config, unet_dtype, manual_cast_dtype, load_device)if model_config.clip_vision_prefix is not None:if output_clipvision:@@ -2000,7 +2191,7 @@ def load_diffusion_model_state_dict(sd, model_options={}, metadata=None, disablemanual_cast_dtype = model_management.unet_manual_cast(None, load_device, model_config.supported_inference_dtypes)else:manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device, model_config.supported_inference_dtypes)- model_config.set_inference_dtype(unet_dtype, manual_cast_dtype)+ _set_model_config_inference_dtype(model_config, unet_dtype, manual_cast_dtype, load_device)if custom_operations is not None:model_config.custom_operations = custom_operationsdiff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex 7cf9c133b..fa95003cc 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -1672,6 +1672,35 @@ class Chroma(supported_models_base.BASE):t5_detect = comfy.text_encoders.sd3_clip.t5_xxl_detect(state_dict, "{}t5xxl.transformer.".format(pref))return supported_models_base.ClipTarget(comfy.text_encoders.pixart_t5.PixArtTokenizer, comfy.text_encoders.pixart_t5.pixart_te(**t5_detect))+class SeedVR2(supported_models_base.BASE):+ unet_config = {+ "image_model": "seedvr2"+ }+ latent_format = comfy.latent_formats.SeedVR2++ vae_key_prefix = ["vae."]+ text_encoder_key_prefix = ["text_encoders."]+ supported_inference_dtypes = [torch.bfloat16, torch.float16, torch.float32]+ sampling_settings = {+ "shift": 1.0,+ }++ def set_inference_dtype(self, dtype, manual_cast_dtype, device=None):+ if (+ dtype == torch.float16+ and manual_cast_dtype is None+ and comfy.model_management.should_use_bf16(device)+ ):+ manual_cast_dtype = torch.bfloat16+ super().set_inference_dtype(dtype, manual_cast_dtype, device=device)++ def get_model(self, state_dict, prefix="", device=None):+ out = model_base.SeedVR2(self, device=device)+ return out++ def clip_target(self, state_dict={}):+ return None+class ChromaRadiance(Chroma):unet_config = {"image_model": "chroma_radiance",@@ -2029,7 +2058,6 @@ class LongCatImage(supported_models_base.BASE):hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen25_7b.transformer.".format(pref))return supported_models_base.ClipTarget(comfy.text_encoders.longcat_image.LongCatImageTokenizer, comfy.text_encoders.longcat_image.te(**hunyuan_detect))-class RT_DETR_v4(supported_models_base.BASE):unet_config = {"image_model": "RT_DETR_v4",@@ -2267,6 +2295,7 @@ models = [HiDream,HiDreamO1,Chroma,+ SeedVR2,ChromaRadiance,ACEStep,ACEStep15,diff --git a/comfy/supported_models_base.py b/comfy/supported_models_base.pyindex 0e7a829ba..572f9984e 100644--- a/comfy/supported_models_base.py+++ b/comfy/supported_models_base.py@@ -115,7 +115,7 @@ class BASE:replace_prefix = {"": self.vae_key_prefix[0]}return utils.state_dict_prefix_replace(state_dict, replace_prefix)- def set_inference_dtype(self, dtype, manual_cast_dtype):+ def set_inference_dtype(self, dtype, manual_cast_dtype, device=None):self.unet_config['dtype'] = dtypeself.manual_cast_dtype = manual_cast_dtypediff --git a/comfy_extras/nodes_seedvr.py b/comfy_extras/nodes_seedvr.pynew file mode 100644index 000000000..d5cd029ba--- /dev/null+++ b/comfy_extras/nodes_seedvr.py@@ -0,0 +1,1015 @@+from typing_extensions import override+from comfy_api.latest import ComfyExtension, io+import torch+import math+import logging+from einops import rearrange++import gc+import comfy.model_management+import comfy.sample+import comfy.samplers+from comfy.ldm.seedvr.color_fix import (+ adain_color_transfer,+ lab_color_transfer,+ wavelet_color_transfer,+)+from comfy.ldm.seedvr.constants import (+ BYTEDANCE_IMG_SHIFT_FIT,+ BYTEDANCE_SCHEDULE_T,+ BYTEDANCE_VID_SHIFT_FIT,+ SEEDVR2_ADAIN_SCALE_MULTIPLIER,+ SEEDVR2_COLOR_MEM_HEADROOM,+ SEEDVR2_COND_CHANNELS,+ SEEDVR2_DTYPE_BYTES_FLOOR,+ SEEDVR2_LAB_SCALE_MULTIPLIER,+ SEEDVR2_LATENT_CHANNELS,+ SEEDVR2_OOM_BACKOFF_DIVISOR,+ SEEDVR2_WAVELET_SCALE_MULTIPLIER,+)++from torchvision.transforms import functional as TVF+from torchvision.transforms import Lambda+from torchvision.transforms.functional import InterpolationMode+++_SEEDVR2_INVALID_MODEL_MSG_PREFIX = (+ "SeedVR2Conditioning: model object does not match expected SeedVR2 structure"+)++# Private sentinel for getattr default: distinguishes "attribute missing"+# from "attribute present but None" so the failure message is accurate.+_ATTR_MISSING = object()+++def _seedvr2_auto_chunk_attempts(t_latent, t_pixel, frames_per_chunk):+ """Return stricter 4n+1 frame chunk sizes for auto OOM retries."""+ attempts = [frames_per_chunk]+ current_chunk_latent = (+ t_latent if t_pixel <= frames_per_chunk+ else (frames_per_chunk - 1) // 4 + 1+ )+ current_chunk_count = max(1, math.ceil(t_latent / current_chunk_latent))+ seen = {frames_per_chunk}++ for target_chunks in range(max(2, current_chunk_count + 1), t_latent + 1):+ chunk_latent = max(1, math.ceil(t_latent / target_chunks))+ candidate = 4 * (chunk_latent - 1) + 1+ if candidate in seen:+ continue+ if candidate >= attempts[-1]:+ continue+ attempts.append(candidate)+ seen.add(candidate)++ return attempts+++def _resolve_seedvr2_diffusion_model(model):+ """Resolve ``model.model.diffusion_model``, failing loud via the ``_ATTR_MISSING`` sentinel so each of the four modes (model/diffusion_model missing vs None) gives an accurate message."""+ inner = getattr(model, "model", _ATTR_MISSING)+ if inner is _ATTR_MISSING:+ raise RuntimeError(+ f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: input has no 'model' attribute "+ f"(got type {type(model).__name__})."+ )+ if inner is None:+ raise RuntimeError(+ f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: input.model is None "+ f"(input type {type(model).__name__})."+ )+ diffusion_model = getattr(inner, "diffusion_model", _ATTR_MISSING)+ if diffusion_model is _ATTR_MISSING:+ raise RuntimeError(+ f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: 'model.model' has no "+ f"'diffusion_model' attribute (got type {type(inner).__name__})."+ )+ if diffusion_model is None:+ raise RuntimeError(+ f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: 'model.model.diffusion_model' "+ f"is None (model.model type {type(inner).__name__})."+ )+ return diffusion_model+++def _apply_rope_freqs_float32_cast(diffusion_model):+ """Cast every module's ``rope.freqs`` to float32; the per-tensor dtype check (not a sentinel attr) self-corrects across Comfy's unload/reload, which would otherwise restore the archived fp16/bf16 dtype."""+ for module in diffusion_model.modules():+ if hasattr(module, 'rope') and hasattr(module.rope, 'freqs'):+ if module.rope.freqs.data.dtype != torch.float32:+ module.rope.freqs.data = module.rope.freqs.data.to(torch.float32)+++def clear_vae_memory(vae_model):+ for module in vae_model.modules():+ if hasattr(module, "memory"):+ module.memory = None+ gc.collect()+ comfy.model_management.soft_empty_cache()++def expand_dims(tensor, ndim):+ shape = tensor.shape + (1,) * (ndim - tensor.ndim)+ return tensor.reshape(shape)++def get_conditions(latent, latent_blur):+ t, h, w, c = latent.shape+ cond = torch.ones([t, h, w, c + 1], device=latent.device, dtype=latent.dtype)+ cond[:, ..., :-1] = latent_blur[:]+ cond[:, ..., -1:] = 1.0+ return cond++def timestep_transform(timesteps, latents_shapes):+ vt = 4+ vs = 8+ frames = (latents_shapes[:, 0] - 1) * vt + 1+ heights = latents_shapes[:, 1] * vs+ widths = latents_shapes[:, 2] * vs++ # Compute shift factor.+ def get_lin_function(x1, y1, x2, y2):+ m = (y2 - y1) / (x2 - x1)+ b = y1 - m * x1+ return lambda x: m * x + b++ img_shift_fn = get_lin_function(*BYTEDANCE_IMG_SHIFT_FIT)+ vid_shift_fn = get_lin_function(*BYTEDANCE_VID_SHIFT_FIT)+ shift = torch.where(+ frames > 1,+ vid_shift_fn(heights * widths * frames),+ img_shift_fn(heights * widths),+ ).to(timesteps.device)++ # Shift timesteps.+ T = BYTEDANCE_SCHEDULE_T+ timesteps = timesteps / T+ timesteps = shift * timesteps / (1 + (shift - 1) * timesteps)+ timesteps = timesteps * T+ return timesteps++def inter(x_0, x_T, t):+ t = expand_dims(t, x_0.ndim)+ T = BYTEDANCE_SCHEDULE_T+ B = lambda t: t / T+ A = lambda t: 1 - (t / T)+ return A(t) * x_0 + B(t) * x_T++def div_pad(image, factor):++ height_factor, width_factor = factor+ height, width = image.shape[-2:]++ pad_height = (height_factor - (height % height_factor)) % height_factor+ pad_width = (width_factor - (width % width_factor)) % width_factor++ if pad_height == 0 and pad_width == 0:+ return image++ if isinstance(image, torch.Tensor):+ padding = (0, pad_width, 0, pad_height)+ image = torch.nn.functional.pad(image, padding, mode='constant', value=0.0)++ return image++def cut_videos(videos):+ t = videos.size(1)+ if t == 1:+ return videos+ if t <= 4 :+ padding = [videos[:, -1].unsqueeze(1)] * (4 - t + 1)+ padding = torch.cat(padding, dim=1)+ videos = torch.cat([videos, padding], dim=1)+ return videos+ if (t - 1) % (4) == 0:+ return videos+ else:+ padding = [videos[:, -1].unsqueeze(1)] * (+ 4 - ((t - 1) % (4))+ )+ padding = torch.cat(padding, dim=1)+ videos = torch.cat([videos, padding], dim=1)+ assert (videos.size(1) - 1) % (4) == 0+ return videos++def _seedvr2_input_shorter_edge(images, node_name):+ if images.dim() == 4:+ return min(images.shape[1], images.shape[2])+ if images.dim() == 5:+ return min(images.shape[2], images.shape[3])+ raise ValueError(+ f"{node_name}: expected 4-D or 5-D IMAGE tensor, "+ f"got shape {tuple(images.shape)}"+ )+++def _seedvr2_pad(images, upscaled_shorter_edge, node_name):+ if upscaled_shorter_edge < 2:+ raise ValueError(+ f"{node_name}: input shorter edge must be at least 2 pixels; "+ f"got {upscaled_shorter_edge}."+ )+ if images.shape[-1] > 3:+ images = images[..., :3]+ if images.dim() == 4:+ # Comfy video components arrive as a 4-D IMAGE frame sequence:+ # (frames, H, W, C). SeedVR2 consumes that as one video.+ images = images.unsqueeze(0)+ elif images.dim() != 5:+ raise ValueError(+ f"{node_name}: expected 4-D or 5-D IMAGE tensor, "+ f"got shape {tuple(images.shape)}"+ )+ images = images.permute(0, 1, 4, 2, 3)++ b, t, c, h, w = images.shape+ images = images.reshape(b * t, c, h, w)++ clip = Lambda(lambda x: torch.clamp(x, 0.0, 1.0))+ images = clip(images)+ images = div_pad(images, (16, 16))+ _, _, new_h, new_w = images.shape++ images = images.reshape(b, t, c, new_h, new_w)+ images = cut_videos(images)+ images_bthwc = rearrange(images, "b t c h w -> b t h w c")++ return io.NodeOutput(images_bthwc)+++class SeedVR2Preprocess(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="SeedVR2Preprocess",+ display_name="Pre-Process SeedVR2 Input",+ category="image/upscaling",+ description="Pad a resized image for SeedVR2 model. Alpha channel is dropped. The node Post-Process SeedVR2 Output re-applies it from the original resized image.",+ inputs=[+ io.Image.Input("resized_images", tooltip="The resized image to process."),+ ],+ outputs=[+ io.Image.Output("images"),+ ]+ )++ @classmethod+ def execute(cls, resized_images):+ upscaled_shorter_edge = _seedvr2_input_shorter_edge(resized_images, "SeedVR2Preprocess")+ return _seedvr2_pad(+ resized_images, upscaled_shorter_edge, "SeedVR2Preprocess",+ )+++class SeedVR2PostProcessing(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="SeedVR2PostProcessing",+ display_name="Post-Process SeedVR2 Output",+ category="image/upscaling",+ description="Align the generated image with the original resized image and apply color correction.",+ inputs=[+ io.Image.Input("images", tooltip="The generated image to process."),+ io.Image.Input("original_resized_images", tooltip="The original resized image before pre-processing, used as reference."),+ io.Combo.Input("color_correction_method", options=["lab", "wavelet", "adain", "none"], default="lab", tooltip="Method to match the generated image colors to the original image. lab: transfer color in CIELAB space, preserving detail (most faithful). wavelet: transfer low-frequency color, keeping upscaled high-frequency detail. adain: match per-channel mean/std (fastest, global tint). none: skip color transfer (geometry alignment only)."),+ ],+ outputs=[io.Image.Output(display_name="images")],+ )++ @classmethod+ def execute(cls, images, original_resized_images, color_correction_method):+ alpha_input = None+ if original_resized_images.shape[-1] == 4:+ alpha_input = original_resized_images[..., 3:4]+ original_resized_images = original_resized_images[..., :3]+ decoded_5d, decoded_was_4d = cls._as_bthwc(images)+ reference_full, _ = cls._as_bthwc(original_resized_images)+ decoded_5d = cls._restore_reference_batch_time(decoded_5d, reference_full)++ b = min(decoded_5d.shape[0], reference_full.shape[0])+ t = min(decoded_5d.shape[1], reference_full.shape[1])+ reference_h = reference_full.shape[2]+ reference_w = reference_full.shape[3]++ decoded_5d = decoded_5d[:b, :t, :, :, :]+ target_h = min(decoded_5d.shape[2], reference_h)+ target_w = min(decoded_5d.shape[3], reference_w)+ decoded_5d = decoded_5d[:, :, :target_h, :target_w, :]+ if color_correction_method in ("lab", "wavelet", "adain"):+ reference_5d = reference_full[:b, :t, :, :, :]+ reference_5d = cls._resize_reference(reference_5d, target_h, target_w)+ output_device = decoded_5d.device+ decoded_raw = cls._to_seedvr2_raw(decoded_5d)+ reference_raw = cls._to_seedvr2_raw(reference_5d)+ decoded_flat = rearrange(decoded_raw, "b t h w c -> (b t) c h w")+ reference_flat = rearrange(reference_raw, "b t h w c -> (b t) c h w")+ output = cls._color_transfer_chunked(+ decoded_flat, reference_flat, output_device, color_correction_method,+ )+ output = rearrange(output, "(b t) c h w -> b t h w c", b=b, t=t)+ output = output.add(1.0).div(2.0).clamp(0.0, 1.0)+ elif color_correction_method == "none":+ output = decoded_5d+ else:+ raise ValueError(f"SeedVR2PostProcessing: unknown color_correction_method {color_correction_method!r}")++ if alpha_input is not None:+ alpha_5d, _ = cls._as_bthwc(alpha_input)+ alpha_5d = alpha_5d[:output.shape[0], :output.shape[1], :output.shape[2], :output.shape[3], :]+ output = torch.cat([output, alpha_5d.to(dtype=output.dtype, device=output.device)], dim=-1)+ h2 = output.shape[-3] - (output.shape[-3] % 2)+ w2 = output.shape[-2] - (output.shape[-2] % 2)+ output = output[:, :, :h2, :w2, :]+ if decoded_was_4d:+ output = output.reshape(-1, output.shape[-3], output.shape[-2], output.shape[-1])+ return io.NodeOutput(output)++ @staticmethod+ def _as_bthwc(images):+ if images.ndim == 4:+ return images.unsqueeze(0), True+ if images.ndim == 5:+ return images, False+ raise ValueError(+ f"SeedVR2PostProcessing: expected 4-D or 5-D IMAGE tensor, got shape {tuple(images.shape)}"+ )++ @staticmethod+ def _restore_reference_batch_time(decoded, reference):+ if decoded.shape[0] != 1:+ return decoded+ ref_b, ref_t = reference.shape[:2]+ if ref_b < 1 or decoded.shape[1] % ref_b != 0:+ return decoded+ decoded_t = decoded.shape[1] // ref_b+ if decoded_t < ref_t:+ return decoded+ return decoded.reshape(ref_b, decoded_t, decoded.shape[2], decoded.shape[3], decoded.shape[4])++ @staticmethod+ def _to_seedvr2_raw(images):+ return images.mul(2.0).sub(1.0)++ @staticmethod+ def _color_transfer_on_vae_device(decoded_flat, reference_flat, output_device, transfer_fn):+ color_device = comfy.model_management.vae_device()+ decoded_flat = decoded_flat.to(device=color_device)+ reference_flat = reference_flat.to(device=color_device)+ output = transfer_fn(decoded_flat, reference_flat)+ return output.to(device=output_device)++ @staticmethod+ def _lab_color_transfer_on_vae_device(decoded_flat, reference_flat, output_device):+ color_device = comfy.model_management.vae_device()+ result = None+ for start in range(decoded_flat.shape[0]):+ decoded_frame = decoded_flat[start:start + 1].to(device=color_device).clone()+ reference_frame = reference_flat[start:start + 1].to(device=color_device).clone()+ output = lab_color_transfer(decoded_frame, reference_frame).to(device=output_device)+ if result is None:+ result = torch.empty(+ (decoded_flat.shape[0],) + tuple(output.shape[1:]),+ device=output_device,+ dtype=output.dtype,+ )+ result[start:start + 1].copy_(output)+ if result is None:+ raise ValueError("SeedVR2PostProcessing: LAB color correction requires at least one frame.")+ return result++ @classmethod+ def _color_transfer_chunked(cls, decoded_flat, reference_flat, output_device, color_correction_method):+ chunk_size = cls._estimate_color_correction_chunk_size(decoded_flat, color_correction_method)+ while True:+ next_chunk_size = None+ try:+ return cls._run_color_transfer_chunks(+ decoded_flat, reference_flat, output_device, color_correction_method, chunk_size,+ )+ except Exception as e:+ comfy.model_management.raise_non_oom(e)+ if chunk_size <= 1:+ raise RuntimeError(+ "SeedVR2PostProcessing: color correction OOM at one frame; "+ f"color_correction_method={color_correction_method}, shape={tuple(decoded_flat.shape)}."+ ) from e+ next_chunk_size = max(1, chunk_size // SEEDVR2_OOM_BACKOFF_DIVISOR)++ comfy.model_management.soft_empty_cache()+ chunk_size = next_chunk_size++ @classmethod+ def _run_color_transfer_chunks(cls, decoded_flat, reference_flat, output_device, color_correction_method, chunk_size):+ result = None+ for start in range(0, decoded_flat.shape[0], chunk_size):+ end = min(start + chunk_size, decoded_flat.shape[0])+ decoded_chunk = decoded_flat[start:end]+ reference_chunk = reference_flat[start:end]+ if color_correction_method == "lab":+ output = cls._lab_color_transfer_on_vae_device(decoded_chunk, reference_chunk, output_device)+ elif color_correction_method == "wavelet":+ output = cls._color_transfer_on_vae_device(+ decoded_chunk, reference_chunk, output_device, wavelet_color_transfer,+ )+ else:+ output = cls._color_transfer_on_vae_device(+ decoded_chunk, reference_chunk, output_device, adain_color_transfer,+ )+ if result is None:+ result = torch.empty(+ (decoded_flat.shape[0],) + tuple(output.shape[1:]),+ device=output_device,+ dtype=output.dtype,+ )+ result[start:end].copy_(output)+ if result is None:+ raise ValueError("SeedVR2PostProcessing: color correction requires at least one frame.")+ return result++ @classmethod+ def _estimate_color_correction_chunk_size(cls, decoded_flat, color_correction_method):+ multiplier = cls._color_correction_memory_multiplier(color_correction_method)+ frames = decoded_flat.shape[0]+ _, channels, height, width = decoded_flat.shape+ dtype_bytes = max(decoded_flat.element_size(), SEEDVR2_DTYPE_BYTES_FLOOR)+ bytes_per_frame = height * width * channels * dtype_bytes * multiplier+ if bytes_per_frame <= 0:+ return frames+ color_device = comfy.model_management.vae_device()+ free_memory = comfy.model_management.get_free_memory(color_device)+ chunk_size = int((free_memory * SEEDVR2_COLOR_MEM_HEADROOM) // bytes_per_frame)+ return max(1, min(frames, chunk_size))++ @staticmethod+ def _color_correction_memory_multiplier(color_correction_method):+ if color_correction_method == "lab":+ return SEEDVR2_LAB_SCALE_MULTIPLIER+ if color_correction_method == "wavelet":+ return SEEDVR2_WAVELET_SCALE_MULTIPLIER+ if color_correction_method == "adain":+ return SEEDVR2_ADAIN_SCALE_MULTIPLIER+ raise ValueError(f"SeedVR2PostProcessing: unknown color_correction_method {color_correction_method!r}")++ @staticmethod+ def _resize_reference(reference, height, width):+ if reference.shape[2] == height and reference.shape[3] == width:+ return reference+ b, t = reference.shape[:2]+ reference_flat = rearrange(reference, "b t h w c -> (b t) c h w")+ resized = TVF.resize(+ reference_flat,+ size=(height, width),+ interpolation=InterpolationMode.BICUBIC,+ antialias=not (isinstance(reference_flat, torch.Tensor) and reference_flat.device.type == "mps"),+ )+ return rearrange(resized, "(b t) c h w -> b t h w c", b=b, t=t)+++class SeedVR2Conditioning(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="SeedVR2Conditioning",+ display_name="Apply SeedVR2 Conditioning",+ category="conditioning",+ description="Build SeedVR2 positive/negative conditioning from a VAE latent.",+ inputs=[+ io.Model.Input("model", tooltip="The SeedVR2 model."),+ io.Latent.Input("vae_conditioning", display_name="latent"),+ ],+ outputs=[+ io.Model.Output(display_name = "model"),+ io.Conditioning.Output(display_name = "positive"),+ io.Conditioning.Output(display_name = "negative"),+ io.Latent.Output(display_name = "latent"),+ ],+ )++ @classmethod+ def execute(cls, model, vae_conditioning) -> io.NodeOutput:++ vae_conditioning = vae_conditioning["samples"]+ if vae_conditioning.ndim != 5:+ raise ValueError(+ "SeedVR2Conditioning expects a 5-D VAE latent in Comfy "+ f"channel-first layout; got shape {tuple(vae_conditioning.shape)}."+ )+ if vae_conditioning.shape[-1] == SEEDVR2_LATENT_CHANNELS and vae_conditioning.shape[1] != SEEDVR2_LATENT_CHANNELS:+ raise ValueError(+ "SeedVR2Conditioning expects SeedVR2 VAE latents in Comfy "+ f"channel-first layout (B, {SEEDVR2_LATENT_CHANNELS}, T, H, W); "+ f"got channel-last shape {tuple(vae_conditioning.shape)}."+ )+ vae_conditioning = vae_conditioning.movedim(1, -1).contiguous()+ model_patcher = model+ model = _resolve_seedvr2_diffusion_model(model_patcher)+ pos_cond = model.positive_conditioning+ neg_cond = model.negative_conditioning++ # Fail-loud guard against silently-wrong output when a+ # DiT-only ``.safetensors`` (no ``positive_conditioning`` /+ # ``negative_conditioning`` keys) is loaded via ``UNETLoader``.+ # ``NaDiT.__init__`` zero-fills the buffers via ``torch.zeros`` (see+ # ``comfy/ldm/seedvr/model.py``); ``load_state_dict(strict=False)``+ # leaves them at zero when the keys are absent. Detect that state+ # here rather than at ``BaseModel.extra_conds`` (per sampling step,+ # wasteful) or at the resolver helper (mixes structural shape with+ # semantic content). Both buffers must be checked together — partial+ # bake regressions could populate one but not the other.+ if (+ pos_cond.float().abs().sum().item() == 0+ and neg_cond.float().abs().sum().item() == 0+ ):+ raise RuntimeError(+ f"{_SEEDVR2_INVALID_MODEL_MSG_PREFIX}: positive_conditioning "+ f"and negative_conditioning buffers are zero-valued — model "+ f"file appears to be a DiT-only export missing "+ f"the SeedVR2 conditioning tensors. "+ f"Re-bake the file with ``positive_conditioning`` (58, 5120) "+ f"and ``negative_conditioning`` (64, 5120) keys at top level, "+ f"or load via CheckpointLoaderSimple from a bundled "+ f"checkpoint."+ )++ _apply_rope_freqs_float32_cast(model)++ condition = torch.stack([get_conditions(c, c) for c in vae_conditioning])+ condition = condition.movedim(-1, 1)+ latent = vae_conditioning.movedim(-1, 1)++ latent = rearrange(latent, "b c t h w -> b (c t) h w")+ condition = rearrange(condition, "b c t h w -> b (c t) h w")++ negative = [[neg_cond.unsqueeze(0), {"condition": condition}]]+ positive = [[pos_cond.unsqueeze(0), {"condition": condition}]]++ return io.NodeOutput(model_patcher, positive, negative, {"samples": latent})++def _slice_collapsed_4d_along_t(tensor_4d: torch.Tensor, t_start: int,+ t_end: int, channels: int) -> torch.Tensor:+ """Slice collapsed ``(B, channels*T, H, W)`` along latent T: reshape (accepts non-contiguous inputs), slice, ``.contiguous()`` (T-slice of 5D is a non-contiguous view; re-collapse needs contiguous), re-collapse."""+ B, CT, H, W = tensor_4d.shape+ if CT % channels != 0:+ raise ValueError(+ f"_slice_collapsed_4d_along_t: collapsed channel dim {CT} is not "+ f"divisible by channels={channels}; tensor shape {tuple(tensor_4d.shape)}."+ )+ T = CT // channels+ if not (0 <= t_start < t_end <= T):+ raise ValueError(+ f"_slice_collapsed_4d_along_t: slice [{t_start}:{t_end}] out of "+ f"range for T={T}."+ )+ new_T = t_end - t_start+ sliced = tensor_4d.reshape(B, channels, T, H, W)[:, :, t_start:t_end, :, :].contiguous()+ return sliced.reshape(B, channels * new_T, H, W)+++def _slice_seedvr2_cond_along_t(cond_list, t_start: int, t_end: int):+ """Return a new conditioning list with each entry's ``options["condition"]`` (collapsed ``(B, 17*T, H, W)``) sliced along latent T; text tensors, other option keys, and condition-less entries pass through unchanged and inputs are not mutated."""+ new_list = []+ for entry in cond_list:+ text_cond, options = entry[0], entry[1]+ if "condition" not in options:+ new_list.append(entry)+ continue+ new_options = options.copy()+ new_options["condition"] = _slice_collapsed_4d_along_t(+ new_options["condition"], t_start, t_end,+ SEEDVR2_COND_CHANNELS,+ )+ new_list.append([text_cond, new_options])+ return new_list+++def _slice_seedvr2_noise_mask_along_t(noise_mask: torch.Tensor,+ samples_4d: torch.Tensor,+ t_start: int,+ t_end: int):+ """Slice only masks already expanded to collapsed ``(B, 16*T, H, W)``; pass standard ``(B, 1, H, W)`` ``SetLatentNoiseMask`` outputs through for KSampler to expand."""+ if noise_mask.ndim == samples_4d.ndim and noise_mask.shape[1] == samples_4d.shape[1]:+ return _slice_collapsed_4d_along_t(+ noise_mask, t_start, t_end, SEEDVR2_LATENT_CHANNELS,+ )+ return noise_mask+++def _concat_chunks_along_t(chunks_4d, channels: int) -> torch.Tensor:+ """Concatenate collapsed ``(B, channels*T_i, H, W)`` chunks along latent T: un-collapse to 5D, cat on ``dim=2``, re-collapse to 4D."""+ if len(chunks_4d) == 0:+ raise ValueError("_concat_chunks_along_t: empty chunk list.")+ fives = []+ for ch in chunks_4d:+ B, CT, H, W = ch.shape+ if CT % channels != 0:+ raise ValueError(+ f"_concat_chunks_along_t: chunk shape {tuple(ch.shape)} "+ f"channel dim {CT} not divisible by channels={channels}."+ )+ T = CT // channels+ fives.append(ch.reshape(B, channels, T, H, W))+ cat = torch.cat(fives, dim=2).contiguous()+ B, C, T_total, H, W = cat.shape+ return cat.reshape(B, C * T_total, H, W)+++def _hann_blend_weights_1d(overlap: int, device, dtype) -> torch.Tensor:+ """1D length-``overlap`` crossfade weights for the previous chunk (current = ``1 - w_prev``):+ Hann window with a ``[1/3, 2/3]`` dead-band for ``overlap >= 3``, linear ramp for ``overlap < 3``+ (dead-band would collapse a tiny transition). Window shape matched to the reference+ overlapping-frame blend for parity; caller broadcasts across ``(B, C, T_overlap, H, W)``.+ """+ if overlap < 1:+ raise ValueError(+ f"_hann_blend_weights_1d: overlap must be >= 1; got {overlap}."+ )+ if overlap >= 3:+ t = torch.linspace(0.0, 1.0, steps=overlap, device=device, dtype=dtype)+ blend_start = 1.0 / 3.0+ blend_end = 2.0 / 3.0+ u = ((t - blend_start) / (blend_end - blend_start)).clamp(0.0, 1.0)+ return 0.5 + 0.5 * torch.cos(torch.pi * u)+ return torch.linspace(1.0, 0.0, steps=overlap, device=device, dtype=dtype)+++def _blend_overlap_region(prev_tail_5d: torch.Tensor,+ cur_head_5d: torch.Tensor) -> torch.Tensor:+ """Blend two equal-shape 5D ``(B, C, T_overlap, H, W)`` tensors with a 1D Hann/linear T-ramp: ``prev_tail_5d`` takes the descending weight, ``cur_head_5d`` takes ``1 - w_prev`` (caller ensures matching shape/dtype/device)."""+ if prev_tail_5d.shape != cur_head_5d.shape:+ raise ValueError(+ f"_blend_overlap_region: shape mismatch "+ f"prev {tuple(prev_tail_5d.shape)} vs "+ f"cur {tuple(cur_head_5d.shape)}."+ )+ overlap = int(prev_tail_5d.shape[2])+ w_prev_1d = _hann_blend_weights_1d(+ overlap, prev_tail_5d.device, prev_tail_5d.dtype,+ )+ # Reshape to (1, 1, overlap, 1, 1) for broadcast across B, C, H, W.+ w_prev = w_prev_1d.view(1, 1, overlap, 1, 1)+ w_cur = 1.0 - w_prev+ return prev_tail_5d * w_prev + cur_head_5d * w_cur+++def _concat_chunks_with_overlap_blend(chunk_specs, channels: int,+ overlap_latent: int) -> torch.Tensor:+ """Concatenate overlapping ``(t_start, t_end, chunk_4d)`` specs (source-latent T coords) into one collapsed 4D tensor, Hann/linear-blending overlaps; ``overlap_latent == 0`` fast-paths to plain concat (bit-identical to ``_concat_chunks_along_t``). Each blend uses the actual width ``min(prev_end - cur_start, chunk length)``, smaller than ``overlap_latent`` for a runt final chunk."""+ if len(chunk_specs) == 0:+ raise ValueError("_concat_chunks_with_overlap_blend: empty chunk list.")+ if overlap_latent < 0:+ raise ValueError(+ f"_concat_chunks_with_overlap_blend: overlap_latent must be "+ f">= 0; got {overlap_latent}."+ )++ # Validate channel divisibility once and capture per-chunk T.+ chunk_5d = []+ for t_start, t_end, ch in chunk_specs:+ B, CT, H, W = ch.shape+ if CT % channels != 0:+ raise ValueError(+ f"_concat_chunks_with_overlap_blend: chunk shape "+ f"{tuple(ch.shape)} channel dim {CT} not divisible "+ f"by channels={channels}."+ )+ T = CT // channels+ if t_end - t_start != T:+ raise ValueError(+ f"_concat_chunks_with_overlap_blend: chunk T={T} mismatches "+ f"declared range [{t_start}:{t_end}]."+ )+ chunk_5d.append((t_start, t_end, ch.reshape(B, channels, T, H, W)))++ if overlap_latent == 0:+ # Fast path: pure concat in the caller-provided chunk order.+ return _concat_chunks_along_t(+ [c.reshape(c.shape[0], channels * c.shape[2], c.shape[3], c.shape[4])+ for _, _, c in chunk_5d],+ channels,+ )++ T_total = max(t_end for _, t_end, _ in chunk_5d)+ first_5d = chunk_5d[0][2]+ B = first_5d.shape[0]+ H = first_5d.shape[3]+ W = first_5d.shape[4]+ result = torch.empty(+ (B, channels, T_total, H, W),+ device=first_5d.device, dtype=first_5d.dtype,+ )+ filled_until = 0+ for i, (cs, ce, ct_5d) in enumerate(chunk_5d):+ chunk_T = int(ct_5d.shape[2])+ if i == 0:+ result[:, :, cs:ce, :, :] = ct_5d+ filled_until = ce+ continue+ # Overlap region width is bounded by both the previous fill+ # frontier and the current chunk's actual length (for runt+ # final chunks shorter than the configured overlap).+ overlap_len = min(filled_until - cs, chunk_T)+ if overlap_len > 0:+ prev_tail = result[:, :, cs:cs + overlap_len, :, :].contiguous()+ cur_head = ct_5d[:, :, :overlap_len, :, :].contiguous()+ blended = _blend_overlap_region(prev_tail, cur_head)+ result[:, :, cs:cs + overlap_len, :, :] = blended+ tail_start = cs + overlap_len+ tail_end = ce+ if tail_end > tail_start:+ result[:, :, tail_start:tail_end, :, :] = (+ ct_5d[:, :, overlap_len:, :, :]+ )+ else:+ # Disjoint chunks (overlap_latent set but this pair did not+ # actually overlap, e.g. step_latent equal to chunk_latent+ # in a degenerate config). Treat as concat.+ result[:, :, cs:ce, :, :] = ct_5d+ filled_until = ce++ return result.contiguous().reshape(B, channels * T_total, H, W)+++def _run_standard_sample(model, seed: int, steps: int, cfg: float,+ sampler_name: str, scheduler: str,+ positive, negative, latent: dict,+ denoise: float) -> dict:+ """Single-shot mirror of ``nodes.py:common_ksampler`` (seed -> noise, ``comfy.sample.sample``, latent dict); used by the ProgressiveSampler short-circuit when the whole sequence fits one chunk."""+ samples_in = latent["samples"]+ samples_in = comfy.sample.fix_empty_latent_channels(+ model, samples_in, latent.get("downscale_ratio_spacial", None),+ )+ batch_inds = latent.get("batch_index", None)+ noise = comfy.sample.prepare_noise(samples_in, seed, batch_inds)+ noise_mask = latent.get("noise_mask", None)+ samples = comfy.sample.sample(+ model, noise, steps, cfg, sampler_name, scheduler,+ positive, negative, samples_in,+ denoise=denoise, noise_mask=noise_mask, seed=seed,+ )+ out = latent.copy()+ out.pop("downscale_ratio_spacial", None)+ out["samples"] = samples+ return out+++class SeedVR2ProgressiveSampler(io.ComfyNode):+ """Sequential temporal chunking sampler for SeedVR2 native.++ Drop-in replacement for ``KSampler`` in SeedVR2 native workflows that+ OOM on long sequences. The latent enters the sampler in SeedVR2's+ collapsed form ``(B, 16*T, H, W)`` (collapsed by ``SeedVR2Conditioning``+ at ``rearrange(b c t h w -> b (c t) h w)``); this node slices that+ tensor along the temporal axis, runs the configured inner sampler+ sequentially per chunk against the standard ``comfy.sample.sample``+ entry point, and concatenates per-chunk outputs back into a single+ ``(B, 16*T_total, H, W)`` latent.++ ``frames_per_chunk`` is expressed in pixel-frame units to match the+ SeedVR2 4n+1 constraint enforced upstream by ``cut_videos`` and the+ VAE's ``temporal_downsample_factor=4``. A pixel chunk size ``F``+ maps to ``(F - 1) // 4 + 1`` latent-frame chunks.++ Determinism contract: a single noise tensor is generated once from+ the user seed and sliced per chunk (rather than re-seeding each+ chunk), so a workflow that fits in a single chunk produces output+ identical to a workflow that fits in N chunks at the same seed,+ modulo the inherent T-axis chunk-boundary independence of the model.+ """++ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="SeedVR2ProgressiveSampler",+ display_name="Sample SeedVR2 (Progressive)",+ category="sampling",+ description="Sample a SeedVR2 latent in sequential temporal chunks to allow longer videos to fit into VRAM via frame blending the resulting upscaled latents.",+ inputs=[+ io.Model.Input("model", tooltip="The model used for denoising the input latent."),+ io.Int.Input("seed", default=0, min=0,+ max=0xffffffffffffffff,+ control_after_generate=True,+ tooltip="The random seed used for creating the noise."),+ io.Int.Input("steps", default=20, min=1, max=10000,+ tooltip="The number of steps used in the denoising process."),+ io.Float.Input("cfg", default=1.0, min=0.0, max=100.0,+ step=0.1, round=0.01,+ tooltip="The Classifier-Free Guidance scale balances creativity and adherence to the prompt. Higher values result in images more closely matching the prompt however too high values will negatively impact quality."),+ io.Combo.Input("sampler_name",+ options=comfy.samplers.SAMPLER_NAMES,+ tooltip="The algorithm used when sampling, this can affect the quality, speed, and style of the generated output."),+ io.Combo.Input("scheduler",+ options=comfy.samplers.SCHEDULER_NAMES,+ tooltip="The scheduler controls how noise is gradually removed to form the image."),+ io.Conditioning.Input("positive",+ tooltip="The conditioning describing the attributes you want to include in the image."),+ io.Conditioning.Input("negative",+ tooltip="The conditioning describing the attributes you want to exclude from the image."),+ io.Latent.Input("latent",+ tooltip="The latent image to denoise."),+ io.Float.Input("denoise", default=1.0, min=0.0, max=1.0,+ step=0.01,+ tooltip="The amount of denoising applied, lower values will maintain the structure of the initial image allowing for image to image sampling."),+ io.Int.Input("frames_per_chunk", default=21, min=1,+ max=16384, step=4,+ tooltip="Pixel frames per temporal chunk (4n+1: 1, 5, 9, 13, ...)."),+ io.Int.Input("temporal_overlap", default=0, min=0,+ max=16384,+ tooltip="Latent frames blended between adjacent chunks to hide the seam; 0 = no blend."),+ io.Combo.Input("chunking_mode",+ options=["manual", "auto"],+ default="manual",+ tooltip="manual = use frames_per_chunk exactly; auto = shrink the chunk until it fits in VRAM."),+ ],+ outputs=[io.Latent.Output(display_name="latent")],+ )++ @classmethod+ def execute(cls, model, seed, steps, cfg, sampler_name, scheduler,+ positive, negative, latent, denoise,+ frames_per_chunk, temporal_overlap,+ chunking_mode="manual") -> io.NodeOutput:+ # 4n+1 validation in pixel-frame domain. The SeedVR2 native pipeline+ # requires pixel-frame counts of the form 4n+1 (1, 5, 9, 13, ...),+ # imposed at ``cut_videos`` upstream and propagated through the VAE's+ # temporal_downsample_factor=4. Reject violations explicitly before+ # any model invocation; a silent rounding would mis-align chunk+ # boundaries with the 4n+1 lattice.+ if frames_per_chunk < 1 or (frames_per_chunk - 1) % 4 != 0:+ raise ValueError(+ f"SeedVR2ProgressiveSampler: frames_per_chunk must be a "+ f"4n+1 pixel-frame count (1, 5, 9, 13, 17, 21, ...); "+ f"got {frames_per_chunk}."+ )++ samples_4d = latent["samples"]+ samples_4d = comfy.sample.fix_empty_latent_channels(+ model, samples_4d,+ latent.get("downscale_ratio_spacial", None),+ )+ if samples_4d.ndim != 4:+ raise ValueError(+ f"SeedVR2ProgressiveSampler: expected 4D collapsed latent "+ f"(B, 16*T, H, W); got shape {tuple(samples_4d.shape)}."+ )+ B, CT, H, W = samples_4d.shape+ if CT % SEEDVR2_LATENT_CHANNELS != 0:+ raise ValueError(+ f"SeedVR2ProgressiveSampler: collapsed channel dim {CT} is "+ f"not divisible by SeedVR2 latent channels "+ f"{SEEDVR2_LATENT_CHANNELS}; latent does not appear to be "+ f"SeedVR2-shaped."+ )+ T_latent = CT // SEEDVR2_LATENT_CHANNELS+ T_pixel = 4 * (T_latent - 1) + 1++ if chunking_mode not in ("manual", "auto"):+ raise ValueError(+ f"SeedVR2ProgressiveSampler: chunking_mode must be "+ f"'manual' or 'auto'; got {chunking_mode!r}."+ )++ if chunking_mode == "auto":+ attempts = _seedvr2_auto_chunk_attempts(+ T_latent, T_pixel, frames_per_chunk,+ )+ for i, attempt_frames_per_chunk in enumerate(attempts):+ retry = False+ try:+ return cls.execute(+ model=model, seed=seed, steps=steps, cfg=cfg,+ sampler_name=sampler_name, scheduler=scheduler,+ positive=positive, negative=negative,+ latent=latent, denoise=denoise,+ frames_per_chunk=attempt_frames_per_chunk,+ temporal_overlap=temporal_overlap,+ chunking_mode="manual",+ )+ except Exception as e:+ comfy.model_management.raise_non_oom(e)+ if i == len(attempts) - 1:+ raise RuntimeError(+ "SeedVR2ProgressiveSampler: exhausted auto "+ "chunking attempts after OOM. Tried "+ f"frames_per_chunk values {attempts}."+ ) from e+ retry = True++ if retry:+ logging.warning(+ "SeedVR2ProgressiveSampler auto chunking OOM at "+ "frames_per_chunk=%s; retrying with "+ "frames_per_chunk=%s.",+ attempt_frames_per_chunk, attempts[i + 1],+ )+ comfy.model_management.soft_empty_cache()++ # Short-circuit: total fits in one chunk -> standard path with no+ # chunking overhead. Output of this branch is byte-identical to the+ # built-in KSampler given the same (model, seed, steps, cfg,+ # sampler_name, scheduler, positive, negative, latent,+ # denoise) tuple.+ if T_pixel <= frames_per_chunk:+ return io.NodeOutput(_run_standard_sample(+ model, seed, steps, cfg, sampler_name, scheduler,+ positive, negative, latent, denoise,+ ))++ # Map pixel chunk -> latent chunk. Each chunk's latent length is+ # at most ``chunk_latent``; the final chunk may be a runt that+ # is automatically 4n+1-aligned in the pixel domain by the+ # T_pixel = 4*(T_latent-1) + 1 mapping (every positive integer+ # T_latent corresponds to a valid 4n+1 pixel count).+ chunk_latent = (frames_per_chunk - 1) // 4 + 1++ # ``temporal_overlap`` is exposed in latent-frame units, but users+ # do not know the derived latent chunk length. Treat oversized+ # values as "maximum valid overlap" while preserving a strictly+ # positive chunk-loop stride.+ if temporal_overlap < 0:+ raise ValueError(+ f"SeedVR2ProgressiveSampler: temporal_overlap must be >= 0; "+ f"got {temporal_overlap}."+ )+ temporal_overlap = min(temporal_overlap, chunk_latent - 1)+ step_latent = chunk_latent - temporal_overlap++ # Generate full noise once from the user seed, then slice along T+ # per chunk. Using one global noise tensor (rather than re-seeding+ # per chunk) preserves seed-determinism across chunk-count+ # variations: the same (seed, total T_latent) always produces the+ # same noise samples regardless of how the work is partitioned.+ batch_inds = latent.get("batch_index", None)+ noise_full = comfy.sample.prepare_noise(samples_4d, seed, batch_inds)++ noise_mask = latent.get("noise_mask", None)++ # Build the flat list of chunk ranges first so the chunking+ # geometry is fully known before any sample call.+ chunk_ranges = []+ for chunk_start in range(0, T_latent, step_latent):+ chunk_end = min(chunk_start + chunk_latent, T_latent)+ if chunk_start >= chunk_end:+ # The final iteration of a stride that lands exactly on+ # T_latent produces a zero-length chunk; skip it.+ break+ chunk_ranges.append((chunk_start, chunk_end))+ if chunk_end >= T_latent:+ break++ def _sample_one_chunk(chunk_start, chunk_end):+ samples_chunk = _slice_collapsed_4d_along_t(+ samples_4d, chunk_start, chunk_end,+ SEEDVR2_LATENT_CHANNELS,+ )+ noise_chunk = _slice_collapsed_4d_along_t(+ noise_full, chunk_start, chunk_end,+ SEEDVR2_LATENT_CHANNELS,+ )+ positive_chunk = _slice_seedvr2_cond_along_t(+ positive, chunk_start, chunk_end,+ )+ negative_chunk = _slice_seedvr2_cond_along_t(+ negative, chunk_start, chunk_end,+ )++ # Per-chunk noise_mask handling: standard masks are passed+ # through for KSampler expansion; pre-expanded collapsed+ # masks are sliced.+ chunk_noise_mask = None+ if noise_mask is not None:+ chunk_noise_mask = _slice_seedvr2_noise_mask_along_t(+ noise_mask, samples_4d, chunk_start, chunk_end,+ )++ return comfy.sample.sample(+ model, noise_chunk, steps, cfg, sampler_name, scheduler,+ positive_chunk, negative_chunk, samples_chunk,+ denoise=denoise, noise_mask=chunk_noise_mask, seed=seed,+ )++ chunk_specs = []+ for chunk_start, chunk_end in chunk_ranges:+ chunk_samples = _sample_one_chunk(chunk_start, chunk_end)+ chunk_specs.append((chunk_start, chunk_end, chunk_samples))++ final = _concat_chunks_with_overlap_blend(+ chunk_specs, SEEDVR2_LATENT_CHANNELS, temporal_overlap,+ )++ out = latent.copy()+ out.pop("downscale_ratio_spacial", None)+ out["samples"] = final+ return io.NodeOutput(out)+++class SeedVRExtension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[io.ComfyNode]]:+ return [+ SeedVR2Conditioning,+ SeedVR2Preprocess,+ SeedVR2PostProcessing,+ SeedVR2ProgressiveSampler,+ ]++async def comfy_entrypoint() -> SeedVRExtension:+ return SeedVRExtension()diff --git a/nodes.py b/nodes.pyindex 2f5a478b5..d9ac53ede 100644--- a/nodes.py+++ b/nodes.py@@ -47,14 +47,18 @@ import node_helpersif args.enable_manager:import comfyui_manager+def before_node_execution():comfy.model_management.throw_exception_if_processing_interrupted()+def interrupt_processing(value=True):comfy.model_management.interrupt_current_processing(value)+MAX_RESOLUTION=16384+class CLIPTextEncode(ComfyNodeABC):@classmethoddef INPUT_TYPES(s) -> InputTypeDict:@@ -323,8 +327,8 @@ class VAEDecodeTiled:return {"required": {"samples": ("LATENT", ), "vae": ("VAE", ),"tile_size": ("INT", {"default": 512, "min": 64, "max": 4096, "step": 32, "advanced": True}),"overlap": ("INT", {"default": 64, "min": 0, "max": 4096, "step": 32, "advanced": True}),- "temporal_size": ("INT", {"default": 64, "min": 8, "max": 4096, "step": 4, "tooltip": "Only used for video VAEs: Amount of frames to decode at a time.", "advanced": True}),- "temporal_overlap": ("INT", {"default": 8, "min": 4, "max": 4096, "step": 4, "tooltip": "Only used for video VAEs: Amount of frames to overlap.", "advanced": True}),+ "temporal_size": ("INT", {"default": 64, "min": 0, "max": 4096, "step": 4, "tooltip": "Only used for video VAEs: Amount of frames to decode at a time. SeedVR2 allows 0 to disable temporal slicing.", "advanced": True}),+ "temporal_overlap": ("INT", {"default": 8, "min": 0, "max": 4096, "step": 4, "tooltip": "Only used for video VAEs: Amount of frames to overlap.", "advanced": True}),}}RETURN_TYPES = ("IMAGE",)FUNCTION = "decode"@@ -334,18 +338,32 @@ class VAEDecodeTiled:def decode(self, vae, samples, tile_size, overlap=64, temporal_size=64, temporal_overlap=8):if tile_size < overlap * 4:overlap = tile_size // 4- if temporal_size < temporal_overlap * 2:- temporal_overlap = temporal_overlap // 2temporal_compression = vae.temporal_compression_decode()if temporal_compression is not None:- temporal_size = max(2, temporal_size // temporal_compression)- temporal_overlap = max(1, min(temporal_size // 2, temporal_overlap // temporal_compression))+ if temporal_size <= 0:+ temporal_size = 0+ temporal_overlap = 0+ else:+ requested_temporal_overlap = temporal_overlap+ if temporal_size < temporal_overlap * 2:+ temporal_overlap = temporal_overlap // 2+ temporal_size = max(2, temporal_size // temporal_compression)+ temporal_overlap = min(temporal_size // 2, temporal_overlap // temporal_compression)+ if requested_temporal_overlap > 0:+ temporal_overlap = max(1, temporal_overlap)else:temporal_size = Nonetemporal_overlap = Nonecompression = vae.spacial_compression_decode()- images = vae.decode_tiled(samples["samples"], tile_x=tile_size // compression, tile_y=tile_size // compression, overlap=overlap // compression, tile_t=temporal_size, overlap_t=temporal_overlap)+ images = vae.decode_tiled(+ samples["samples"],+ tile_x=tile_size // compression,+ tile_y=tile_size // compression,+ overlap=overlap // compression,+ tile_t=temporal_size,+ overlap_t=temporal_overlap,+ )if len(images.shape) == 5: #Combine batchesimages = images.reshape(-1, images.shape[-3], images.shape[-2], images.shape[-1])return (images, )@@ -362,7 +380,7 @@ class VAEEncode:def encode(self, vae, pixels):t = vae.encode(pixels)- return ({"samples":t}, )+ return ({"samples": t}, )class VAEEncodeTiled:@classmethod@@ -370,8 +388,8 @@ class VAEEncodeTiled:return {"required": {"pixels": ("IMAGE", ), "vae": ("VAE", ),"tile_size": ("INT", {"default": 512, "min": 64, "max": 4096, "step": 64, "advanced": True}),"overlap": ("INT", {"default": 64, "min": 0, "max": 4096, "step": 32, "advanced": True}),- "temporal_size": ("INT", {"default": 64, "min": 8, "max": 4096, "step": 4, "tooltip": "Only used for video VAEs: Amount of frames to encode at a time.", "advanced": True}),- "temporal_overlap": ("INT", {"default": 8, "min": 4, "max": 4096, "step": 4, "tooltip": "Only used for video VAEs: Amount of frames to overlap.", "advanced": True}),+ "temporal_size": ("INT", {"default": 64, "min": 0, "max": 4096, "step": 4, "tooltip": "Only used for video VAEs: Amount of frames to encode at a time. SeedVR2 allows 0 to disable temporal slicing.", "advanced": True}),+ "temporal_overlap": ("INT", {"default": 8, "min": 0, "max": 4096, "step": 4, "tooltip": "Only used for video VAEs: Amount of frames to overlap.", "advanced": True}),}}RETURN_TYPES = ("LATENT",)FUNCTION = "encode"@@ -379,6 +397,9 @@ class VAEEncodeTiled:CATEGORY = "experimental"def encode(self, vae, pixels, tile_size, overlap, temporal_size=64, temporal_overlap=8):+ if temporal_size <= 0:+ temporal_size = 0+ temporal_overlap = 0t = vae.encode_tiled(pixels, tile_x=tile_size, tile_y=tile_size, overlap=overlap, tile_t=temporal_size, overlap_t=temporal_overlap)return ({"samples": t}, )@@ -2418,6 +2439,7 @@ async def init_builtin_extra_nodes():"nodes_camera_trajectory.py","nodes_edit_model.py","nodes_tcfg.py",+ "nodes_seedvr.py","nodes_context_windows.py","nodes_qwen.py","nodes_chroma_radiance.py",diff --git a/tests-unit/comfy_extras_test/test_seedvr2_conditioning.py b/tests-unit/comfy_extras_test/test_seedvr2_conditioning.pynew file mode 100644index 000000000..2a6e3d430--- /dev/null+++ b/tests-unit/comfy_extras_test/test_seedvr2_conditioning.py@@ -0,0 +1,213 @@+"""Consolidated SeedVR2 conditioning and refactor regression tests.++Merges the prior test_seedvr2_refactor_nodes.py and+test_seedvr_conditioning_hardening.py modules. Refactor tests use the+top-level comfy_extras.nodes_seedvr import; conditioning-hardening tests+use _import_nodes_seedvr_isolated() for sys.modules isolation when+mocking comfy.model_management.+"""++import importlib+import sys+from unittest.mock import MagicMock++import pytest+import torch+import torch.nn as nn++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True+++_SENTINEL = object()+_TARGETS = (+ ("comfy.model_management", "comfy"),+ ("comfy_extras.nodes_seedvr", "comfy_extras"),+)+++def _import_nodes_seedvr_isolated():+ """Import comfy_extras.nodes_seedvr with comfy.model_management mocked."""+ priors = []+ for mod_name, parent_name in _TARGETS:+ prior_mod = sys.modules.get(mod_name, _SENTINEL)+ parent = sys.modules.get(parent_name)+ attr = mod_name.split(".")[-1]+ prior_attr = (+ getattr(parent, attr, _SENTINEL) if parent is not None else _SENTINEL+ )+ priors.append((mod_name, parent_name, attr, prior_mod, prior_attr))++ mock_mm = MagicMock()+ for fn in (+ "xformers_enabled", "xformers_enabled_vae",+ "pytorch_attention_enabled", "pytorch_attention_enabled_vae",+ "sage_attention_enabled", "flash_attention_enabled",+ "is_intel_xpu",+ ):+ getattr(mock_mm, fn).return_value = False+ tv = torch.version.__version__.split(".")+ mock_mm.torch_version_numeric = (int(tv[0]), int(tv[1]))+ mock_mm.WINDOWS = False+ sys.modules["comfy.model_management"] = mock_mm+ if sys.modules.get("comfy") is None:+ import comfy as _comfy_pkg # noqa: F401+ comfy_pkg = sys.modules.get("comfy")+ if comfy_pkg is not None:+ setattr(comfy_pkg, "model_management", mock_mm)+ nodes_seedvr = sys.modules.get("comfy_extras.nodes_seedvr") or (+ importlib.import_module("comfy_extras.nodes_seedvr")+ )++ def _restore():+ for mod_name, parent_name, attr, prior_mod, prior_attr in priors:+ if prior_mod is _SENTINEL:+ sys.modules.pop(mod_name, None)+ else:+ sys.modules[mod_name] = prior_mod+ parent = sys.modules.get(parent_name)+ if parent is None:+ continue+ if prior_attr is _SENTINEL:+ if hasattr(parent, attr):+ delattr(parent, attr)+ else:+ setattr(parent, attr, prior_attr)++ return nodes_seedvr, _restore+++class _Rope(nn.Module):+ """Minimal RoPE stub exposing a `freqs` parameter."""+ def __init__(self):+ super().__init__()+ self.freqs = nn.Parameter(torch.zeros(4))+++class _Block(nn.Module):+ """Minimal transformer block stub holding a `_Rope`."""+ def __init__(self):+ super().__init__()+ self.rope = _Rope()+++class _DiffusionModel(nn.Module):+ """Stub diffusion model with N blocks and pos/neg conditioning buffers."""+ def __init__(self, n_blocks=3, zero_conditioning=False, conditioning_dtype=torch.float32):+ super().__init__()+ self.blocks = nn.ModuleList([_Block() for _ in range(n_blocks)])+ pos = torch.zeros if zero_conditioning else torch.ones+ self.register_buffer("positive_conditioning", pos((2, 4), dtype=conditioning_dtype))+ self.register_buffer("negative_conditioning", torch.zeros((3, 4), dtype=conditioning_dtype))+++class _ModelInner:+ """Inner model wrapper exposing `.diffusion_model`."""+ def __init__(self, diffusion_model):+ self.diffusion_model = diffusion_model+++class _ModelPatcher:+ """ModelPatcher stub exposing `.model._ModelInner`."""+ def __init__(self, diffusion_model):+ self.model = _ModelInner(diffusion_model)+++def test_seedvr2_conditioning_schema_exposes_model_passthrough_output():+ nodes_seedvr, restore = _import_nodes_seedvr_isolated()+ try:+ schema = nodes_seedvr.SeedVR2Conditioning.define_schema()+ assert [input_item.id for input_item in schema.inputs] == [+ "model",+ "vae_conditioning",+ ]+ assert schema.inputs[1].display_name == "latent"+ assert [output.display_name for output in schema.outputs] == [+ "model",+ "positive",+ "negative",+ "latent",+ ]+ finally:+ restore()+++def test_seedvr2_conditioning_returns_packed_input_latent_deterministically():+ nodes_seedvr, restore = _import_nodes_seedvr_isolated()+ try:+ diffusion_model = _DiffusionModel()+ patcher = _ModelPatcher(diffusion_model)+ samples = torch.arange(1, 25, dtype=torch.float32).reshape(1, 2, 3, 2, 2)+ vae_conditioning = {"samples": samples}++ _, first_positive, first_negative, first_latent = (+ nodes_seedvr.SeedVR2Conditioning.execute(+ patcher,+ vae_conditioning,+ )+ )+ _, second_positive, second_negative, second_latent = (+ nodes_seedvr.SeedVR2Conditioning.execute(+ patcher,+ vae_conditioning,+ )+ )++ expected_latent = samples.reshape(1, 6, 2, 2)+ channel_last = samples.movedim(1, -1).contiguous()+ expected_condition = torch.cat(+ [+ channel_last,+ torch.ones((*channel_last.shape[:-1], 1)),+ ],+ dim=-1,+ ).movedim(-1, 1).reshape(1, 9, 2, 2)++ assert torch.equal(first_latent["samples"], expected_latent)+ assert torch.equal(second_latent["samples"], expected_latent)+ assert torch.equal(+ first_positive[0][1]["condition"],+ expected_condition,+ )+ assert torch.equal(+ second_positive[0][1]["condition"],+ expected_condition,+ )+ assert torch.equal(+ first_negative[0][1]["condition"],+ expected_condition,+ )+ assert torch.equal(+ second_negative[0][1]["condition"],+ expected_condition,+ )+ finally:+ restore()+++def test_seedvr2_conditioning_fails_loud_on_zero_buffers():+ nodes_seedvr, restore = _import_nodes_seedvr_isolated()+ try:+ diffusion_model = _DiffusionModel(zero_conditioning=True)+ patcher = _ModelPatcher(diffusion_model)+ vae_conditioning = {"samples": torch.zeros((1, 2, 1, 1, 1))}++ with pytest.raises(RuntimeError) as excinfo:+ nodes_seedvr.SeedVR2Conditioning.execute(+ patcher, vae_conditioning,+ )++ message = str(excinfo.value)+ assert message.startswith(+ nodes_seedvr._SEEDVR2_INVALID_MODEL_MSG_PREFIX+ ), (+ "Fail-loud message must use the standard "+ "_SEEDVR2_INVALID_MODEL_MSG_PREFIX so callers/log scrapers "+ f"can match it. Got: {message!r}"+ )+ assert "positive_conditioning" in message+ assert "negative_conditioning" in message+ finally:+ restore()diff --git a/tests-unit/comfy_extras_test/test_seedvr2_nodes.py b/tests-unit/comfy_extras_test/test_seedvr2_nodes.pynew file mode 100644index 000000000..f7d9a4f65--- /dev/null+++ b/tests-unit/comfy_extras_test/test_seedvr2_nodes.py@@ -0,0 +1,55 @@+import importlib+import inspect+import sys+from unittest.mock import MagicMock, patch++import torch++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True+++def test_seedvr_node_signature_matches_schema():+ mock_mm = MagicMock()+ mock_mm.xformers_enabled.return_value = False+ mock_mm.xformers_enabled_vae.return_value = False+ mock_mm.sage_attention_enabled.return_value = False+ mock_mm.flash_attention_enabled.return_value = False++ sentinel = object()+ prior_cpu = cli_args.cpu+ cli_args.cpu = True+ prior_module = sys.modules.get("comfy_extras.nodes_seedvr", sentinel)+ comfy_pkg = sys.modules.get("comfy")+ prior_mm_attr = getattr(comfy_pkg, "model_management", sentinel) if comfy_pkg else sentinel++ with patch.dict(sys.modules, {"comfy.model_management": mock_mm}):+ if comfy_pkg is not None:+ setattr(comfy_pkg, "model_management", mock_mm)+ sys.modules.pop("comfy_extras.nodes_seedvr", None)+ try:+ nodes_seedvr = importlib.import_module("comfy_extras.nodes_seedvr")+ for node_cls in (nodes_seedvr.SeedVR2Preprocess, nodes_seedvr.SeedVR2PostProcessing, nodes_seedvr.SeedVR2Conditioning, nodes_seedvr.SeedVR2ProgressiveSampler):+ schema_ids = [i.id for i in node_cls.define_schema().inputs]+ exec_params = [+ p for p in inspect.signature(node_cls.execute).parameters.keys()+ if p != "cls"+ ]+ assert schema_ids == exec_params, (+ f"{node_cls.__name__} schema/execute drift: "+ f"schema_ids={schema_ids}, exec_params={exec_params}"+ )+ finally:+ cli_args.cpu = prior_cpu+ if prior_module is sentinel:+ sys.modules.pop("comfy_extras.nodes_seedvr", None)+ else:+ sys.modules["comfy_extras.nodes_seedvr"] = prior_module+ if comfy_pkg is not None:+ if prior_mm_attr is sentinel:+ if hasattr(comfy_pkg, "model_management"):+ delattr(comfy_pkg, "model_management")+ else:+ setattr(comfy_pkg, "model_management", prior_mm_attr)diff --git a/tests-unit/comfy_extras_test/test_seedvr2_post_processing.py b/tests-unit/comfy_extras_test/test_seedvr2_post_processing.pynew file mode 100644index 000000000..a27a8f8df--- /dev/null+++ b/tests-unit/comfy_extras_test/test_seedvr2_post_processing.py@@ -0,0 +1,57 @@+from unittest.mock import patch++import torch++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True++from comfy_extras import nodes_seedvr # noqa: E402+++def _schema_ids(items):+ return [item.id for item in items]+++def test_seedvr2_post_processing_schema():+ schema = nodes_seedvr.SeedVR2PostProcessing.define_schema()++ assert _schema_ids(schema.inputs) == ["images", "original_resized_images", "color_correction_method"]+ assert schema.inputs[2].options == ["lab", "wavelet", "adain", "none"]+ assert schema.inputs[2].default == "lab"+ assert schema.outputs[0].get_io_type() == "IMAGE"+++def test_seedvr2_post_processing_oom_error_uses_color_correction_method(monkeypatch):+ decoded = torch.full((1, 3, 4, 4), 0.25)+ reference = torch.full((1, 3, 4, 4), 0.75)++ def _lab(content, style):+ raise torch.cuda.OutOfMemoryError("CUDA out of memory")++ monkeypatch.setattr(nodes_seedvr.comfy.model_management, "vae_device", lambda: torch.device("cpu"))+ monkeypatch.setattr(nodes_seedvr.comfy.model_management, "get_free_memory", lambda device: 1_000_000)+ monkeypatch.setattr(nodes_seedvr.comfy.model_management, "soft_empty_cache", lambda: None)++ with patch.object(nodes_seedvr, "lab_color_transfer", _lab):+ try:+ nodes_seedvr.SeedVR2PostProcessing._color_transfer_chunked(+ decoded, reference, torch.device("cpu"), "lab",+ )+ except RuntimeError as exc:+ assert "color_correction_method=lab" in str(exc)+ assert " method=lab" not in str(exc)+ else:+ raise AssertionError("expected RuntimeError for one-frame LAB OOM")+++def test_seedvr2_post_processing_unknown_color_correction_method_raises():+ decoded = torch.zeros(1, 2, 4, 4, 3)+ original = torch.zeros(1, 2, 4, 4, 3)+ try:+ nodes_seedvr.SeedVR2PostProcessing.execute(decoded, original, "bogus")+ except ValueError as exc:+ assert "color_correction_method" in str(exc)+ else:+ raise AssertionError("expected ValueError for unknown color_correction_method")diff --git a/tests-unit/comfy_test/model_detection_test.py b/tests-unit/comfy_test/model_detection_test.pyindex 4e9350602..c63f69a0d 100644--- a/tests-unit/comfy_test/model_detection_test.py+++ b/tests-unit/comfy_test/model_detection_test.py@@ -73,6 +73,24 @@ def _make_flux_schnell_comfyui_sd():return sd+def _make_seedvr2_7b_separate_mm_sd():+ return {+ "blocks.35.mlp.vid.proj_in.weight": torch.empty(1, 3072),+ }+++def _make_seedvr2_7b_shared_mm_sd():+ return {+ "blocks.35.mlp.all.proj_in_gate.weight": torch.empty(1, 1),+ }+++def _make_seedvr2_3b_shared_mm_sd():+ return {+ "blocks.31.mlp.all.proj_in_gate.weight": torch.empty(1, 1),+ }++class TestModelDetection:"""Verify that first-match model detection selects the correct modelbased on list ordering and unet_config specificity."""@@ -125,6 +143,48 @@ class TestModelDetection:assert model_config is not Noneassert type(model_config).__name__ == "FluxSchnell"+ def test_seedvr2_7b_separate_mm_detection_config(self):+ sd = _make_seedvr2_7b_separate_mm_sd()+ unet_config = detect_unet_config(sd, "")++ assert unet_config is not None+ assert unet_config["image_model"] == "seedvr2"+ assert unet_config["vid_dim"] == 3072+ assert unet_config["heads"] == 24+ assert unet_config["num_layers"] == 36+ assert unet_config["mm_layers"] == 36+ assert unet_config["mlp_type"] == "normal"+ assert unet_config["qk_rope"] is True+ assert unet_config["rope_type"] == "rope3d"+ assert unet_config["rope_dim"] == 64++ def test_seedvr2_7b_shared_mm_detection_config(self):+ sd = _make_seedvr2_7b_shared_mm_sd()+ unet_config = detect_unet_config(sd, "")++ assert unet_config is not None+ assert unet_config["image_model"] == "seedvr2"+ assert unet_config["vid_dim"] == 3072+ assert unet_config["heads"] == 24+ assert unet_config["num_layers"] == 36+ assert unet_config["mm_layers"] == 10+ assert unet_config["mlp_type"] == "swiglu"+ assert unet_config["qk_rope"] is True+ assert unet_config["rope_type"] == "rope3d"+ assert unet_config["rope_dim"] == 64++ def test_seedvr2_3b_shared_mm_detection_config(self):+ sd = _make_seedvr2_3b_shared_mm_sd()+ unet_config = detect_unet_config(sd, "")++ assert unet_config is not None+ assert unet_config["image_model"] == "seedvr2"+ assert unet_config["vid_dim"] == 2560+ assert unet_config["heads"] == 20+ assert unet_config["num_layers"] == 32+ assert unet_config["mlp_type"] == "swiglu"+ assert unet_config["qk_rope"] is None+def test_unet_config_and_required_keys_combination_is_unique(self):"""Each model in the registry must have a unique combination of``unet_config`` and ``required_keys``. If two models share the samediff --git a/tests-unit/comfy_test/seedvr_vae_forward_test.py b/tests-unit/comfy_test/seedvr_vae_forward_test.pynew file mode 100644index 000000000..f9dbd6890--- /dev/null+++ b/tests-unit/comfy_test/seedvr_vae_forward_test.py@@ -0,0 +1,90 @@+"""Regression: ``comfy.ldm.seedvr.vae.VideoAutoencoderKL.forward`` must+honor the actual tensor/tuple return contract of ``encode()`` and+``decode_()`` and must NOT dereference diffusers-style ``.latent_dist``+or ``.sample`` attributes on those returns.++The pre-fix body raised ``AttributeError: 'Tensor' object has no+attribute 'latent_dist'`` for ``mode in {"encode", "all"}`` and+``AttributeError: 'VideoAutoencoderKL' object has no attribute 'decode'``+for ``mode == "decode"`` (the class only defines ``decode_`` with a+trailing underscore). The post-fix body unwraps the optional one-element+tuple shape that ``return_dict=False`` produces and returns the tensor+directly.++Tests construct a stub subclass of ``VideoAutoencoderKL`` that bypasses+the heavy ``__init__`` via ``torch.nn.Module.__init__(self)`` and+overrides ``encode``/``decode_`` with known tensors so the contract can+be probed without loading any real VAE weights.+"""++import torch+import torch.nn as nn++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True++from comfy.ldm.seedvr.vae import VideoAutoencoderKL # noqa: E402+++_LATENT_SHAPE = (1, 16, 2, 2, 2)+_DECODED_SHAPE = (1, 3, 5, 16, 16)+_INPUT_ENCODE_SHAPE = (1, 3, 5, 16, 16)+_INPUT_DECODE_SHAPE = (1, 16, 2, 2, 2)+++class _StubVAE(VideoAutoencoderKL):+ def __init__(self):+ nn.Module.__init__(self)+ self._encode_out = torch.zeros(*_LATENT_SHAPE)+ self._decode_out = torch.zeros(*_DECODED_SHAPE)++ def encode(self, x, return_dict=True):+ return self._encode_out++ def decode_(self, z, return_dict=True):+ return self._decode_out+++def test_forward_encode_returns_tensor():+ vae = _StubVAE()+ x = torch.zeros(*_INPUT_ENCODE_SHAPE)+ result = vae.forward(x, mode="encode")+ assert type(result) is torch.Tensor+ assert result.shape == torch.Size(_LATENT_SHAPE)+++def test_forward_decode_returns_tensor():+ vae = _StubVAE()+ z = torch.zeros(*_INPUT_DECODE_SHAPE)+ result = vae.forward(z, mode="decode")+ assert type(result) is torch.Tensor+ assert result.shape == torch.Size(_DECODED_SHAPE)+++class _TupleReturningStubVAE(VideoAutoencoderKL):+ """Stub variant whose ``encode``/``decode_`` return the+ ``(tensor,)`` one-element tuple shape ``return_dict=False`` produces+ in the parent class. Exercises the unwrap branch of+ ``VideoAutoencoderKL.forward``.+ """++ def __init__(self):+ nn.Module.__init__(self)+ self._encode_tensor = torch.zeros(*_LATENT_SHAPE)+ self._decode_tensor = torch.zeros(*_DECODED_SHAPE)++ def encode(self, x, return_dict=True):+ return (self._encode_tensor,)++ def decode_(self, z, return_dict=True):+ return (self._decode_tensor,)+++def test_forward_all_unwraps_one_tuple_at_each_step():+ vae = _TupleReturningStubVAE()+ x = torch.zeros(*_INPUT_ENCODE_SHAPE)+ result = vae.forward(x, mode="all")+ assert type(result) is torch.Tensor+ assert result.shape == torch.Size(_DECODED_SHAPE)diff --git a/tests-unit/comfy_test/test_seedvr2_dtype.py b/tests-unit/comfy_test/test_seedvr2_dtype.pynew file mode 100644index 000000000..e5d79a306--- /dev/null+++ b/tests-unit/comfy_test/test_seedvr2_dtype.py@@ -0,0 +1,47 @@+import torch++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True++import comfy.sd+import comfy.supported_models+import comfy.ldm.seedvr.model as seedvr_model+++def test_seedvr2_fp16_manual_cast_only_for_bf16_device(monkeypatch):+ bf16_device = object()+ fp16_device = object()++ monkeypatch.setattr(+ comfy.supported_models.comfy.model_management,+ "should_use_bf16",+ lambda device=None: device is bf16_device,+ )++ bf16_config = comfy.supported_models.SeedVR2({"image_model": "seedvr2"})+ bf16_config.set_inference_dtype(torch.float16, None, device=bf16_device)+ assert bf16_config.manual_cast_dtype is torch.bfloat16++ fp16_config = comfy.supported_models.SeedVR2({"image_model": "seedvr2"})+ fp16_config.set_inference_dtype(torch.float16, None, device=fp16_device)+ assert fp16_config.manual_cast_dtype is None+++def test_seedvr2_text_conditioning_accepts_cfg1_single_branch():+ context = torch.arange(6, dtype=torch.float32).reshape(1, 3, 2)++ txt, txt_shape = seedvr_model.NaDiT._resolve_text_conditioning(object(), context, [0])++ torch.testing.assert_close(txt, context.squeeze(0))+ torch.testing.assert_close(txt_shape, torch.tensor([[3]], device=context.device))+++def test_seedvr2_vae_decode_memory_covers_full_frame_lab_transfer():+ estimate = comfy.sd._seedvr2_vae_decode_memory_used((1, 16, 26, 120, 160))+ old_estimate = 16 * 120 * 160 * (4 * 8 * 8) * 2++ assert estimate == 101 * 960 * 1280 * 160+ assert estimate > 15 * 1024 ** 3+ assert estimate > old_estimate * 100diff --git a/tests-unit/comfy_test/test_seedvr2_internals.py b/tests-unit/comfy_test/test_seedvr2_internals.pynew file mode 100644index 000000000..5b008ea6e--- /dev/null+++ b/tests-unit/comfy_test/test_seedvr2_internals.py@@ -0,0 +1,341 @@+"""Consolidated SeedVR2 internals regression tests.++Sources (all merged verbatim, helper names disambiguated where colliding):++ * RoPE rewrite — NaMMRotaryEmbedding3d.forward must match the legacy+ apply_rotary_emb wrapper oracle at fp32.+ * GroupNorm limit gate — causal_norm_wrapper at vae.py:509 must compare+ memory_occupy against get_norm_limit(), not float('inf').+ * SeedVR2 variable-length attention split-loop contract.++Pre-import CPU-only guard is required because comfy.ldm.seedvr.model and+comfy.ldm.modules.attention transitively pull in comfy.model_management,+which probes torch.cuda.current_device() at import time unless args.cpu is+set first.+"""++from __future__ import annotations++from unittest.mock import patch++import pytest+import torch++from comfy.cli_args import args++if not torch.cuda.is_available():+ args.cpu = True++import comfy.ldm.seedvr.model as seedvr_model # noqa: E402+import comfy.ldm.seedvr.vae as vae_mod # noqa: E402+import comfy.ldm.modules.attention as attention # noqa: E402+import comfy.ops as comfy_ops # noqa: E402+from comfy.ldm.seedvr.model import ( # noqa: E402+ Cache,+ NaMMRotaryEmbedding3d,+)+from comfy.ldm.seedvr.vae import ( # noqa: E402+ causal_norm_wrapper,+ set_norm_limit,+)+from comfy.ldm.modules.attention import var_attention_optimized_split # noqa: E402+++# ---------------------------------------------------------------------------+# RoPE rewrite tests (test_seedvr_rope_rewrite.py)+# ---------------------------------------------------------------------------++# Test rig dimensions. dim=192 → per-axis rope dim = 64 (even, lucidrains+# requirement). vid_shape=(2,4,4) → L_vid = 32. txt_shape=(8,) → L_txt = 8.+_DIM = 192+_HEADS = 4+_VID_T, _VID_H, _VID_W = 2, 4, 4+_TXT_L = 8+_L_VID = _VID_T * _VID_H * _VID_W+_SEED = 0+++def _make_inputs(dtype=torch.float32, device="cpu"):+ """Construct the 6 forward inputs + cache. Deterministic via local+ Generator so global RNG state is not mutated.+ """+ g = torch.Generator(device=device).manual_seed(_SEED)+ vid_q = torch.randn(_L_VID, _HEADS, _DIM, dtype=dtype, device=device, generator=g)+ vid_k = torch.randn(_L_VID, _HEADS, _DIM, dtype=dtype, device=device, generator=g)+ txt_q = torch.randn(_TXT_L, _HEADS, _DIM, dtype=dtype, device=device, generator=g)+ txt_k = torch.randn(_TXT_L, _HEADS, _DIM, dtype=dtype, device=device, generator=g)+ vid_shape = torch.tensor([[_VID_T, _VID_H, _VID_W]], dtype=torch.long, device=device)+ txt_shape = torch.tensor([[_TXT_L]], dtype=torch.long, device=device)+ cache = Cache(disable=True)+ return vid_q, vid_k, vid_shape, txt_q, txt_k, txt_shape, cache+++def _legacy_get_freqs(rope: NaMMRotaryEmbedding3d, vid_shape, txt_shape):+ """Reproduce the pre-rewrite ``get_freqs`` body verbatim against+ ``self.get_axial_freqs`` (parent ``RotaryEmbeddingBase`` method,+ unchanged by the rewrite).+ """+ max_temporal = 0+ max_height = 0+ max_width = 0+ max_txt_len = 0+ for (f, h, w), l in zip(vid_shape.tolist(), txt_shape[:, 0].tolist()):+ max_temporal = max(max_temporal, l + f)+ max_height = max(max_height, h)+ max_width = max(max_width, w)+ max_txt_len = max(max_txt_len, l)+ with torch.amp.autocast(device_type="cuda", enabled=False):+ vid_freqs_full = rope.get_axial_freqs(+ min(max_temporal + 16, 1024),+ min(max_height + 4, 128),+ min(max_width + 4, 128),+ ).float()+ txt_freqs_full = rope.get_axial_freqs(min(max_txt_len + 16, 1024))+ vid_freq_list, txt_freq_list = [], []+ for (f, h, w), l in zip(vid_shape.tolist(), txt_shape[:, 0].tolist()):+ vid_freq = vid_freqs_full[l : l + f, :h, :w].reshape(-1, vid_freqs_full.size(-1))+ txt_freq = txt_freqs_full[:l].repeat(1, 3).reshape(-1, vid_freqs_full.size(-1))+ vid_freq_list.append(vid_freq)+ txt_freq_list.append(txt_freq)+ return torch.cat(vid_freq_list, dim=0), torch.cat(txt_freq_list, dim=0)+++def _legacy_forward(rope: NaMMRotaryEmbedding3d, vid_q, vid_k, vid_shape,+ txt_q, txt_k, txt_shape):+ """Compute expected forward output via the unchanged+ ``apply_rotary_emb`` wrapper fed with legacy-shape freqs. This is the+ oracle. The wrapper itself is out of scope for the rewrite (Shape B).+ """+ vid_freqs, txt_freqs = _legacy_get_freqs(rope, vid_shape, txt_shape)+ vid_freqs = vid_freqs.to(vid_q.device)+ txt_freqs = txt_freqs.to(txt_q.device)++ from einops import rearrange++ vid_q = rearrange(vid_q, "L h d -> h L d")+ vid_k = rearrange(vid_k, "L h d -> h L d")+ vid_q_out = seedvr_model.apply_rotary_emb(vid_freqs, vid_q.float()).to(vid_q.dtype)+ vid_k_out = seedvr_model.apply_rotary_emb(vid_freqs, vid_k.float()).to(vid_k.dtype)+ vid_q_out = rearrange(vid_q_out, "h L d -> L h d")+ vid_k_out = rearrange(vid_k_out, "h L d -> L h d")++ txt_q = rearrange(txt_q, "L h d -> h L d")+ txt_k = rearrange(txt_k, "L h d -> h L d")+ txt_q_out = seedvr_model.apply_rotary_emb(txt_freqs, txt_q.float()).to(txt_q.dtype)+ txt_k_out = seedvr_model.apply_rotary_emb(txt_freqs, txt_k.float()).to(txt_k.dtype)+ txt_q_out = rearrange(txt_q_out, "h L d -> L h d")+ txt_k_out = rearrange(txt_k_out, "h L d -> L h d")+ return vid_q_out, vid_k_out, txt_q_out, txt_k_out+++def test_namm_forward_output_tensor_equal_against_legacy_oracle():+ rope = NaMMRotaryEmbedding3d(dim=_DIM)+ vid_q, vid_k, vid_shape, txt_q, txt_k, txt_shape, cache = _make_inputs()++ expected_vid_q, expected_vid_k, expected_txt_q, expected_txt_k = _legacy_forward(+ rope,+ vid_q.clone(), vid_k.clone(), vid_shape,+ txt_q.clone(), txt_k.clone(), txt_shape,+ )++ actual_vid_q, actual_vid_k, actual_txt_q, actual_txt_k = rope.forward(+ vid_q.clone(), vid_k.clone(), vid_shape,+ txt_q.clone(), txt_k.clone(), txt_shape, cache,+ )++ torch.testing.assert_close(actual_vid_q, expected_vid_q, rtol=0, atol=0,+ msg="vid_q output diverges from wrapper oracle")+ torch.testing.assert_close(actual_vid_k, expected_vid_k, rtol=0, atol=0,+ msg="vid_k output diverges from wrapper oracle")+ torch.testing.assert_close(actual_txt_q, expected_txt_q, rtol=0, atol=0,+ msg="txt_q output diverges from wrapper oracle")+ torch.testing.assert_close(actual_txt_k, expected_txt_k, rtol=0, atol=0,+ msg="txt_k output diverges from wrapper oracle")+++# ---------------------------------------------------------------------------+# GroupNorm limit tests (test_seedvr_groupnorm_limit.py)+# ---------------------------------------------------------------------------++_NUM_CHANNELS = 8+_NUM_GROUPS = 4+_TENSOR_SHAPE = (1, 8, 2, 4, 4)++_GROUPNORM_SUBCLASSES = [+ pytest.param(comfy_ops.disable_weight_init.GroupNorm, id="disable_weight_init"),+ pytest.param(comfy_ops.manual_cast.GroupNorm, id="manual_cast"),+]++[email protected]("groupnorm_cls", _GROUPNORM_SUBCLASSES)+def test_seedvr_groupnorm_low_limit_uses_chunked_groupnorm_path(groupnorm_cls):+ real_group_norm = vae_mod.F.group_norm+ set_norm_limit(1e-9)+ try:+ gn = groupnorm_cls(num_channels=_NUM_CHANNELS, num_groups=_NUM_GROUPS)+ gn.eval()++ forward_hook_calls = []++ def _hook(module, inputs, output):+ forward_hook_calls.append(tuple(inputs[0].shape))++ spy_calls = []++ def _group_norm_spy(input_tensor, num_groups_arg, *args, **kwargs):+ spy_calls.append({"num_groups": int(num_groups_arg)})+ return real_group_norm(input_tensor, num_groups_arg, *args, **kwargs)++ handle = gn.register_forward_hook(_hook)+ try:+ with patch.object(vae_mod.F, "group_norm", side_effect=_group_norm_spy):+ out_tensor = causal_norm_wrapper(gn, torch.randn(*_TENSOR_SHAPE))+ finally:+ handle.remove()++ full_calls = len(forward_hook_calls)+ chunked_calls = sum(1 for entry in spy_calls if entry["num_groups"] < _NUM_GROUPS)++ assert tuple(int(s) for s in out_tensor.shape) == _TENSOR_SHAPE+ assert full_calls == 0, (+ f"low-limit GroupNorm gate must NOT take the full-forward path; got full_calls={full_calls}"+ )+ assert chunked_calls > 0, (+ f"low-limit GroupNorm gate must take the chunked path; got chunked_calls={chunked_calls}"+ )+ finally:+ set_norm_limit(None)+++# ---------------------------------------------------------------------------+# SeedVR2 var_attention split-loop tests+# ---------------------------------------------------------------------------++def test_var_attention_registry_contains_always_available_entries():+ assert (+ attention.REGISTERED_ATTENTION_FUNCTIONS["var_attention_optimized_split"]+ is attention.var_attention_optimized_split+ )+++def test_seedvr2_7b_swin_attention_forward_uses_optimized_var_attention(monkeypatch):+ dim = 8+ heads = 2+ head_dim = 4+ attn = seedvr_model.NaSwinAttention(+ vid_dim=dim,+ txt_dim=dim,+ heads=heads,+ head_dim=head_dim,+ qk_bias=False,+ qk_norm=seedvr_model.CustomRMSNorm,+ qk_norm_eps=1e-6,+ rope_type=None,+ rope_dim=head_dim,+ shared_weights=False,+ window=(2, 1, 1),+ window_method="720pwin_by_size_bysize",+ version=True,+ device="cpu",+ dtype=torch.float32,+ operations=comfy_ops.disable_weight_init,+ )+ generator = torch.Generator(device="cpu").manual_seed(11)+ vid = torch.randn(8, dim, generator=generator)+ txt = torch.randn(3, dim, generator=generator)+ vid_shape = torch.tensor([[2, 2, 2]], dtype=torch.long)+ txt_shape = torch.tensor([[3]], dtype=torch.long)+ calls = []++ def fake_optimized_var_attention(**kwargs):+ calls.append(kwargs)+ return kwargs["q"]++ monkeypatch.setattr(seedvr_model, "optimized_var_attention", fake_optimized_var_attention)++ vid_out, txt_out = attn(vid, txt, vid_shape, txt_shape, seedvr_model.Cache(disable=True))++ assert tuple(vid_out.shape) == (8, dim)+ assert tuple(txt_out.shape) == (3, dim)+ assert len(calls) == 1+ call = calls[0]+ assert tuple(call["q"].shape) == (14, heads, head_dim)+ assert tuple(call["k"].shape) == (14, heads, head_dim)+ assert tuple(call["v"].shape) == (14, heads, head_dim)+ assert call["heads"] == heads+ assert call["skip_reshape"] is True+ assert call["skip_output_reshape"] is True+ torch.testing.assert_close(+ call["cu_seqlens_q"],+ torch.tensor([0, 7, 14], dtype=torch.int32),+ rtol=0,+ atol=0,+ )+ torch.testing.assert_close(+ call["cu_seqlens_k"],+ torch.tensor([0, 7, 14], dtype=torch.int32),+ rtol=0,+ atol=0,+ )+++def test_var_attention_optimized_split_calls_dense_backend_per_window(monkeypatch):+ heads = 2+ head_dim = 3+ q = torch.arange(30, dtype=torch.float32).reshape(5, heads, head_dim)+ k = q + 100+ v = q + 200+ cu = torch.tensor([0, 2, 5], dtype=torch.int32)+ calls = []++ def fake_optimized_attention(q_arg, k_arg, v_arg, heads_arg, **kwargs):+ calls.append(+ {+ "q_shape": tuple(q_arg.shape),+ "k_shape": tuple(k_arg.shape),+ "v_shape": tuple(v_arg.shape),+ "heads": heads_arg,+ "kwargs": kwargs,+ }+ )+ return q_arg + v_arg++ monkeypatch.setattr(attention, "optimized_attention", fake_optimized_attention)++ out = var_attention_optimized_split(+ q,+ k,+ v,+ heads,+ cu,+ cu,+ skip_reshape=True,+ skip_output_reshape=True,+ )++ assert tuple(out.shape) == (5, heads, head_dim)+ assert len(calls) == 2+ assert calls[0]["q_shape"] == (1, heads, 2, head_dim)+ assert calls[1]["q_shape"] == (1, heads, 3, head_dim)+ assert all(call["heads"] == heads for call in calls)+ assert all(call["kwargs"]["skip_reshape"] is True for call in calls)+ assert all(call["kwargs"]["skip_output_reshape"] is True for call in calls)+ torch.testing.assert_close(out, q + v, rtol=0, atol=0)+++def test_var_attention_optimized_split_rejects_bad_offsets():+ q = torch.randn(5, 2, 3)+ cu_bad = torch.tensor([0, 2, 6], dtype=torch.int32)+ cu_ok = torch.tensor([0, 2, 5], dtype=torch.int32)++ with pytest.raises(ValueError, match="cu_seqlens_q does not match token count"):+ var_attention_optimized_split(+ q,+ q,+ q,+ 2,+ cu_bad,+ cu_ok,+ skip_reshape=True,+ skip_output_reshape=True,+ )diff --git a/tests-unit/comfy_test/test_seedvr2_model.py b/tests-unit/comfy_test/test_seedvr2_model.pynew file mode 100644index 000000000..f2b9bcbbe--- /dev/null+++ b/tests-unit/comfy_test/test_seedvr2_model.py@@ -0,0 +1,308 @@+"""Consolidated SeedVR2 model/graph/forward regression tests.++Merged from:+- seedvr_model_test.py+- test_seedvr_7b_final_block_text_path.py+- test_seedvr_forward_no_device_cast.py+- test_seedvr_latent_format.py+- test_seedvr2_vae_graph_boundaries.py+"""++from __future__ import annotations++from unittest.mock import MagicMock++import torch+from torch import nn++from comfy.cli_args import args++if not torch.cuda.is_available():+ args.cpu = True++import comfy # noqa: E402+import comfy.latent_formats # noqa: E402+import comfy.ldm.seedvr.model # noqa: E402+import comfy.ldm.seedvr.model as seedvr_model # noqa: E402+import comfy.ldm.seedvr.vae as seedvr_vae_mod # noqa: E402+import comfy.model_management # noqa: E402+import comfy.sample # noqa: E402+import comfy.sd as sd_mod # noqa: E402+import nodes as nodes_mod # noqa: E402+from comfy.ldm.seedvr.model import NaDiT # noqa: E402+++# ---------------------------------------------------------------------------+# Helpers from seedvr_model_test.py+# ---------------------------------------------------------------------------+++def _make_standin(positive_conditioning):+ class _StandIn(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.register_buffer(+ "positive_conditioning", positive_conditioning+ )++ _resolve_text_conditioning = NaDiT._resolve_text_conditioning++ return _StandIn()+++# ---------------------------------------------------------------------------+# Helpers from test_seedvr_7b_final_block_text_path.py+# ---------------------------------------------------------------------------+++class _StubModule(nn.Module):+ def __init__(self, *args, **kwargs):+ super().__init__()+++def _capture_last_layer_flags(monkeypatch, vid_dim: int, txt_in_dim: int) -> list[bool]:+ flags = []++ class _Block(_StubModule):+ def __init__(self, *args, **kwargs):+ flags.append(kwargs["is_last_layer"])+ super().__init__()++ monkeypatch.setattr(seedvr_model, "NaPatchIn", _StubModule)+ monkeypatch.setattr(seedvr_model, "NaPatchOut", _StubModule)+ monkeypatch.setattr(seedvr_model, "TimeEmbedding", _StubModule)+ monkeypatch.setattr(seedvr_model, "NaMMSRTransformerBlock", _Block)++ seedvr_model.NaDiT(+ norm_eps=1e-5,+ qk_rope=None,+ num_layers=4,+ mlp_type="normal",+ vid_dim=vid_dim,+ txt_in_dim=txt_in_dim,+ heads=24,+ mm_layers=3,+ )++ return flags+++# ---------------------------------------------------------------------------+# Helpers from test_seedvr_latent_format.py+# ---------------------------------------------------------------------------+++class _Model:+ def __init__(self, latent_format):+ self._latent_format = latent_format++ def get_model_object(self, name):+ assert name == "latent_format"+ return self._latent_format+++# ---------------------------------------------------------------------------+# Helpers from test_seedvr2_vae_graph_boundaries.py+# ---------------------------------------------------------------------------+++class _Patcher:+ def get_free_memory(self, device):+ return 1024 * 1024 * 1024+++class _EncodeWrapper(seedvr_vae_mod.VideoAutoencoderKLWrapper):+ def __init__(self, encoded):+ nn.Module.__init__(self)+ self.encoded = encoded+ self.spatial_downsample_factor = 8+ self.temporal_downsample_factor = 4+ self.seen = []++ def encode(self, x):+ self.seen.append(tuple(x.shape))+ return self.encoded.to(device=x.device, dtype=x.dtype)+++class _DecodeWrapper(seedvr_vae_mod.VideoAutoencoderKLWrapper):+ def __init__(self):+ nn.Module.__init__(self)+ self.spatial_downsample_factor = 8+ self.temporal_downsample_factor = 4+ self.calls = []++ def decode(self, z, seedvr2_tiling=None):+ self.calls.append({"shape": tuple(z.shape), "seedvr2_tiling": seedvr2_tiling})+ if z.ndim == 4:+ b, tc, h, w = z.shape+ t = tc // 16+ else:+ b, _, t, h, w = z.shape+ return torch.zeros(b, 3, t, h * 8, w * 8, dtype=z.dtype, device=z.device)+++def _make_vae(wrapper):+ vae = sd_mod.VAE.__new__(sd_mod.VAE)+ vae.first_stage_model = wrapper+ vae.device = torch.device("cpu")+ vae.output_device = torch.device("cpu")+ vae.vae_dtype = torch.float32+ vae.latent_channels = 16+ vae.latent_dim = 3+ vae.downscale_ratio = (lambda a: max(0, (a + 3) // 4), 8, 8)+ vae.upscale_ratio = (lambda a: max(0, a * 4 - 3), 8, 8)+ vae.output_channels = 3+ vae.disable_offload = True+ vae.extra_1d_channel = None+ vae.crop_input = False+ vae.not_video = False+ vae.patcher = _Patcher()+ vae.process_input = lambda image: image+ vae.process_output = lambda image: image.add(1.0).div(2.0).clamp(0.0, 1.0)+ vae.vae_output_dtype = lambda: torch.float32+ vae.memory_used_encode = lambda shape, dtype: 1+ vae.memory_used_decode = lambda shape, dtype: 1+ vae.throw_exception_if_invalid = lambda: None+ vae.vae_encode_crop_pixels = lambda pixels: pixels+ vae.spacial_compression_decode = lambda: 8+ vae.temporal_compression_decode = lambda: 4+ return vae+++# ---------------------------------------------------------------------------+# Tests from seedvr_model_test.py+# ---------------------------------------------------------------------------+++def test_missing_context_falls_back_to_positive_buffer():+ """AC: ``context is None`` falls back to the registered+ ``positive_conditioning`` buffer and runs to completion — no+ silent zero substitution, no raised exception.+ """+ pos_buffer = torch.full((58, 5120), 7.0)+ standin = _make_standin(pos_buffer)+ txt, txt_shape = standin._resolve_text_conditioning(None)+ assert txt.shape == (58, 5120)+ assert (txt == 7.0).all(), (+ "fallback path must use the positive_conditioning buffer "+ "verbatim, not a zero tensor"+ )+ assert txt_shape.shape == (1, 1)+ assert txt_shape[0, 0].item() == 58+++# ---------------------------------------------------------------------------+# Tests from test_seedvr_7b_final_block_text_path.py+# ---------------------------------------------------------------------------+++def test_seedvr2_7b_keeps_final_block_text_path(monkeypatch):+ assert _capture_last_layer_flags(monkeypatch, vid_dim=3072, txt_in_dim=3072) == [+ False,+ False,+ False,+ False,+ ]+++def test_seedvr2_7b_rope3d_matches_wrapper_oracle():+ rope = seedvr_model.get_na_rope("rope3d", dim=64)+ generator = torch.Generator(device="cpu").manual_seed(0)+ q = torch.randn(4, 2, 128, generator=generator)+ k = torch.randn(4, 2, 128, generator=generator)+ shape = torch.tensor([[1, 2, 2]], dtype=torch.long)+ freqs = rope.get_axial_freqs(1, 2, 2).reshape(4, -1)++ expected_q = seedvr_model._apply_seedvr2_rotary_emb(+ freqs,+ q.permute(1, 0, 2).float(),+ ).to(q.dtype).permute(1, 0, 2)+ expected_k = seedvr_model._apply_seedvr2_rotary_emb(+ freqs,+ k.permute(1, 0, 2).float(),+ ).to(k.dtype).permute(1, 0, 2)++ actual_q, actual_k = rope(q.clone(), k.clone(), shape, seedvr_model.Cache(disable=True))++ torch.testing.assert_close(actual_q, expected_q, rtol=0, atol=0)+ torch.testing.assert_close(actual_k, expected_k, rtol=0, atol=0)+++# ---------------------------------------------------------------------------+# Tests from test_seedvr_latent_format.py+# ---------------------------------------------------------------------------+++def test_seedvr2_latent_format_uses_16_channels_without_3d_empty_latent_expansion():+ latent_format = comfy.latent_formats.SeedVR2()+ latent_image = torch.zeros(1, 1, 4, 5)++ fixed = comfy.sample.fix_empty_latent_channels(_Model(latent_format), latent_image)++ assert latent_format.latent_channels == 16+ assert latent_format.latent_dimensions == 2+ assert fixed.shape == (1, 16, 4, 5)+++# ---------------------------------------------------------------------------+# Tests from test_seedvr2_vae_graph_boundaries.py+# ---------------------------------------------------------------------------+++def test_seedvr2_encode_and_encode_tiled_preserve_native_latent_contract(monkeypatch):+ monkeypatch.setattr(sd_mod.model_management, "load_models_gpu", lambda *a, **k: None)++ encoded = torch.full((1, 16, 2, 4, 5), 2.0)+ vae = _make_vae(_EncodeWrapper(encoded))+ pixels = torch.zeros(1, 5, 32, 40, 3)++ node_output = nodes_mod.VAEEncode().encode(vae, pixels)[0]+ node_latent = node_output["samples"]+ assert set(node_output) == {"samples"}+ assert tuple(node_latent.shape) == (1, 16, 2, 4, 5)+ assert node_latent.dtype == torch.float32+ assert node_latent.stride()[-1] == 1+ assert torch.equal(node_latent, torch.full_like(node_latent, 2.0 * 0.9152))++ tiled = torch.full((1, 16, 2, 4, 5), 3.0)+ monkeypatch.setattr(seedvr_vae_mod, "tiled_vae", MagicMock(return_value=tiled))+ tiled_output = nodes_mod.VAEEncodeTiled().encode(+ vae,+ pixels,+ tile_size=512,+ overlap=64,+ temporal_size=16,+ temporal_overlap=4,+ )[0]+ tiled_latent = tiled_output["samples"]+ assert set(tiled_output) == {"samples"}+ assert tuple(tiled_latent.shape) == (1, 16, 2, 4, 5)+ assert tiled_latent.dtype == torch.float32+ assert torch.equal(tiled_latent, torch.full_like(tiled_latent, 3.0 * 0.9152))+++def test_vaedecode_tiled_visible_inputs_are_seedvr2_decode_tiling_authority(monkeypatch):+ monkeypatch.setattr(sd_mod.model_management, "load_models_gpu", lambda *a, **k: None)+ vae = _make_vae(_DecodeWrapper())++ nodes_mod.VAEDecodeTiled().decode(+ vae,+ {"samples": torch.zeros(1, 16, 2, 4, 5)},+ tile_size=512,+ overlap=64,+ temporal_size=16,+ temporal_overlap=4,+ )++ assert vae.first_stage_model.calls == [+ {+ "shape": (1, 16, 2, 4, 5),+ "seedvr2_tiling": {+ "enable_tiling": True,+ "tile_size": (512, 512),+ "tile_overlap": (64, 64),+ "temporal_size": 16,+ "temporal_overlap": 4,+ },+ }+ ]diff --git a/tests-unit/comfy_test/test_seedvr2_vae_decode.py b/tests-unit/comfy_test/test_seedvr2_vae_decode.pynew file mode 100644index 000000000..ea9f978f3--- /dev/null+++ b/tests-unit/comfy_test/test_seedvr2_vae_decode.py@@ -0,0 +1,91 @@+from unittest.mock import patch++import pytest+import torch+import torch.nn as nn++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True++import comfy.ldm.seedvr.vae as vae_mod # noqa: E402+from comfy_extras import nodes_seedvr # noqa: E402+++def _make_wrapper() -> vae_mod.VideoAutoencoderKLWrapper:+ wrapper = vae_mod.VideoAutoencoderKLWrapper.__new__(+ vae_mod.VideoAutoencoderKLWrapper+ )+ nn.Module.__init__(wrapper)+ return wrapper+++def _fingerprint_decode_(self, z, return_dict=True):+ b = int(z.shape[0])+ t = int(z.shape[2])+ h = int(z.shape[3])+ w = int(z.shape[4])+ out = torch.empty(b, 3, t, h * 8, w * 8)+ for batch_idx in range(b):+ out[batch_idx].fill_(float(batch_idx + 1))+ return out+++def _decode_with_patches(wrapper, z):+ with patch.object(vae_mod.VideoAutoencoderKL, "decode_", _fingerprint_decode_):+ return wrapper.decode(z)+++def test_decode_b2_t3_multi_frame_batch_unchanged():+ wrapper = _make_wrapper()++ out = _decode_with_patches(wrapper, torch.zeros(2, 16 * 3, 2, 2))++ assert tuple(out.shape) == (2, 3, 3, 16, 16)+++class _Wrapper(vae_mod.VideoAutoencoderKLWrapper):+ def __init__(self):+ nn.Module.__init__(self)+ self.calls = []++ def parameters(self):+ return iter([torch.nn.Parameter(torch.zeros(()))])++def _decode_stub(self, latent):+ self.calls.append(tuple(latent.shape))+ return torch.zeros(latent.shape[0], 3, latent.shape[2], latent.shape[3] * 8, latent.shape[4] * 8)+++def test_seedvr2_wrapper_decode_accepts_5d_channel_first_latents_without_preprocessor_state():+ wrapper = _Wrapper()++ with patch.object(vae_mod.VideoAutoencoderKL, "decode_", _decode_stub):+ out = wrapper.decode(torch.zeros(1, 16, 2, 4, 5))++ assert tuple(out.shape) == (1, 3, 2, 32, 40)+ assert wrapper.calls == [(1, 16, 2, 4, 5)]+++def test_seedvr2_wrapper_decode_rejects_wrong_rank_latents():+ wrapper = _Wrapper()++ with pytest.raises(RuntimeError, match=r"latent input must be 4-D collapsed .* or 5-D"):+ wrapper.decode(torch.zeros(1, 16, 4))+++def _t_padded(t_in: int) -> int:+ if t_in == 1:+ return 1+ if t_in <= 4:+ return 5+ if (t_in - 1) % 4 == 0:+ return t_in+ return t_in + (4 - ((t_in - 1) % 4))++[email protected]("t_in", [1, 5, 9])+def test_t_padded_matches_cut_videos(t_in):+ dummy = torch.zeros(1, t_in, 1, 1, 1)+ assert nodes_seedvr.cut_videos(dummy).shape[1] == _t_padded(t_in)diff --git a/tests-unit/comfy_test/test_seedvr2_vae_tiled.py b/tests-unit/comfy_test/test_seedvr2_vae_tiled.pynew file mode 100644index 000000000..40079bbe2--- /dev/null+++ b/tests-unit/comfy_test/test_seedvr2_vae_tiled.py@@ -0,0 +1,347 @@+from contextlib import ExitStack+from unittest.mock import MagicMock, patch++import torch+import torch.nn as nn++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True++import comfy.ldm.seedvr.vae as vae_mod # noqa: E402+import comfy.ldm.seedvr.vae as seedvr_vae_mod # noqa: E402+import comfy.sd as sd_mod # noqa: E402+from comfy.ldm.seedvr.vae import MemoryState, tiled_vae # noqa: E402+++# ---------------------------------------------------------------------------+# From test_seedvr_vae_tiled_decode_latent_min_size_override.py+# ---------------------------------------------------------------------------+++def test_runtime_decode_zero_temporal_size_disables_slicing_for_call():+ from comfy.ldm.seedvr.vae import MemoryState, VideoAutoencoderKL, tiled_vae++ class StubVAEModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.slicing_latent_min_size = 2+ self.spatial_downsample_factor = 8+ self.temporal_downsample_factor = 4+ self.device = torch.device("cpu")+ self.use_slicing = True+ self._dummy = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32))+ self.decode_min_sizes = []+ self.memory_states = []++ def decode_(self, t_chunk):+ self.decode_min_sizes.append(self.slicing_latent_min_size)+ return VideoAutoencoderKL.slicing_decode(self, t_chunk)++ def _decode(self, z, memory_state=MemoryState.DISABLED):+ self.memory_states.append(memory_state)+ b, c, d, h, w = z.shape+ return torch.zeros((b, 3, d, h * 8, w * 8), dtype=z.dtype)++ vae = StubVAEModel()+ z = torch.zeros((1, 16, 5, 8, 8), dtype=torch.float32)++ tiled_vae(+ z,+ vae,+ tile_size=(64, 64),+ tile_overlap=(0, 0),+ temporal_size=0,+ temporal_overlap=0,+ encode=False,+ )++ assert vae.decode_min_sizes == [5]+ assert vae.memory_states == [MemoryState.DISABLED]+ assert vae.slicing_latent_min_size == 2+++# ---------------------------------------------------------------------------+# From test_seedvr_vae_tiled_encode_runt_slice_override.py+# ---------------------------------------------------------------------------+++def test_zero_temporal_size_preserves_min_size_when_encode_raises():+ from comfy.ldm.seedvr.vae import tiled_vae++ class RaisingVAEModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.slicing_sample_min_size = 4+ self.spatial_downsample_factor = 8+ self.temporal_downsample_factor = 4+ self.device = torch.device("cpu")+ self._dummy = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32))++ def encode(self, t_chunk):+ raise RuntimeError("simulated encode failure")++ vae = RaisingVAEModel()+ x = torch.zeros((1, 3, 12, 64, 64), dtype=torch.float32)++ raised = False+ try:+ tiled_vae(+ x,+ vae,+ tile_size=(64, 64),+ tile_overlap=(0, 0),+ temporal_size=0,+ temporal_overlap=0,+ encode=True,+ )+ except RuntimeError as exc:+ if "simulated encode failure" not in str(exc):+ raise+ raised = True++ assert raised+ assert vae.slicing_sample_min_size == 4+++# ---------------------------------------------------------------------------+# From test_seedvr_vae_tiled_temporal_slicing.py+# ---------------------------------------------------------------------------+++class _SlicingDecodeVAE(nn.Module):+ def __init__(self, slicing_latent_min_size):+ super().__init__()+ self.slicing_latent_min_size = slicing_latent_min_size+ self.spatial_downsample_factor = 8+ self.temporal_downsample_factor = 4+ self.device = torch.device("cpu")+ self.use_slicing = True+ self._dummy = nn.Parameter(torch.zeros(1, dtype=torch.float32))+ self.decode_min_sizes = []+ self.memory_states = []++ def decode_(self, z):+ self.decode_min_sizes.append(self.slicing_latent_min_size)+ return vae_mod.VideoAutoencoderKL.slicing_decode(self, z)++ def _decode(self, z, memory_state=MemoryState.DISABLED):+ self.memory_states.append(memory_state)+ x = z[:, :1].repeat(+ 1,+ 3,+ 1,+ self.spatial_downsample_factor,+ self.spatial_downsample_factor,+ )+ return x+++def test_decode_tiled_vae_maps_temporal_args_to_latent_slicing_min_size():+ vae = _SlicingDecodeVAE(slicing_latent_min_size=2)+ z = torch.arange(1 * 16 * 5 * 8 * 8, dtype=torch.float32).reshape(1, 16, 5, 8, 8)++ tiled_vae(+ z,+ vae,+ tile_size=(64, 64),+ tile_overlap=(0, 0),+ temporal_size=12,+ temporal_overlap=4,+ encode=False,+ )++ assert vae.decode_min_sizes == [2]+ assert vae.memory_states == [MemoryState.INITIALIZING, MemoryState.ACTIVE]+ assert vae.slicing_latent_min_size == 2++ wrapper = vae_mod.VideoAutoencoderKLWrapper.__new__(+ vae_mod.VideoAutoencoderKLWrapper+ )+ nn.Module.__init__(wrapper)+ seedvr2_tiling = {+ "enable_tiling": True,+ "tile_size": (64, 64),+ "tile_overlap": (0, 0),+ "temporal_size": 8,+ "temporal_overlap": 7,+ }++ captured = {}++ def _fake_tiled_vae(latent, model, **kwargs):+ captured.update(kwargs)+ return torch.zeros(1, 3, 1, 16, 16)++ with patch.object(vae_mod, "tiled_vae", side_effect=_fake_tiled_vae):+ wrapper.decode(torch.zeros(1, 16, 2, 2), seedvr2_tiling=seedvr2_tiling)++ assert captured["temporal_overlap"] == 7+++# ---------------------------------------------------------------------------+# From test_vae_decode_tiled_dispatcher_seedvr2_4d.py+# ---------------------------------------------------------------------------+++def _force_oom(*a, **k):+ raise torch.cuda.OutOfMemoryError("forced OOM for dispatcher test")+++def _make_vae(first_stage_model, latent_channels, latent_dim):+ vae = sd_mod.VAE.__new__(sd_mod.VAE)+ vae.first_stage_model = first_stage_model+ vae.patcher = MagicMock()+ vae.patcher.get_free_memory = MagicMock(return_value=8 * 1024 * 1024 * 1024)+ vae.device = vae.output_device = torch.device("cpu")+ vae.vae_dtype = torch.float32+ vae.disable_offload = True+ vae.extra_1d_channel = None+ vae.upscale_ratio = vae.downscale_ratio = 8+ vae.upscale_index_formula = vae.downscale_index_formula = None+ vae.output_channels = 3+ vae.latent_channels = latent_channels+ vae.latent_dim = latent_dim+ vae.vae_output_dtype = lambda: torch.float32+ vae.spacial_compression_decode = lambda: 8+ vae.process_input = lambda x: x+ vae.process_output = lambda x: x+ vae.throw_exception_if_invalid = lambda: None+ vae.memory_used_decode = lambda *a, **k: 1+ return vae+++def _dispatch(vae, samples, seedvr2_call, generic_call, patch_wrapper_decode):+ mm = sd_mod.model_management+ with ExitStack() as stack:+ stack.enter_context(patch.object(mm, "raise_non_oom", lambda e: None))+ stack.enter_context(patch.object(mm, "load_models_gpu", lambda *a, **k: None))+ stack.enter_context(patch.object(mm, "soft_empty_cache", lambda: None))+ stack.enter_context(patch.object(sd_mod.VAE, "decode_tiled_seedvr2", seedvr2_call))+ stack.enter_context(patch.object(sd_mod.VAE, "decode_tiled_", generic_call))+ if patch_wrapper_decode:+ stack.enter_context(patch.object(+ seedvr_vae_mod.VideoAutoencoderKLWrapper, "decode",+ side_effect=_force_oom))+ vae.decode(samples)+++def test_4d_seedvr2_latent_routes_to_decode_tiled_seedvr2():+ wrapper = seedvr_vae_mod.VideoAutoencoderKLWrapper.__new__(+ seedvr_vae_mod.VideoAutoencoderKLWrapper)+ vae = _make_vae(wrapper, latent_channels=16, latent_dim=3)+ seedvr2_call = MagicMock(return_value=torch.zeros(1, 3, 9, 64, 64))+ generic_call = MagicMock(return_value=torch.zeros(1, 3, 64, 64))+ _dispatch(vae, torch.zeros(1, 16 * 3, 8, 8), seedvr2_call, generic_call, True)+ assert seedvr2_call.call_count == 1+ assert generic_call.call_count == 0+++def test_4d_non_seedvr2_latent_still_routes_to_generic_decode_tiled():+ first_stage = MagicMock()+ first_stage.decode = MagicMock(side_effect=_force_oom)+ vae = _make_vae(first_stage, latent_channels=4, latent_dim=2)+ seedvr2_call = MagicMock(return_value=torch.zeros(1, 3, 9, 64, 64))+ generic_call = MagicMock(return_value=torch.zeros(1, 3, 64, 64))+ _dispatch(vae, torch.zeros(1, 4, 8, 8), seedvr2_call, generic_call, False)+ assert generic_call.call_count == 1+ assert seedvr2_call.call_count == 0+++# ---------------------------------------------------------------------------+# From test_vae_encode_tiled_fallback_dispatcher_seedvr2.py+# ---------------------------------------------------------------------------+++def _populate_common_vae_attrs_fallback(vae):+ vae.patcher = MagicMock()+ vae.patcher.get_free_memory = MagicMock(return_value=8 * 1024 * 1024 * 1024)+ vae.device = torch.device("cpu")+ vae.output_device = torch.device("cpu")+ vae.vae_dtype = torch.float32+ vae.disable_offload = True+ vae.extra_1d_channel = None+ vae.upscale_ratio = 8+ vae.upscale_index_formula = None+ vae.output_channels = 3+ vae.latent_channels = 16+ vae.latent_dim = 3+ vae.downscale_ratio = 8+ vae.downscale_index_formula = None+ vae.not_video = False+ vae.crop_input = False+ vae.pad_channel_value = None++ vae.vae_output_dtype = lambda: torch.float32+ vae.spacial_compression_encode = lambda: 8+ vae.process_input = lambda x: x+ vae.process_output = lambda x: x+ vae.throw_exception_if_invalid = lambda: None+ vae.memory_used_encode = lambda *a, **k: 1+++def _make_seedvr2_vae_fallback():+ vae = sd_mod.VAE.__new__(sd_mod.VAE)+ wrapper = seedvr_vae_mod.VideoAutoencoderKLWrapper.__new__(+ seedvr_vae_mod.VideoAutoencoderKLWrapper+ )+ vae.first_stage_model = wrapper+ _populate_common_vae_attrs_fallback(vae)+ return vae+++def _make_non_seedvr2_vae_fallback():+ vae = sd_mod.VAE.__new__(sd_mod.VAE)+ vae.first_stage_model = MagicMock()+ _populate_common_vae_attrs_fallback(vae)+ return vae+++def _force_regular_encode_oom(*args, **kwargs):+ raise torch.cuda.OutOfMemoryError("forced OOM for dispatcher test")+++def test_seedvr2_3d_routes_to_encode_tiled_seedvr2_on_oom():+ vae = _make_seedvr2_vae_fallback()+ pixel_samples = torch.zeros((1, 8, 64, 64, 3))++ seedvr2_call = MagicMock(return_value=torch.zeros(1, 16, 2, 8, 8))+ generic_call = MagicMock(return_value=torch.zeros(1, 16, 2, 8, 8))++ with patch.object(sd_mod.model_management, "raise_non_oom",+ lambda e: None), \+ patch.object(sd_mod.model_management, "load_models_gpu",+ lambda *a, **k: None), \+ patch.object(sd_mod.model_management, "soft_empty_cache",+ lambda: None), \+ patch.object(seedvr_vae_mod.VideoAutoencoderKLWrapper, "encode",+ side_effect=_force_regular_encode_oom), \+ patch.object(sd_mod.VAE, "encode_tiled_seedvr2", seedvr2_call,+ create=True), \+ patch.object(sd_mod.VAE, "encode_tiled_3d", generic_call):+ vae.encode(pixel_samples)++ assert seedvr2_call.call_count == 1, (+ f"Expected encode_tiled_seedvr2 to be called once for a SeedVR2 3D "+ f"input under OOM fallback; got {seedvr2_call.call_count} calls."+ )+ assert generic_call.call_count == 0, (+ f"encode_tiled_3d must NOT be called for a SeedVR2 input; got "+ f"{generic_call.call_count} calls."+ )+++def test_non_seedvr2_encode_tiled_3d_default_overlap_is_concrete():+ vae = _make_non_seedvr2_vae_fallback()+ vae.downscale_ratio = (lambda a: max(1, a // 4), 8, 8)+ vae.upscale_ratio = (lambda a: a * 4, 8, 8)+ generic_call = MagicMock(return_value=torch.zeros(1, 16, 2, 8, 8))+ pixel_samples = torch.zeros((1, 8, 64, 64, 3))++ with patch.object(sd_mod.model_management, "load_models_gpu",+ lambda *a, **k: None), \+ patch.object(sd_mod.VAE, "encode_tiled_3d", generic_call):+ vae.encode_tiled(pixel_samples)++ assert generic_call.call_args.kwargs["overlap"] == (1, 64, 64)diff --git a/tests-unit/comfy_test/test_seedvr_progressive_sampler.py b/tests-unit/comfy_test/test_seedvr_progressive_sampler.pynew file mode 100644index 000000000..05291989e--- /dev/null+++ b/tests-unit/comfy_test/test_seedvr_progressive_sampler.py@@ -0,0 +1,126 @@+"""Unit tests for ``comfy_extras.nodes_seedvr.SeedVR2ProgressiveSampler``."""++from unittest.mock import patch++import pytest+import torch++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True++import comfy.sample # noqa: E402+import comfy_extras.nodes_seedvr as nodes_seedvr_mod # noqa: E402+from comfy_extras.nodes_seedvr import SeedVR2ProgressiveSampler # noqa: E402++_LAT_C = 16+_COND_C = 17+++def _make_inputs(B: int = 1, T: int = 5, H: int = 8, W: int = 8):+ """Build minimal SeedVR2-shaped sampling inputs."""+ samples_5d = torch.arange(+ B * _LAT_C * T * H * W, dtype=torch.float32+ ).reshape(B, _LAT_C, T, H, W)+ samples = samples_5d.reshape(B, _LAT_C * T, H, W).contiguous()++ cond_5d = torch.arange(+ B * _COND_C * T * H * W, dtype=torch.float32+ ).reshape(B, _COND_C, T, H, W) + 10000.0+ cond = cond_5d.reshape(B, _COND_C * T, H, W).contiguous()++ text_pos = torch.zeros(1, 4, 32)+ text_neg = torch.zeros(1, 4, 32)+ positive = [[text_pos, {"condition": cond.clone()}]]+ negative = [[text_neg, {"condition": cond.clone()}]]+ latent_image = {"samples": samples}+ return latent_image, positive, negative, samples_5d, cond_5d+++def _identity_fix_empty(model, latent_image, downscale_ratio_spacial=None):+ return latent_image+++def _fingerprinted_prepare_noise(latent_image, seed, batch_inds=None):+ """Return a tensor whose values encode ``(seed, position)``."""+ base = torch.arange(+ latent_image.numel(), dtype=torch.float32+ ).reshape(latent_image.shape)+ return base + float(seed) * 1e6+++def test_progressive_sampler_schema_exposes_manual_default_auto_chunking():+ schema = SeedVR2ProgressiveSampler.define_schema()+ inputs = {item.id: item for item in schema.inputs}++ assert inputs["chunking_mode"].options == ["manual", "auto"]+ assert inputs["chunking_mode"].default == "manual"+++def test_auto_chunking_walks_two_three_four_chunk_ladder():+ """Auto mode must walk 2-, 3-, then 4-chunk geometries on OOM."""+ latent, pos, neg, _, _ = _make_inputs(T=17)+ calls = []++ def _oom_until_four_chunks(model, noise, steps, cfg, sampler_name,+ scheduler, positive, negative,+ latent_image, denoise=1.0,+ noise_mask=None, seed=None):+ calls.append(tuple(latent_image.shape))+ if latent_image.shape[1] > _LAT_C * 5:+ raise torch.cuda.OutOfMemoryError("chunk too large")+ return latent_image.clone()++ with patch.object(comfy.sample, "sample",+ side_effect=_oom_until_four_chunks), \+ patch.object(comfy.sample, "fix_empty_latent_channels",+ side_effect=_identity_fix_empty), \+ patch.object(comfy.sample, "prepare_noise",+ side_effect=_fingerprinted_prepare_noise), \+ patch.object(nodes_seedvr_mod.comfy.model_management,+ "soft_empty_cache") as soft_empty:+ out = SeedVR2ProgressiveSampler.execute(+ model=None, seed=0, steps=2, cfg=1.0,+ sampler_name="euler", scheduler="simple",+ positive=pos, negative=neg, latent=latent,+ denoise=1.0, frames_per_chunk=65, temporal_overlap=0,+ chunking_mode="auto",+ )++ assert calls[:4] == [+ (1, _LAT_C * 17, 8, 8),+ (1, _LAT_C * 9, 8, 8),+ (1, _LAT_C * 6, 8, 8),+ (1, _LAT_C * 5, 8, 8),+ ]+ assert torch.equal(out.result[0]["samples"], latent["samples"])+ assert soft_empty.call_count == 3++[email protected]("bad_chunk", [0, -1, 2])+def test_t3_invalid_frames_per_chunk_raises_value_error(bad_chunk):+ """``frames_per_chunk`` violating 4n+1 (or <1) must raise ``ValueError`` before any model invocation."""+ latent, pos, neg, _, _ = _make_inputs(T=5)++ sampler_called = {"n": 0}++ def _should_not_be_called(*args, **kwargs):+ sampler_called["n"] += 1+ return torch.zeros(1)++ with patch.object(comfy.sample, "sample",+ side_effect=_should_not_be_called), \+ patch.object(comfy.sample, "fix_empty_latent_channels",+ side_effect=_identity_fix_empty), \+ patch.object(comfy.sample, "prepare_noise",+ side_effect=_fingerprinted_prepare_noise):+ with pytest.raises(ValueError) as excinfo:+ SeedVR2ProgressiveSampler.execute(+ model=None, seed=0, steps=2, cfg=1.0,+ sampler_name="euler", scheduler="simple",+ positive=pos, negative=neg, latent=latent,+ denoise=1.0, frames_per_chunk=bad_chunk, temporal_overlap=0,+ )+ assert str(bad_chunk) in str(excinfo.value)+ assert sampler_called["n"] == 0