The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Optimize Seedvr2 (#16530)
Jukka Seppänen84131a647167
diff --git a/comfy/ldm/seedvr/attention.py b/comfy/ldm/seedvr/attention.pyindex 11b4c1e4a..a39f0ba9d 100644--- a/comfy/ldm/seedvr/attention.py+++ b/comfy/ldm/seedvr/attention.py@@ -22,29 +22,41 @@ def _var_attention_output(out, heads, head_dim, skip_output_reshape):return out.reshape(-1, heads * head_dim)+def _equal_length_runs(lengths):+ """(start, stop) sequence spans over which the sequence length is constant."""+ runs = []+ start = 0+ for i in range(1, len(lengths) + 1):+ if i == len(lengths) or lengths[i] != lengths[start]:+ runs.append((start, i))+ start = i+ return runs++def var_attention_optimized_split(q, k, v, heads, cu_seqlens_q, cu_seqlens_k, *args, skip_reshape=False, skip_output_reshape=False, **kwargs):q, k, v, head_dim = _var_attention_qkv(q, k, v, heads, skip_reshape)- q_split_indices = cu_seqlens_q[1:-1]- k_split_indices = cu_seqlens_k[1:-1]if k.shape[0] != v.shape[0]:raise ValueError("cu_seqlens_k does not match v token count")-- q_splits = torch.tensor_split(q, q_split_indices, dim=0)- k_splits = torch.tensor_split(k, k_split_indices, dim=0)- v_splits = torch.tensor_split(v, k_split_indices, dim=0)- if len(q_splits) != len(k_splits) or len(q_splits) != len(v_splits):+ if len(cu_seqlens_q) != len(cu_seqlens_k):raise ValueError("cu_seqlens_q and cu_seqlens_k must describe the same sequence count")- out = []- for q_i, k_i, v_i in zip(q_splits, k_splits, v_splits):- q_i = q_i.permute(1, 0, 2).unsqueeze(0)- k_i = k_i.permute(1, 0, 2).unsqueeze(0)- v_i = v_i.permute(1, 0, 2).unsqueeze(0)+ q_lens = [end - start for start, end in zip(cu_seqlens_q, cu_seqlens_q[1:])]+ k_lens = [end - start for start, end in zip(cu_seqlens_k, cu_seqlens_k[1:])]++ out = torch.empty((cu_seqlens_q[-1], heads, head_dim), device=q.device, dtype=q.dtype)+ # Windows of matching length are adjacent, so each run becomes one batched attention call.+ for first, last in _equal_length_runs(list(zip(q_lens, k_lens))):+ n = last - first+ len_q, len_k = q_lens[first], k_lens[first]+ q_slice = slice(cu_seqlens_q[first], cu_seqlens_q[last])+ k_slice = slice(cu_seqlens_k[first], cu_seqlens_k[last])+ q_i = q[q_slice].reshape(n, len_q, heads, head_dim).transpose(1, 2)+ k_i = k[k_slice].reshape(n, len_k, heads, head_dim).transpose(1, 2)+ v_i = v[k_slice].reshape(n, len_k, heads, head_dim).transpose(1, 2)out_i = _attention.optimized_attention(q_i, k_i, v_i, heads, skip_reshape=True, skip_output_reshape=True)- out.append(out_i.squeeze(0).permute(1, 0, 2))+ out[q_slice] = out_i.transpose(1, 2).reshape(n * len_q, heads, head_dim)- out = torch.cat(out, dim=0)return _var_attention_output(out, heads, head_dim, skip_output_reshape)diff --git a/comfy/ldm/seedvr/constants.py b/comfy/ldm/seedvr/constants.pyindex 12c4b4bef..98fb5057b 100644--- a/comfy/ldm/seedvr/constants.py+++ b/comfy/ldm/seedvr/constants.py@@ -13,10 +13,35 @@ SEEDVR2_CHUNK_SIGMA_K = 4SEEDVR2_7B_VID_DIM = 3072SEEDVR2_OOM_BACKOFF_DIVISOR = 2SEEDVR2_DTYPE_BYTES_FLOOR = 4-SEEDVR2_7B_MLP_CHUNK = 8192+SEEDVR2_MLP_CHUNK = 8192SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS = 4096 # partial-RoPE application token-chunk.SEEDVR2_LATENT_CHANNELS = 16+# VAE temporal caches: int8-packed (rotated, per-token scales) between slices.+SEEDVR2_VAE_CACHE_QUANT_BYTES = 64 * 1024 ** 2 # tails at or above this size are packed.+SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL = 6200 # host-held caches and parked frames, per output-frame pixel.++# VAE decode estimate: the free VRAM an untiled decode needs under cudaMallocAsync (ComfyUI's default),+# fitted 2-7% above the measured floor from 480p to 4K. Flat in clip length: the decoded frames go+# to sd.py's output buffer off the GPU slice by slice (1088p: 5.33 GiB allocated at 61 and 121 frames).+SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL = 3150 # working set, per output-frame pixel.+SEEDVR2_DECODE_FIXED_BYTES = 384 * 1024 ** 2+SEEDVR2_DECODE_LAB_BYTES_PER_OUTPUT_PIXEL = 160 # colour correction is per frame in the node.++# VAE encode estimate, likewise fitted 3-23% above the measured floor; slicing keeps it flat in clip length.+SEEDVR2_ENCODE_BYTES_PER_PIXEL = 2550 # per input-frame pixel.+SEEDVR2_ENCODE_FIXED_BYTES = 576 * 1024 ** 2++# The fits cover the fp16 channels-last path; other dtypes run the eager path, measured at about this+# many times the estimate per two bytes of element.+SEEDVR2_EAGER_DECODE_FACTOR = 5+SEEDVR2_EAGER_ENCODE_FACTOR = 3++# Tiled decode, in latent units (8 output pixels each).+SEEDVR2_TILE_MEM_HEADROOM = 0.6+SEEDVR2_MIN_TILE_LATENT = 32+SEEDVR2_MAX_TILE_LATENT = 96 # 768 px; larger tiles measured no faster.+SEEDVR2_COLOR_MEM_HEADROOM = 0.75SEEDVR2_LAB_SCALE_MULTIPLIER = 13SEEDVR2_WAVELET_SCALE_MULTIPLIER = 10 # per-frame byte multiplier, wavelet path.@@ -26,8 +51,6 @@ BYTEDANCE_VAE_SCALING_FACTOR = 0.9152 # configs_3b/main.yaml:57.BYTEDANCE_VAE_SHIFTING_FACTOR = 0.0BYTEDANCE_VAE_CONV_MEM_GIB = 0.5BYTEDANCE_VAE_NORM_MEM_GIB = 0.5-BYTEDANCE_LOGVAR_CLAMP_MIN = -30.0 # video_vae_v3/modules/types.py:28.-BYTEDANCE_LOGVAR_CLAMP_MAX = 20.0 # video_vae_v3/modules/types.py:28.BYTEDANCE_GN_CHUNKS_FP16 = 4 # causal_inflation_lib.py:351 (GroupNorm chunk count, fp16).BYTEDANCE_GN_CHUNKS_FP32 = 2 # causal_inflation_lib.py:351 (GroupNorm chunk count, fp32).BYTEDANCE_BLOCK_OUT_CHANNELS = (128, 256, 512, 512) # s8_c16_t4_inflation_sd3.yaml:7-11.diff --git a/comfy/ldm/seedvr/model.py b/comfy/ldm/seedvr/model.pyindex a978698d5..16bbaa494 100644--- a/comfy/ldm/seedvr/model.py+++ b/comfy/ldm/seedvr/model.py@@ -16,13 +16,14 @@ from comfy.ldm.seedvr.constants import (BYTEDANCE_ROPE_MAX_FREQ,BYTEDANCE_SINUSOIDAL_DIM,ROPE_THETA,- SEEDVR2_7B_MLP_CHUNK,+ SEEDVR2_MLP_CHUNK,SEEDVR2_7B_VID_DIM,SEEDVR2_LATENT_CHANNELS,SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS,)import comfy.model_managementimport comfy.ops+import comfy.quant_opsclass Cache:def __init__(self, disable=False, prefix="", cache=None):@@ -47,19 +48,6 @@ class Cache:cache=self.cache,)-def repeat_concat(- vid: torch.FloatTensor, # (VL ... c)- txt: torch.FloatTensor, # (TL ... c)- vid_len: torch.LongTensor, # (n*b)- txt_len: torch.LongTensor, # (b)- txt_repeat: List, # (n)-) -> torch.FloatTensor: # (L ... c)- vid = torch.split(vid, vid_len.tolist())- txt = torch.split(txt, txt_len.tolist())- txt = [[x] * n for x, n in zip(txt, txt_repeat)]- txt = list(chain(*txt))- return torch.cat(list(chain(*zip(vid, txt))))-def repeat_concat_idx(vid_len: torch.LongTensor, # (n*b)txt_len: torch.LongTensor, # (b)@@ -67,12 +55,18 @@ def repeat_concat_idx() -> Tuple[Callable,Callable,+ List,]:device = vid_len.devicevid_idx = torch.arange(vid_len.sum(), device=device)txt_idx = torch.arange(len(vid_idx), len(vid_idx) + txt_len.sum(), device=device)txt_repeat_list = txt_repeat.tolist()- tgt_idx = repeat_concat(vid_idx, txt_idx, vid_len, txt_len, txt_repeat_list)+ vid_blocks = torch.split(vid_idx, vid_len.tolist())+ txt_blocks = list(chain(*([x] * n for x, n in zip(torch.split(txt_idx, txt_len.tolist()), txt_repeat_list))))+ seq_len = [vid.numel() + txt.numel() for vid, txt in zip(vid_blocks, txt_blocks)]+ # Stable sort by length so equal-length windows end up adjacent and can share one attention call.+ order = sorted(range(len(seq_len)), key=seq_len.__getitem__)+ tgt_idx = torch.cat([block for i in order for block in (vid_blocks[i], txt_blocks[i])])src_idx = torch.argsort(tgt_idx)txt_idx_len = len(tgt_idx) - len(vid_idx)repeat_txt_len = (txt_len * txt_repeat).tolist()@@ -88,6 +82,7 @@ def repeat_concat_idx(return (lambda vid, txt: torch.cat([vid, txt])[tgt_idx],lambda all: unconcat_coalesce(all),+ cumulative_lengths([seq_len[i] for i in order]),)def cumulative_lengths(lengths):@@ -274,12 +269,8 @@ class NaRotaryEmbedding3d(RotaryEmbedding3d):]:freqs = cache("rope_freqs_3d", lambda: self.get_freqs(shape))freqs = freqs.to(device=q.device)- q = q.transpose(0, 1)- k = k.transpose(0, 1)- q = _apply_seedvr2_rotary_emb(freqs, q.float()).to(q.dtype)- k = _apply_seedvr2_rotary_emb(freqs, k.float()).to(k.dtype)- q = q.transpose(0, 1)- k = k.transpose(0, 1)+ q = _apply_rope1_partial(q, freqs)+ k = _apply_rope1_partial(k, freqs)return q, k@torch._dynamo.disable@@ -300,7 +291,7 @@ class NaRotaryEmbedding3d(RotaryEmbedding3d):for f, h, w in shape.tolist():freqs = plain_rope.get_axial_freqs(f, h, w)freq_list.append(freqs.view(-1, freqs.size(-1)))- return torch.cat(freq_list, dim=0)+ return _to_flux_freqs_cis(torch.cat(freq_list, dim=0))class MMRotaryEmbeddingBase(RotaryEmbeddingBase):@@ -313,48 +304,9 @@ class MMRotaryEmbeddingBase(RotaryEmbeddingBase):)self.mm = True-def slice_at_dim(t, dim_slice: slice, *, dim):- dim += (t.ndim if dim < 0 else 0)- colons = [slice(None)] * t.ndim- colons[dim] = dim_slice- return t[tuple(colons)]--def rotate_half(x):- x = x.reshape(*x.shape[:-1], x.shape[-1] // 2, 2)- x1, x2 = x.unbind(dim = -1)- x = torch.stack((-x2, x1), dim = -1)- return x.flatten(-2)def exists(val):return val is not None-def _apply_seedvr2_rotary_emb(- freqs: torch.Tensor,- t: torch.Tensor,- start_index: int = 0,- scale: float = 1.0,- seq_dim: int = -2,- freqs_seq_dim: int | None = None,-) -> torch.Tensor:- dtype = t.dtype- if freqs_seq_dim is None and (freqs.ndim == 2 or t.ndim == 3):- freqs_seq_dim = 0-- if t.ndim == 3 or freqs_seq_dim is not None:- seq_len = t.shape[seq_dim]- freqs = slice_at_dim(freqs, slice(-seq_len, None), dim=freqs_seq_dim)-- rot_feats = freqs.shape[-1]- end_index = start_index + rot_feats-- t_left = t[..., :start_index]- t_middle = t[..., start_index:end_index]- t_right = t[..., end_index:]-- freqs = freqs.to(device=t_middle.device, dtype=t_middle.dtype)- cos = freqs.cos() * scale- sin = freqs.sin() * scale- t_middle = (t_middle * cos) + (rotate_half(t_middle) * sin)- return torch.cat((t_left, t_middle, t_right), dim=-1).to(dtype)def _to_flux_freqs_cis(freqs_interleaved: torch.Tensor) -> torch.Tensor:angles = freqs_interleaved[..., ::2].float()@@ -365,16 +317,26 @@ def _to_flux_freqs_cis(freqs_interleaved: torch.Tensor) -> torch.Tensor:def _apply_rope1_partial(t: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor:- out = t.clone() if t.requires_grad or comfy.model_management.in_training else t+ """Rotate the leading ``2 * freqs_cis.shape[-3]`` head features of an (L, heads, dim) tensor.++ The 4-D token-major view with 6-D frequencies is what the fused kernel dispatches on;+ a 3-D tensor silently falls back to the elementwise reference path.+ """+ training = comfy.model_management.in_training or t.requires_grad+ out = t.clone() if training else trot_d = 2 * freqs_cis.shape[-3]- seq_len = out.shape[-2]+ view = out.unsqueeze(0)+ if rot_d != view.shape[-1]:+ view = view[..., :rot_d]+ freqs_cis = freqs_cis[None, :, None]+ seq_len = out.shape[0]for start in range(0, seq_len, SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS):end = min(start + SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS, seq_len)- freqs_chunk = freqs_cis[start:end]- if rot_d == out.shape[-1]:- out[..., start:end, :] = apply_rope1(out[..., start:end, :], freqs_chunk).to(out.dtype)+ chunk, freqs_chunk = view[:, start:end], freqs_cis[:, start:end]+ if training:+ chunk.copy_(apply_rope1(chunk, freqs_chunk))else:- out[..., start:end, :rot_d] = apply_rope1(out[..., start:end, :rot_d], freqs_chunk).to(out.dtype)+ comfy.quant_ops.ck.apply_rope1_(chunk, freqs_chunk)return out@@ -390,6 +352,7 @@ class NaMMRotaryEmbedding3d(MMRotaryEmbeddingBase):txt_q: torch.FloatTensor, # L h dtxt_k: torch.FloatTensor, # L h dtxt_shape: torch.LongTensor, # B 1+ vid_txt_shape: torch.LongTensor, # B 1, text length of the sample each vid entry belongs tocache: Cache,) -> Tuple[torch.FloatTensor,@@ -399,26 +362,17 @@ class NaMMRotaryEmbedding3d(MMRotaryEmbeddingBase):]:vid_freqs, txt_freqs = cache("mmrope_freqs_3d",- lambda: self.get_freqs(vid_shape, txt_shape),+ lambda: self.get_freqs(vid_shape, txt_shape, vid_txt_shape),)target_device = vid_q.deviceif vid_freqs.device != target_device:vid_freqs = vid_freqs.to(target_device)if txt_freqs.device != target_device:txt_freqs = txt_freqs.to(target_device)- vid_q = vid_q.transpose(0, 1)- vid_k = vid_k.transpose(0, 1)vid_q = _apply_rope1_partial(vid_q, vid_freqs)vid_k = _apply_rope1_partial(vid_k, vid_freqs)- vid_q = vid_q.transpose(0, 1)- vid_k = vid_k.transpose(0, 1)-- txt_q = txt_q.transpose(0, 1)- txt_k = txt_k.transpose(0, 1)txt_q = _apply_rope1_partial(txt_q, txt_freqs)txt_k = _apply_rope1_partial(txt_k, txt_freqs)- txt_q = txt_q.transpose(0, 1)- txt_k = txt_k.transpose(0, 1)return vid_q, vid_k, txt_q, txt_k@torch._dynamo.disable # Disable compilation: .tolist() is data-dependent and causes graph breaks@@ -426,6 +380,7 @@ class NaMMRotaryEmbedding3d(MMRotaryEmbeddingBase):self,vid_shape: torch.LongTensor,txt_shape: torch.LongTensor,+ vid_txt_shape: torch.LongTensor,) -> Tuple[torch.Tensor,torch.Tensor,@@ -434,9 +389,9 @@ class NaMMRotaryEmbedding3d(MMRotaryEmbeddingBase):max_temporal = 0max_height = 0max_width = 0- max_txt_len = 0+ max_txt_len = max(txt_shape[:, 0].tolist())- for (f, h, w), l in zip(vid_shape.tolist(), txt_shape[:, 0].tolist()):+ for (f, h, w), l in zip(vid_shape.tolist(), vid_txt_shape[:, 0].tolist()):max_temporal = max(max_temporal, l + f)max_height = max(max_height, h)max_width = max(max_width, w)@@ -451,12 +406,14 @@ class NaMMRotaryEmbedding3d(MMRotaryEmbeddingBase):).float()txt_freqs = self.get_axial_freqs(max_txt_len + 16)- vid_freq_list, txt_freq_list = [], []- for (f, h, w), l in zip(vid_shape.tolist(), txt_shape[:, 0].tolist()):- vid_freq = vid_freqs[l : l + f, :h, :w].reshape(-1, vid_freqs.size(-1))- txt_freq = txt_freqs[:l].repeat(1, 3).reshape(-1, vid_freqs.size(-1))- vid_freq_list.append(vid_freq)- txt_freq_list.append(txt_freq)+ vid_freq_list = [+ vid_freqs[l : l + f, :h, :w].reshape(-1, vid_freqs.size(-1))+ for (f, h, w), l in zip(vid_shape.tolist(), vid_txt_shape[:, 0].tolist())+ ]+ txt_freq_list = [+ txt_freqs[:l].repeat(1, 3).reshape(-1, vid_freqs.size(-1))+ for l in txt_shape[:, 0].tolist()+ ]vid_freqs_interleaved = torch.cat(vid_freq_list, dim=0)txt_freqs_interleaved = torch.cat(txt_freq_list, dim=0)@@ -579,7 +536,7 @@ def window_idx(window_fn: Callable[[torch.Tensor], List[torch.Tensor]],):hid_idx = torch.arange(hid_shape.prod(-1).sum(), device=hid_shape.device).unsqueeze(-1)- tgt_idx, tgt_shape, tgt_windows, tgt_len_list, tgt_windows_list = window(hid_idx, hid_shape, window_fn)+ tgt_idx, tgt_shape, tgt_windows, _, _ = window(hid_idx, hid_shape, window_fn)tgt_idx = tgt_idx.squeeze(-1)src_idx = torch.argsort(tgt_idx)return (@@ -587,8 +544,6 @@ def window_idx(lambda hid: torch.index_select(hid, 0, src_idx),tgt_shape,tgt_windows,- tgt_len_list,- tgt_windows_list,)class NaSwinAttention(NaMMAttention):@@ -630,11 +585,12 @@ class NaSwinAttention(NaMMAttention):window_slices = self.window_op((t, h, w), self.window)return [x[st, sh, sw] for (st, sh, sw) in window_slices]- window_partition, window_reverse, window_shape, window_count, vid_len_win_list, window_count_list = cache_win(+ window_partition, window_reverse, window_shape, window_count = cache_win("win_transform",lambda: window_idx(vid_shape, make_window),)vid_qkv_win = window_partition(vid_qkv)+ del vid_qkvvid_qkv_win = vid_qkv_win.reshape(vid_qkv_win.shape[0], 3, self.heads, self.head_dim)txt_qkv = txt_qkv.reshape(txt_qkv.shape[0], 3, self.heads, self.head_dim)@@ -654,44 +610,35 @@ class NaSwinAttention(NaMMAttention):if self.version_7b:vid_q, vid_k = self.rope(vid_q, vid_k, window_shape, cache_win)elif self.rope.mm:- _, num_h, _ = txt_q.shape- txt_q_repeat = txt_q.flatten(1, 2)- txt_q_repeat = unflatten(txt_q_repeat, txt_shape)- txt_q_repeat = [[x] * n for x, n in zip(txt_q_repeat, window_count_list)]- txt_q_repeat = list(chain(*txt_q_repeat))- txt_q_repeat, txt_shape_repeat = flatten(txt_q_repeat)- txt_q_repeat = txt_q_repeat.reshape(txt_q_repeat.shape[0], num_h, self.head_dim)-- txt_k_repeat = txt_k.flatten(1, 2)- txt_k_repeat = unflatten(txt_k_repeat, txt_shape)- txt_k_repeat = [[x] * n for x, n in zip(txt_k_repeat, window_count_list)]- txt_k_repeat = list(chain(*txt_k_repeat))- txt_k_repeat, _ = flatten(txt_k_repeat)- txt_k_repeat = txt_k_repeat.reshape(txt_k_repeat.shape[0], num_h, self.head_dim)-+ # Each window carries the text length of the sample it came from, which offsets its vid freqs.+ txt_shape_win = cache_win(+ "txt_shape_win",+ lambda: torch.repeat_interleave(txt_shape, window_count.to(txt_shape.device), dim=0),+ )vid_q, vid_k, txt_q, txt_k = self.rope(- vid_q, vid_k, window_shape, txt_q_repeat, txt_k_repeat, txt_shape_repeat, cache_win+ vid_q, vid_k, window_shape, txt_q, txt_k, txt_shape, txt_shape_win, cache_win)else:vid_q, vid_k = self.rope(vid_q, vid_k, window_shape, cache_win)- txt_len_win_list = cache_win(- "txt_len_list",- lambda: [txt_len for txt_len, window_count in zip(txt_len.tolist(), window_count_list) for _ in range(window_count)],- )- all_len_win = cache_win("all_len", lambda: [vid_len + txt_len for vid_len, txt_len in zip(vid_len_win_list, txt_len_win_list)])- concat_win, unconcat_win = cache_win(+ concat_win, unconcat_win, cu_seqlens_win = cache_win("mm_pnp", lambda: repeat_concat_idx(vid_len_win, txt_len, window_count))+ q = concat_win(vid_q, txt_q)+ del vid_q, txt_q+ k = concat_win(vid_k, txt_k)+ del vid_k, txt_k+ v = concat_win(vid_v, txt_v)+ del vid_v, txt_v, vid_qkv_win, txt_qkvout = optimized_var_attention(- q=concat_win(vid_q, txt_q),- k=concat_win(vid_k, txt_k),- v=concat_win(vid_v, txt_v),+ q=q, k=k, v=v,heads=self.heads, skip_reshape=True, skip_output_reshape=True,- cu_seqlens_q=cache_win("vid_seqlens_q", lambda: cumulative_lengths(all_len_win)),- cu_seqlens_k=cache_win("vid_seqlens_k", lambda: cumulative_lengths(all_len_win)),+ cu_seqlens_q=cu_seqlens_win,+ cu_seqlens_k=cu_seqlens_win,)+ del q, k, vvid_out, txt_out = unconcat_win(out)+ del outvid_out = vid_out.flatten(1, 2)txt_out = txt_out.flatten(1, 2)@@ -736,7 +683,11 @@ class SwiGLUMLP(nn.Module):self.proj_in = operations.Linear(dim, hidden_dim, bias=False, device=device, dtype=dtype)def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:- return self.proj_out(F.silu(self.proj_in_gate(x)) * self.proj_in(x))+ gate = self.proj_in_gate(x)+ up = self.proj_in(x)+ if comfy.model_management.in_training or x.requires_grad:+ return self.proj_out(F.silu(gate) * up)+ return self.proj_out(F.silu(gate, inplace=True).mul_(up))def get_mlp(mlp_type: Optional[str] = "normal"):if mlp_type == "normal":@@ -803,8 +754,21 @@ class NaMMSRTransformerBlock(nn.Module):self.ada = MMModule(ada, dim=dim, emb_dim=emb_dim, layers=["attn", "mlp"], shared_weights=shared_weights, vid_only=is_last_layer, device=device, dtype=dtype)self.is_last_layer = is_last_layerself.version = version+ self.norm_eps = norm_eps++ def _norm_ada_in(self, norm, vid, txt, layer, ada_kwargs):+ """Normalize then modulate as one fused kernel, only where norm and ada own the same+ branches (the last block's attn_norm covers txt, its ada does not)."""+ fusable = (+ not (comfy.model_management.in_training or vid.requires_grad)+ and norm.vid_only == self.ada.vid_only+ )+ if not fusable:+ vid, txt = norm(vid, txt)+ return self.ada(vid, txt, layer=layer, mode="in", **ada_kwargs)+ return self.ada(vid, txt, layer=layer, mode="in", norm_eps=self.norm_eps, **ada_kwargs)- def _seedvr2_7b_mlp(+ def _chunked_mlp(self,vid: torch.FloatTensor,txt: torch.FloatTensor,@@ -813,12 +777,14 @@ class NaMMSRTransformerBlock(nn.Module):torch.FloatTensor,]:vid_module = self.mlp.vid if not self.mlp.shared_weights else self.mlp.all- if comfy.model_management.in_training or vid.requires_grad:- vid = torch.cat([vid_module(chunk) for chunk in vid.split(SEEDVR2_7B_MLP_CHUNK, dim=0)], dim=0)+ if vid.shape[0] <= SEEDVR2_MLP_CHUNK:+ vid = vid_module(vid)+ elif comfy.model_management.in_training or vid.requires_grad:+ vid = torch.cat([vid_module(chunk) for chunk in vid.split(SEEDVR2_MLP_CHUNK, dim=0)], dim=0)else:vid_out = Noneoffset = 0- for chunk in vid.split(SEEDVR2_7B_MLP_CHUNK, dim=0):+ for chunk in vid.split(SEEDVR2_MLP_CHUNK, dim=0):chunk_out = vid_module(chunk)if vid_out is None:vid_out = chunk_out.new_empty((vid.shape[0], *chunk_out.shape[1:]))@@ -856,18 +822,13 @@ class NaMMSRTransformerBlock(nn.Module):"branch_tag": MMArg("vid", "txt"),}- vid_attn, txt_attn = self.attn_norm(vid, txt)- vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="in", **ada_kwargs)+ vid_attn, txt_attn = self._norm_ada_in(self.attn_norm, vid, txt, "attn", ada_kwargs)vid_attn, txt_attn = self.attn(vid_attn, txt_attn, vid_shape, txt_shape, cache)vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="out", **ada_kwargs)vid_attn, txt_attn = (vid_attn + vid), (txt_attn + txt)- vid_mlp, txt_mlp = self.mlp_norm(vid_attn, txt_attn)- vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, layer="mlp", mode="in", **ada_kwargs)- if self.version:- vid_mlp, txt_mlp = self._seedvr2_7b_mlp(vid_mlp, txt_mlp)- else:- vid_mlp, txt_mlp = self.mlp(vid_mlp, txt_mlp)+ vid_mlp, txt_mlp = self._norm_ada_in(self.mlp_norm, vid_attn, txt_attn, "mlp", ada_kwargs)+ vid_mlp, txt_mlp = self._chunked_mlp(vid_mlp, txt_mlp)vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, layer="mlp", mode="out", **ada_kwargs)vid_mlp, txt_mlp = (vid_mlp + vid_attn), (txt_mlp + txt_attn)@@ -1021,36 +982,62 @@ class AdaSingle(nn.Module):cache: Optional[Cache] = None,branch_tag: str = "",hid_len: Optional[torch.LongTensor] = None, # b+ norm_eps: Optional[float] = None,) -> torch.FloatTensor:if cache is None:cache = Cache(disable=True)idx = self.layers.index(layer)- emb = emb.reshape(emb.shape[0], -1, len(self.layers), 3)[:, :, idx, :]- emb = expand_dims(emb, 1, hid.ndim + 1)- if hid_len is not None:- emb = cache(- f"emb_repeat_{idx}_{branch_tag}",- lambda: torch.repeat_interleave(emb, hid_len, dim=0),- )+ def sliced():+ e = emb.reshape(emb.shape[0], -1, len(self.layers), 3)[:, :, idx, :]+ return expand_dims(e, 1, hid.ndim + 1)- shiftA, scaleA, gateA = emb.unbind(-1)- shiftB, scaleB, gateB = (- getattr(self, f"{layer}_shift", None),- getattr(self, f"{layer}_scale", None),- getattr(self, f"{layer}_gate", None),- )+ if hid_len is None:+ mod = sliced()+ else:+ # Keyed as the reference: vid_out_ada (layers=["out"], idx 0) aliases the blocks' "attn" entry+ # and takes their modulation; slicing its own would be 2*dim wide. The 3B weights expect this.+ mod = cache(f"emb_repeat_{idx}_{branch_tag}", sliced)++ shiftA, scaleA, gateA = mod.unbind(-1)+ if hid_len is None or mod.shape[0] == 1:+ # A single sample's modulation broadcasts over every token, so it never needs materializing.+ return self._modulate(hid, layer, mode, shiftA, scaleA, gateA, norm_eps=norm_eps)++ # The fused norm is out of place, so its per-sample slices land in a fresh buffer.+ out = torch.empty_like(hid) if norm_eps is not None else hid+ offset = 0+ for i, length in enumerate(cache(f"hid_len_list_{branch_tag}", hid_len.tolist)):+ modulated = self._modulate(+ hid[offset:offset + length], layer, mode, shiftA[i], scaleA[i], gateA[i], norm_eps=norm_eps,+ )+ if norm_eps is not None:+ out[offset:offset + length] = modulated+ offset += length+ return out+ def _modulate(+ self,+ hid: torch.FloatTensor,+ layer: str,+ mode: str,+ shiftA: torch.FloatTensor,+ scaleA: torch.FloatTensor,+ gateA: torch.FloatTensor,+ norm_eps: Optional[float] = None,+ ) -> torch.FloatTensor:if mode == "in":- shiftB = comfy.ops.cast_to_input(shiftB, hid)- scaleB = comfy.ops.cast_to_input(scaleB, hid)+ shiftB = comfy.ops.cast_to_input(getattr(self, f"{layer}_shift"), hid)+ scaleB = comfy.ops.cast_to_input(getattr(self, f"{layer}_scale"), hid)+ if norm_eps is not None:+ # rms_adaln applies rmsnorm(x) * (1 + scale) + shift, so hand it scale - 1.+ return comfy.quant_ops.ck.rms_adaln(hid, scaleA + scaleB - 1.0, shiftA + shiftB, norm_eps)return hid.mul_(scaleA + scaleB).add_(shiftA + shiftB)if mode == "out":+ gateB = getattr(self, f"{layer}_gate", None)if gateB is not None:- gateB = comfy.ops.cast_to_input(gateB, hid)- return hid.mul_(gateA + gateB)- else:- return hid.mul_(gateA)+ return hid.mul_(gateA + comfy.ops.cast_to_input(gateB, hid))+ return hid.mul_(gateA)raise ValueError(f"Unknown AdaSingle mode: {mode}")diff --git a/comfy/ldm/seedvr/vae.py b/comfy/ldm/seedvr/vae.pyindex 7a8070b65..6c8032855 100644--- a/comfy/ldm/seedvr/vae.py+++ b/comfy/ldm/seedvr/vae.py@@ -1,17 +1,14 @@-from typing import Literal, Optional, Tuple+from typing import Optionalimport torchimport torch.nn as nnimport torch.nn.functional as Ffrom torch import Tensor-from contextlib import contextmanagerfrom comfy.utils import ProgressBarfrom comfy.ldm.seedvr.constants import (BYTEDANCE_BLOCK_OUT_CHANNELS,BYTEDANCE_GN_CHUNKS_FP16,BYTEDANCE_GN_CHUNKS_FP32,- BYTEDANCE_LOGVAR_CLAMP_MAX,- BYTEDANCE_LOGVAR_CLAMP_MIN,BYTEDANCE_SLICING_SAMPLE_MIN,BYTEDANCE_VAE_CONV_MEM_GIB,BYTEDANCE_VAE_NORM_MEM_GIB,@@ -20,32 +17,63 @@ from comfy.ldm.seedvr.constants import (BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE,BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE,SEEDVR2_LATENT_CHANNELS,+ SEEDVR2_VAE_CACHE_QUANT_BYTES,+ SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL,+ SEEDVR2_DECODE_FIXED_BYTES,+ SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL,+ SEEDVR2_DECODE_LAB_BYTES_PER_OUTPUT_PIXEL,+ SEEDVR2_ENCODE_BYTES_PER_PIXEL,+ SEEDVR2_ENCODE_FIXED_BYTES,+ SEEDVR2_EAGER_DECODE_FACTOR,+ SEEDVR2_EAGER_ENCODE_FACTOR,+ SEEDVR2_MAX_TILE_LATENT,+ SEEDVR2_MIN_TILE_LATENT,+ SEEDVR2_TILE_MEM_HEADROOM,)-from comfy.ldm.modules.attention import optimized_attentionfrom comfy.ldm.modules.diffusionmodules.model import vae_attention+import collections+import dataclassesimport mathfrom enum import Enum-import loggingimport comfy.model_managementimport comfy.ops-ops = comfy.ops.manual_cast+import comfy.quant_ops+import comfy_kitchen++_CL = torch.channels_last_3d+++def _ck_eligible(x):+ """fp16 on CUDA: the channels_last_3d path; the kitchen conv itself needs the fp16-accumulate opt-in."""+ return x.is_cuda and x.dtype == torch.float16 and x.dim() == 5+++def _fp16_accumulate_wanted(x):+ """The kitchen conv accumulates in fp16 (1.4-1.9x over cuDNN on this decoder, 73.9 -> 58 dB+ against fp32); it is taken only when the user opted into fp16 accumulation, as the MiniMax H3+ VAE and the fp16 GEMMs do."""+ return comfy.ops._fp16_linear_wanted(x)-def _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap, temporal_scale=1):- if temporal_size is None:- return None+def _ndhwc_like(t):+ """channels_last_3d, or a frame window of such a tensor (batch of one)."""+ return t.is_contiguous(memory_format=_CL) or (t.dim() == 5 and t.stride(1) == 1 and t.stride(4) == t.size(1))- temporal_size = int(temporal_size)- if temporal_size <= 0:- return None- temporal_overlap = max(0, int(temporal_overlap or 0))- temporal_overlap = min(temporal_overlap, temporal_size - 1)- temporal_step = temporal_size - temporal_overlap- temporal_scale = max(1, int(temporal_scale))- return max(1, math.ceil(temporal_step / temporal_scale))+def _match_layout(t, ref):+ """Return ``t`` in ``ref``'s memory format, so a cat that lists it first keeps the layout."""+ if ref.is_contiguous(memory_format=_CL) and not t.is_contiguous(memory_format=_CL):+ return t.contiguous(memory_format=_CL)+ return t+++def _ndhwc_filter(conv):+ """Store a resident filter NDHWC once, so cast_bias_weight hands it back without a per-call copy."""+ if conv.weight.is_cuda and not conv.weight.is_contiguous(memory_format=_CL):+ conv.weight.data = conv.weight.data.contiguous(memory_format=_CL)+ops = comfy.ops.manual_castdef _seedvr2_clamped_spatial_overlap(overlap, tile_size):@@ -54,78 +82,35 @@ def _seedvr2_clamped_spatial_overlap(overlap, tile_size):return min(overlap, tile_size - 1)-def tiled_vae(- x,- vae_model,- tile_size=(512, 512),- tile_overlap=(64, 64),- temporal_size=16,- temporal_overlap=0,- encode=True,-):- if x.ndim != 5:- x = x.unsqueeze(2)-+def tiled_vae(x, vae_model, tile_size=(512, 512), tile_overlap=(64, 64), encode=True):+ """Spatial tiles, each through the model's own temporally sliced encode or decode, blended with+ cosine ramps over the overlaps in fp32 on the intermediate device, as comfy.utils.tiled_scale does:+ kept on the GPU, the full-size blend buffer was 12 bytes per output pixel per frame (6 GiB at 4K/61f)."""_, _, d, h, w = x.shape-- sf_s = getattr(vae_model, "spatial_downsample_factor", BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE)- sf_t = getattr(vae_model, "temporal_downsample_factor", BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE)- if encode:- slicing_attr = "slicing_sample_min_size"- slicing_min_size = _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap)- else:- slicing_attr = "slicing_latent_min_size"- slicing_min_size = _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap, sf_t)+ sf_s, sf_t = vae_model.spatial_downsample_factor, vae_model.temporal_downsample_factorif encode:ti_h, ti_w = tile_sizeov_h = _seedvr2_clamped_spatial_overlap(tile_overlap[0], ti_h)ov_w = _seedvr2_clamped_spatial_overlap(tile_overlap[1], ti_w)- blend_ov_h = max(0, ov_h // sf_s)- blend_ov_w = max(0, ov_w // sf_s)+ blend_ov_h, blend_ov_w = ov_h // sf_s, ov_w // sf_starget_d = (d + sf_t - 1) // sf_t- target_h = (h + sf_s - 1) // sf_s- target_w = (w + sf_s - 1) // sf_s+ target_h, target_w = (h + sf_s - 1) // sf_s, (w + sf_s - 1) // sf_selse:- ti_h = max(1, tile_size[0] // sf_s)- ti_w = max(1, tile_size[1] // sf_s)+ ti_h, ti_w = max(1, tile_size[0] // sf_s), max(1, tile_size[1] // sf_s)ov_h = _seedvr2_clamped_spatial_overlap(tile_overlap[0] // sf_s, ti_h)ov_w = _seedvr2_clamped_spatial_overlap(tile_overlap[1] // sf_s, ti_w)- blend_ov_h = ov_h * sf_s- blend_ov_w = ov_w * sf_s-+ blend_ov_h, blend_ov_w = ov_h * sf_s, ov_w * sf_starget_d = max(1, d * sf_t - (sf_t - 1))- target_h = h * sf_s- target_w = w * sf_s-- stride_h = max(1, ti_h - ov_h)- stride_w = max(1, ti_w - ov_w)-- storage_device = vae_model.device- result = None- count = None- def run_temporal_chunks(spatial_tile, model=vae_model):- t_chunk = spatial_tile.contiguous()- old_device = getattr(model, "device", None)- model.device = t_chunk.device- old_slicing_min_size = getattr(model, slicing_attr, None)- if old_slicing_min_size is not None and slicing_min_size is not None:- if slicing_min_size <= 0:- setattr(model, slicing_attr, t_chunk.shape[2])- else:- setattr(model, slicing_attr, slicing_min_size)- try:- if encode:- out = model.encode(t_chunk)- else:- out = model.decode_(t_chunk)- finally:- if old_slicing_min_size is not None and slicing_min_size is not None:- setattr(model, slicing_attr, old_slicing_min_size)- if old_device is not None:- model.device = old_device- if out.ndim == 4:+ target_h, target_w = h * sf_s, w * sf_s+ stride_h, stride_w = max(1, ti_h - ov_h), max(1, ti_w - ov_w)+ storage_device = comfy.model_management.intermediate_device()++ def run(tile):+ tile = tile.contiguous()+ out = vae_model.encode(tile) if encode else vae_model.slicing_decode(tile)+ if out.ndim == 4: # encode drops the time axis of a single-frame latentout = out.unsqueeze(2)- return out.to(storage_device)+ return out.to(storage_device).float()ramp_cache = {}def get_ramp(steps):@@ -145,87 +130,51 @@ def tiled_vae(continuetile_ranges.append((y_idx, y_end, x_idx, x_end))- total_tiles = len(tile_ranges)- bar = ProgressBar(total_tiles)- single_spatial_tile = h <= ti_h and w <= ti_w-- def run_tile(tile_index, tile_range):- y_idx, y_end, x_idx, x_end = tile_range- tile_x = x[:, :, :, y_idx:y_end, x_idx:x_end]- tile_out = run_temporal_chunks(tile_x)- return tile_index, y_idx, y_end, x_idx, x_end, tile_out-- ordered_tile_outputs = (- run_tile(tile_index, tile_range)- for tile_index, tile_range in enumerate(tile_ranges)- )-- for _, y_idx, y_end, x_idx, x_end, tile_out in ordered_tile_outputs:-- if single_spatial_tile:- result = tile_out[:, :, :target_d, :target_h, :target_w]- if result.device != x.device or result.dtype != x.dtype:- result = result.to(device=x.device, dtype=x.dtype)- if x.shape[2] == 1 and sf_t == 1:- result = result.squeeze(2)- bar.update(1)- return result+ bar = ProgressBar(len(tile_ranges))+ if h <= ti_h and w <= ti_w:+ result = run(x)[:, :, :target_d, :target_h, :target_w]+ bar.update(1)+ return result.to(dtype=x.dtype)+ result = count = None+ for y_idx, y_end, x_idx, x_end in tile_ranges:+ tile_out = run(x[:, :, :, y_idx:y_end, x_idx:x_end])if result is None:- b_out, c_out = tile_out.shape[0], tile_out.shape[1]- result = torch.zeros((b_out, c_out, target_d, target_h, target_w), device=storage_device, dtype=torch.float32)+ result = torch.zeros((tile_out.shape[0], tile_out.shape[1], target_d, target_h, target_w), device=storage_device, dtype=torch.float32)count = torch.zeros((1, 1, 1, target_h, target_w), device=storage_device, dtype=torch.float32)- if encode:- ys, ye = y_idx // sf_s, (y_idx // sf_s) + tile_out.shape[3]- xs, xe = x_idx // sf_s, (x_idx // sf_s) + tile_out.shape[4]- cur_ov_h = max(0, min(blend_ov_h, tile_out.shape[3] // 2))- cur_ov_w = max(0, min(blend_ov_w, tile_out.shape[4] // 2))- else:- ys, ye = y_idx * sf_s, (y_idx * sf_s) + tile_out.shape[3]- xs, xe = x_idx * sf_s, (x_idx * sf_s) + tile_out.shape[4]- cur_ov_h = max(0, min(blend_ov_h, tile_out.shape[3] // 2))- cur_ov_w = max(0, min(blend_ov_w, tile_out.shape[4] // 2))+ ys = y_idx // sf_s if encode else y_idx * sf_s+ xs = x_idx // sf_s if encode else x_idx * sf_s+ ye, xe = ys + tile_out.shape[3], xs + tile_out.shape[4]+ cur_ov_h = min(blend_ov_h, tile_out.shape[3] // 2)+ cur_ov_w = min(blend_ov_w, tile_out.shape[4] // 2)w_h = torch.ones((tile_out.shape[3],), device=storage_device)w_w = torch.ones((tile_out.shape[4],), device=storage_device)-if cur_ov_h > 0:r = get_ramp(cur_ov_h)if y_idx > 0:w_h[:cur_ov_h] = rif y_end < h:w_h[-cur_ov_h:] = 1.0 - r-if cur_ov_w > 0:r = get_ramp(cur_ov_w)if x_idx > 0:w_w[:cur_ov_w] = rif x_end < w:w_w[-cur_ov_w:] = 1.0 - r-- final_weight = w_h.view(1,1,1,-1,1) * w_w.view(1,1,1,1,-1)+ final_weight = w_h.view(1, 1, 1, -1, 1) * w_w.view(1, 1, 1, 1, -1)valid_d = min(tile_out.shape[2], result.shape[2])- tile_out = tile_out[:, :, :valid_d, :, :]-+ tile_out = tile_out[:, :, :valid_d]tile_out.mul_(final_weight)-result[:, :, :valid_d, ys:ye, xs:xe] += tile_outcount[:, :, :, ys:ye, xs:xe] += final_weight-del tile_out, final_weight, w_h, w_wbar.update(1)result.div_(count.clamp(min=1e-6))-- if result.device != x.device or result.dtype != x.dtype:- result = result.to(device=x.device, dtype=x.dtype)-- if x.shape[2] == 1 and sf_t == 1:- result = result.squeeze(2)-- return result+ return result.to(dtype=x.dtype)_NORM_LIMIT = float("inf")def get_norm_limit():@@ -238,14 +187,235 @@ def set_norm_limit(value: Optional[float] = None):value = float("inf")_NORM_LIMIT = value-@contextmanager-def ignore_padding(model):- orig_padding = model.padding- model.padding = (0, 0, 0)- try:- yield- finally:- model.padding = orig_padding++class CausalMemoryCache:+ """Per-convolution temporal tails, kept between the slices of one encode or decode.++ Large channels_last CUDA tails are int8-packed (ConvRot, per-token scales); with ``offload`` the+ packed tails live in host memory and the next reader's tail is prefetched while this one runs.+ A tail can be a ring of per-frame entries, so extending it packs one frame. free() ends the pass.+ """++ CONVROT_GROUPSIZE = 256++ def __init__(self, offload=False):+ self.offload = offload+ self.plain = {}+ self.packed = {} # key -> (qdata, params, shape), on the host when offloaded+ self._order = [] # keys in packing order, which is the order the next slice reads them+ self._staged = {} # key -> (qdata, params, event) on its way back to the GPU+ self._spare = {} # (shape, dtype) -> host buffers of dropped entries+ self._pinned = []+ self._landing = None # (event, sources) of the copy to the host still in flight+ self._stream = None+ self._device = None+ self._rings = {} # key -> deque of per-frame subkeys, oldest first+ self._ring_serial = 0++ def _transfer_stream(self, device):+ """The one stream this cache copies on, so copies reusing a host buffer stay ordered;+ it starts after the compute queued so far. None copies on the compute stream."""+ if self._stream is None:+ self._stream = comfy.model_management.get_offload_stream(device)+ if self._stream is not None:+ self._stream.wait_stream(comfy.model_management.current_stream(device))+ return self._stream++ def _host_like(self, t):+ spare = self._spare.get((t.shape, t.dtype))+ if spare:+ return spare.pop()+ host = torch.empty(t.shape, dtype=t.dtype, device="cpu")+ if comfy.model_management.pin_memory(host, evict_active=False):+ self._pinned.append(host)+ return host++ def _to_host(self, *tensors):+ """Start copying ``tensors`` to host buffers; they stay alive until the copy lands.+ One copy in flight at a time: sources held for copies the CPU ran ahead of are what grows+ the peak (2x at 720p on either allocator); record_stream instead grows cudaMallocAsync's pool."""+ if self._landing is not None:+ self._landing[0].synchronize()+ self._device = tensors[0].device+ stream = self._transfer_stream(self._device)+ host = [comfy.model_management.cast_to(t, None, torch.device("cpu"), non_blocking=True, stream=stream, r=self._host_like(t))+ for t in tensors]+ event = torch.cuda.Event()+ event.record(stream)+ self._landing = (event, tensors)+ return host++ def park(self, owner, frames):+ """Hold channels_last ``frames`` unpacked in host memory until popped, the first already on+ its way back; returns their keys."""+ keys = [(owner, "parked", i) for i in range(len(frames))]+ host = self._to_host(*(f.permute(0, 2, 3, 4, 1) for f in frames))+ for key, h, f in zip(keys, host, frames):+ self.packed[key] = (h, None, f.shape)+ if key not in self._order:+ self._order.append(key)+ if keys:+ self._staged[keys[0]] = self._bring_back(keys[0], self._device)+ return keys++ def _bring_back(self, key, device):+ """Start the copy of an offloaded entry back to the GPU; returns (data, params, event)."""+ hq, params, _ = self.packed[key]+ # allocated on the compute stream, which the copy then waits on: queued work may still be releasing the block+ qdata = torch.empty_like(hq, device=device)+ stream = self._transfer_stream(device)+ comfy.model_management.cast_to(hq, None, device, non_blocking=True, stream=stream, r=qdata)+ if params is not None:+ scale = torch.empty_like(params.scale, device=device)+ comfy.model_management.cast_to(params.scale, None, device, non_blocking=True, stream=stream, r=scale)+ params = dataclasses.replace(params, scale=scale)+ event = torch.cuda.Event()+ event.record(stream)+ return qdata, params, event++ def _top(self, key):+ return key[0] if isinstance(key, tuple) and len(key) == 2 and key[0] in self._rings else key++ def _prefetch_after(self, top, device):+ """The reader after this one is next in line; have its tail on the way while this one runs."""+ try:+ nxt = self._order[self._order.index(top) + 1]+ except (ValueError, IndexError):+ return+ for sub in self._rings.get(nxt, (nxt,)):+ if sub in self.packed and sub not in self._staged:+ self._staged[sub] = self._bring_back(sub, device)++ def push_frames(self, key, frames, keep):+ """Extend ``key``'s tail by ``frames``, keeping the last ``keep``; each frame is its own entry,+ stored as given (the buffered convs pass frames of their padded buffer, border included)."""+ ring = self._rings.setdefault(key, collections.deque())+ for j in range(frames.size(2)):+ self._ring_serial += 1+ sub = (key, self._ring_serial)+ frame = frames[:, :, j:j + 1]+ if not self._packable(frame):+ frame = frame.clone(memory_format=_CL) # held or sent to the host: not as a view of the caller's buffer+ self[sub] = frame+ ring.append(sub)+ self.keep_last(key, keep)+ if key not in self._order:+ self._order.append(key)++ def keep_last(self, key, keep):+ """Drop all but the newest ``keep`` frames of ``key``'s ring."""+ ring = self._rings.get(key, ())+ while len(ring) > keep:+ self.discard(ring.popleft())++ def _packable(self, value):+ return (+ torch.is_tensor(value)+ and value.dim() == 5+ and value.is_cuda+ and value.is_contiguous(memory_format=_CL)+ and value.shape[1] % 4 == 0 # ConvRot rotates power-of-4 channel groups+ and value.numel() * value.element_size() >= SEEDVR2_VAE_CACHE_QUANT_BYTES+ )++ def __setitem__(self, key, value):+ self.discard(key) # a whole tail replaces a ring too+ if self._packable(value):+ c = value.shape[1]+ g = self.CONVROT_GROUPSIZE+ while g > 4 and c % g:+ g //= 4+ # rows of the NDHWC view are tokens: the rotation spreads each token's outliers over its channels+ qdata, params = comfy.quant_ops.TensorWiseINT8Layout.quantize(+ value.permute(0, 2, 3, 4, 1).reshape(-1, c), is_weight=True, per_channel=True, convrot=True, convrot_groupsize=g)+ elif self.offload and value.is_cuda and value.dim() == 5 and value.is_contiguous(memory_format=_CL):+ # too small to be worth packing, but off the GPU all the same: long-lived small blocks+ # fragment the big activations' segments (720p needed 2 GiB more free VRAM)+ qdata, params = value.permute(0, 2, 3, 4, 1), None+ else:+ self.plain[key] = value+ return+ if self.offload:+ if params is None:+ qdata, = self._to_host(qdata)+ else:+ qdata, scale = self._to_host(qdata, params.scale)+ params = dataclasses.replace(params, scale=scale)+ top = self._top(key)+ if top not in self._order:+ self._order.append(top)+ self.packed[key] = (qdata, params, value.shape)++ def get(self, key, default=None):+ """A whole entry; a ring's frames are read with for_each_frame."""+ if key not in self.plain and key not in self.packed:+ return default+ out = self._get_entry(key)+ if self.offload and self._device is not None:+ self._prefetch_after(key, self._device)+ return out++ def for_each_frame(self, key, fn, count=None):+ """``fn(i, frame)`` over the first ``count`` frames of ``key``'s ring one at a time, never holding+ the whole unpacked tail."""+ for i, sub in enumerate(list(self._rings[key])[:count]):+ fn(i, self._get_entry(sub))+ if self.offload and self._device is not None:+ self._prefetch_after(key, self._device)++ def _get_entry(self, key):+ if key in self.plain:+ return self.plain[key]+ qdata, params, (b, c, t, h, w) = self.packed[key]+ if self.offload:+ qdata, params, event = self._staged.pop(key, None) or self._bring_back(key, self._device)+ comfy.model_management.current_stream(self._device).wait_event(event)+ if params is None: # parked unpacked, NDHWC+ return qdata.permute(0, 4, 1, 2, 3)+ flat = comfy.quant_ops.TensorWiseINT8Layout.dequantize(qdata, params)+ return flat.view(b, t, h, w, c).permute(0, 4, 1, 2, 3)++ def pop(self, key, default=None):+ value = self.get(key, default)+ self.discard(key)+ return value++ def discard(self, key):+ """Drop an entry, or a ring's frames, without bringing it back; its host buffers are reused."""+ for sub in self._rings.pop(key, ()):+ self.discard(sub)+ self.plain.pop(key, None)+ entry = self.packed.pop(key, None)+ staged = self._staged.pop(key, None)+ if staged is not None: # its GPU copy is freed on the compute stream+ comfy.model_management.current_stream(self._device).wait_event(staged[2])+ if entry is not None and self.offload:+ for t in (entry[0],) if entry[1] is None else (entry[0], entry[1].scale):+ self._spare.setdefault((t.shape, t.dtype), []).append(t)++ def free(self):+ """Unpin the host buffers at the end of the pass; the cache is empty afterwards."""+ if self._pinned:+ torch.cuda.synchronize(self._device) # no copy may still be using a buffer when it is unpinned+ for t in self._pinned:+ comfy.model_management.unpin_memory(t)+ self._pinned, self._spare, self._staged, self._landing = [], {}, {}, None+ self.plain, self.packed, self._rings, self._order = {}, {}, {}, []++ def __contains__(self, key):+ return key in self.plain or key in self.packed or key in self._rings+++def _eager_factor(dtype, factor):+ """The memory figures are fitted on the fp16 channels-last path; other dtypes take the eager path."""+ return 1 if dtype == torch.float16 else factor * dtype.itemsize / 2+++def _offload_caches_for(frame_pixels):+ """Whether a pass over this frame area keeps its packed caches in host memory: when there is+ room for them, reclaiming RAM from inactive models if needed."""+ return comfy.model_management.ensure_pin_budget(frame_pixels * SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL, evict_active=False)+class MemoryState(Enum):DISABLED = 0@@ -253,6 +423,38 @@ class MemoryState(Enum):ACTIVE = 2UNSET = 3++def _frame_chunked(owner, run, source, chunk, memory_state, memory_cache, frame_pixels):+ """``run(source, memory_state, memory_cache)`` over a causal, spatial-only stage ``chunk`` frames at+ a time, the cache carrying state between chunks: exact, and only a chunk's activations are live.+ ``source`` is a 1-list the stage pops, so no caller keeps its input alive (the input of a module+ call is held until that module returns)."""+ if not chunk or source[0].size(2) <= chunk:+ return run(source, memory_state, memory_cache)+ local_cache = memory_state == MemoryState.DISABLED or memory_cache is None+ if local_cache:+ # under the same host-memory policy as a sliced pass+ memory_cache = CausalMemoryCache(offload=_offload_caches_for(frame_pixels))+ memory_state = MemoryState.INITIALIZING+ outs = []+ try:+ pieces = list(source.pop().split(chunk, dim=2))+ if memory_cache.offload and pieces[0].is_contiguous(memory_format=_CL):+ # the rest waits in host memory, fetched a chunk ahead: resident, the whole input sat+ # under every chunk's peak (2.5 GiB for the decoder's tail at 1080p)+ pieces[0] = pieces[0].clone(memory_format=_CL)+ pieces[1:] = memory_cache.park(owner, pieces[1:])++ def take(i):+ piece, pieces[i] = pieces[i], None+ return [piece if torch.is_tensor(piece) else memory_cache.pop(piece)]+ for i in range(len(pieces)):+ outs.append(run(take(i), memory_state if i == 0 else MemoryState.ACTIVE, memory_cache))+ finally:+ if local_cache:+ memory_cache.free()+ return torch.cat(outs, dim=2)+def get_cache_size(conv_module, input_len, pad_len, dim=0):dilated_kernel_size = conv_module.dilation[dim] * (conv_module.kernel_size[dim] - 1) + 1output_len = (input_len + pad_len - dilated_kernel_size) // conv_module.stride[dim] + 1@@ -268,225 +470,300 @@ def get_cache_size(conv_module, input_len, pad_len, dim=0):)return cache_len-class DiagonalGaussianDistribution(object):- def __init__(self, parameters: torch.Tensor):- self.parameters = parameters- self.mean, self.logvar = torch.chunk(parameters, 2, dim=1)- self.logvar = torch.clamp(self.logvar, BYTEDANCE_LOGVAR_CLAMP_MIN, BYTEDANCE_LOGVAR_CLAMP_MAX)-- def mode(self):- return self.mean--class SpatialNorm(nn.Module):- def __init__(- self,- f_channels: int,- zq_channels: int,- ):- super().__init__()- self.norm_layer = ops.GroupNorm(num_channels=f_channels, num_groups=32, eps=1e-6, affine=True)- self.conv_y = ops.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0)- self.conv_b = ops.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0)-- def forward(self, f: torch.Tensor, zq: torch.Tensor) -> torch.Tensor:- f_size = f.shape[-2:]- zq = F.interpolate(zq, size=f_size, mode="nearest")- norm_f = self.norm_layer(f)- new_f = norm_f * self.conv_y(zq) + self.conv_b(zq)- return new_f-class Attention(nn.Module):- def __init__(- self,- query_dim: int,- heads: int = 8,- dim_head: int = 64,- bias: bool = False,- norm_num_groups: Optional[int] = None,- spatial_norm_dim: Optional[int] = None,- out_bias: bool = True,- eps: float = 1e-5,- rescale_output_factor: float = 1.0,- residual_connection: bool = False,- ):+ """The mid-block's single-head self-attention over each (c, h, w) frame, residual included."""+ def __init__(self, query_dim, norm_num_groups, eps):super().__init__()-- self.inner_dim = dim_head * heads- self.rescale_output_factor = rescale_output_factor- self.residual_connection = residual_connection- self.out_dim = query_dim- self.heads = heads-- if norm_num_groups is not None:- self.group_norm = ops.GroupNorm(num_channels=query_dim, num_groups=norm_num_groups, eps=eps, affine=True)- else:- self.group_norm = None-- if spatial_norm_dim is not None:- self.spatial_norm = SpatialNorm(f_channels=query_dim, zq_channels=spatial_norm_dim)- else:- self.spatial_norm = None-- self.to_q = ops.Linear(query_dim, self.inner_dim, bias=bias)- self.to_k = ops.Linear(query_dim, self.inner_dim, bias=bias)- self.to_v = ops.Linear(query_dim, self.inner_dim, bias=bias)- self.to_out = nn.ModuleList([])- self.to_out.append(ops.Linear(self.inner_dim, self.out_dim, bias=out_bias))- self.to_out.append(nn.Identity())-+ self.group_norm = ops.GroupNorm(num_channels=query_dim, num_groups=norm_num_groups, eps=eps, affine=True)+ self.to_q = ops.Linear(query_dim, query_dim)+ self.to_k = ops.Linear(query_dim, query_dim)+ self.to_v = ops.Linear(query_dim, query_dim)+ self.to_out = nn.ModuleList([ops.Linear(query_dim, query_dim)])self.optimized_vae_attention = vae_attention()- def forward(- self,- hidden_states: torch.Tensor,- temb: Optional[torch.Tensor] = None,- ) -> torch.Tensor:-- residual = hidden_states- if self.spatial_norm is not None:- hidden_states = self.spatial_norm(hidden_states, temb)-- input_ndim = hidden_states.ndim-- if input_ndim == 4:- batch_size, channel, height, width = hidden_states.shape- hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)-- batch_size = hidden_states.shape[0]-- if self.group_norm is not None:- hidden_states = self.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)-- query = self.to_q(hidden_states)- key = self.to_k(hidden_states)- value = self.to_v(hidden_states)-- inner_dim = key.shape[-1]- head_dim = inner_dim // self.heads-- query = query.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)-- key = key.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)- value = value.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)-- if input_ndim == 4 and self.heads == 1:- query = query.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width)- key = key.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width)- value = value.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width)- hidden_states = self.optimized_vae_attention(query, key, value).reshape(batch_size, self.heads, head_dim, height * width).transpose(2, 3)- else:- hidden_states = optimized_attention(query, key, value, heads = self.heads, skip_reshape=True, skip_output_reshape=True)-- hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, self.heads * head_dim)- hidden_states = hidden_states.to(query.dtype)-- hidden_states = self.to_out[0](hidden_states)- hidden_states = self.to_out[1](hidden_states)-- if input_ndim == 4:- hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)-- if self.residual_connection:- hidden_states = hidden_states + residual-- hidden_states = hidden_states / self.rescale_output_factor+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:+ b, c, h, w = hidden_states.shape+ x = self.group_norm(hidden_states.view(b, c, h * w)).transpose(1, 2)+ q, k, v = (proj(x).transpose(1, 2).reshape(b, c, h, w) for proj in (self.to_q, self.to_k, self.to_v))+ x = self.optimized_vae_attention(q, k, v).reshape(b, c, h * w).transpose(1, 2)+ return self.to_out[0](x).transpose(1, 2).reshape(b, c, h, w) + hidden_states- return hidden_states--def causal_norm_wrapper(norm_layer: nn.Module, x: torch.Tensor) -> torch.Tensor:- input_dtype = x.dtype- if isinstance(norm_layer, (nn.LayerNorm, nn.RMSNorm)):- if x.ndim == 4:- x = x.permute(0, 2, 3, 1)- x = norm_layer(x)- x = x.permute(0, 3, 1, 2)- return x.to(input_dtype)- if x.ndim == 5:- x = x.permute(0, 2, 3, 4, 1)- x = norm_layer(x)- x = x.permute(0, 4, 1, 2, 3)- return x.to(input_dtype)- if isinstance(norm_layer, (nn.GroupNorm, nn.BatchNorm2d, nn.SyncBatchNorm)):- if x.ndim <= 4:- return norm_layer(x).to(input_dtype)- if x.ndim == 5:- b, c, t, h, w = x.shape- x = x.transpose(1, 2).reshape(b * t, c, h, w)- memory_occupy = x.numel() * x.element_size() / 1024**3- if isinstance(norm_layer, nn.GroupNorm) and memory_occupy > get_norm_limit():- num_chunks = min(BYTEDANCE_GN_CHUNKS_FP16 if x.element_size() == 2 else BYTEDANCE_GN_CHUNKS_FP32, norm_layer.num_groups)- if norm_layer.num_groups % num_chunks != 0:- raise ValueError(- f"SeedVR2 VAE GroupNorm groups must divide chunks: groups={norm_layer.num_groups}, chunks={num_chunks}."- )- num_groups_per_chunk = norm_layer.num_groups // num_chunks-- weights = comfy.ops.cast_to_input(norm_layer.weight, x).chunk(num_chunks, dim=0)- biases = comfy.ops.cast_to_input(norm_layer.bias, x).chunk(num_chunks, dim=0)- x = list(x.chunk(num_chunks, dim=1))- for i, (w, bias) in enumerate(zip(weights, biases)):- x[i] = F.group_norm(x[i], num_groups_per_chunk, w, bias, norm_layer.eps)- x[i] = x[i].to(input_dtype)- x = torch.cat(x, dim=1)- else:- x = norm_layer(x)- x = x.reshape((b, t, x.size(1), x.size(2), x.size(3))).transpose(1, 2)- return x.to(input_dtype)- raise TypeError(f"SeedVR2 VAE unsupported norm layer type: {type(norm_layer).__name__}")--_receptive_field_t = Literal["half", "full"]--def extend_head(tensor, times: int = 2, memory = None):- if memory is not None:- return torch.cat((memory.to(tensor), tensor), dim=2)- if times < 0:- raise ValueError(f"SeedVR2 VAE extend_head expected times >= 0, got {times}.")- if times == 0:- return tensor+def causal_norm_wrapper(norm_layer: nn.GroupNorm, x: torch.Tensor, silu: bool = False) -> torch.Tensor:+ """Per-frame GroupNorm of a (b, c, t, h, w) tensor, with the SiLU folded in where the kernel does both."""+ if _ck_eligible(x) and norm_layer.affine:+ # accepts either layout and emits channels_last_3d, which is where the convolution wants its input+ weight, bias, offload_stream = comfy.ops.cast_bias_weight(norm_layer, x, offloadable=True)+ try:+ return comfy_kitchen.group_norm_silu_pad3d(+ x, weight, bias, norm_layer.num_groups, norm_layer.eps, (0, 0, 0, 0, 0), silu,+ )+ finally:+ comfy.ops.uncast_bias_weight(norm_layer, weight, bias, offload_stream)+ b, c, t, h, w = x.shape+ x = x.transpose(1, 2).reshape(b * t, c, h, w)+ if x.numel() * x.element_size() / 1024**3 > get_norm_limit():+ num_chunks = min(BYTEDANCE_GN_CHUNKS_FP16 if x.element_size() == 2 else BYTEDANCE_GN_CHUNKS_FP32, norm_layer.num_groups)+ if norm_layer.num_groups % num_chunks != 0:+ raise ValueError(+ f"SeedVR2 VAE GroupNorm groups must divide chunks: groups={norm_layer.num_groups}, chunks={num_chunks}."+ )+ num_groups_per_chunk = norm_layer.num_groups // num_chunks++ weights = comfy.ops.cast_to_input(norm_layer.weight, x).chunk(num_chunks, dim=0)+ biases = comfy.ops.cast_to_input(norm_layer.bias, x).chunk(num_chunks, dim=0)+ x = list(x.chunk(num_chunks, dim=1))+ for i, (w, bias) in enumerate(zip(weights, biases)):+ x[i] = F.group_norm(x[i], num_groups_per_chunk, w, bias, norm_layer.eps)+ x = torch.cat(x, dim=1)else:- tile_repeat = [1] * tensor.ndim- tile_repeat[2] = times- return torch.cat(tensors=(torch.tile(tensor[:, :, :1], tile_repeat), tensor), dim=2)--def cache_send_recv(tensor, cache_size, times, memory=None):- recv_buffer = None+ x = norm_layer(x)+ x = x.reshape((b, t, x.size(1), x.size(2), x.size(3))).transpose(1, 2)+ if silu:+ x = F.silu(x)+ return x- if memory is not None:- recv_buffer = memory.to(tensor[0])- elif times > 0:- tile_repeat = [1] * tensor[0].ndim- tile_repeat[2] = times- recv_buffer = torch.tile(tensor[0][:, :, :1], tile_repeat)-- return recv_bufferclass InflatedCausalConv3d(ops.Conv3d):- def __init__(- self,- *args,- inflation_mode,- **kwargs,- ):- self.inflation_mode = inflation_mode+ def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)self.temporal_padding = self.padding[0]self.padding = (0, *self.padding[1:])- self.memory_limit = float("inf")- self.logged_once = False+ self.memory_limit = float("inf") # GiB of padded input above which slicing_forward splits- def set_memory_limit(self, value: float):- self.memory_limit = value+ def _conv_padded(self, x, padding):+ """The plain conv with ``padding`` in place of the module's own, through comfy's _conv_forward."""+ orig_padding, self.padding = self.padding, padding+ try:+ return torch.nn.Conv3d.forward(self, x)+ finally:+ self.padding = orig_padding++ def _conv_without_padding_copy(self, x, prev_cache, padding):+ """Convolve with native symmetric padding and slice off the surplus; None with a halo or stride."""+ if prev_cache is not None or any(st != 1 for st in self.stride):+ return None++ pads = ((padding[4], padding[5]), (padding[2], padding[3]), (padding[0], padding[1]))+ native = tuple(max(lo, hi) for lo, hi in pads)+ out = self._conv_padded(x, native)++ for i, (dim, (lo, hi)) in enumerate(zip((2, 3, 4), pads)):+ width = self.dilation[i] * (self.kernel_size[i] - 1)+ length = x.size(dim) + lo + hi - width+ start = native[i] - lo+ if start or out.size(dim) != length:+ out = out.narrow(dim, start, length)+ return out++ @staticmethod+ def _padded_input(x, prev_cache, padding, concat_dim, channels_last=None):+ """The conv's padded input (halo included) in one buffer, written once; ``copy_`` also casts the halo."""+ pads = ((padding[4], padding[5]), (padding[2], padding[3]), (padding[0], padding[1]))+ cache_len = 0 if prev_cache is None else prev_cache.size(concat_dim)+ if cache_len == 0 and not any(lo or hi for lo, hi in pads):+ return x.contiguous(memory_format=_CL) if channels_last else x++ shape = list(x.shape)+ for dim, (lo, hi) in zip((2, 3, 4), pads):+ shape[dim] += lo + hi+ shape[concat_dim] += cache_len++ if channels_last is None:+ channels_last = x.is_contiguous(memory_format=_CL)+ buf = torch.empty(shape, dtype=x.dtype, device=x.device,+ memory_format=_CL if channels_last else torch.contiguous_format)+ for dim, (lo, hi) in zip((2, 3, 4), pads):+ if lo:+ buf.narrow(dim, 0, lo).zero_()+ if hi:+ buf.narrow(dim, shape[dim] - hi, hi).zero_()++ core = buf+ for dim, (lo, hi) in zip((2, 3, 4), pads):+ core = core.narrow(dim, lo, shape[dim] - lo - hi)+ if cache_len:+ core.narrow(concat_dim, 0, cache_len).copy_(prev_cache)+ core.narrow(concat_dim, cache_len, x.size(concat_dim)).copy_(x)+ else:+ core.copy_(x)+ return buf++ def _ck_applies(self, x):+ """Whether this conv runs on the channels_last_3d path; any input layout, the pad copy lands it there."""+ return (_ck_eligible(x) and self.padding_mode == "zeros" and self.groups == 1+ and all(d == 1 for d in self.dilation))++ def _ck_conv(self, padded):+ """Run an already-padded channels_last_3d input through the conv, kitchen or cuDNN."""+ _ndhwc_filter(self)+ weight, bias, offload_stream = comfy.ops.cast_bias_weight(self, padded, offloadable=True)+ try:+ weight_cl = weight.contiguous(memory_format=_CL)+ if _fp16_accumulate_wanted(padded):+ return comfy_kitchen.fp16_conv3d(padded, weight_cl, bias, stride=tuple(self.stride))+ # cuDNN keeps channels_last_3d in and out, so the layout still saves its transposes+ return F.conv3d(padded, weight_cl, bias, stride=tuple(self.stride))+ finally:+ comfy.ops.uncast_bias_weight(self, weight, bias, offload_stream)++ def _symmetric_padding(self):+ return tuple(p for p in reversed(self.padding) for _ in range(2))++ # The ck kernels index a buffer with 32-bit byte offsets; a view handed to one call stays+ # under this many elements (2 GiB of fp16), which is one call at 1080p and two at 1440p.+ _CK_MAX_VIEW_ELEMENTS = 2 ** 30++ # _ck_taps runs each halo tap in this many bands of output rows: on cuDNN, which cannot accumulate+ # into the conv's output, a tap's own output is then that fraction of a frame; the kitchen conv+ # accumulates in place, and the bands keep its views under the 32-bit offsets+ _TAP_BANDS = 4++ def _ck_buffered(self, like, x_shape, memory_state, memory_cache, write_body, residual=None, pad=None):+ """Run this conv from one padded buffer: halo in front, ``write_body`` writes the padded frames+ behind it, and the conv reads and writes frame-window views. ``residual`` goes into the kitchen+ conv's epilogue (added explicitly on the cuDNN path). ``pad`` is the buffer's spatial border+ (top, bottom, left, right), the conv's own padding by default. Replicates forward()'s cache handling.+ """+ b, c, t, h, w = x_shape+ _, ph, pw = self.padding+ pad = pad or (ph, ph, pw, pw)+ kt = self.kernel_size[0]+ cache_size = kt - self.stride[0]+ if memory_cache is None:+ memory_cache = CausalMemoryCache() # DISABLED: nothing is kept anyway+ elif memory_state != MemoryState.ACTIVE:+ memory_cache.pop(self, None)+ has_memory = self in memory_cache+ halo = cache_size if has_memory else self.temporal_padding * 2+ if t == 1 and halo == 2 and kt == 3 and tuple(self.stride) == (1, 1, 1) and pad == (ph, ph, pw, pw):+ return self._ck_taps(like, x_shape, memory_state, memory_cache, write_body, residual, has_memory)+ hp, wp = h + pad[0] + pad[1], w + pad[2] + pad[3]++ buf = torch.empty((b, c, halo + t, hp, wp), dtype=like.dtype, device=like.device, memory_format=_CL)+ write_body(buf[:, :, halo:])+ del write_body # and with it the source it captured: nothing but the buffer feeds the conv+ if halo:+ front = buf[:, :, :halo]+ if has_memory:+ # after the body, whose source is freed by then; the frames come with their zero border+ memory_cache.for_each_frame(self, lambda i, frame: front[:, :, i:i + 1].copy_(frame))+ else:+ # the causal head: the first frame repeated, already padded+ front.copy_(buf[:, :, halo:halo + 1].expand(-1, -1, halo, -1, -1))+ if cache_size and memory_state in (MemoryState.INITIALIZING, MemoryState.ACTIVE):+ if has_memory:+ # the halo frames are already the cache's newest entries: push only the new ones+ memory_cache.push_frames(self, buf[:, :, halo:][:, :, -min(t, cache_size):], cache_size)+ else:+ memory_cache.push_frames(self, buf[:, :, -cache_size:], cache_size)- def _conv_forward(self, input, weight, bias, *args, **kwargs):+ _ndhwc_filter(self)+ weight, bias, offload_stream = comfy.ops.cast_bias_weight(self, buf, offloadable=True)+ try:+ weight_cl = weight.contiguous(memory_format=_CL)+ stride = sd, sh, sw = tuple(self.stride)+ out_hw = ((hp - self.kernel_size[1]) // sh + 1) * ((wp - self.kernel_size[2]) // sw + 1)+ fused_residual = None+ # the kitchen's views must stay under its 32-bit offsets; when even one output frame's window+ # does not, cuDNN takes the whole buffer (the kitchen would decline and copy a second output)+ if (not _fp16_accumulate_wanted(buf) or kt * c * hp * wp > self._CK_MAX_VIEW_ELEMENTS+ or weight.size(0) * out_hw > self._CK_MAX_VIEW_ELEMENTS):+ out = F.conv3d(buf, weight_cl, bias, stride=stride) # cuDNN, NDHWC in and out+ else:+ zt = (buf.size(2) - kt) // sd + 1+ out = torch.empty((b, weight.size(0), zt, (hp - self.kernel_size[1]) // sh + 1, (wp - self.kernel_size[2]) // sw + 1),+ dtype=buf.dtype, device=buf.device, memory_format=_CL)+ chunk = min(zt, (self._CK_MAX_VIEW_ELEMENTS // (c * hp * wp) - kt) // sd + 1,+ self._CK_MAX_VIEW_ELEMENTS // (weight.size(0) * out_hw))+ if residual is not None and residual.shape == out.shape and residual.dtype == out.dtype and _ndhwc_like(residual):+ fused_residual = residual # otherwise added after: the epilogue wants the output's shape and layout+ for z0 in range(0, zt, chunk):+ z1 = min(zt, z0 + chunk)+ comfy_kitchen.fp16_conv3d(buf[:, :, z0 * sd:(z1 - 1) * sd + kt], weight_cl, bias,+ residual=None if fused_residual is None else fused_residual[:, :, z0:z1],+ stride=stride, out=out[:, :, z0:z1])+ finally:+ comfy.ops.uncast_bias_weight(self, weight, bias, offload_stream)+ if residual is not None and fused_residual is None:+ out += residual+ return out++ def _ck_taps(self, like, x_shape, memory_state, memory_cache, write_body, residual, has_memory):+ """A one-frame chunk against its two-frame halo as three single-frame convs summed into the+ output: the 3-frame buffer was the tail's peak (1.5 GiB lower floor at 1080p). The halo taps+ run straight from the unpacked frames, their padding in place of a buffer; with fp16+ accumulation the kitchen conv adds each tap, and the residual, into the output in its+ epilogue (1.7x over cuDNN plus an add). Without a cache the halo is the frame itself, so the+ taps sum first."""+ b, c, _, h, w = x_shape+ _, ph, pw = self.padding+ buf = torch.empty((b, c, 1, h + 2 * ph, w + 2 * pw), dtype=like.dtype, device=like.device, memory_format=_CL)+ write_body(buf)+ del write_body+ _ndhwc_filter(self)+ weight, bias, offload_stream = comfy.ops.cast_bias_weight(self, buf, offloadable=True)try:- return super()._conv_forward(input, weight, bias, *args, **kwargs)- except NotImplementedError:- # for: Could not run 'aten::cudnn_convolution' with arguments from the 'CPU' backend- if not self.logged_once:- logging.warning("VAE is on CPU for decoding. This is most likely due to not enough memory")- self.logged_once = True- return F.conv3d(input, weight, bias, *args, **kwargs)+ tap = weight[:, :, 2:] if has_memory else weight.sum(dim=2, keepdim=True)+ tap = tap.contiguous(memory_format=_CL)+ fp16_acc = _fp16_accumulate_wanted(buf)+ if fp16_acc:+ fused = (residual is not None and residual.shape == (b, weight.size(0), 1, h, w)+ and residual.dtype == buf.dtype and _ndhwc_like(residual))+ out = comfy_kitchen.fp16_conv3d(buf, tap, bias, residual=residual if fused else None)+ if fused:+ residual = None+ else:+ out = F.conv3d(buf, tap, bias)+ if memory_state in (MemoryState.INITIALIZING, MemoryState.ACTIVE):+ # pushed now so the buffer can go; three kept, as the oldest is still this call's first tap+ memory_cache.push_frames(self, buf if has_memory else buf.expand(-1, -1, 2, -1, -1), 3)+ del buf+ if has_memory:+ rows = -(-out.size(3) // self._TAP_BANDS)++ def halo_tap(i, frame):+ tap_i = weight[:, :, i:i + 1].contiguous(memory_format=_CL)+ for r0 in range(0, out.size(3), rows):+ r1 = min(out.size(3), r0 + rows)+ band, x = out[:, :, :, r0:r1], frame[:, :, :, r0:r1 + 2 * ph]+ if fp16_acc:+ comfy_kitchen.fp16_conv3d(x, tap_i, residual=band, out=band)+ else:+ band.add_(F.conv3d(x, tap_i))+ memory_cache.for_each_frame(self, halo_tap, count=2)+ memory_cache.keep_last(self, 2)+ finally:+ comfy.ops.uncast_bias_weight(self, weight, bias, offload_stream)+ if residual is not None:+ out += residual+ return out++ def _ck_buffered_applies(self, x, memory_state):+ """Whether _ck_buffered can take this call: the frame-offset write needs a batch of one."""+ return self._ck_applies(x) and memory_state != MemoryState.UNSET and x.size(0) == 1++ def fused_norm_conv(self, source, norm, memory_state, memory_cache, residual=None):+ """GroupNorm + SiLU written straight into the conv's buffer; ``source`` (a 1-list) is emptied+ once read so the activation is freed early. None when the fused path does not apply."""+ x = source[0]+ if not (self._ck_buffered_applies(x, memory_state) and isinstance(norm, nn.GroupNorm)+ and norm.affine):+ return None+ _, ph, pw = self.padding+ like, shape = x.new_empty(0), x.shape+ del x+ gw, gb, offload_stream = comfy.ops.cast_bias_weight(norm, like, offloadable=True)++ def write_body(dst):+ comfy_kitchen.group_norm_silu_pad3d(+ source.pop(), gw, gb, norm.num_groups, norm.eps, (pw, pw, ph, ph, 0), True,+ zero_pad=True, out=dst,+ )+ try:+ return self._ck_buffered(like, shape, memory_state, memory_cache, write_body, residual=residual)+ finally:+ comfy.ops.uncast_bias_weight(norm, gw, gb, offload_stream)def memory_limit_conv(self,@@ -496,11 +773,6 @@ class InflatedCausalConv3d(ops.Conv3d):padding=(0, 0, 0, 0, 0, 0),prev_cache=None,):- if math.isinf(self.memory_limit):- if prev_cache is not None:- x = torch.cat([prev_cache, x], dim=split_dim - 1)- return super().forward(x)-shape = list(x.size())if prev_cache is not None:shape[split_dim - 1] += prev_cache.size(split_dim - 1)@@ -508,18 +780,13 @@ class InflatedCausalConv3d(ops.Conv3d):shape[-3 + i] += pad_summemory_occupy = math.prod(shape) * x.element_size() / 1024**3 # GiBif memory_occupy < self.memory_limit or split_dim == x.ndim:- x_concat = x- if prev_cache is not None:- x_concat = torch.cat([prev_cache, x], dim=split_dim - 1)-- def pad_and_forward():- padded = F.pad(x_concat, padding, mode='constant', value=0.0)- if not padded.is_contiguous():- padded = padded.contiguous()- with ignore_padding(self):- return torch.nn.Conv3d.forward(self, padded)-- return pad_and_forward()+ if self._ck_applies(x):+ return self._ck_conv(self._padded_input(x, prev_cache, padding, split_dim - 1, channels_last=True))+ out = self._conv_without_padding_copy(x, prev_cache, padding)+ if out is not None:+ return out+ padded = self._padded_input(x, prev_cache, padding, split_dim - 1)+ return self._conv_padded(padded, (0, 0, 0))num_splits = math.ceil(memory_occupy / self.memory_limit)size_per_split = x.size(split_dim) // num_splits@@ -583,8 +850,9 @@ class InflatedCausalConv3d(ops.Conv3d):memory_cache = {}if memory_state != MemoryState.ACTIVE:memory_cache.pop(self, None)+ # a 1x1x1 conv has no halo to save by splitting: split and concatenated, its output is held twiceif (- math.isinf(self.memory_limit)+ (math.isinf(self.memory_limit) or tuple(self.kernel_size) == (1, 1, 1))and torch.is_tensor(input)):return self.basic_forward(input, memory_state, memory_cache)@@ -594,9 +862,26 @@ class InflatedCausalConv3d(ops.Conv3d):mem_size = self.stride[0] - self.kernel_size[0]memory = memory_cache.get(self) if memory_cache is not None else Noneif (memory is not None) and (memory_state == MemoryState.ACTIVE):- input = extend_head(input, memory=memory, times=-1)+ head = memory.to(input)+ elif self.temporal_padding:+ head = torch.tile(input[:, :, :1], [1, 1, self.temporal_padding * 2, 1, 1])else:- input = extend_head(input, times=self.temporal_padding * 2)+ head = None+ if self._ck_applies(input):+ # The halo and the zero border share one buffer: the kernel needs a padded copy+ # anyway, so the cat with the head is folded into it. On the largest+ # activations that is one full pass and one activation's worth of peak.+ padded = self._padded_input(input, head, self._symmetric_padding(), 2, channels_last=True)+ if mem_size != 0 and memory_state != MemoryState.DISABLED and memory_cache is not None:+ # From the halo'd buffer, head included: a one-frame slice+ # still leaves a two-frame cache. The spatial border is cropped back off.+ _, ph, pw = self.padding+ tail = padded[:, :, mem_size:, ph:padded.size(3) - ph or None, pw:padded.size(4) - pw or None]+ memory_cache[self] = tail.detach().clone(memory_format=_CL)+ elif memory_cache is not None and memory_state != MemoryState.DISABLED:+ memory_cache.pop(self, None)+ return self._ck_conv(padded)+ input = input if head is None else torch.cat((_match_layout(head, input), input), dim=2)next_memory = (input[:, :, mem_size:].detach()if (mem_size != 0 and memory_state != MemoryState.DISABLED)@@ -609,967 +894,516 @@ class InflatedCausalConv3d(ops.Conv3d):memory_cache[self] = next_memoryreturn super().forward(input)- def slicing_forward(- self,- input,- memory_state: MemoryState = MemoryState.UNSET,- memory_cache = None,- ) -> Tensor:- if memory_cache is None:- memory_cache = {}- squeeze_out = False- if torch.is_tensor(input):- input = [input]- squeeze_out = True-+ def slicing_forward(self, input, memory_state, memory_cache):+ """forward() for a finite memory_limit: the halo is handed to memory_limit_conv, which splits+ the conv along H, then W, while its padded input exceeds the limit."""cache_size = self.kernel_size[0] - self.stride[0]- memory = memory_cache.get(self) if memory_cache is not None else None- cache = cache_send_recv(- input, cache_size=cache_size, memory=memory, times=self.temporal_padding * 2- )+ memory = memory_cache.get(self)+ if memory is not None:+ cache = _match_layout(memory.to(input), input)+ elif self.temporal_padding:+ cache = _match_layout(torch.tile(input[:, :, :1], [1, 1, self.temporal_padding * 2, 1, 1]), input)+ else:+ cache = None- if (- memory_state in [MemoryState.INITIALIZING, MemoryState.ACTIVE]- and cache_size != 0- ):- if cache_size > input[-1].size(2) and cache is not None and len(input) == 1:- input[0] = torch.cat([cache, input[0]], dim=2)+ if memory_state in (MemoryState.INITIALIZING, MemoryState.ACTIVE) and cache_size != 0:+ if cache_size > input.size(2) and cache is not None:+ input = torch.cat([cache, input], dim=2)cache = None- if cache_size <= input[-1].size(2):- memory_cache[self] = input[-1][:, :, -cache_size:].detach().contiguous()-- padding = tuple(x for x in reversed(self.padding) for _ in range(2))- for i in range(len(input)):- next_cache = None- cache_size = 0- if i < len(input) - 1:- cache_len = cache.size(2) if cache is not None else 0- cache_size = get_cache_size(self, input[i].size(2) + cache_len, pad_len=0)- if cache_size != 0:- if cache_size > input[i].size(2) and cache is not None:- input[i] = torch.cat([cache, input[i]], dim=2)- cache = None- if cache_size > input[i].size(2):- raise ValueError(f"SeedVR2 VAE cache size {cache_size} exceeds input length {input[i].size(2)}.")- next_cache = input[i][:, :, -cache_size:]-- input[i] = self.memory_limit_conv(- input[i],- padding=padding,- prev_cache=cache- )-- cache = next_cache-- return input[0] if squeeze_out else input+ if cache_size <= input.size(2):+ # clone, not contiguous(): for a batch of one a frame slice of an NDHWC tensor+ # already satisfies the channels_last_3d check, and contiguous() would return the+ # view, pinning the whole activation for the next slice instead of two frames.+ tail = input[:, :, -cache_size:].detach()+ memory_cache[self] = tail.clone(+ memory_format=_CL if input.is_contiguous(memory_format=_CL) else torch.contiguous_format)-def remove_head(tensor: Tensor, times: int = 1) -> Tensor:- if times == 0:- return tensor- return torch.cat(tensors=(tensor[:, :, :1], tensor[:, :, times + 1 :]), dim=2)+ return self.memory_limit_conv(input, padding=self._symmetric_padding(), prev_cache=cache)class Upsample3D(nn.Module):+ """2x spatial (and 2x temporal) upsampling: a 1x1 expansion, its pixel shuffle, a causal 3x3x3 conv."""- def __init__(- self,- channels,- out_channels = None,- inflation_mode = "tail",- temporal_up: bool = False,- spatial_up: bool = True,- ):+ def __init__(self, channels, temporal_up: bool = False):super().__init__()- self.channels = channels- self.out_channels = out_channels or channels-- conv = InflatedCausalConv3d(- self.channels,- self.out_channels,- 3,- padding=1,- inflation_mode=inflation_mode,- )-self.temporal_up = temporal_up- self.spatial_up = spatial_upself.temporal_ratio = 2 if temporal_up else 1- self.spatial_ratio = 2 if spatial_up else 1-- upscale_ratio = (self.spatial_ratio**2) * self.temporal_ratio- self.upscale_conv = ops.Conv3d(- self.channels, self.channels * upscale_ratio, kernel_size=1, padding=0- )-- self.conv = conv-- def forward(- self,- hidden_states: torch.FloatTensor,- memory_state=None,- memory_cache=None,- ) -> torch.FloatTensor:- if hidden_states.shape[1] != self.channels:- raise ValueError(f"SeedVR2 upsample expected {self.channels} channels, got {hidden_states.shape[1]}.")-- hidden_states = self.upscale_conv(hidden_states)+ self.spatial_ratio = 2+ self.upscale_conv = ops.Conv3d(channels, channels * self.spatial_ratio ** 2 * self.temporal_ratio, kernel_size=1, padding=0)+ self.conv = InflatedCausalConv3d(channels, channels, 3, padding=1)++ def _expand(self, x, weight, bias, i, j, t):+ """One shuffle phase of the 1x1 expansion; its output channels are (sr, sr, tr, c). A helper+ so the weight slice's copy is freed before the phase is written out."""+ c = x.size(1)+ k = ((i * self.spatial_ratio + j) * self.temporal_ratio + t) * c+ wk = weight[k:k + c].contiguous(memory_format=_CL)+ bk = None if bias is None else bias[k:k + c]+ if _fp16_accumulate_wanted(x):+ return comfy_kitchen.fp16_conv3d(x, wk, bk)+ return F.conv3d(x, wk, bk)++ def _expand_into(self, dst, x, phases, drop_head):+ """Write the 1x1 expansion of ``x`` for temporal ``phases`` into the conv's padded buffer ``dst``,+ one shuffle phase at a time at its strided place: a phase's output is a quarter (an eighth) of+ the expanded tensor. With ``drop_head`` the first input frame keeps only its first phase, as+ the video's first frame is not doubled."""+ sr, tr = self.spatial_ratio, self.temporal_ratio+ _, ph, pw = self.conv.padding+ h_out, w_out = x.size(3) * sr, x.size(4) * sr+ for dim, lo, hi in ((3, ph, ph + h_out), (4, pw, pw + w_out)):+ if lo:+ dst.narrow(dim, 0, lo).zero_()+ dst.narrow(dim, hi, dst.size(dim) - hi).zero_()+ core = dst[:, :, :, ph:ph + h_out, pw:pw + w_out]+ _ndhwc_filter(self.upscale_conv)+ weight, bias, offload_stream = comfy.ops.cast_bias_weight(self.upscale_conv, x, offloadable=True)+ try:+ for i in range(sr):+ for j in range(sr):+ for t in phases:+ y = self._expand(x, weight, bias, i, j, t)+ frames = core[:, :, :, i::sr, j::sr]+ if len(phases) == 1:+ frames.copy_(y)+ elif not drop_head:+ frames[:, :, t::tr].copy_(y)+ elif t == 0:+ frames[:, :, 0].copy_(y[:, :, 0])+ frames[:, :, 1::tr].copy_(y[:, :, 1:])+ else:+ frames[:, :, t + 1::tr].copy_(y[:, :, 1:])+ del y+ finally:+ comfy.ops.uncast_bias_weight(self.upscale_conv, weight, bias, offload_stream)++ def frames_apply(self, x, memory_state):+ return x.size(2) == 1 and x.is_contiguous(memory_format=_CL) and self.conv._ck_buffered_applies(x, memory_state)++ def frames(self, source, memory_state, memory_cache):+ """This upsampler over a one-frame ``source`` (a 1-list), an output frame at a time: each+ temporal phase's expansion goes straight into the conv's buffer and the conv runs on that frame+ alone. Each output is yielded boxed, so nothing here holds it while the caller uses it."""+ x = source.pop()+ b, c, _, h, w = x.shape+ sr = self.spatial_ratio+ phases = [0] if self.temporal_up and memory_state != MemoryState.ACTIVE else range(self.temporal_ratio)+ for n, t in enumerate(phases):+ state = memory_state if n == 0 else MemoryState.ACTIVE+ box = [self.conv._ck_buffered(x.new_empty(0), (b, c, 1, h * sr, w * sr), state, memory_cache,+ lambda dst, t=t: self._expand_into(dst, x, [t], False))]+ yield box+ del box++ def run(self, source, memory_state, memory_cache):+ """The upsampler on a 1-list ``source`` it empties: a module call would hold its input until it returns."""+ sr, tr = self.spatial_ratio, self.temporal_ratio+ drop_head = self.temporal_up and memory_state != MemoryState.ACTIVE+ x = source[0]+ if x.is_contiguous(memory_format=_CL) and self.conv._ck_buffered_applies(x, memory_state):+ b, c, f, h, w = x.shape+ like = x.new_empty(0) # dtype/device only, shares nothing+ del x+ return self.conv._ck_buffered(like, (b, c, f * tr - int(drop_head), h * sr, w * sr), memory_state, memory_cache,+ lambda dst: self._expand_into(dst, source.pop(), range(tr), drop_head))+ del x++ hidden_states = self.upscale_conv(source.pop())b, channels, f, h, w = hidden_states.shape- c = channels // (self.spatial_ratio * self.spatial_ratio * self.temporal_ratio)- hidden_states = hidden_states.view(b, self.spatial_ratio, self.spatial_ratio, self.temporal_ratio, c, f, h, w)- hidden_states = hidden_states.permute(0, 4, 5, 3, 6, 1, 7, 2).reshape(- b,- c,- f * self.temporal_ratio,- h * self.spatial_ratio,- w * self.spatial_ratio,- )-- if self.temporal_up and memory_state != MemoryState.ACTIVE:- hidden_states = remove_head(hidden_states)-- hidden_states = self.conv(hidden_states, memory_state=memory_state, memory_cache=memory_cache)-- return hidden_states+ c = channels // (sr * sr * tr)+ hidden_states = hidden_states.view(b, sr, sr, tr, c, f, h, w).permute(0, 4, 5, 3, 6, 1, 7, 2).reshape(+ b, c, f * tr, h * sr, w * sr)+ if drop_head:+ hidden_states = torch.cat((hidden_states[:, :, :1], hidden_states[:, :, 2:]), dim=2)+ return self.conv(hidden_states, memory_state=memory_state, memory_cache=memory_cache)class Downsample3D(nn.Module):- def __init__(- self,- channels,- out_channels = None,- inflation_mode = "tail",- spatial_down: bool = False,- temporal_down: bool = False,- ):- super().__init__()- self.channels = channels- self.out_channels = out_channels or channels- self.temporal_down = temporal_down- self.spatial_down = spatial_down-- self.temporal_ratio = 2 if temporal_down else 1- self.spatial_ratio = 2 if spatial_down else 1-- self.temporal_kernel = 3 if temporal_down else 1- self.spatial_kernel = 3 if spatial_down else 1+ """2x spatial (and 2x temporal) downsampling: a strided conv over a right/bottom zero border."""+ def __init__(self, channels, temporal_down: bool = False):+ super().__init__()self.conv = InflatedCausalConv3d(- self.channels,- self.out_channels,- kernel_size=(self.temporal_kernel, self.spatial_kernel, self.spatial_kernel),- stride=(self.temporal_ratio, self.spatial_ratio, self.spatial_ratio),- padding=(1 if self.temporal_down else 0, 0, 0),- inflation_mode=inflation_mode,+ channels,+ channels,+ kernel_size=(3 if temporal_down else 1, 3, 3),+ stride=(2 if temporal_down else 1, 2, 2),+ padding=(1 if temporal_down else 0, 0, 0),)+ def run(self, source, memory_state, memory_cache):+ """The downsampler on a 1-list ``source`` it empties: a module call would hold its input until it returns."""+ x = source[0]+ if x.is_contiguous(memory_format=_CL) and self.conv._ck_buffered_applies(x, memory_state):+ # straight into the conv's buffer, its right and bottom zero border in place of the pad copy+ b, c, t, h, w = x.shape+ like = x.new_empty(0)+ del x- def forward(- self,- hidden_states: torch.FloatTensor,- memory_state = None,- memory_cache = None,- ) -> torch.FloatTensor:-- if hidden_states.shape[1] != self.channels:- raise ValueError(f"SeedVR2 downsample expected {self.channels} channels, got {hidden_states.shape[1]}.")-- if self.spatial_down:- pad = (0, 1, 0, 1)- hidden_states = F.pad(hidden_states, pad, mode="constant", value=0)-- if hidden_states.shape[1] != self.channels:- raise ValueError(f"SeedVR2 downsample expected {self.channels} channels after padding, got {hidden_states.shape[1]}.")-- hidden_states = self.conv(hidden_states, memory_state=memory_state, memory_cache=memory_cache)+ def write_body(dst):+ dst[:, :, :, h:].zero_()+ dst[:, :, :, :h, w:].zero_()+ dst[:, :, :, :h, :w].copy_(source.pop())+ return self.conv._ck_buffered(like, (b, c, t, h, w), memory_state, memory_cache, write_body, pad=(0, 1, 0, 1))+ del x- return hidden_states+ hidden_states = F.pad(source.pop(), (0, 1, 0, 1), mode="constant", value=0)+ return self.conv(hidden_states, memory_state=memory_state, memory_cache=memory_cache)class ResnetBlock3D(nn.Module):- def __init__(- self,- in_channels: int,- out_channels: Optional[int] = None,- temb_channels: int = 512,- groups: int = 32,- groups_out: Optional[int] = None,- eps: float = 1e-6,- output_scale_factor: float = 1.0,- skip_time_act: bool = False,- inflation_mode = "tail",- time_receptive_field: _receptive_field_t = "half",- ):+ def __init__(self, in_channels: int, out_channels: int, groups: int = 32, eps: float = 1e-6):super().__init__()- self.in_channels = in_channels- self.out_channels = in_channels if out_channels is None else out_channels- self.output_scale_factor = output_scale_factor- self.skip_time_act = skip_time_act- self.nonlinearity = nn.SiLU()- if temb_channels is not None:- self.time_emb_proj = ops.Linear(temb_channels, self.out_channels)- else:- self.time_emb_proj = Noneself.norm1 = ops.GroupNorm(num_groups=groups, num_channels=in_channels, eps=eps, affine=True)- if groups_out is None:- groups_out = groups- self.norm2 = ops.GroupNorm(num_groups=groups_out, num_channels=self.out_channels, eps=eps, affine=True)- self.use_in_shortcut = self.in_channels != self.out_channels- self.conv1 = InflatedCausalConv3d(- self.in_channels,- self.out_channels,- kernel_size=(1, 3, 3) if time_receptive_field == "half" else (3, 3, 3),- stride=1,- padding=(0, 1, 1) if time_receptive_field == "half" else (1, 1, 1),- inflation_mode=inflation_mode,- )-- self.conv2 = InflatedCausalConv3d(- self.out_channels,- self.out_channels,- kernel_size=3,- stride=1,- padding=1,- inflation_mode=inflation_mode,- )-+ self.norm2 = ops.GroupNorm(num_groups=groups, num_channels=out_channels, eps=eps, affine=True)+ self.conv1 = InflatedCausalConv3d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)+ self.conv2 = InflatedCausalConv3d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)self.conv_shortcut = None- if self.use_in_shortcut:- self.conv_shortcut = InflatedCausalConv3d(- self.in_channels,- self.out_channels,- kernel_size=1,- stride=1,- padding=0,- bias=True,- inflation_mode=inflation_mode,- )-- def forward(self, input_tensor, temb, memory_state = None, memory_cache = None):- hidden_states = input_tensor-- hidden_states = causal_norm_wrapper(self.norm1, hidden_states)-- hidden_states = self.nonlinearity(hidden_states)-- hidden_states = self.conv1(hidden_states, memory_state=memory_state, memory_cache=memory_cache)-- if self.time_emb_proj is not None:- if not self.skip_time_act:- temb = self.nonlinearity(temb)- temb = self.time_emb_proj(temb)[:, :, None, None]-- if temb is not None:- hidden_states = hidden_states + temb-- hidden_states = causal_norm_wrapper(self.norm2, hidden_states)-- hidden_states = self.nonlinearity(hidden_states)-- hidden_states = self.conv2(hidden_states, memory_state=memory_state, memory_cache=memory_cache)-+ if in_channels != out_channels:+ self.conv_shortcut = InflatedCausalConv3d(in_channels, out_channels, kernel_size=1, stride=1, padding=0, bias=True)++ def forward(self, input_tensor, memory_state, memory_cache):+ hidden_states = self.conv1.fused_norm_conv([input_tensor], self.norm1, memory_state, memory_cache)+ if hidden_states is None:+ hidden_states = causal_norm_wrapper(self.norm1, input_tensor, silu=True)+ hidden_states = self.conv1(hidden_states, memory_state=memory_state, memory_cache=memory_cache)++ # An identity skip rides in conv2's epilogue, sparing a pass over the output. A 1x1 shortcut+ # runs after conv2 instead: computed first, its output would sit under conv2's buffer and+ # output (the input is held either way).+ residual = input_tensor if self.conv_shortcut is None else None+ source = [hidden_states]+ del hidden_states+ hidden_states = self.conv2.fused_norm_conv(source, self.norm2, memory_state, memory_cache, residual=residual)+ if hidden_states is None:+ hidden_states = causal_norm_wrapper(self.norm2, source.pop(), silu=True)+ hidden_states = self.conv2(hidden_states, memory_state=memory_state, memory_cache=memory_cache)+ elif residual is not None:+ return hidden_statesif self.conv_shortcut is not None:input_tensor = self.conv_shortcut(input_tensor, memory_state=memory_state, memory_cache=memory_cache)+ return hidden_states.add_(input_tensor)- output_tensor = (input_tensor + hidden_states) / self.output_scale_factor- return output_tensor+def _run_block(resnets, samplers, source, memory_state, memory_cache):+ """A down/up block's layers one by one on a 1-list ``source`` it empties, each sampler handed its+ input boxed: a module call holds its input until it returns, so nothing may hold a layer's input+ past the layer that consumes it."""+ sample = source.pop()+ for resnet in resnets:+ sample = resnet(sample, memory_state, memory_cache)+ for sampler in samplers or ():+ box = [sample]+ del sample+ sample = sampler.run(box, memory_state, memory_cache)+ return sampleclass DownEncoderBlock3D(nn.Module):- def __init__(- self,- in_channels: int,- out_channels: int,- num_layers: int = 1,- resnet_eps: float = 1e-6,- resnet_groups: int = 32,- output_scale_factor: float = 1.0,- add_downsample: bool = True,- inflation_mode = "tail",- time_receptive_field: _receptive_field_t = "half",- temporal_down: bool = True,- spatial_down: bool = True,- ):+ def __init__(self, in_channels: int, out_channels: int, num_layers: int, resnet_groups: int,+ add_downsample: bool, temporal_down: bool):super().__init__()- resnets = []-- for i in range(num_layers):- in_channels = in_channels if i == 0 else out_channels- resnets.append(- ResnetBlock3D(- in_channels=in_channels,- out_channels=out_channels,- temb_channels=None,- eps=resnet_eps,- groups=resnet_groups,- output_scale_factor=output_scale_factor,- inflation_mode=inflation_mode,- time_receptive_field=time_receptive_field,- )- )-- self.resnets = nn.ModuleList(resnets)-- if add_downsample:- self.downsamplers = nn.ModuleList(- [- Downsample3D(- out_channels,- out_channels=out_channels,- temporal_down=temporal_down,- spatial_down=spatial_down,- inflation_mode=inflation_mode,- )- ]- )- else:- self.downsamplers = None-- def forward(- self,- hidden_states: torch.FloatTensor,- memory_state = None,- memory_cache = None,- ) -> torch.FloatTensor:- for resnet in self.resnets:- hidden_states = resnet(hidden_states, temb=None, memory_state=memory_state, memory_cache=memory_cache)-- if self.downsamplers is not None:- for downsampler in self.downsamplers:- hidden_states = downsampler(hidden_states, memory_state=memory_state, memory_cache=memory_cache)+ self.resnets = nn.ModuleList([+ ResnetBlock3D(in_channels if i == 0 else out_channels, out_channels, groups=resnet_groups)+ for i in range(num_layers)+ ])+ self.downsamplers = nn.ModuleList([Downsample3D(out_channels, temporal_down=temporal_down)]) if add_downsample else None- return hidden_states+ def run(self, source, memory_state, memory_cache):+ return _run_block(self.resnets, self.downsamplers, source, memory_state, memory_cache)class UpDecoderBlock3D(nn.Module):- def __init__(- self,- in_channels: int,- out_channels: int,- num_layers: int = 1,- resnet_eps: float = 1e-6,- resnet_groups: int = 32,- output_scale_factor: float = 1.0,- add_upsample: bool = True,- temb_channels: Optional[int] = None,- inflation_mode = "tail",- time_receptive_field: _receptive_field_t = "half",- temporal_up: bool = True,- spatial_up: bool = True,- ):+ def __init__(self, in_channels: int, out_channels: int, num_layers: int, resnet_groups: int,+ add_upsample: bool, temporal_up: bool):super().__init__()- resnets = []-- for i in range(num_layers):- input_channels = in_channels if i == 0 else out_channels-- resnets.append(- ResnetBlock3D(- in_channels=input_channels,- out_channels=out_channels,- temb_channels=temb_channels,- eps=resnet_eps,- groups=resnet_groups,- output_scale_factor=output_scale_factor,- inflation_mode=inflation_mode,- time_receptive_field=time_receptive_field,- )- )-- self.resnets = nn.ModuleList(resnets)-- if add_upsample:- self.upsamplers = nn.ModuleList(- [- Upsample3D(- out_channels,- out_channels=out_channels,- temporal_up=temporal_up,- spatial_up=spatial_up,- inflation_mode=inflation_mode,- )- ]- )- else:- self.upsamplers = None+ self.resnets = nn.ModuleList([+ ResnetBlock3D(in_channels if i == 0 else out_channels, out_channels, groups=resnet_groups)+ for i in range(num_layers)+ ])+ self.upsamplers = nn.ModuleList([Upsample3D(out_channels, temporal_up=temporal_up)]) if add_upsample else None- def forward(- self,- hidden_states: torch.FloatTensor,- temb: Optional[torch.FloatTensor] = None,- memory_state=None,- memory_cache=None,- ) -> torch.FloatTensor:- for resnet in self.resnets:- hidden_states = resnet(hidden_states, temb=None, memory_state=memory_state, memory_cache=memory_cache)-- if self.upsamplers is not None:- for upsampler in self.upsamplers:- hidden_states = upsampler(hidden_states, memory_state=memory_state, memory_cache=memory_cache)-- return hidden_states+ def run(self, source, memory_state, memory_cache):+ return _run_block(self.resnets, self.upsamplers, source, memory_state, memory_cache)class UNetMidBlock3D(nn.Module):- def __init__(- self,- in_channels: int,- temb_channels: int,- num_layers: int = 1,- resnet_eps: float = 1e-6,- resnet_time_scale_shift: str = "default", # default, spatial- resnet_groups: int = 32,- add_attention: bool = True,- attention_head_dim: int = 1,- output_scale_factor: float = 1.0,- inflation_mode = "tail",- time_receptive_field: _receptive_field_t = "half",- ):- super().__init__()- resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)- self.add_attention = add_attention-- resnets = [- ResnetBlock3D(- in_channels=in_channels,- out_channels=in_channels,- temb_channels=temb_channels,- eps=resnet_eps,- groups=resnet_groups,- output_scale_factor=output_scale_factor,- inflation_mode=inflation_mode,- time_receptive_field=time_receptive_field,- )- ]- attentions = []-- if attention_head_dim is None:- attention_head_dim = in_channels-- for _ in range(num_layers):- if self.add_attention:- attentions.append(- Attention(- in_channels,- heads=in_channels // attention_head_dim,- dim_head=attention_head_dim,- rescale_output_factor=output_scale_factor,- eps=resnet_eps,- norm_num_groups=(- resnet_groups if resnet_time_scale_shift == "default" else None- ),- spatial_norm_dim=(- temb_channels if resnet_time_scale_shift == "spatial" else None- ),- residual_connection=True,- bias=True,- )- )- else:- attentions.append(None)-- resnets.append(- ResnetBlock3D(- in_channels=in_channels,- out_channels=in_channels,- temb_channels=temb_channels,- eps=resnet_eps,- groups=resnet_groups,- output_scale_factor=output_scale_factor,- inflation_mode=inflation_mode,- time_receptive_field=time_receptive_field,- )- )+ """A resnet, single-head self-attention within each frame, a resnet."""- self.attentions = nn.ModuleList(attentions)- self.resnets = nn.ModuleList(resnets)-- def forward(self, hidden_states, temb=None, memory_state=None, memory_cache=None):- video_length = hidden_states.size(2)- hidden_states = self.resnets[0](hidden_states, temb, memory_state=memory_state, memory_cache=memory_cache)- for attn, resnet in zip(self.attentions, self.resnets[1:]):- if attn is not None:- b, c, f, h, w = hidden_states.shape- hidden_states = hidden_states.transpose(1, 2).reshape(b * f, c, h, w)- hidden_states = attn(hidden_states, temb=temb)- hidden_states = hidden_states.reshape(b, video_length, c, h, w).transpose(1, 2)- hidden_states = resnet(hidden_states, temb, memory_state=memory_state, memory_cache=memory_cache)+ def __init__(self, channels: int, resnet_groups: int):+ super().__init__()+ self.attentions = nn.ModuleList([Attention(channels, norm_num_groups=resnet_groups, eps=1e-6)])+ self.resnets = nn.ModuleList([ResnetBlock3D(channels, channels, groups=resnet_groups) for _ in range(2)])- return hidden_states+ def forward(self, hidden_states, memory_state, memory_cache):+ hidden_states = self.resnets[0](hidden_states, memory_state, memory_cache)+ b, c, f, h, w = hidden_states.shape+ hidden_states = self.attentions[0](hidden_states.transpose(1, 2).reshape(b * f, c, h, w))+ hidden_states = hidden_states.reshape(b, f, c, h, w).transpose(1, 2)+ return self.resnets[1](hidden_states, memory_state, memory_cache)class Encoder3D(nn.Module):- def __init__(- self,- in_channels: int = 3,- out_channels: int = 3,- down_block_types: Tuple[str, ...] = ("DownEncoderBlock3D",),- block_out_channels: Tuple[int, ...] = (64,),- layers_per_block: int = 2,- norm_num_groups: int = 32,- mid_block_add_attention=True,- temporal_down_num: int = 2,- inflation_mode = "tail",- time_receptive_field: _receptive_field_t = "half",- ):+ def __init__(self, in_channels: int, out_channels: int, block_out_channels, layers_per_block: int,+ norm_num_groups: int, temporal_down_num: int):super().__init__()- self.layers_per_block = layers_per_block- self.temporal_down_num = temporal_down_num-- self.conv_in = InflatedCausalConv3d(- in_channels,- block_out_channels[0],- kernel_size=3,- stride=1,- padding=1,- inflation_mode=inflation_mode,- )-- self.mid_block = None+ self.head_blocks = max(0, len(block_out_channels) - temporal_down_num - 1) # the spatial-only down blocks+ self.conv_in = InflatedCausalConv3d(in_channels, block_out_channels[0], kernel_size=3, stride=1, padding=1)self.down_blocks = nn.ModuleList([])-output_channel = block_out_channels[0]- for i, down_block_type in enumerate(down_block_types):+ for i in range(len(block_out_channels)):input_channel = output_channeloutput_channel = block_out_channels[i]- is_final_block = i == len(block_out_channels) - 1- is_temporal_down_block = i >= len(block_out_channels) - self.temporal_down_num - 1-- if down_block_type != "DownEncoderBlock3D":- raise ValueError(f"SeedVR2 encoder only supports DownEncoderBlock3D, got {down_block_type}.")-- down_block = DownEncoderBlock3D(- num_layers=self.layers_per_block,- in_channels=input_channel,- out_channels=output_channel,- add_downsample=not is_final_block,- resnet_eps=1e-6,+ self.down_blocks.append(DownEncoderBlock3D(+ input_channel,+ output_channel,+ num_layers=layers_per_block,resnet_groups=norm_num_groups,- temporal_down=is_temporal_down_block,- spatial_down=True,- inflation_mode=inflation_mode,- time_receptive_field=time_receptive_field,- )- self.down_blocks.append(down_block)-- self.mid_block = UNetMidBlock3D(- in_channels=block_out_channels[-1],- resnet_eps=1e-6,- output_scale_factor=1,- resnet_time_scale_shift="default",- attention_head_dim=block_out_channels[-1],- resnet_groups=norm_num_groups,- temb_channels=None,- add_attention=mid_block_add_attention,- inflation_mode=inflation_mode,- time_receptive_field=time_receptive_field,- )-- self.conv_norm_out = ops.GroupNorm(- num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6- )- self.conv_act = nn.SiLU()-- conv_out_channels = 2 * out_channels- self.conv_out = InflatedCausalConv3d(- block_out_channels[-1], conv_out_channels, 3, padding=1, inflation_mode=inflation_mode- )--- def forward(- self,- sample: torch.FloatTensor,- memory_state = None,- memory_cache = None,- ) -> torch.FloatTensor:- sample = sample.to(next(self.parameters()).device)- sample = self.conv_in(sample, memory_state=memory_state, memory_cache=memory_cache)- for down_block in self.down_blocks:- sample = down_block(sample, memory_state=memory_state, memory_cache=memory_cache)-- sample = self.mid_block(sample, memory_state=memory_state, memory_cache=memory_cache)-- sample = causal_norm_wrapper(self.conv_norm_out, sample)- sample = self.conv_act(sample)- sample = self.conv_out(sample, memory_state=memory_state, memory_cache=memory_cache)-- return sample+ add_downsample=i < len(block_out_channels) - 1,+ temporal_down=i >= len(block_out_channels) - temporal_down_num - 1,+ ))+ self.mid_block = UNetMidBlock3D(block_out_channels[-1], resnet_groups=norm_num_groups)+ self.conv_norm_out = ops.GroupNorm(num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6)+ self.conv_out = InflatedCausalConv3d(block_out_channels[-1], 2 * out_channels, 3, padding=1)++ # Frames per pass through the full-resolution, spatial-only head; None runs a slice's frames at once.+ head_frames = 1++ def _head(self, source, memory_state, memory_cache):+ box = [self.conv_in(source.pop(), memory_state=memory_state, memory_cache=memory_cache)]+ for down_block in self.down_blocks[:self.head_blocks]:+ box = [down_block.run(box, memory_state, memory_cache)]+ return box.pop()++ def forward(self, sample, memory_state, memory_cache):+ frame_pixels = sample.size(-2) * sample.size(-1)+ source = [sample]+ del sample+ box = [_frame_chunked(self, self._head, source, self.head_frames, memory_state, memory_cache, frame_pixels)]+ for down_block in self.down_blocks[self.head_blocks:]:+ box = [down_block.run(box, memory_state, memory_cache)]+ sample = self.mid_block(box.pop(), memory_state, memory_cache)+ sample = causal_norm_wrapper(self.conv_norm_out, sample, silu=True)+ return self.conv_out(sample, memory_state=memory_state, memory_cache=memory_cache)class Decoder3D(nn.Module):-- def __init__(- self,- in_channels: int = 3,- out_channels: int = 3,- up_block_types: Tuple[str, ...] = ("UpDecoderBlock3D",),- block_out_channels: Tuple[int, ...] = (64,),- layers_per_block: int = 2,- norm_num_groups: int = 32,- mid_block_add_attention=True,- inflation_mode = "tail",- time_receptive_field: _receptive_field_t = "half",- temporal_up_num: int = 2,- ):+ def __init__(self, in_channels: int, out_channels: int, block_out_channels, layers_per_block: int,+ norm_num_groups: int, temporal_up_num: int):super().__init__()- self.layers_per_block = layers_per_blockself.temporal_up_num = temporal_up_num-- self.conv_in = InflatedCausalConv3d(- in_channels,- block_out_channels[-1],- kernel_size=3,- stride=1,- padding=1,- inflation_mode=inflation_mode,- )-- self.mid_block = None+ self.conv_in = InflatedCausalConv3d(in_channels, block_out_channels[-1], kernel_size=3, stride=1, padding=1)+ self.mid_block = UNetMidBlock3D(block_out_channels[-1], resnet_groups=norm_num_groups)self.up_blocks = nn.ModuleList([])-- temb_channels = None-- self.mid_block = UNetMidBlock3D(- in_channels=block_out_channels[-1],- resnet_eps=1e-6,- output_scale_factor=1,- resnet_time_scale_shift="default",- attention_head_dim=block_out_channels[-1],- resnet_groups=norm_num_groups,- temb_channels=temb_channels,- add_attention=mid_block_add_attention,- inflation_mode=inflation_mode,- time_receptive_field=time_receptive_field,- )-reversed_block_out_channels = list(reversed(block_out_channels))output_channel = reversed_block_out_channels[0]- for i, up_block_type in enumerate(up_block_types):+ for i in range(len(block_out_channels)):prev_output_channel = output_channeloutput_channel = reversed_block_out_channels[i]-- is_final_block = i == len(block_out_channels) - 1- is_temporal_up_block = i < self.temporal_up_num- if up_block_type != "UpDecoderBlock3D":- raise ValueError(f"SeedVR2 decoder only supports UpDecoderBlock3D, got {up_block_type}.")- up_block = UpDecoderBlock3D(- num_layers=self.layers_per_block + 1,- in_channels=prev_output_channel,- out_channels=output_channel,- add_upsample=not is_final_block,- resnet_eps=1e-6,+ self.up_blocks.append(UpDecoderBlock3D(+ prev_output_channel,+ output_channel,+ num_layers=layers_per_block + 1,resnet_groups=norm_num_groups,- temb_channels=temb_channels,- temporal_up=is_temporal_up_block,- inflation_mode=inflation_mode,- time_receptive_field=time_receptive_field,- )- self.up_blocks.append(up_block)- prev_output_channel = output_channel-- self.conv_norm_out = ops.GroupNorm(- num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6- )- self.conv_act = nn.SiLU()- self.conv_out = InflatedCausalConv3d(- block_out_channels[0], out_channels, 3, padding=1, inflation_mode=inflation_mode- )--- def forward(- self,- sample: torch.FloatTensor,- latent_embeds: Optional[torch.FloatTensor] = None,- memory_state = None,- memory_cache = None,- ) -> torch.FloatTensor:-- sample = sample.to(next(self.parameters()).device)+ add_upsample=i < len(block_out_channels) - 1,+ temporal_up=i < temporal_up_num,+ ))+ self.conv_norm_out = ops.GroupNorm(num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6)+ self.conv_out = InflatedCausalConv3d(block_out_channels[0], out_channels, 3, padding=1)++ # Frames per pass through the spatial-only tail; None runs a slice's frames at once.+ tail_frames = 1++ def _tail(self, source, memory_state, memory_cache):+ for up_block in self.up_blocks[self.temporal_up_num:]:+ source = [up_block.run(source, memory_state, memory_cache)]+ out = self.conv_out.fused_norm_conv(source, self.conv_norm_out, memory_state, memory_cache)+ if out is None:+ sample = causal_norm_wrapper(self.conv_norm_out, source.pop(), silu=True)+ out = self.conv_out(sample, memory_state=memory_state, memory_cache=memory_cache)+ return out++ def forward(self, sample, memory_state, memory_cache):sample = self.conv_in(sample, memory_state=memory_state, memory_cache=memory_cache)-- upscale_dtype = next(iter(self.up_blocks.parameters())).dtype- sample = self.mid_block(sample, latent_embeds, memory_state=memory_state, memory_cache=memory_cache)- sample = sample.to(upscale_dtype)-- for up_block in self.up_blocks:- sample = up_block(sample, latent_embeds, memory_state=memory_state, memory_cache=memory_cache)-- sample = causal_norm_wrapper(self.conv_norm_out, sample)- sample = self.conv_act(sample)- sample = self.conv_out(sample, memory_state=memory_state, memory_cache=memory_cache)-- return sample--class VideoAutoencoderKL(nn.Module):- def __init__(- self,- in_channels: int = 3,- out_channels: int = 3,- layers_per_block: int = 2,- latent_channels: int = SEEDVR2_LATENT_CHANNELS,- norm_num_groups: int = 32,- temporal_scale_num: int = 2,- inflation_mode = "pad",- time_receptive_field: _receptive_field_t = "full",- slicing_sample_min_size = BYTEDANCE_SLICING_SAMPLE_MIN,- ):- self.slicing_sample_min_size = slicing_sample_min_size- self.slicing_latent_min_size = slicing_sample_min_size // (2**temporal_scale_num)- block_out_channels = BYTEDANCE_BLOCK_OUT_CHANNELS- down_block_types = ("DownEncoderBlock3D",) * 4- up_block_types = ("UpDecoderBlock3D",) * 4+ sample = self.mid_block(sample, memory_state, memory_cache)++ *head, last = self.up_blocks[:self.temporal_up_num]+ box = [sample]+ del sample+ for up_block in head:+ box = [up_block.run(box, memory_state, memory_cache)]+ sample = box.pop()+ for resnet in last.resnets:+ sample = resnet(sample, memory_state, memory_cache)++ # From the last temporal upsampler on, everything is causal and per frame: it runs an input+ # frame at a time, each of its output frames going on through the spatial-only tail as it is made.+ ups = sum(1 for blk in self.up_blocks[self.temporal_up_num - 1:] if blk.upsamplers is not None)+ frame_pixels = sample.size(-2) * sample.size(-1) * 4 ** ups+ up = last.upsamplers[0]++ def upsample_and_tail(source, state, cache):+ if up.frames_apply(source[0], state):+ outs = [self._tail(box, state if n == 0 else MemoryState.ACTIVE, cache)+ for n, box in enumerate(up.frames(source, state, cache))]+ return torch.cat(outs, dim=2)+ box = [up.run(source, state, cache)]+ return _frame_chunked(self, self._tail, box, self.tail_frames, state, cache, frame_pixels)+ source = [sample]+ del sample+ return _frame_chunked(last, upsample_and_tail, source, self.tail_frames and max(1, self.tail_frames // up.temporal_ratio),+ memory_state, memory_cache, frame_pixels)+++class VideoAutoencoderKLWrapper(nn.Module):+ """The SeedVR2 causal video VAE, run a temporal slice at a time with the causal caches carried+ across slices."""+ spatial_downsample_factor = BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE+ temporal_downsample_factor = BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE+ slicing_sample_min_size = BYTEDANCE_SLICING_SAMPLE_MIN+ slicing_latent_min_size = BYTEDANCE_SLICING_SAMPLE_MIN // BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE++ def __init__(self):super().__init__()+ self.encoder = Encoder3D(3, SEEDVR2_LATENT_CHANNELS, BYTEDANCE_BLOCK_OUT_CHANNELS, layers_per_block=2,+ norm_num_groups=32, temporal_down_num=2)+ self.decoder = Decoder3D(SEEDVR2_LATENT_CHANNELS, 3, BYTEDANCE_BLOCK_OUT_CHANNELS, layers_per_block=2,+ norm_num_groups=32, temporal_up_num=2)+ set_norm_limit(BYTEDANCE_VAE_NORM_MEM_GIB)+ for m in self.modules():+ if isinstance(m, InflatedCausalConv3d):+ m.memory_limit = BYTEDANCE_VAE_CONV_MEM_GIB- self.encoder = Encoder3D(- in_channels=in_channels,- out_channels=latent_channels,- down_block_types=down_block_types,- block_out_channels=block_out_channels,- layers_per_block=layers_per_block,- norm_num_groups=norm_num_groups,- temporal_down_num=temporal_scale_num,- inflation_mode=inflation_mode,- time_receptive_field=time_receptive_field,- )-- self.decoder = Decoder3D(- in_channels=latent_channels,- out_channels=out_channels,- up_block_types=up_block_types,- block_out_channels=block_out_channels,- layers_per_block=layers_per_block,- norm_num_groups=norm_num_groups,- temporal_up_num=temporal_scale_num,- inflation_mode=inflation_mode,- time_receptive_field=time_receptive_field,- )-- self.use_slicing = True-- def encode(self, x: torch.FloatTensor, return_dict: bool = True):- h = self.slicing_encode(x)- posterior = DiagonalGaussianDistribution(h).mode()-- if not return_dict:- return (posterior,)-- return posterior-- def decode_(- self, z: torch.Tensor, return_dict: bool = True- ):- decoded = self.slicing_decode(z)-- if not return_dict:- return (decoded,)-- return decoded-- def _encode(- self, x, memory_state = MemoryState.DISABLED, memory_cache = None- ) -> torch.Tensor:- _x = x.to(self.device)- h = self.encoder(_x, memory_state=memory_state, memory_cache=memory_cache)+ def _encode(self, x, memory_state=MemoryState.DISABLED, memory_cache=None):+ h = self.encoder(x.to(self.device), memory_state=memory_state, memory_cache=memory_cache)return h.to(x.device)- def _decode(- self, z, memory_state = MemoryState.DISABLED, memory_cache = None- ) -> torch.Tensor:- _z = z.to(self.device)- output = self.decoder(_z, memory_state=memory_state, memory_cache=memory_cache)+ def _decode(self, z, memory_state=MemoryState.DISABLED, memory_cache=None):+ output = self.decoder(z.to(self.device), memory_state=memory_state, memory_cache=memory_cache)return output.to(z.device)def slicing_encode(self, x: torch.Tensor) -> torch.Tensor:- if self.use_slicing and (x.shape[2] - 1) > self.slicing_sample_min_size:- memory_cache = {}- split_size = max(- self.slicing_sample_min_size,- getattr(self, "temporal_downsample_factor", 1),- )- x_slices = list(x[:, :, 1:].split(split_size=split_size, dim=2))- min_active_len = getattr(self, "temporal_downsample_factor", 1)- if len(x_slices) > 1 and x_slices[-1].shape[2] < min_active_len:- x_slices[-2] = torch.cat((x_slices[-2], x_slices[-1]), dim=2)- x_slices.pop()- encoded_slices = [- self._encode(- torch.cat((x[:, :, :1], x_slices[0]), dim=2),- memory_state=MemoryState.INITIALIZING,- memory_cache=memory_cache,- )- ]- for x_idx in range(1, len(x_slices)):- encoded_slices.append(- self._encode(x_slices[x_idx], memory_state=MemoryState.ACTIVE, memory_cache=memory_cache)- )- out = torch.cat(encoded_slices, dim=2)- return out- else:- return self._encode(x)-- def slicing_decode(self, z: torch.Tensor) -> torch.Tensor:- if self.use_slicing and (z.shape[2] - 1) > self.slicing_latent_min_size:- memory_cache = {}- z_slices = z[:, :, 1:].split(split_size=self.slicing_latent_min_size, dim=2)- decoded_slices = [- self._decode(- torch.cat((z[:, :, :1], z_slices[0]), dim=2),- memory_state=MemoryState.INITIALIZING,- memory_cache=memory_cache,- )- ]- for z_idx in range(1, len(z_slices)):- decoded_slices.append(- self._decode(z_slices[z_idx], memory_state=MemoryState.ACTIVE, memory_cache=memory_cache)- )- out = torch.cat(decoded_slices, dim=2)- return out- else:- return self._decode(z)-- def forward(self, x: torch.FloatTensor, mode: Literal["encode", "decode", "all"] = "all"):- def _unwrap(value):- return value[0] if isinstance(value, tuple) else value-- if mode == "encode":- return _unwrap(self.encode(x))- if mode == "decode":- return _unwrap(self.decode_(x))- if mode == "all":- latent = _unwrap(self.encode(x))- return _unwrap(self.decode_(latent))- raise ValueError(f"Unknown SeedVR2 VAE forward mode: {mode}")--class VideoAutoencoderKLWrapper(VideoAutoencoderKL):- def __init__(- self,- spatial_downsample_factor = 8,- temporal_downsample_factor = 4,- ):- self.spatial_downsample_factor = spatial_downsample_factor- self.temporal_downsample_factor = temporal_downsample_factor- super().__init__()- self.set_memory_limit(BYTEDANCE_VAE_CONV_MEM_GIB, BYTEDANCE_VAE_NORM_MEM_GIB)+ if x.size(0) > 1: # a video at a time: the buffered path and the memory estimate are per video+ return torch.cat([self.slicing_encode(x[i:i + 1]) for i in range(x.size(0))])+ if (x.shape[2] - 1) <= self.slicing_sample_min_size:+ return self._encode(x.to(self.device))+ memory_cache = CausalMemoryCache(offload=_offload_caches_for(x.shape[-2] * x.shape[-1]))+ split_size = max(self.slicing_sample_min_size, self.temporal_downsample_factor)+ x_slices = list(x[:, :, 1:].split(split_size=split_size, dim=2)) # _encode moves each to the device+ if len(x_slices) > 1 and x_slices[-1].shape[2] < self.temporal_downsample_factor:+ x_slices[-2] = torch.cat((x_slices[-2], x_slices[-1]), dim=2)+ x_slices.pop()+ try:+ encoded_slices = [self._encode(torch.cat((x[:, :, :1], x_slices[0]), dim=2),+ memory_state=MemoryState.INITIALIZING, memory_cache=memory_cache)]+ for x_slice in x_slices[1:]:+ encoded_slices.append(self._encode(x_slice, memory_state=MemoryState.ACTIVE, memory_cache=memory_cache))+ finally:+ memory_cache.free()+ return torch.cat(encoded_slices, dim=2)++ def slicing_decode(self, z: torch.Tensor, output_buffer=None) -> torch.Tensor:+ """Decode slice by slice; with ``output_buffer`` each slice is written into it (no cat, no accumulation)."""+ if z.size(0) > 1: # a video at a time: the buffered path and the memory estimate are per video+ outs = [self.slicing_decode(z[i:i + 1], None if output_buffer is None else output_buffer[i:i + 1])+ for i in range(z.size(0))]+ return output_buffer if output_buffer is not None else torch.cat(outs)+ if (z.shape[2] - 1) <= self.slicing_latent_min_size:+ decoded = self._decode(z)+ return decoded if output_buffer is None else output_buffer.copy_(decoded)+ memory_cache = CausalMemoryCache(offload=_offload_caches_for(z.shape[-2] * z.shape[-1] * self.spatial_downsample_factor ** 2))+ z_slices = z[:, :, 1:].split(split_size=self.slicing_latent_min_size, dim=2)+ decoded_slices = []+ write_pos = 0++ def emit(decoded):+ nonlocal write_pos+ if output_buffer is None:+ decoded_slices.append(decoded)+ return+ output_buffer[:, :, write_pos:write_pos + decoded.size(2)].copy_(decoded)+ write_pos += decoded.size(2)- def forward(self, x: torch.FloatTensor):- z, p = self._encode_with_raw_latent(x)- x = self.decode(z)- return x, z, p+ try:+ emit(self._decode(torch.cat((z[:, :, :1], z_slices[0]), dim=2),+ memory_state=MemoryState.INITIALIZING, memory_cache=memory_cache))+ for z_slice in z_slices[1:]:+ emit(self._decode(z_slice, memory_state=MemoryState.ACTIVE, memory_cache=memory_cache))+ finally:+ memory_cache.free()+ return output_buffer if output_buffer is not None else torch.cat(decoded_slices, dim=2)- def _encode_with_raw_latent(self, x):+ def encode(self, x, device=None):+ """The latent mean of (b, c, t, h, w) pixels, or of an image batch; a single latent frame loses+ its time axis. sd.py leaves the pixels wherever they were when it uses the chunked-io protocol:+ the slicing moves each slice to ``device`` as it encodes, so the whole clip never sits on the GPU."""if x.ndim == 4:x = x.unsqueeze(2)- self.device = x.device- p = super().encode(x)- z = p.squeeze(2)- return z, p-- def encode(self, x):- z, _ = self._encode_with_raw_latent(x)- return z-- def decode(self, z, seedvr2_tiling=None):- seedvr2_tiling = {} if seedvr2_tiling is None else seedvr2_tiling- if not isinstance(seedvr2_tiling, dict):- raise RuntimeError(- "SeedVR2 VideoAutoencoderKLWrapper.decode: `seedvr2_tiling` must be a dict; "- f"got {type(seedvr2_tiling).__name__} with value {seedvr2_tiling!r}."- )+ self.device = x.device if device is None else device+ return self.slicing_encode(x).chunk(2, dim=1)[0].squeeze(2)- if z.ndim == 5:- _, c, _, _, _ = z.shape- if c != SEEDVR2_LATENT_CHANNELS:- raise RuntimeError(- "SeedVR2 VideoAutoencoderKLWrapper.decode: 5-D latent input must "- f"have {SEEDVR2_LATENT_CHANNELS} channels; got shape {tuple(z.shape)}."- )- latent = z- elif z.ndim == 4:- b, tc, h, w = z.shape- if tc % SEEDVR2_LATENT_CHANNELS != 0:- raise RuntimeError(- "SeedVR2 VideoAutoencoderKLWrapper.decode: 4-D latent input must "- f"use collapsed channel layout (B, {SEEDVR2_LATENT_CHANNELS}*T, H, W); "- f"got shape {tuple(z.shape)}."- )- latent = z.reshape(b, SEEDVR2_LATENT_CHANNELS, -1, h, w)- else:- raise RuntimeError(- "SeedVR2 VideoAutoencoderKLWrapper.decode: latent input must be "- f"4-D collapsed (B, {SEEDVR2_LATENT_CHANNELS}*T, H, W) or "- f"5-D (B, {SEEDVR2_LATENT_CHANNELS}, T, H, W); "- f"got shape {tuple(z.shape)}."- )- scale = BYTEDANCE_VAE_SCALING_FACTOR- shift = BYTEDANCE_VAE_SHIFTING_FACTOR- latent = latent / scale + shift+ # sd.py preallocates the output on its device/dtype and hands slices of it to decode(); the+ # decoded frames then never accumulate on the GPU (see slicing_decode).+ comfy_has_chunked_io = True- self.device = latent.device- enable_tiling = seedvr2_tiling.get("enable_tiling", False)-- if enable_tiling:- decode_seedvr2_args = dict(seedvr2_tiling)- decode_seedvr2_args.pop("enable_tiling", None)- tile_h, tile_w = decode_seedvr2_args.get("tile_size", (512, 512))- ov_h, ov_w = decode_seedvr2_args.get("tile_overlap", (64, 64))- decode_seedvr2_args["tile_overlap"] = (- min(ov_h, max(0, tile_h - 8)),- min(ov_w, max(0, tile_w - 8)),- )- x = tiled_vae(latent, self, **decode_seedvr2_args, encode=False)- if x.ndim == 4:- # tiled_vae squeezes the temporal axis when- # temporal_downsample_factor == 1 AND latent T == 1- # (see tiled_vae line 179-180); re-add it so the post-decode- # pipeline can keep batch and time distinct on the tiled path.- x = x.unsqueeze(2)- else:- x = super().decode_(latent)+ def decode_output_shape(self, input_shape):+ b, c, t, h, w = self._latent_dims(input_shape)+ frames = max(1, (t - 1) * self.temporal_downsample_factor + 1)+ return (b, 3, frames, h * self.spatial_downsample_factor, w * self.spatial_downsample_factor)- h, w = x.shape[-2:]- w2 = w - (w % 2)- h2 = h - (h % 2)- x = x[..., :h2, :w2]+ @staticmethod+ def _latent_dims(shape):+ """(b, c, t, h, w) of a 5-D latent or the collapsed 4-D (B, 16*T, H, W) form."""+ if len(shape) == 5:+ return tuple(shape)+ b, tc, h, w = shape+ return b, SEEDVR2_LATENT_CHANNELS, tc // SEEDVR2_LATENT_CHANNELS, h, w- return x+ def _unscaled(self, z):+ """A scaled latent, in either form, as the decoder's (b, c, t, h, w) input."""+ return z.reshape(self._latent_dims(z.shape)) / BYTEDANCE_VAE_SCALING_FACTOR + BYTEDANCE_VAE_SHIFTING_FACTOR++ def decode(self, z, output_buffer=None):+ latent = self._unscaled(z)+ self.device = latent.device+ return self.slicing_decode(latent, output_buffer=output_buffer)++ # comfy_memory_used_decode is fitted against measured peaks, so a decode it says will not fit+ # can be tiled without trying first. See preferred_decode_tile for the sizing that follows.+ comfy_decode_estimate_is_reliable = True++ def preferred_decode_tile(self, free_memory, dtype):+ """Largest square tile, in latent units, whose decode fits ``free_memory``, from the same per-pixel+ figures as the estimate: the temporal caches count too when there is no room to pin them."""+ factor = _eager_factor(dtype, SEEDVR2_EAGER_DECODE_FACTOR)+ budget = free_memory * SEEDVR2_TILE_MEM_HEADROOM - SEEDVR2_DECODE_FIXED_BYTES * factor+ if budget <= 0:+ return SEEDVR2_MIN_TILE_LATENT+ area = budget / (SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL * factor)+ if not _offload_caches_for(area):+ area = budget / ((SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL + SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL) * factor)+ side = int(math.sqrt(max(area, 1.0))) // self.spatial_downsample_factor+ side = (side // 8) * 8 # keep tiles a whole number of latent blocks+ return max(SEEDVR2_MIN_TILE_LATENT, min(SEEDVR2_MAX_TILE_LATENT, side))def decode_tiled(self, z, tile_x=32, tile_y=32, overlap=8, tile_t=None, overlap_t=None):# SeedVR2's causal VAE owns temporal via the MemoryState cache; external# temporal tiling breaks that continuity, so only spatial tiling is applied.+ latent = self._unscaled(z)+ self.device = latent.devicesf = self.spatial_downsample_factor- seedvr2_tiling = {- "enable_tiling": True,- "tile_size": (tile_y * sf, tile_x * sf),- "tile_overlap": (overlap * sf, overlap * sf),- "temporal_size": None,- "temporal_overlap": None,- }- return self.decode(z, seedvr2_tiling=seedvr2_tiling)+ tile_h, tile_w, ov = tile_y * sf, tile_x * sf, overlap * sf+ return tiled_vae(latent, self, tile_size=(tile_h, tile_w),+ tile_overlap=(min(ov, max(0, tile_h - 8)), min(ov, max(0, tile_w - 8))), encode=False)def encode_tiled(self, x, tile_x=None, tile_y=None, overlap=None, tile_t=None, overlap_t=None):# External temporal tiling knobs are discarded; the causal VAE keeps its# own internal MemoryState slicing.- if tile_y is None:- tile_y = 512- if tile_x is None:- tile_x = 512- if overlap is None:- overlap_y = 64- overlap_x = 64- else:- overlap_y = overlap- overlap_x = overlap- overlap_y = min(overlap_y, max(0, tile_y - 8))- overlap_x = min(overlap_x, max(0, tile_x - 8))+ tile_y = 512 if tile_y is None else tile_y+ tile_x = 512 if tile_x is None else tile_x+ overlap = 64 if overlap is None else overlapself.device = x.device- return tiled_vae(- x,- self,- tile_size=(tile_y, tile_x),- tile_overlap=(overlap_y, overlap_x),- temporal_size=None,- temporal_overlap=None,- encode=True,- )+ return tiled_vae(x, self, tile_size=(tile_y, tile_x),+ tile_overlap=(min(overlap, max(0, tile_y - 8)), min(overlap, max(0, tile_x - 8))), encode=True)def comfy_format_encoded(self, samples):if samples.ndim == 4:@@ -1578,33 +1412,22 @@ class VideoAutoencoderKLWrapper(VideoAutoencoderKL):samples = samples * BYTEDANCE_VAE_SCALING_FACTORreturn samples- def comfy_memory_used_decode(self, shape):- bytes_per_output_pixel = 160-- def output_pixels(latent_t, latent_h, latent_w):- output_t = max(1, (latent_t - 1) * 4 + 1)- return output_t * latent_h * 8 * latent_w * 8-- # SeedVR2 decode performs full-frame LAB histogram matching: fp32 channels- # plus int64 sort indices dominate peak memory, not the VAE weight dtype.- if len(shape) == 5:- candidates = []- if shape[1] == SEEDVR2_LATENT_CHANNELS:- candidates.append((shape[2], shape[3], shape[4]))- if shape[-1] == SEEDVR2_LATENT_CHANNELS:- candidates.append((shape[1], shape[2], shape[3]))- if len(candidates) == 0:- candidates.append((shape[2], shape[3], shape[4]))- pixels = max(output_pixels(*candidate) for candidate in candidates)- elif len(shape) == 4:- latent_t = max(1, (shape[1] + SEEDVR2_LATENT_CHANNELS - 1) // SEEDVR2_LATENT_CHANNELS)- pixels = output_pixels(latent_t, shape[2], shape[3])- else:- pixels = output_pixels(1, shape[-2], shape[-1])- return pixels * bytes_per_output_pixel-- def set_memory_limit(self, conv_max_mem: Optional[float], norm_max_mem: Optional[float]):- set_norm_limit(norm_max_mem)- for m in self.modules():- if isinstance(m, InflatedCausalConv3d):- m.set_memory_limit(conv_max_mem if conv_max_mem is not None else float("inf"))+ def comfy_memory_used_encode(self, shape, dtype):+ """Peak device memory of an encode of (b, c, t, h, w) pixels, in bytes: one frame's area sets it+ (the videos of a batch are encoded one at a time)."""+ peak = shape[-2] * shape[-1] * SEEDVR2_ENCODE_BYTES_PER_PIXEL + SEEDVR2_ENCODE_FIXED_BYTES+ return int(peak * _eager_factor(dtype, SEEDVR2_EAGER_ENCODE_FACTOR))++ def comfy_memory_used_decode(self, shape, dtype):+ """Peak device memory of a decode, in bytes: one frame's area sets it, not the clip length+ (the videos of a batch are decoded one at a time)."""+ _, _, _, latent_h, latent_w = self._latent_dims(shape)+ area = latent_h * self.spatial_downsample_factor * latent_w * self.spatial_downsample_factor+ decode_peak = area * SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL + SEEDVR2_DECODE_FIXED_BYTES+ if not _offload_caches_for(area):+ decode_peak += area * SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL # the temporal caches stay resident+ # The node's colour correction runs after the decode, one frame (LAB) or a free-memory+ # sized chunk (wavelet, adain) at a time with its own OOM back-off, so it never coincides+ # with the decode's working set and only a frame of it is worth reserving.+ colour_transfer = area * SEEDVR2_DECODE_LAB_BYTES_PER_OUTPUT_PIXEL+ return int(max(decode_peak * _eager_factor(dtype, SEEDVR2_EAGER_DECODE_FACTOR), colour_transfer))diff --git a/comfy/sd.py b/comfy/sd.pyindex 1023ad9ea..73a35aaa4 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -604,8 +604,8 @@ class VAE:self.latent_channels = comfy.ldm.seedvr.vae.SEEDVR2_LATENT_CHANNELSself.latent_dim = 3self.disable_offload = True- self.memory_used_decode = lambda shape, dtype: self.first_stage_model.comfy_memory_used_decode(shape)- self.memory_used_encode = lambda shape, dtype: (max(shape[2], 5) * shape[3] * shape[4] * 64) * model_management.dtype_size(dtype)+ self.memory_used_decode = self.first_stage_model.comfy_memory_used_decode+ self.memory_used_encode = self.first_stage_model.comfy_memory_used_encodeself.working_dtypes = [torch.float16, torch.bfloat16, torch.float32]self.handles_tiling = Trueself.format_encoded = self.first_stage_model.comfy_format_encoded@@ -1270,6 +1270,14 @@ class VAE:batch_number = int(free_memory / memory_used)batch_number = max(1, batch_number)+ # A VAE with a reliable estimate skips the doomed untiled attempt.+ if getattr(self.first_stage_model, "comfy_decode_estimate_is_reliable", False):+ if memory_used > free_memory:+ logging.info("VAE decode needs more than the free VRAM; tiling directly.")+ raise model_management.OOM_EXCEPTION(+ "decode estimate exceeds free memory; going straight to tiled"+ )+# Pre-allocate output for VAEs that support direct buffer writespreallocated = Falseif getattr(self.first_stage_model, 'comfy_has_chunked_io', False):@@ -1313,6 +1321,10 @@ class VAE:pixel_samples = self.decode_tiled_(samples_in)elif dims == 3:tile = 256 // self.spacial_compression_decode()+ # A VAE that can size its own tiles raises the 256-pixel floor.+ prefers_tile = getattr(self.first_stage_model, "preferred_decode_tile", None)+ if prefers_tile is not None:+ tile = max(tile, prefers_tile(self.patcher.get_free_memory(self.device), self.vae_dtype))overlap = tile // 4if self.handles_tiling:memory_used = self.memory_used_decode(self._tile_bounded_shape(samples_in.shape, tile, tile, None), self.vae_dtype)diff --git a/tests-unit/comfy_test/seedvr_vae_forward_test.py b/tests-unit/comfy_test/seedvr_vae_forward_test.pydeleted file mode 100644index 7ea7a143e..000000000--- a/tests-unit/comfy_test/seedvr_vae_forward_test.py+++ /dev/null@@ -1,74 +0,0 @@-"""Regression tests for the SeedVR2 VAE forward return contract."""--import pytest-import torch-import torch.nn as nn--from comfy.cli_args import args as cli_args--if not torch.cuda.is_available():- cli_args.cpu = True--from comfy.ldm.seedvr.vae import SEEDVR2_LATENT_CHANNELS, VideoAutoencoderKL # noqa: E402---_LATENT_SHAPE = (1, SEEDVR2_LATENT_CHANNELS, 2, 2, 2)-_DECODED_SHAPE = (1, 3, 5, 16, 16)-_INPUT_ENCODE_SHAPE = (1, 3, 5, 16, 16)-_INPUT_DECODE_SHAPE = _LATENT_SHAPE---class _StubVAE(VideoAutoencoderKL):- def __init__(self):- nn.Module.__init__(self)- self._encode_out = torch.zeros(*_LATENT_SHAPE)- self._decode_out = torch.zeros(*_DECODED_SHAPE)-- def encode(self, x, return_dict=True):- return self._encode_out-- def decode_(self, z, return_dict=True):- return self._decode_out---def test_forward_encode_returns_tensor():- vae = _StubVAE()- x = torch.zeros(*_INPUT_ENCODE_SHAPE)- result = vae.forward(x, mode="encode")- assert type(result) is torch.Tensor- assert result.shape == torch.Size(_LATENT_SHAPE)---def test_forward_decode_returns_tensor():- vae = _StubVAE()- z = torch.zeros(*_INPUT_DECODE_SHAPE)- result = vae.forward(z, mode="decode")- assert type(result) is torch.Tensor- assert result.shape == torch.Size(_DECODED_SHAPE)---class _TupleReturningStubVAE(VideoAutoencoderKL):- def __init__(self):- nn.Module.__init__(self)- self._encode_tensor = torch.zeros(*_LATENT_SHAPE)- self._decode_tensor = torch.zeros(*_DECODED_SHAPE)-- def encode(self, x, return_dict=True):- return (self._encode_tensor,)-- def decode_(self, z, return_dict=True):- return (self._decode_tensor,)---def test_forward_all_unwraps_one_tuple_at_each_step():- vae = _TupleReturningStubVAE()- x = torch.zeros(*_INPUT_ENCODE_SHAPE)- result = vae.forward(x, mode="all")- assert type(result) is torch.Tensor- assert result.shape == torch.Size(_DECODED_SHAPE)---def test_forward_rejects_unknown_mode():- vae = _StubVAE()- with pytest.raises(ValueError, match="Unknown SeedVR2 VAE forward mode"):- vae.forward(torch.zeros(*_INPUT_ENCODE_SHAPE), mode="bogus")diff --git a/tests-unit/comfy_test/test_seedvr2_dtype.py b/tests-unit/comfy_test/test_seedvr2_dtype.pyindex d743cc848..4dd6939f9 100644--- a/tests-unit/comfy_test/test_seedvr2_dtype.py+++ b/tests-unit/comfy_test/test_seedvr2_dtype.py@@ -40,39 +40,28 @@ def test_seedvr2_text_conditioning_accepts_cfg1_single_branch():torch.testing.assert_close(txt_shape, torch.tensor([[3]], device=context.device))-def test_seedvr2_vae_decode_memory_covers_full_frame_lab_transfer():- wrapper = seedvr_vae.VideoAutoencoderKLWrapper.__new__(seedvr_vae.VideoAutoencoderKLWrapper)- latent_channels = seedvr_vae.SEEDVR2_LATENT_CHANNELS- estimate = wrapper.comfy_memory_used_decode((1, latent_channels, 26, 120, 160))- old_estimate = latent_channels * 120 * 160 * (4 * 8 * 8) * 2-- assert estimate == 101 * 960 * 1280 * 160- assert estimate > 15 * 1024 ** 3- assert estimate > old_estimate * 100--def test_seedvr2_vae_encode_preserves_compute_dtype(monkeypatch):wrapper = seedvr_vae.VideoAutoencoderKLWrapper.__new__(seedvr_vae.VideoAutoencoderKLWrapper)nn.Module.__init__(wrapper)wrapper._dummy = nn.Parameter(torch.empty(1, dtype=torch.float16))input_dtype = None- def encode(self, x):+ def slicing_encode(self, x):nonlocal input_dtypeinput_dtype = x.dtypereturn x- monkeypatch.setattr(seedvr_vae.VideoAutoencoderKL, "encode", encode)+ monkeypatch.setattr(seedvr_vae.VideoAutoencoderKLWrapper, "slicing_encode", slicing_encode)x = torch.zeros((1, 3, 1, 8, 8), dtype=torch.float32)- wrapper._encode_with_raw_latent(x)+ wrapper.encode(x)assert input_dtype == torch.float32def test_seedvr2_vae_ops_cast_weights_to_compute_dtype():- attention = seedvr_vae.Attention(query_dim=4, heads=1, dim_head=4).to(torch.float16)- hidden_states = torch.zeros((1, 2, 4), dtype=torch.float32)+ attention = seedvr_vae.Attention(query_dim=4, norm_num_groups=2, eps=1e-6).to(torch.float16)+ hidden_states = torch.zeros((1, 4, 2, 2), dtype=torch.float32)output = attention(hidden_states)diff --git a/tests-unit/comfy_test/test_seedvr2_internals.py b/tests-unit/comfy_test/test_seedvr2_internals.pyindex fe4bde1c4..9e620a215 100644--- a/tests-unit/comfy_test/test_seedvr2_internals.py+++ b/tests-unit/comfy_test/test_seedvr2_internals.py@@ -2,6 +2,7 @@from __future__ import annotations+import inspectfrom unittest.mock import patchimport pytest@@ -27,6 +28,27 @@ _NUM_CHANNELS = 8_NUM_GROUPS = 4_TENSOR_SHAPE = (1, 8, 2, 4, 4)+[email protected](autouse=True)+def _isolate_weight_patches():+ """Keep these tests independent of leaked weight patches.++ ``comfy.ops.CastWeightBiasOp`` holds ``weight_function``/``bias_function`` as *class* level+ lists, so any code that appends to one without first giving the module its own list mutates the+ shared default and every comfy layer created afterwards silently inherits the patch. That is+ what happens when these tests run after tests-unit/comfy_quant in the same process.+ """+ cls = comfy_ops.CastWeightBiasOp+ saved = (list(cls.weight_function), list(cls.bias_function))+ cls.weight_function.clear()+ cls.bias_function.clear()+ try:+ yield+ finally:+ cls.weight_function[:] = saved[0]+ cls.bias_function[:] = saved[1]++_GROUPNORM_SUBCLASSES = [pytest.param(comfy_ops.disable_weight_init.GroupNorm, id="disable_weight_init"),pytest.param(comfy_ops.manual_cast.GroupNorm, id="manual_cast"),@@ -124,6 +146,78 @@ def test_seedvr2_7b_swin_attention_forward_uses_optimized_var_attention(monkeypaassert call["cu_seqlens_k"] == [0, 7, 14]+def _make_swin_attention(rope_type, dim=16, heads=2, head_dim=8):+ torch.manual_seed(0)+ attn = seedvr_model.NaSwinAttention(+ vid_dim=dim,+ txt_dim=dim,+ heads=heads,+ head_dim=head_dim,+ qk_bias=False,+ qk_norm=comfy_ops.disable_weight_init.RMSNorm,+ qk_norm_eps=1e-6,+ rope_type=rope_type,+ rope_dim=head_dim,+ shared_weights=False,+ window=(2, 2, 2),+ window_method="720pwin_by_size_bysize",+ version=(rope_type == "rope3d"),+ device="cpu",+ dtype=torch.float32,+ operations=comfy_ops.disable_weight_init,+ )+ for param in attn.parameters():+ torch.nn.init.normal_(param, std=0.5)+ return attn++[email protected]("rope_type", [None, "rope3d", "mmrope3d"])+def test_seedvr2_swin_attention_batched_samples_match_one_at_a_time(rope_type):+ """A batched cond+uncond forward must give each sample its own text tokens."""+ attn = _make_swin_attention(rope_type)+ generator = torch.Generator(device="cpu").manual_seed(3)+ vids = [torch.randn(2, 6, 6, 16, generator=generator), torch.randn(3, 6, 8, 16, generator=generator)]+ txts = [torch.randn(5, 16, generator=generator), torch.randn(7, 16, generator=generator)]++ vid, vid_shape = seedvr_model.flatten(vids)+ txt, txt_shape = seedvr_model.flatten(txts)+ batched_vid, batched_txt = attn(vid, txt, vid_shape, txt_shape, seedvr_model.Cache())++ single_vid, single_txt = [], []+ for one_vid, one_txt in zip(vids, txts):+ vid_i, vid_shape_i = seedvr_model.flatten([one_vid])+ txt_i, txt_shape_i = seedvr_model.flatten([one_txt])+ out_vid, out_txt = attn(vid_i, txt_i, vid_shape_i, txt_shape_i, seedvr_model.Cache())+ single_vid.append(out_vid)+ single_txt.append(out_txt)++ torch.testing.assert_close(batched_vid, torch.cat(single_vid), rtol=1e-4, atol=1e-5)+ torch.testing.assert_close(batched_txt, torch.cat(single_txt), rtol=1e-4, atol=1e-5)+++def test_var_attention_optimized_split_batches_equal_length_windows(monkeypatch):+ heads = 2+ head_dim = 3+ q = torch.arange(36, dtype=torch.float32).reshape(6, heads, head_dim)+ k = q + 100+ v = q + 200+ cu = [0, 2, 4, 6]+ calls = []++ def fake_optimized_attention(q_arg, k_arg, v_arg, heads_arg, **kwargs):+ calls.append(tuple(q_arg.shape))+ return q_arg + v_arg++ monkeypatch.setattr(attention, "optimized_attention", fake_optimized_attention)++ out = var_attention_optimized_split(q, k, v, heads, cu, cu, skip_reshape=True, skip_output_reshape=True)++ assert calls == [(3, heads, 2, head_dim)], (+ f"equal-length windows must share one batched attention call; got {calls}"+ )+ torch.testing.assert_close(out, q + v, rtol=0, atol=0)++def test_var_attention_optimized_split_calls_dense_backend_per_window(monkeypatch):heads = 2head_dim = 3@@ -167,3 +261,462 @@ def test_var_attention_optimized_split_calls_dense_backend_per_window(monkeypatcassert all(call["kwargs"]["skip_output_reshape"] is True for call in calls)torch.testing.assert_close(out, q + v, rtol=0, atol=0)++class _AlwaysPackedCache(vae_mod.CausalMemoryCache):+ """Drops the size gate so small tails exercise the packing."""++ def _packable(self, value):+ return torch.is_tensor(value) and value.is_cuda and value.is_contiguous(memory_format=torch.channels_last_3d)+++def _cache_tail(channels, frames=2):+ torch.manual_seed(0)+ tail = torch.randn(1, channels, frames, 48, 64, device="cuda") * 2+ tail[:, ::5] *= 20.0 # the per-channel outliers the rotation exists to spread+ return tail.half().contiguous(memory_format=torch.channels_last_3d)+++def _rel_err(a, b):+ return ((a.float() - b.float()).norm() / b.float().norm()).item()+++def test_causal_memory_cache_holds_unpackable_tails_as_they_are():+ cache = vae_mod.CausalMemoryCache()+ assert not cache._packable(torch.zeros(1, 128, 2, 4, 4)), "cpu tails stay unpacked"+ assert not cache._packable(torch.zeros(4, 8)) and not cache._packable("not a tensor")+ tail = torch.randn(1, 96, 2, 4, 4)+ cache["conv"] = tail+ assert cache.get("conv") is tail and "conv" in cache+ assert cache.pop("conv") is tail and "conv" not in cache+ assert cache.pop("conv", "fallback") == "fallback"+ assert cache.get("missing") is None++[email protected](not torch.cuda.is_available(), reason="tails pack on CUDA")[email protected]("channels", [128, 256, 512])+def test_causal_memory_cache_packing_round_trips(channels):+ tail = _cache_tail(channels)+ cache = _AlwaysPackedCache()+ cache["conv"] = tail+ assert "conv" in cache.packed+ restored = cache.get("conv")+ assert restored.shape == tail.shape and restored.dtype == tail.dtype+ assert restored.is_contiguous(memory_format=torch.channels_last_3d)+ assert _rel_err(restored, tail) < 1e-2+ plain = torch.randn(1, 96, 2, 4, 4)+ cache["conv"] = plain # replaces the packed entry+ assert cache.get("conv") is plain and "conv" not in cache.packed++[email protected](not torch.cuda.is_available(), reason="offload moves tails between CUDA and host memory")+def test_causal_memory_cache_offload_round_trips_across_slices():+ """Offloaded tails come back exactly as the resident packing returns them, in the order the+ next slice reads them (which the one-ahead prefetch assumes), and free() unpins the host buffers."""+ keys = [f"conv{i}" for i in range(6)]+ tails = {k: _cache_tail(128) for k in keys}+ pinned_before = vae_mod.comfy.model_management.TOTAL_PINNED_MEMORY+ resident = _AlwaysPackedCache()+ offloaded = _AlwaysPackedCache(offload=True)+ for slice_idx in range(3):+ for k in keys:+ resident[k] = tails[k] * (slice_idx + 1)+ offloaded[k] = tails[k] * (slice_idx + 1)+ assert not any(q.is_cuda for q, *_ in offloaded.packed.values()), "offloaded tails stay off the GPU between slices"+ for k in keys:+ a, b = resident.get(k), offloaded.get(k)+ assert b.is_contiguous(memory_format=torch.channels_last_3d)+ assert torch.equal(a, b), f"{k} slice {slice_idx}: offloaded tail differs"+ assert offloaded.pop("conv0") is not None and "conv0" not in offloaded+ offloaded.free()+ assert vae_mod.comfy.model_management.TOTAL_PINNED_MEMORY == pinned_before++[email protected](not torch.cuda.is_available(), reason="offload moves tails between CUDA and host memory")+def test_causal_memory_cache_offloads_small_tails_and_parked_frames_exactly():+ """Below the packing threshold an offloaded tail goes to the host as it is, like parked frames:+ both come back bit-exact."""+ cache = vae_mod.CausalMemoryCache(offload=True)+ tail = _cache_tail(64)+ cache["conv"] = tail+ assert not cache.packed["conv"][0].is_cuda+ assert torch.equal(cache.pop("conv"), tail)+ frames = list(_cache_tail(128, frames=3).split(1, dim=2))+ keys = cache.park("tail", frames)+ assert all(torch.equal(cache.pop(k), f) for k, f in zip(keys, frames))+ cache.free()+++def _ring(cache, key):+ frames = []+ cache.for_each_frame(key, lambda i, frame: frames.append(frame.clone()))+ return torch.cat(frames, dim=2)++[email protected](not torch.cuda.is_available(), reason="ring entries pack and offload on CUDA")+def test_causal_memory_cache_frame_ring_keeps_the_last_frames():+ """push_frames extends a key's tail one frame at a time; for_each_frame reads the last `keep`+ frames in order, retired frames are dropped, and a plain assignment replaces the ring."""+ cache = _AlwaysPackedCache(offload=True)+ frames = [f.contiguous(memory_format=torch.channels_last_3d) for f in _cache_tail(128, frames=4).split(1, dim=2)]+ cache.push_frames("conv", torch.cat(frames[:2], dim=2), keep=2)+ assert "conv" in cache+ assert _rel_err(_ring(cache, "conv"), torch.cat(frames[:2], dim=2)) < 1e-2+ cache.push_frames("conv", frames[2], keep=2) # one new frame: one packed entry+ assert len(cache._rings["conv"]) == 2+ assert _rel_err(_ring(cache, "conv"), torch.cat(frames[1:3], dim=2)) < 1e-2+ cache.push_frames("conv", frames[3], keep=2)+ assert len(cache.plain) + len(cache.packed) == 2, "retired frames must be dropped, not accumulated"+ cache["conv"] = frames[0] # plain assignment replaces the ring+ assert "conv" not in cache._rings and cache.get("conv").shape == frames[0].shape+ assert len(cache.plain) + len(cache.packed) == 1+ cache.free()++[email protected](not torch.cuda.is_available(), reason="ring entries are CUDA frames")[email protected]("offload", [False, True])+def test_causal_memory_cache_push_frames_copies_out_of_the_callers_buffer(offload):+ """Frames too small to pack are pushed as views of the conv's padded buffer; held as views they+ would keep the whole buffer alive, and the caller reuses it once the push returns."""+ cache = vae_mod.CausalMemoryCache(offload=offload)+ buf = _cache_tail(64, frames=4)+ expected = buf[:, :, -2:].clone()+ cache.push_frames("conv", buf[:, :, -2:], keep=2)+ buf.zero_()+ if not offload:+ assert all(t.untyped_storage().data_ptr() != buf.untyped_storage().data_ptr() for t in cache.plain.values())+ assert torch.equal(_ring(cache, "conv"), expected)+ cache.free()+++def _half_cuda(module):+ torch.manual_seed(0)+ for p in module.parameters():+ torch.nn.init.normal_(p, std=0.1)+ return module.to("cuda", torch.float16)+++def _slices(channels, frames, h, w):+ """fp16 activations for consecutive slices; the reference runs them NCDHW, which takes the eager+ path. Sized so the fp16-accumulate runs reach the kitchen conv, which leaves small launches to cuDNN."""+ torch.manual_seed(1)+ return [torch.randn(1, channels, t, h, w, device="cuda", dtype=torch.float16) for t in frames]+++def _states(n):+ return [vae_mod.MemoryState.INITIALIZING] + [vae_mod.MemoryState.ACTIVE] * (n - 1)++[email protected](not torch.cuda.is_available(), reason="the buffered convs run fp16 on CUDA")[email protected]("fp16acc", [False, True])[email protected]("temporal_down", [False, True])+def test_seedvr2_downsample_buffered_matches_padded_conv(temporal_down, fp16acc, monkeypatch):+ """The buffered downsampler writes into the conv's buffer with its own right/bottom border in+ place of F.pad; across slices it must match the pad-then-conv path and its cache."""+ down = _half_cuda(vae_mod.Downsample3D(16, temporal_down=temporal_down))+ slices = _slices(16, (9, 8) if temporal_down else (5, 4), h=160, w=160)+ ref_cache, cache = vae_mod.CausalMemoryCache(), vae_mod.CausalMemoryCache()+ for x, state in zip(slices, _states(len(slices))):+ ref = down.run([x], state, ref_cache)+ with monkeypatch.context() as m:+ m.setattr(torch.backends.cuda.matmul, "allow_fp16_accumulation", fp16acc, raising=False)+ got = down.run([x.contiguous(memory_format=torch.channels_last_3d)], state, cache)+ assert got.shape == ref.shape+ assert _rel_err(got, ref) < (2e-3 if fp16acc else 1e-3)++[email protected](not torch.cuda.is_available(), reason="the buffered convs run fp16 on CUDA")[email protected]("fp16acc", [False, True])+def test_seedvr2_upsample_per_frame_matches_whole_slice(fp16acc, monkeypatch):+ """The decoder's last temporal upsampler runs a latent frame at a time through frames(); its+ output frames, and the whole-slice buffered run, must match the eager shuffle-then-conv path."""+ up = _half_cuda(vae_mod.Upsample3D(16, temporal_up=True))+ slices = _slices(16, (1, 1, 1), h=64, w=64)+ caches = [vae_mod.CausalMemoryCache() for _ in range(3)]+ for x, state in zip(slices, _states(len(slices))):+ ref = up.run([x], state, caches[0])+ x = x.contiguous(memory_format=torch.channels_last_3d)+ with monkeypatch.context() as m:+ m.setattr(torch.backends.cuda.matmul, "allow_fp16_accumulation", fp16acc, raising=False)+ whole = up.run([x], state, caches[1])+ per_frame = torch.cat([box.pop() for box in up.frames([x], state, caches[2])], dim=2)+ for got in (whole, per_frame):+ assert got.shape == ref.shape+ assert _rel_err(got, ref) < (2e-3 if fp16acc else 1e-3)+++def _make_block(is_last_layer, dim=16, heads=2, head_dim=8):+ torch.manual_seed(0)+ block = seedvr_model.NaMMSRTransformerBlock(+ vid_dim=dim, txt_dim=dim, emb_dim=dim * 6, heads=heads, head_dim=head_dim,+ expand_ratio=2, norm=comfy_ops.disable_weight_init.RMSNorm, norm_eps=1e-6,+ ada=seedvr_model.AdaSingle, qk_bias=False, qk_norm=comfy_ops.disable_weight_init.RMSNorm,+ mlp_type="normal", shared_weights=False, rope_type="mmrope3d", rope_dim=head_dim,+ is_last_layer=is_last_layer, window=(2, 2, 2), window_method="720pwin_by_size_bysize",+ version=False, device="cpu", dtype=torch.float32,+ operations=comfy_ops.disable_weight_init,+ )+ for param in block.parameters():+ torch.nn.init.normal_(param, std=0.5)+ return block++[email protected]("is_last_layer", [False, True])+def test_seedvr2_norm_ada_in_matches_unfused_norm_then_modulate(is_last_layer):+ """The fused norm+modulate must agree with norm() then ada() on BOTH branches.++ The last block's attn_norm normalizes txt while its ada modulates vid alone, so a fused path+ that lets ada carry the norm hands attention an unnormalized txt to read as K/V.+ """+ block = _make_block(is_last_layer)+ generator = torch.Generator(device="cpu").manual_seed(5)+ vid = torch.randn(12, 16, generator=generator) * 30.0+ txt = torch.randn(7, 16, generator=generator) * 30.0+ emb = torch.randn(1, 16 * 6, generator=generator)++ for norm, layer in ((block.attn_norm, "attn"), (block.mlp_norm, "mlp")):+ ada_kwargs = {+ "emb": emb,+ "hid_len": seedvr_model.MMArg(torch.tensor([12]), torch.tensor([7])),+ "cache": seedvr_model.Cache(),+ "branch_tag": seedvr_model.MMArg("vid", "txt"),+ }+ fused_vid, fused_txt = block._norm_ada_in(norm, vid.clone(), txt.clone(), layer, ada_kwargs)++ ada_kwargs["cache"] = seedvr_model.Cache()+ ref_vid, ref_txt = norm(vid.clone(), txt.clone())+ ref_vid, ref_txt = block.ada(ref_vid, ref_txt, layer=layer, mode="in", **ada_kwargs)++ torch.testing.assert_close(fused_vid, ref_vid, rtol=1e-4, atol=1e-4)+ torch.testing.assert_close(+ fused_txt, ref_txt, rtol=1e-4, atol=1e-4,+ msg=lambda m: f"{layer} txt branch diverged (is_last_layer={is_last_layer}):\n{m}",+ )+++def test_seedvr2_vid_out_ada_reuses_the_block_attn_modulation():+ """``vid_out_ada`` has layers=["out"], so its own slice of a 6*dim embedding is twice as wide+ as hid. Upstream survives that because its cache key collides with the blocks' "attn" entry and+ it silently takes theirs; the released 3B weights were exported against that aliasing."""+ dim = 16+ torch.manual_seed(0)+ block_ada = seedvr_model.AdaSingle(dim=dim, emb_dim=dim * 6, layers=["attn", "mlp"])+ out_ada = seedvr_model.AdaSingle(dim=dim, emb_dim=dim * 6, layers=["out"], modes=["in"])+ for module in (block_ada, out_ada):+ for param in module.parameters():+ torch.nn.init.normal_(param, std=0.1)++ generator = torch.Generator(device="cpu").manual_seed(1)+ emb = torch.randn(1, dim * 6, generator=generator)+ hid_len = torch.tensor([9])+ cache = seedvr_model.Cache()++ block_ada(torch.randn(9, dim, generator=generator), emb=emb, layer="attn", mode="in",+ cache=cache, branch_tag="vid", hid_len=hid_len)+ assert "emb_repeat_0_vid" in cache.cache++ hid = torch.randn(9, dim, generator=generator)+ out = out_ada(hid.clone(), emb=emb, layer="out", mode="in",+ cache=cache, branch_tag="vid", hid_len=hid_len)++ shiftA, scaleA, _ = cache.cache["emb_repeat_0_vid"].unbind(-1)+ expected = hid * (scaleA + out_ada.out_scale) + (shiftA + out_ada.out_shift)+ torch.testing.assert_close(out, expected, rtol=1e-5, atol=1e-6)++[email protected]("padding", [(0, 0, 0, 0, 0, 0), (1, 1, 1, 1, 0, 0), (1, 2, 0, 1, 3, 0)])[email protected]("concat_dim", [2, 3])[email protected]("with_cache", [False, True])+def test_seedvr2_padded_input_matches_cat_then_pad(padding, concat_dim, with_cache):+ """``_padded_input`` replaces cat-then-F.pad with one buffer; it must be value-identical, a+ no-op when there is nothing to add, and cast a cache of another dtype where cat would raise."""+ generator = torch.Generator(device="cpu").manual_seed(7)+ x = torch.randn(1, 8, 4, 5, 6, generator=generator)+ cache = None+ if with_cache:+ shape = list(x.shape)+ shape[concat_dim] = 2+ cache = torch.randn(*shape, generator=generator).to(torch.float16)++ got = vae_mod.InflatedCausalConv3d._padded_input(x, cache, padding, concat_dim)++ reference = x if cache is None else torch.cat([cache.to(x.dtype), x], dim=concat_dim)+ reference = torch.nn.functional.pad(reference, padding, mode="constant", value=0.0)+ assert got.dtype == x.dtype and got.shape == reference.shape+ torch.testing.assert_close(got, reference, rtol=0, atol=0)+ if cache is None and not any(padding):+ assert got is x+++def _reference_group_norm(norm, x, silu):+ """What causal_norm_wrapper computes: per-frame statistics via the (b*t, c, h, w) view."""+ b, c, t, h, w = x.shape+ out = torch.nn.functional.group_norm(+ x.transpose(1, 2).reshape(b * t, c, h, w), norm.num_groups, norm.weight, norm.bias, norm.eps,+ )+ if silu:+ out = torch.nn.functional.silu(out)+ return out.reshape(b, t, c, h, w).transpose(1, 2)++[email protected]("silu", [False, True])+def test_seedvr2_causal_norm_wrapper_silu_matches_norm_then_silu(silu):+ """The silu flag must fold in exactly, on whichever path this machine takes."""+ torch.manual_seed(0)+ norm = comfy_ops.disable_weight_init.GroupNorm(4, 16, eps=1e-6, affine=True)+ torch.nn.init.normal_(norm.weight, mean=1.0, std=0.2)+ torch.nn.init.normal_(norm.bias, std=0.2)+ x = torch.randn(1, 16, 3, 5, 7)++ got = causal_norm_wrapper(norm, x, silu=silu)++ torch.testing.assert_close(got, _reference_group_norm(norm, x, silu), rtol=1e-5, atol=1e-6)+++def _decode_estimate(frames, height, width, dtype=torch.float16, batch=1, offload=True):+ """Whether the caches can be pinned in host memory depends on the machine's RAM; the measured+ floors are with them offloaded, so the test decides it instead of the machine."""+ wrapper = vae_mod.VideoAutoencoderKLWrapper.__new__(vae_mod.VideoAutoencoderKLWrapper)+ latent_t = (frames - 1) // 4 + 1+ with patch.object(vae_mod, "_offload_caches_for", lambda frame_pixels: offload):+ return wrapper.comfy_memory_used_decode((batch, 16, latent_t, height // 8, width // 8), dtype)+++def test_seedvr2_decode_estimate_charges_resident_caches_without_offload():+ """Without room to pin the caches in host memory they stay on the GPU, and the estimate says so."""+ resident = _decode_estimate(21, 1080, 1920, offload=False) - _decode_estimate(21, 1080, 1920)+ assert resident == 1080 * 1920 * vae_mod.SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL+++def _encode_estimate(frames, height, width, dtype=torch.float16, batch=1):+ wrapper = vae_mod.VideoAutoencoderKLWrapper.__new__(vae_mod.VideoAutoencoderKLWrapper)+ return wrapper.comfy_memory_used_encode((batch, 3, frames, height, width), dtype)+++def test_seedvr2_decode_estimate_is_flat_in_clip_length():+ """Slicing bounds the working set and the decoded frames go straight to sd.py's output buffer+ off the GPU, so clip length must not move the estimate: charging it per frame sent long clips to+ the reliable-estimate tiled path (about 1.75x slower) for memory they never use."""+ assert _decode_estimate(241, 1080, 1920) == _decode_estimate(9, 1080, 1920)++[email protected]("frames, height, width, measured_gib", [+ (9, 480, 864, 1.51),+ (21, 720, 1280, 3.01),+ (9, 864, 1536, 4.01),+ (21, 1080, 1920, 6.26),+ (9, 2160, 3840, 23.98),+])+def test_seedvr2_decode_estimate_tracks_measured_peak(frames, height, width, measured_gib):+ """The least free VRAM (to 0.5 GiB) an untiled decode ran in under cudaMallocAsync, RTX 5090;+ the estimate should sit just above it, never under it."""+ estimate_gib = _decode_estimate(frames, height, width) / 1024 ** 3+ assert estimate_gib >= measured_gib, (+ f"estimate {estimate_gib:.2f} GiB under-reports the measured {measured_gib:.2f} GiB peak"+ )+ assert estimate_gib < measured_gib * 1.35, (+ f"estimate {estimate_gib:.2f} GiB is far above the measured {measured_gib:.2f} GiB peak"+ )++[email protected]("frames, height, width, measured_gib", [+ (9, 480, 864, 1.51),+ (21, 720, 1280, 2.51),+ (9, 864, 1536, 3.01),+ (21, 1088, 1920, 5.01),+ (9, 2160, 3840, 19.73),+])+def test_seedvr2_encode_estimate_tracks_measured_floor(frames, height, width, measured_gib):+ """Same measurement as the decode table; slicing and the frame-chunked head keep an encode flat+ in clip length, so the estimate must not grow with it."""+ estimate = _encode_estimate(frames, height, width)+ assert measured_gib <= estimate / 1024 ** 3 < measured_gib * 1.35+ assert _encode_estimate(frames * 10, height, width) == estimate++[email protected]("dtype, decode_factor, encode_factor", [(torch.bfloat16, 5, 3), (torch.float32, 10, 6)])+def test_seedvr2_estimates_scale_for_the_eager_path(dtype, decode_factor, encode_factor):+ """Only fp16 runs the channels-last path the figures are fitted on; bf16 and fp32 run the eager+ path, measured at 2.4-5x (bf16) and 4-8.4x (fp32) the fp16 estimate. A batch is run a video at a+ time, so the estimate is per video."""+ assert _decode_estimate(21, 720, 1280, dtype) / _decode_estimate(21, 720, 1280) == pytest.approx(decode_factor)+ assert _encode_estimate(21, 720, 1280, dtype) / _encode_estimate(21, 720, 1280) == pytest.approx(encode_factor)+ assert _decode_estimate(21, 720, 1280, dtype, batch=3) == _decode_estimate(21, 720, 1280, dtype)+ assert _encode_estimate(21, 720, 1280, dtype, batch=3) == _encode_estimate(21, 720, 1280, dtype)+ assert _tile_side(8, dtype) < _tile_side(8)+++def test_seedvr2_batch_runs_a_video_at_a_time():+ """The buffered convs and the estimates are per video: a batch is split before slicing, and the+ slices land back in their video's place (in the preallocated output too)."""+ vae = vae_mod.VideoAutoencoderKLWrapper.__new__(vae_mod.VideoAutoencoderKLWrapper)+ vae.slicing_sample_min_size = vae.slicing_latent_min_size = 2+ seen = []++ def run(x, memory_state=vae_mod.MemoryState.DISABLED, memory_cache=None):+ seen.append(x.size(0))+ return x[:, :3].clone()+ vae._decode = vae._encode = run+ z = torch.randn(2, 16, 7, 4, 4)+ torch.testing.assert_close(vae.slicing_decode(z), z[:, :3], rtol=0, atol=0)+ out = torch.empty(2, 3, 7, 4, 4)+ assert vae.slicing_decode(z, output_buffer=out) is out+ torch.testing.assert_close(out, z[:, :3], rtol=0, atol=0)+ torch.testing.assert_close(vae.slicing_encode(z), z[:, :3], rtol=0, atol=0)+ assert set(seen) == {1}+++def test_seedvr2_encode_accepts_the_chunked_io_device_kwarg():+ """``comfy_has_chunked_io`` is one flag for both directions: sd.py leaves the pixels where they+ are and calls ``encode(x, device=...)``, so encode must take it and move the data itself."""+ sig = inspect.signature(vae_mod.VideoAutoencoderKLWrapper.encode)+ assert "device" in sig.parameters, "encode must accept the chunked-io device kwarg"+ assert sig.parameters["device"].default is None, "device must be optional"+ # the wrapper claims the protocol, so both sides of it have to exist+ assert vae_mod.VideoAutoencoderKLWrapper.comfy_has_chunked_io is True+ assert hasattr(vae_mod.VideoAutoencoderKLWrapper, "decode_output_shape")+ assert "output_buffer" in inspect.signature(vae_mod.VideoAutoencoderKLWrapper.decode).parameters+++def test_seedvr2_decode_output_shape_matches_decode():+ """sd.py preallocates from this; it must equal what decode() returns (frames from the 4n+1+ rule, spatial 8x, cropped to even) for both latent layouts."""+ wrapper = vae_mod.VideoAutoencoderKLWrapper.__new__(vae_mod.VideoAutoencoderKLWrapper)+ wrapper.spatial_downsample_factor = 8+ assert wrapper.decode_output_shape((1, 16, 6, 90, 160)) == (1, 3, 21, 720, 1280)+ assert wrapper.decode_output_shape((1, 16, 1, 135, 240)) == (1, 3, 1, 1080, 1920)+ assert wrapper.decode_output_shape((2, 16 * 3, 45, 81)) == (2, 3, 9, 360, 648)+ assert wrapper.decode_output_shape((1, 16, 2, 13, 13)) == (1, 3, 5, 104, 104)+ assert vae_mod.VideoAutoencoderKLWrapper.comfy_has_chunked_io is True+++def _tile_side(free_gib, dtype=torch.float16, offload=True):+ wrapper = vae_mod.VideoAutoencoderKLWrapper.__new__(vae_mod.VideoAutoencoderKLWrapper)+ wrapper.spatial_downsample_factor = 8+ with patch.object(vae_mod, "_offload_caches_for", lambda frame_pixels: offload):+ return wrapper.preferred_decode_tile(free_gib * 1024 ** 3, dtype)+++def test_seedvr2_tile_side_tracks_free_memory_within_bounds():+ """Grows with free memory, stays inside [min, max] on whole latent blocks, and the tile it+ picks is predicted to fit the memory it was sized against."""+ assert _tile_side(0.1) == vae_mod.SEEDVR2_MIN_TILE_LATENT+ assert _tile_side(10_000) == vae_mod.SEEDVR2_MAX_TILE_LATENT+ assert _tile_side(3) < _tile_side(30)+ assert _tile_side(30) * 8 >= 512+ for free in (2, 4, 8, 16, 24, 32, 80):+ side = _tile_side(free)+ assert vae_mod.SEEDVR2_MIN_TILE_LATENT <= side <= vae_mod.SEEDVR2_MAX_TILE_LATENT and side % 8 == 0+ if side != vae_mod.SEEDVR2_MIN_TILE_LATENT:+ predicted = (side * 8) ** 2 * vae_mod.SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL + vae_mod.SEEDVR2_DECODE_FIXED_BYTES+ assert predicted <= free * 1024 ** 3+++def test_seedvr2_tile_side_counts_resident_caches_without_offload():+ """With no room to pin the caches they stay on the GPU, and the chosen tile must still fit with them."""+ assert _tile_side(8, offload=False) < _tile_side(8), "the resident caches must shrink the tile"+ per_pixel = vae_mod.SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL + vae_mod.SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL+ for free in (8, 16, 32):+ side = _tile_side(free, offload=False)+ assert side <= _tile_side(free)+ predicted = (side * 8) ** 2 * per_pixel + vae_mod.SEEDVR2_DECODE_FIXED_BYTES+ assert predicted <= free * 1024 ** 3 * vae_mod.SEEDVR2_TILE_MEM_HEADROOMdiff --git a/tests-unit/comfy_test/test_seedvr2_model.py b/tests-unit/comfy_test/test_seedvr2_model.pyindex 1d454aaf1..b5aad00ca 100644--- a/tests-unit/comfy_test/test_seedvr2_model.py+++ b/tests-unit/comfy_test/test_seedvr2_model.py@@ -91,11 +91,9 @@ class _EncodeWrapper(seedvr_vae_mod.VideoAutoencoderKLWrapper):def __init__(self, encoded):nn.Module.__init__(self)self.encoded = encoded- self.spatial_downsample_factor = 8- self.temporal_downsample_factor = 4self.seen = []- def encode(self, x):+ def encode(self, x, device=None):self.seen.append(tuple(x.shape))return self.encoded.to(device=x.device, dtype=x.dtype)@@ -103,60 +101,29 @@ class _EncodeWrapper(seedvr_vae_mod.VideoAutoencoderKLWrapper):class _DecodeWrapper(seedvr_vae_mod.VideoAutoencoderKLWrapper):def __init__(self):nn.Module.__init__(self)- self.spatial_downsample_factor = 8- self.temporal_downsample_factor = 4- self.calls = []-- def decode(self, z, seedvr2_tiling=None):- self.calls.append({"shape": tuple(z.shape), "seedvr2_tiling": seedvr2_tiling})- if z.ndim == 4:- b, tc, h, w = z.shape- t = tc // _LATENT_CHANNELS- else:- b, _, t, h, w = z.shape- return torch.zeros(b, 3, t, h * 8, w * 8, dtype=z.dtype, device=z.device)def test_seedvr2_wrapper_public_encode_returns_tensor(monkeypatch):- raw_latent = torch.full((1, _LATENT_CHANNELS, 1, 4, 5), 2.0)+ moments = torch.cat([torch.full((1, _LATENT_CHANNELS, 1, 4, 5), 2.0), torch.full((1, _LATENT_CHANNELS, 1, 4, 5), -1.0)], dim=1)seen_shapes = []- def base_encode(self, x):+ def slicing_encode(self, x):seen_shapes.append(tuple(x.shape))- return raw_latent.to(device=x.device, dtype=x.dtype)+ return moments.to(device=x.device, dtype=x.dtype)- monkeypatch.setattr(seedvr_vae_mod.VideoAutoencoderKL, "encode", base_encode)+ monkeypatch.setattr(seedvr_vae_mod.VideoAutoencoderKLWrapper, "slicing_encode", slicing_encode)vae = seedvr_vae_mod.VideoAutoencoderKLWrapper.__new__(seedvr_vae_mod.VideoAutoencoderKLWrapper)nn.Module.__init__(vae)- vae._dummy = nn.Parameter(torch.zeros((), dtype=torch.float32))latent = vae.encode(torch.zeros(1, 3, 32, 40))assert type(latent) is torch.Tensorassert tuple(latent.shape) == (1, _LATENT_CHANNELS, 4, 5)+ assert torch.equal(latent, torch.full_like(latent, 2.0)), "the latent is the posterior mean"assert seen_shapes == [(1, 3, 1, 32, 40)]-def test_seedvr2_wrapper_private_encode_helper_keeps_raw_latent(monkeypatch):- raw_latent = torch.full((1, _LATENT_CHANNELS, 1, 4, 5), 3.0)-- def base_encode(self, x):- return raw_latent.to(device=x.device, dtype=x.dtype)-- monkeypatch.setattr(seedvr_vae_mod.VideoAutoencoderKL, "encode", base_encode)-- vae = seedvr_vae_mod.VideoAutoencoderKLWrapper.__new__(seedvr_vae_mod.VideoAutoencoderKLWrapper)- nn.Module.__init__(vae)- vae._dummy = nn.Parameter(torch.zeros((), dtype=torch.float32))-- latent, raw = vae._encode_with_raw_latent(torch.zeros(1, 3, 32, 40))-- assert tuple(latent.shape) == (1, _LATENT_CHANNELS, 4, 5)- assert tuple(raw.shape) == (1, _LATENT_CHANNELS, 1, 4, 5)- assert torch.equal(raw, raw_latent)--def _make_vae(wrapper):vae = sd_mod.VAE.__new__(sd_mod.VAE)vae.first_stage_model = wrapper@@ -209,6 +176,20 @@ def test_seedvr2_7b_keeps_final_block_text_path(monkeypatch):]+def _bytedance_interleaved_rope(x, freqs):+ """ByteDance SeedVR2 pixel-RoPE: adjacent feature pairs rotated by the interleaved angles,+ leaving the head-dim tail past the rotary width untouched."""+ rot = freqs.shape[-1]+ angles = freqs[:, ::2].float().unsqueeze(1)+ cos, sin = torch.cos(angles), torch.sin(angles)+ out = x.float().clone()+ even = out[..., 0:rot:2].clone()+ odd = out[..., 1:rot:2].clone()+ out[..., 0:rot:2] = even * cos - odd * sin+ out[..., 1:rot:2] = odd * cos + even * sin+ return out.to(x.dtype)++def test_seedvr2_7b_rope3d_matches_wrapper_oracle():rope = seedvr_model.get_na_rope("rope3d", dim=64)generator = torch.Generator(device="cpu").manual_seed(0)@@ -217,19 +198,13 @@ def test_seedvr2_7b_rope3d_matches_wrapper_oracle():shape = torch.tensor([[1, 2, 2]], dtype=torch.long)freqs = rope.get_axial_freqs(1, 2, 2).reshape(4, -1)- expected_q = seedvr_model._apply_seedvr2_rotary_emb(- freqs,- q.permute(1, 0, 2).float(),- ).to(q.dtype).permute(1, 0, 2)- expected_k = seedvr_model._apply_seedvr2_rotary_emb(- freqs,- k.permute(1, 0, 2).float(),- ).to(k.dtype).permute(1, 0, 2)+ expected_q = _bytedance_interleaved_rope(q, freqs)+ expected_k = _bytedance_interleaved_rope(k, freqs)actual_q, actual_k = rope(q.clone(), k.clone(), shape, seedvr_model.Cache(disable=True))- torch.testing.assert_close(actual_q, expected_q, rtol=0, atol=0)- torch.testing.assert_close(actual_k, expected_k, rtol=0, atol=0)+ torch.testing.assert_close(actual_q, expected_q, rtol=1e-5, atol=1e-6)+ torch.testing.assert_close(actual_k, expected_k, rtol=1e-5, atol=1e-6)def test_seedvr2_forward_requires_conditioning_latents():@@ -293,6 +268,8 @@ def test_seedvr2_encode_and_encode_tiled_preserve_native_latent_contract(monkeypdef test_vaedecode_tiled_spatial_applies_temporal_discarded(monkeypatch):monkeypatch.setattr(sd_mod.model_management, "load_models_gpu", lambda *a, **k: None)+ tiled = MagicMock(return_value=torch.zeros(1, 3, 5, 32, 40))+ monkeypatch.setattr(seedvr_vae_mod, "tiled_vae", tiled)vae = _make_vae(_DecodeWrapper())nodes_mod.VAEDecodeTiled().decode(@@ -306,15 +283,7 @@ def test_vaedecode_tiled_spatial_applies_temporal_discarded(monkeypatch):# Spatial inputs flow through; temporal inputs are discarded as public tiling# knobs, but SeedVR2's internal MemoryState causal slicing is left intact.- assert vae.first_stage_model.calls == [- {- "shape": (1, _LATENT_CHANNELS, 2, 4, 5),- "seedvr2_tiling": {- "enable_tiling": True,- "tile_size": (512, 512),- "tile_overlap": (64, 64),- "temporal_size": None,- "temporal_overlap": None,- },- }- ]+ assert tiled.call_count == 1+ (latent, _), kwargs = tiled.call_args+ assert tuple(latent.shape) == (1, _LATENT_CHANNELS, 2, 4, 5)+ assert kwargs == {"tile_size": (512, 512), "tile_overlap": (64, 64), "encode": False}diff --git a/tests-unit/comfy_test/test_seedvr2_vae_decode.py b/tests-unit/comfy_test/test_seedvr2_vae_decode.pyindex c486b9195..10c195374 100644--- a/tests-unit/comfy_test/test_seedvr2_vae_decode.py+++ b/tests-unit/comfy_test/test_seedvr2_vae_decode.py@@ -24,7 +24,7 @@ def _make_wrapper() -> vae_mod.VideoAutoencoderKLWrapper:return wrapper-def _fingerprint_decode_(self, z, return_dict=True):+def _fingerprint_decode(self, z, output_buffer=None):b = int(z.shape[0])t = int(z.shape[2])h = int(z.shape[3])@@ -36,7 +36,7 @@ def _fingerprint_decode_(self, z, return_dict=True):def _decode_with_patches(wrapper, z):- with patch.object(vae_mod.VideoAutoencoderKL, "decode_", _fingerprint_decode_):+ with patch.object(vae_mod.VideoAutoencoderKLWrapper, "slicing_decode", _fingerprint_decode):return wrapper.decode(z)@@ -53,10 +53,8 @@ class _Wrapper(vae_mod.VideoAutoencoderKLWrapper):nn.Module.__init__(self)self.calls = []- def parameters(self):- return iter([torch.nn.Parameter(torch.zeros(()))])-def _decode_stub(self, latent):+def _decode_stub(self, latent, output_buffer=None):self.calls.append(tuple(latent.shape))return torch.zeros(latent.shape[0], 3, latent.shape[2], latent.shape[3] * 8, latent.shape[4] * 8)@@ -64,20 +62,13 @@ def _decode_stub(self, latent):def test_seedvr2_wrapper_decode_accepts_5d_channel_first_latents_without_preprocessor_state():wrapper = _Wrapper()- with patch.object(vae_mod.VideoAutoencoderKL, "decode_", _decode_stub):+ with patch.object(vae_mod.VideoAutoencoderKLWrapper, "slicing_decode", _decode_stub):out = wrapper.decode(torch.zeros(1, _LATENT_CHANNELS, 2, 4, 5))assert tuple(out.shape) == (1, 3, 2, 32, 40)assert wrapper.calls == [(1, _LATENT_CHANNELS, 2, 4, 5)]-def test_seedvr2_wrapper_decode_rejects_wrong_rank_latents():- wrapper = _Wrapper()-- with pytest.raises(RuntimeError, match=r"latent input must be 4-D collapsed .* or 5-D"):- wrapper.decode(torch.zeros(1, _LATENT_CHANNELS, 4))--def _t_padded(t_in: int) -> int:if t_in == 1:return 1diff --git a/tests-unit/comfy_test/test_seedvr2_vae_tiled.py b/tests-unit/comfy_test/test_seedvr2_vae_tiled.pyindex d64f51918..322f48e28 100644--- a/tests-unit/comfy_test/test_seedvr2_vae_tiled.py+++ b/tests-unit/comfy_test/test_seedvr2_vae_tiled.py@@ -1,7 +1,6 @@from contextlib import ExitStackfrom unittest.mock import MagicMock, patch-import pytestimport torchimport torch.nn as nn@@ -10,85 +9,14 @@ from comfy.cli_args import args as cli_argsif not torch.cuda.is_available():cli_args.cpu = True-import comfy.ldm.seedvr.vae as vae_mod # noqa: E402import comfy.ldm.seedvr.vae as seedvr_vae_mod # noqa: E402import comfy.sd as sd_mod # noqa: E402-from comfy.ldm.seedvr.vae import MemoryState, tiled_vae # noqa: E402+from comfy.ldm.seedvr.vae import tiled_vae # noqa: E402_LATENT_CHANNELS = seedvr_vae_mod.SEEDVR2_LATENT_CHANNELS-def test_runtime_decode_zero_temporal_size_preserves_model_slicing():- class StubVAEModel(torch.nn.Module):- def __init__(self):- super().__init__()- self.slicing_latent_min_size = 2- self.spatial_downsample_factor = 8- self.temporal_downsample_factor = 4- self.device = torch.device("cpu")- self.use_slicing = True- self._dummy = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32))- self.decode_min_sizes = []- self.memory_states = []-- def decode_(self, t_chunk):- self.decode_min_sizes.append(self.slicing_latent_min_size)- return vae_mod.VideoAutoencoderKL.slicing_decode(self, t_chunk)-- def _decode(self, z, memory_state=MemoryState.DISABLED, memory_cache=None):- self.memory_states.append(memory_state)- b, c, d, h, w = z.shape- return torch.zeros((b, 3, d, h * 8, w * 8), dtype=z.dtype)-- vae = StubVAEModel()- z = torch.zeros((1, _LATENT_CHANNELS, 5, 8, 8), dtype=torch.float32)-- tiled_vae(- z,- vae,- tile_size=(64, 64),- tile_overlap=(0, 0),- temporal_size=0,- temporal_overlap=0,- encode=False,- )-- assert vae.decode_min_sizes == [2]- assert vae.memory_states == [MemoryState.INITIALIZING, MemoryState.ACTIVE]- assert vae.slicing_latent_min_size == 2---def test_zero_temporal_size_preserves_min_size_when_encode_raises():- class RaisingVAEModel(torch.nn.Module):- def __init__(self):- super().__init__()- self.slicing_sample_min_size = 4- self.spatial_downsample_factor = 8- self.temporal_downsample_factor = 4- self.device = torch.device("cpu")- self._dummy = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32))-- def encode(self, t_chunk):- raise RuntimeError("simulated encode failure")-- vae = RaisingVAEModel()- x = torch.zeros((1, 3, 12, 64, 64), dtype=torch.float32)-- with pytest.raises(RuntimeError, match="simulated encode failure"):- tiled_vae(- x,- vae,- tile_size=(64, 64),- tile_overlap=(0, 0),- temporal_size=0,- temporal_overlap=0,- encode=True,- )-- assert vae.slicing_sample_min_size == 4--def test_tiled_vae_encode_uses_tensor_return_without_indexing():class TensorEncodeVAEModel(torch.nn.Module):def __init__(self):@@ -113,8 +41,6 @@ def test_tiled_vae_encode_uses_tensor_return_without_indexing():vae,tile_size=(64, 64),tile_overlap=(0, 0),- temporal_size=0,- temporal_overlap=0,encode=True,)@@ -166,87 +92,12 @@ def test_tiled_vae_preserves_input_dtype_on_single_tile():FloatOutputVAEModel(),tile_size=(64, 64),tile_overlap=(0, 0),- temporal_size=0,- temporal_overlap=0,encode=True,)assert out.dtype == torch.float16-class _SlicingDecodeVAE(nn.Module):- def __init__(self, slicing_latent_min_size):- super().__init__()- self.slicing_latent_min_size = slicing_latent_min_size- self.spatial_downsample_factor = 8- self.temporal_downsample_factor = 4- self.device = torch.device("cpu")- self.use_slicing = True- self._dummy = nn.Parameter(torch.zeros(1, dtype=torch.float32))- self.decode_min_sizes = []- self.memory_states = []-- def decode_(self, z):- self.decode_min_sizes.append(self.slicing_latent_min_size)- return vae_mod.VideoAutoencoderKL.slicing_decode(self, z)-- def _decode(self, z, memory_state=MemoryState.DISABLED, memory_cache=None):- self.memory_states.append(memory_state)- x = z[:, :1].repeat(- 1,- 3,- 1,- self.spatial_downsample_factor,- self.spatial_downsample_factor,- )- return x---def test_decode_tiled_vae_maps_temporal_args_to_latent_slicing_min_size():- vae = _SlicingDecodeVAE(slicing_latent_min_size=2)- z = torch.arange(- _LATENT_CHANNELS * 5 * 8 * 8,- dtype=torch.float32,- ).reshape(1, _LATENT_CHANNELS, 5, 8, 8)-- tiled_vae(- z,- vae,- tile_size=(64, 64),- tile_overlap=(0, 0),- temporal_size=12,- temporal_overlap=4,- encode=False,- )-- assert vae.decode_min_sizes == [2]- assert vae.memory_states == [MemoryState.INITIALIZING, MemoryState.ACTIVE]- assert vae.slicing_latent_min_size == 2-- wrapper = vae_mod.VideoAutoencoderKLWrapper.__new__(- vae_mod.VideoAutoencoderKLWrapper- )- nn.Module.__init__(wrapper)- seedvr2_tiling = {- "enable_tiling": True,- "tile_size": (64, 64),- "tile_overlap": (0, 0),- "temporal_size": 8,- "temporal_overlap": 7,- }-- captured = {}-- def _fake_tiled_vae(latent, model, **kwargs):- captured.update(kwargs)- return torch.zeros(1, 3, 1, 16, 16)-- with patch.object(vae_mod, "tiled_vae", side_effect=_fake_tiled_vae):- wrapper.decode(torch.zeros(1, _LATENT_CHANNELS, 2, 2), seedvr2_tiling=seedvr2_tiling)-- assert captured["temporal_overlap"] == 7--def _force_oom(*a, **k):raise torch.cuda.OutOfMemoryError("forced OOM for dispatcher test")