Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Optimize Seedvr2 (#16530)

Jukka Seppänen84131a647167
diff --git a/comfy/ldm/seedvr/attention.py b/comfy/ldm/seedvr/attention.py
index 11b4c1e4a..a39f0ba9d 100644
--- a/comfy/ldm/seedvr/attention.py
+++ b/comfy/ldm/seedvr/attention.py
@@ -22,29 +22,41 @@ def _var_attention_output(out, heads, head_dim, skip_output_reshape):
return out.reshape(-1, heads * head_dim)
+def _equal_length_runs(lengths):
+ """(start, stop) sequence spans over which the sequence length is constant."""
+ runs = []
+ start = 0
+ for i in range(1, len(lengths) + 1):
+ if i == len(lengths) or lengths[i] != lengths[start]:
+ runs.append((start, i))
+ start = i
+ return runs
+
+
def var_attention_optimized_split(q, k, v, heads, cu_seqlens_q, cu_seqlens_k, *args, skip_reshape=False, skip_output_reshape=False, **kwargs):
q, k, v, head_dim = _var_attention_qkv(q, k, v, heads, skip_reshape)
- q_split_indices = cu_seqlens_q[1:-1]
- k_split_indices = cu_seqlens_k[1:-1]
if k.shape[0] != v.shape[0]:
raise ValueError("cu_seqlens_k does not match v token count")
-
- q_splits = torch.tensor_split(q, q_split_indices, dim=0)
- k_splits = torch.tensor_split(k, k_split_indices, dim=0)
- v_splits = torch.tensor_split(v, k_split_indices, dim=0)
- if len(q_splits) != len(k_splits) or len(q_splits) != len(v_splits):
+ if len(cu_seqlens_q) != len(cu_seqlens_k):
raise ValueError("cu_seqlens_q and cu_seqlens_k must describe the same sequence count")
- out = []
- for q_i, k_i, v_i in zip(q_splits, k_splits, v_splits):
- q_i = q_i.permute(1, 0, 2).unsqueeze(0)
- k_i = k_i.permute(1, 0, 2).unsqueeze(0)
- v_i = v_i.permute(1, 0, 2).unsqueeze(0)
+ q_lens = [end - start for start, end in zip(cu_seqlens_q, cu_seqlens_q[1:])]
+ k_lens = [end - start for start, end in zip(cu_seqlens_k, cu_seqlens_k[1:])]
+
+ out = torch.empty((cu_seqlens_q[-1], heads, head_dim), device=q.device, dtype=q.dtype)
+ # Windows of matching length are adjacent, so each run becomes one batched attention call.
+ for first, last in _equal_length_runs(list(zip(q_lens, k_lens))):
+ n = last - first
+ len_q, len_k = q_lens[first], k_lens[first]
+ q_slice = slice(cu_seqlens_q[first], cu_seqlens_q[last])
+ k_slice = slice(cu_seqlens_k[first], cu_seqlens_k[last])
+ q_i = q[q_slice].reshape(n, len_q, heads, head_dim).transpose(1, 2)
+ k_i = k[k_slice].reshape(n, len_k, heads, head_dim).transpose(1, 2)
+ v_i = v[k_slice].reshape(n, len_k, heads, head_dim).transpose(1, 2)
out_i = _attention.optimized_attention(q_i, k_i, v_i, heads, skip_reshape=True, skip_output_reshape=True)
- out.append(out_i.squeeze(0).permute(1, 0, 2))
+ out[q_slice] = out_i.transpose(1, 2).reshape(n * len_q, heads, head_dim)
- out = torch.cat(out, dim=0)
return _var_attention_output(out, heads, head_dim, skip_output_reshape)
diff --git a/comfy/ldm/seedvr/constants.py b/comfy/ldm/seedvr/constants.py
index 12c4b4bef..98fb5057b 100644
--- a/comfy/ldm/seedvr/constants.py
+++ b/comfy/ldm/seedvr/constants.py
@@ -13,10 +13,35 @@ SEEDVR2_CHUNK_SIGMA_K = 4
SEEDVR2_7B_VID_DIM = 3072
SEEDVR2_OOM_BACKOFF_DIVISOR = 2
SEEDVR2_DTYPE_BYTES_FLOOR = 4
-SEEDVR2_7B_MLP_CHUNK = 8192
+SEEDVR2_MLP_CHUNK = 8192
SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS = 4096 # partial-RoPE application token-chunk.
SEEDVR2_LATENT_CHANNELS = 16
+# VAE temporal caches: int8-packed (rotated, per-token scales) between slices.
+SEEDVR2_VAE_CACHE_QUANT_BYTES = 64 * 1024 ** 2 # tails at or above this size are packed.
+SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL = 6200 # host-held caches and parked frames, per output-frame pixel.
+
+# VAE decode estimate: the free VRAM an untiled decode needs under cudaMallocAsync (ComfyUI's default),
+# fitted 2-7% above the measured floor from 480p to 4K. Flat in clip length: the decoded frames go
+# to sd.py's output buffer off the GPU slice by slice (1088p: 5.33 GiB allocated at 61 and 121 frames).
+SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL = 3150 # working set, per output-frame pixel.
+SEEDVR2_DECODE_FIXED_BYTES = 384 * 1024 ** 2
+SEEDVR2_DECODE_LAB_BYTES_PER_OUTPUT_PIXEL = 160 # colour correction is per frame in the node.
+
+# VAE encode estimate, likewise fitted 3-23% above the measured floor; slicing keeps it flat in clip length.
+SEEDVR2_ENCODE_BYTES_PER_PIXEL = 2550 # per input-frame pixel.
+SEEDVR2_ENCODE_FIXED_BYTES = 576 * 1024 ** 2
+
+# The fits cover the fp16 channels-last path; other dtypes run the eager path, measured at about this
+# many times the estimate per two bytes of element.
+SEEDVR2_EAGER_DECODE_FACTOR = 5
+SEEDVR2_EAGER_ENCODE_FACTOR = 3
+
+# Tiled decode, in latent units (8 output pixels each).
+SEEDVR2_TILE_MEM_HEADROOM = 0.6
+SEEDVR2_MIN_TILE_LATENT = 32
+SEEDVR2_MAX_TILE_LATENT = 96 # 768 px; larger tiles measured no faster.
+
SEEDVR2_COLOR_MEM_HEADROOM = 0.75
SEEDVR2_LAB_SCALE_MULTIPLIER = 13
SEEDVR2_WAVELET_SCALE_MULTIPLIER = 10 # per-frame byte multiplier, wavelet path.
@@ -26,8 +51,6 @@ BYTEDANCE_VAE_SCALING_FACTOR = 0.9152 # configs_3b/main.yaml:57.
BYTEDANCE_VAE_SHIFTING_FACTOR = 0.0
BYTEDANCE_VAE_CONV_MEM_GIB = 0.5
BYTEDANCE_VAE_NORM_MEM_GIB = 0.5
-BYTEDANCE_LOGVAR_CLAMP_MIN = -30.0 # video_vae_v3/modules/types.py:28.
-BYTEDANCE_LOGVAR_CLAMP_MAX = 20.0 # video_vae_v3/modules/types.py:28.
BYTEDANCE_GN_CHUNKS_FP16 = 4 # causal_inflation_lib.py:351 (GroupNorm chunk count, fp16).
BYTEDANCE_GN_CHUNKS_FP32 = 2 # causal_inflation_lib.py:351 (GroupNorm chunk count, fp32).
BYTEDANCE_BLOCK_OUT_CHANNELS = (128, 256, 512, 512) # s8_c16_t4_inflation_sd3.yaml:7-11.
diff --git a/comfy/ldm/seedvr/model.py b/comfy/ldm/seedvr/model.py
index a978698d5..16bbaa494 100644
--- a/comfy/ldm/seedvr/model.py
+++ b/comfy/ldm/seedvr/model.py
@@ -16,13 +16,14 @@ from comfy.ldm.seedvr.constants import (
BYTEDANCE_ROPE_MAX_FREQ,
BYTEDANCE_SINUSOIDAL_DIM,
ROPE_THETA,
- SEEDVR2_7B_MLP_CHUNK,
+ SEEDVR2_MLP_CHUNK,
SEEDVR2_7B_VID_DIM,
SEEDVR2_LATENT_CHANNELS,
SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS,
)
import comfy.model_management
import comfy.ops
+import comfy.quant_ops
class Cache:
def __init__(self, disable=False, prefix="", cache=None):
@@ -47,19 +48,6 @@ class Cache:
cache=self.cache,
)
-def repeat_concat(
- vid: torch.FloatTensor, # (VL ... c)
- txt: torch.FloatTensor, # (TL ... c)
- vid_len: torch.LongTensor, # (n*b)
- txt_len: torch.LongTensor, # (b)
- txt_repeat: List, # (n)
-) -> torch.FloatTensor: # (L ... c)
- vid = torch.split(vid, vid_len.tolist())
- txt = torch.split(txt, txt_len.tolist())
- txt = [[x] * n for x, n in zip(txt, txt_repeat)]
- txt = list(chain(*txt))
- return torch.cat(list(chain(*zip(vid, txt))))
-
def repeat_concat_idx(
vid_len: torch.LongTensor, # (n*b)
txt_len: torch.LongTensor, # (b)
@@ -67,12 +55,18 @@ def repeat_concat_idx(
) -> Tuple[
Callable,
Callable,
+ List,
]:
device = vid_len.device
vid_idx = torch.arange(vid_len.sum(), device=device)
txt_idx = torch.arange(len(vid_idx), len(vid_idx) + txt_len.sum(), device=device)
txt_repeat_list = txt_repeat.tolist()
- tgt_idx = repeat_concat(vid_idx, txt_idx, vid_len, txt_len, txt_repeat_list)
+ vid_blocks = torch.split(vid_idx, vid_len.tolist())
+ txt_blocks = list(chain(*([x] * n for x, n in zip(torch.split(txt_idx, txt_len.tolist()), txt_repeat_list))))
+ seq_len = [vid.numel() + txt.numel() for vid, txt in zip(vid_blocks, txt_blocks)]
+ # Stable sort by length so equal-length windows end up adjacent and can share one attention call.
+ order = sorted(range(len(seq_len)), key=seq_len.__getitem__)
+ tgt_idx = torch.cat([block for i in order for block in (vid_blocks[i], txt_blocks[i])])
src_idx = torch.argsort(tgt_idx)
txt_idx_len = len(tgt_idx) - len(vid_idx)
repeat_txt_len = (txt_len * txt_repeat).tolist()
@@ -88,6 +82,7 @@ def repeat_concat_idx(
return (
lambda vid, txt: torch.cat([vid, txt])[tgt_idx],
lambda all: unconcat_coalesce(all),
+ cumulative_lengths([seq_len[i] for i in order]),
)
def cumulative_lengths(lengths):
@@ -274,12 +269,8 @@ class NaRotaryEmbedding3d(RotaryEmbedding3d):
]:
freqs = cache("rope_freqs_3d", lambda: self.get_freqs(shape))
freqs = freqs.to(device=q.device)
- q = q.transpose(0, 1)
- k = k.transpose(0, 1)
- q = _apply_seedvr2_rotary_emb(freqs, q.float()).to(q.dtype)
- k = _apply_seedvr2_rotary_emb(freqs, k.float()).to(k.dtype)
- q = q.transpose(0, 1)
- k = k.transpose(0, 1)
+ q = _apply_rope1_partial(q, freqs)
+ k = _apply_rope1_partial(k, freqs)
return q, k
@torch._dynamo.disable
@@ -300,7 +291,7 @@ class NaRotaryEmbedding3d(RotaryEmbedding3d):
for f, h, w in shape.tolist():
freqs = plain_rope.get_axial_freqs(f, h, w)
freq_list.append(freqs.view(-1, freqs.size(-1)))
- return torch.cat(freq_list, dim=0)
+ return _to_flux_freqs_cis(torch.cat(freq_list, dim=0))
class MMRotaryEmbeddingBase(RotaryEmbeddingBase):
@@ -313,48 +304,9 @@ class MMRotaryEmbeddingBase(RotaryEmbeddingBase):
)
self.mm = True
-def slice_at_dim(t, dim_slice: slice, *, dim):
- dim += (t.ndim if dim < 0 else 0)
- colons = [slice(None)] * t.ndim
- colons[dim] = dim_slice
- return t[tuple(colons)]
-
-def rotate_half(x):
- x = x.reshape(*x.shape[:-1], x.shape[-1] // 2, 2)
- x1, x2 = x.unbind(dim = -1)
- x = torch.stack((-x2, x1), dim = -1)
- return x.flatten(-2)
def exists(val):
return val is not None
-def _apply_seedvr2_rotary_emb(
- freqs: torch.Tensor,
- t: torch.Tensor,
- start_index: int = 0,
- scale: float = 1.0,
- seq_dim: int = -2,
- freqs_seq_dim: int | None = None,
-) -> torch.Tensor:
- dtype = t.dtype
- if freqs_seq_dim is None and (freqs.ndim == 2 or t.ndim == 3):
- freqs_seq_dim = 0
-
- if t.ndim == 3 or freqs_seq_dim is not None:
- seq_len = t.shape[seq_dim]
- freqs = slice_at_dim(freqs, slice(-seq_len, None), dim=freqs_seq_dim)
-
- rot_feats = freqs.shape[-1]
- end_index = start_index + rot_feats
-
- t_left = t[..., :start_index]
- t_middle = t[..., start_index:end_index]
- t_right = t[..., end_index:]
-
- freqs = freqs.to(device=t_middle.device, dtype=t_middle.dtype)
- cos = freqs.cos() * scale
- sin = freqs.sin() * scale
- t_middle = (t_middle * cos) + (rotate_half(t_middle) * sin)
- return torch.cat((t_left, t_middle, t_right), dim=-1).to(dtype)
def _to_flux_freqs_cis(freqs_interleaved: torch.Tensor) -> torch.Tensor:
angles = freqs_interleaved[..., ::2].float()
@@ -365,16 +317,26 @@ def _to_flux_freqs_cis(freqs_interleaved: torch.Tensor) -> torch.Tensor:
def _apply_rope1_partial(t: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor:
- out = t.clone() if t.requires_grad or comfy.model_management.in_training else t
+ """Rotate the leading ``2 * freqs_cis.shape[-3]`` head features of an (L, heads, dim) tensor.
+
+ The 4-D token-major view with 6-D frequencies is what the fused kernel dispatches on;
+ a 3-D tensor silently falls back to the elementwise reference path.
+ """
+ training = comfy.model_management.in_training or t.requires_grad
+ out = t.clone() if training else t
rot_d = 2 * freqs_cis.shape[-3]
- seq_len = out.shape[-2]
+ view = out.unsqueeze(0)
+ if rot_d != view.shape[-1]:
+ view = view[..., :rot_d]
+ freqs_cis = freqs_cis[None, :, None]
+ seq_len = out.shape[0]
for start in range(0, seq_len, SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS):
end = min(start + SEEDVR2_ROPE_PARTIAL_CHUNK_TOKENS, seq_len)
- freqs_chunk = freqs_cis[start:end]
- if rot_d == out.shape[-1]:
- out[..., start:end, :] = apply_rope1(out[..., start:end, :], freqs_chunk).to(out.dtype)
+ chunk, freqs_chunk = view[:, start:end], freqs_cis[:, start:end]
+ if training:
+ chunk.copy_(apply_rope1(chunk, freqs_chunk))
else:
- out[..., start:end, :rot_d] = apply_rope1(out[..., start:end, :rot_d], freqs_chunk).to(out.dtype)
+ comfy.quant_ops.ck.apply_rope1_(chunk, freqs_chunk)
return out
@@ -390,6 +352,7 @@ class NaMMRotaryEmbedding3d(MMRotaryEmbeddingBase):
txt_q: torch.FloatTensor, # L h d
txt_k: torch.FloatTensor, # L h d
txt_shape: torch.LongTensor, # B 1
+ vid_txt_shape: torch.LongTensor, # B 1, text length of the sample each vid entry belongs to
cache: Cache,
) -> Tuple[
torch.FloatTensor,
@@ -399,26 +362,17 @@ class NaMMRotaryEmbedding3d(MMRotaryEmbeddingBase):
]:
vid_freqs, txt_freqs = cache(
"mmrope_freqs_3d",
- lambda: self.get_freqs(vid_shape, txt_shape),
+ lambda: self.get_freqs(vid_shape, txt_shape, vid_txt_shape),
)
target_device = vid_q.device
if vid_freqs.device != target_device:
vid_freqs = vid_freqs.to(target_device)
if txt_freqs.device != target_device:
txt_freqs = txt_freqs.to(target_device)
- vid_q = vid_q.transpose(0, 1)
- vid_k = vid_k.transpose(0, 1)
vid_q = _apply_rope1_partial(vid_q, vid_freqs)
vid_k = _apply_rope1_partial(vid_k, vid_freqs)
- vid_q = vid_q.transpose(0, 1)
- vid_k = vid_k.transpose(0, 1)
-
- txt_q = txt_q.transpose(0, 1)
- txt_k = txt_k.transpose(0, 1)
txt_q = _apply_rope1_partial(txt_q, txt_freqs)
txt_k = _apply_rope1_partial(txt_k, txt_freqs)
- txt_q = txt_q.transpose(0, 1)
- txt_k = txt_k.transpose(0, 1)
return vid_q, vid_k, txt_q, txt_k
@torch._dynamo.disable # Disable compilation: .tolist() is data-dependent and causes graph breaks
@@ -426,6 +380,7 @@ class NaMMRotaryEmbedding3d(MMRotaryEmbeddingBase):
self,
vid_shape: torch.LongTensor,
txt_shape: torch.LongTensor,
+ vid_txt_shape: torch.LongTensor,
) -> Tuple[
torch.Tensor,
torch.Tensor,
@@ -434,9 +389,9 @@ class NaMMRotaryEmbedding3d(MMRotaryEmbeddingBase):
max_temporal = 0
max_height = 0
max_width = 0
- max_txt_len = 0
+ max_txt_len = max(txt_shape[:, 0].tolist())
- for (f, h, w), l in zip(vid_shape.tolist(), txt_shape[:, 0].tolist()):
+ for (f, h, w), l in zip(vid_shape.tolist(), vid_txt_shape[:, 0].tolist()):
max_temporal = max(max_temporal, l + f)
max_height = max(max_height, h)
max_width = max(max_width, w)
@@ -451,12 +406,14 @@ class NaMMRotaryEmbedding3d(MMRotaryEmbeddingBase):
).float()
txt_freqs = self.get_axial_freqs(max_txt_len + 16)
- vid_freq_list, txt_freq_list = [], []
- for (f, h, w), l in zip(vid_shape.tolist(), txt_shape[:, 0].tolist()):
- vid_freq = vid_freqs[l : l + f, :h, :w].reshape(-1, vid_freqs.size(-1))
- txt_freq = txt_freqs[:l].repeat(1, 3).reshape(-1, vid_freqs.size(-1))
- vid_freq_list.append(vid_freq)
- txt_freq_list.append(txt_freq)
+ vid_freq_list = [
+ vid_freqs[l : l + f, :h, :w].reshape(-1, vid_freqs.size(-1))
+ for (f, h, w), l in zip(vid_shape.tolist(), vid_txt_shape[:, 0].tolist())
+ ]
+ txt_freq_list = [
+ txt_freqs[:l].repeat(1, 3).reshape(-1, vid_freqs.size(-1))
+ for l in txt_shape[:, 0].tolist()
+ ]
vid_freqs_interleaved = torch.cat(vid_freq_list, dim=0)
txt_freqs_interleaved = torch.cat(txt_freq_list, dim=0)
@@ -579,7 +536,7 @@ def window_idx(
window_fn: Callable[[torch.Tensor], List[torch.Tensor]],
):
hid_idx = torch.arange(hid_shape.prod(-1).sum(), device=hid_shape.device).unsqueeze(-1)
- tgt_idx, tgt_shape, tgt_windows, tgt_len_list, tgt_windows_list = window(hid_idx, hid_shape, window_fn)
+ tgt_idx, tgt_shape, tgt_windows, _, _ = window(hid_idx, hid_shape, window_fn)
tgt_idx = tgt_idx.squeeze(-1)
src_idx = torch.argsort(tgt_idx)
return (
@@ -587,8 +544,6 @@ def window_idx(
lambda hid: torch.index_select(hid, 0, src_idx),
tgt_shape,
tgt_windows,
- tgt_len_list,
- tgt_windows_list,
)
class NaSwinAttention(NaMMAttention):
@@ -630,11 +585,12 @@ class NaSwinAttention(NaMMAttention):
window_slices = self.window_op((t, h, w), self.window)
return [x[st, sh, sw] for (st, sh, sw) in window_slices]
- window_partition, window_reverse, window_shape, window_count, vid_len_win_list, window_count_list = cache_win(
+ window_partition, window_reverse, window_shape, window_count = cache_win(
"win_transform",
lambda: window_idx(vid_shape, make_window),
)
vid_qkv_win = window_partition(vid_qkv)
+ del vid_qkv
vid_qkv_win = vid_qkv_win.reshape(vid_qkv_win.shape[0], 3, self.heads, self.head_dim)
txt_qkv = txt_qkv.reshape(txt_qkv.shape[0], 3, self.heads, self.head_dim)
@@ -654,44 +610,35 @@ class NaSwinAttention(NaMMAttention):
if self.version_7b:
vid_q, vid_k = self.rope(vid_q, vid_k, window_shape, cache_win)
elif self.rope.mm:
- _, num_h, _ = txt_q.shape
- txt_q_repeat = txt_q.flatten(1, 2)
- txt_q_repeat = unflatten(txt_q_repeat, txt_shape)
- txt_q_repeat = [[x] * n for x, n in zip(txt_q_repeat, window_count_list)]
- txt_q_repeat = list(chain(*txt_q_repeat))
- txt_q_repeat, txt_shape_repeat = flatten(txt_q_repeat)
- txt_q_repeat = txt_q_repeat.reshape(txt_q_repeat.shape[0], num_h, self.head_dim)
-
- txt_k_repeat = txt_k.flatten(1, 2)
- txt_k_repeat = unflatten(txt_k_repeat, txt_shape)
- txt_k_repeat = [[x] * n for x, n in zip(txt_k_repeat, window_count_list)]
- txt_k_repeat = list(chain(*txt_k_repeat))
- txt_k_repeat, _ = flatten(txt_k_repeat)
- txt_k_repeat = txt_k_repeat.reshape(txt_k_repeat.shape[0], num_h, self.head_dim)
-
+ # Each window carries the text length of the sample it came from, which offsets its vid freqs.
+ txt_shape_win = cache_win(
+ "txt_shape_win",
+ lambda: torch.repeat_interleave(txt_shape, window_count.to(txt_shape.device), dim=0),
+ )
vid_q, vid_k, txt_q, txt_k = self.rope(
- vid_q, vid_k, window_shape, txt_q_repeat, txt_k_repeat, txt_shape_repeat, cache_win
+ vid_q, vid_k, window_shape, txt_q, txt_k, txt_shape, txt_shape_win, cache_win
)
else:
vid_q, vid_k = self.rope(vid_q, vid_k, window_shape, cache_win)
- txt_len_win_list = cache_win(
- "txt_len_list",
- lambda: [txt_len for txt_len, window_count in zip(txt_len.tolist(), window_count_list) for _ in range(window_count)],
- )
- all_len_win = cache_win("all_len", lambda: [vid_len + txt_len for vid_len, txt_len in zip(vid_len_win_list, txt_len_win_list)])
- concat_win, unconcat_win = cache_win(
+ concat_win, unconcat_win, cu_seqlens_win = cache_win(
"mm_pnp", lambda: repeat_concat_idx(vid_len_win, txt_len, window_count)
)
+ q = concat_win(vid_q, txt_q)
+ del vid_q, txt_q
+ k = concat_win(vid_k, txt_k)
+ del vid_k, txt_k
+ v = concat_win(vid_v, txt_v)
+ del vid_v, txt_v, vid_qkv_win, txt_qkv
out = optimized_var_attention(
- q=concat_win(vid_q, txt_q),
- k=concat_win(vid_k, txt_k),
- v=concat_win(vid_v, txt_v),
+ q=q, k=k, v=v,
heads=self.heads, skip_reshape=True, skip_output_reshape=True,
- cu_seqlens_q=cache_win("vid_seqlens_q", lambda: cumulative_lengths(all_len_win)),
- cu_seqlens_k=cache_win("vid_seqlens_k", lambda: cumulative_lengths(all_len_win)),
+ cu_seqlens_q=cu_seqlens_win,
+ cu_seqlens_k=cu_seqlens_win,
)
+ del q, k, v
vid_out, txt_out = unconcat_win(out)
+ del out
vid_out = vid_out.flatten(1, 2)
txt_out = txt_out.flatten(1, 2)
@@ -736,7 +683,11 @@ class SwiGLUMLP(nn.Module):
self.proj_in = operations.Linear(dim, hidden_dim, bias=False, device=device, dtype=dtype)
def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:
- return self.proj_out(F.silu(self.proj_in_gate(x)) * self.proj_in(x))
+ gate = self.proj_in_gate(x)
+ up = self.proj_in(x)
+ if comfy.model_management.in_training or x.requires_grad:
+ return self.proj_out(F.silu(gate) * up)
+ return self.proj_out(F.silu(gate, inplace=True).mul_(up))
def get_mlp(mlp_type: Optional[str] = "normal"):
if mlp_type == "normal":
@@ -803,8 +754,21 @@ class NaMMSRTransformerBlock(nn.Module):
self.ada = MMModule(ada, dim=dim, emb_dim=emb_dim, layers=["attn", "mlp"], shared_weights=shared_weights, vid_only=is_last_layer, device=device, dtype=dtype)
self.is_last_layer = is_last_layer
self.version = version
+ self.norm_eps = norm_eps
+
+ def _norm_ada_in(self, norm, vid, txt, layer, ada_kwargs):
+ """Normalize then modulate as one fused kernel, only where norm and ada own the same
+ branches (the last block's attn_norm covers txt, its ada does not)."""
+ fusable = (
+ not (comfy.model_management.in_training or vid.requires_grad)
+ and norm.vid_only == self.ada.vid_only
+ )
+ if not fusable:
+ vid, txt = norm(vid, txt)
+ return self.ada(vid, txt, layer=layer, mode="in", **ada_kwargs)
+ return self.ada(vid, txt, layer=layer, mode="in", norm_eps=self.norm_eps, **ada_kwargs)
- def _seedvr2_7b_mlp(
+ def _chunked_mlp(
self,
vid: torch.FloatTensor,
txt: torch.FloatTensor,
@@ -813,12 +777,14 @@ class NaMMSRTransformerBlock(nn.Module):
torch.FloatTensor,
]:
vid_module = self.mlp.vid if not self.mlp.shared_weights else self.mlp.all
- if comfy.model_management.in_training or vid.requires_grad:
- vid = torch.cat([vid_module(chunk) for chunk in vid.split(SEEDVR2_7B_MLP_CHUNK, dim=0)], dim=0)
+ if vid.shape[0] <= SEEDVR2_MLP_CHUNK:
+ vid = vid_module(vid)
+ elif comfy.model_management.in_training or vid.requires_grad:
+ vid = torch.cat([vid_module(chunk) for chunk in vid.split(SEEDVR2_MLP_CHUNK, dim=0)], dim=0)
else:
vid_out = None
offset = 0
- for chunk in vid.split(SEEDVR2_7B_MLP_CHUNK, dim=0):
+ for chunk in vid.split(SEEDVR2_MLP_CHUNK, dim=0):
chunk_out = vid_module(chunk)
if vid_out is None:
vid_out = chunk_out.new_empty((vid.shape[0], *chunk_out.shape[1:]))
@@ -856,18 +822,13 @@ class NaMMSRTransformerBlock(nn.Module):
"branch_tag": MMArg("vid", "txt"),
}
- vid_attn, txt_attn = self.attn_norm(vid, txt)
- vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="in", **ada_kwargs)
+ vid_attn, txt_attn = self._norm_ada_in(self.attn_norm, vid, txt, "attn", ada_kwargs)
vid_attn, txt_attn = self.attn(vid_attn, txt_attn, vid_shape, txt_shape, cache)
vid_attn, txt_attn = self.ada(vid_attn, txt_attn, layer="attn", mode="out", **ada_kwargs)
vid_attn, txt_attn = (vid_attn + vid), (txt_attn + txt)
- vid_mlp, txt_mlp = self.mlp_norm(vid_attn, txt_attn)
- vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, layer="mlp", mode="in", **ada_kwargs)
- if self.version:
- vid_mlp, txt_mlp = self._seedvr2_7b_mlp(vid_mlp, txt_mlp)
- else:
- vid_mlp, txt_mlp = self.mlp(vid_mlp, txt_mlp)
+ vid_mlp, txt_mlp = self._norm_ada_in(self.mlp_norm, vid_attn, txt_attn, "mlp", ada_kwargs)
+ vid_mlp, txt_mlp = self._chunked_mlp(vid_mlp, txt_mlp)
vid_mlp, txt_mlp = self.ada(vid_mlp, txt_mlp, layer="mlp", mode="out", **ada_kwargs)
vid_mlp, txt_mlp = (vid_mlp + vid_attn), (txt_mlp + txt_attn)
@@ -1021,36 +982,62 @@ class AdaSingle(nn.Module):
cache: Optional[Cache] = None,
branch_tag: str = "",
hid_len: Optional[torch.LongTensor] = None, # b
+ norm_eps: Optional[float] = None,
) -> torch.FloatTensor:
if cache is None:
cache = Cache(disable=True)
idx = self.layers.index(layer)
- emb = emb.reshape(emb.shape[0], -1, len(self.layers), 3)[:, :, idx, :]
- emb = expand_dims(emb, 1, hid.ndim + 1)
- if hid_len is not None:
- emb = cache(
- f"emb_repeat_{idx}_{branch_tag}",
- lambda: torch.repeat_interleave(emb, hid_len, dim=0),
- )
+ def sliced():
+ e = emb.reshape(emb.shape[0], -1, len(self.layers), 3)[:, :, idx, :]
+ return expand_dims(e, 1, hid.ndim + 1)
- shiftA, scaleA, gateA = emb.unbind(-1)
- shiftB, scaleB, gateB = (
- getattr(self, f"{layer}_shift", None),
- getattr(self, f"{layer}_scale", None),
- getattr(self, f"{layer}_gate", None),
- )
+ if hid_len is None:
+ mod = sliced()
+ else:
+ # Keyed as the reference: vid_out_ada (layers=["out"], idx 0) aliases the blocks' "attn" entry
+ # and takes their modulation; slicing its own would be 2*dim wide. The 3B weights expect this.
+ mod = cache(f"emb_repeat_{idx}_{branch_tag}", sliced)
+
+ shiftA, scaleA, gateA = mod.unbind(-1)
+ if hid_len is None or mod.shape[0] == 1:
+ # A single sample's modulation broadcasts over every token, so it never needs materializing.
+ return self._modulate(hid, layer, mode, shiftA, scaleA, gateA, norm_eps=norm_eps)
+
+ # The fused norm is out of place, so its per-sample slices land in a fresh buffer.
+ out = torch.empty_like(hid) if norm_eps is not None else hid
+ offset = 0
+ for i, length in enumerate(cache(f"hid_len_list_{branch_tag}", hid_len.tolist)):
+ modulated = self._modulate(
+ hid[offset:offset + length], layer, mode, shiftA[i], scaleA[i], gateA[i], norm_eps=norm_eps,
+ )
+ if norm_eps is not None:
+ out[offset:offset + length] = modulated
+ offset += length
+ return out
+ def _modulate(
+ self,
+ hid: torch.FloatTensor,
+ layer: str,
+ mode: str,
+ shiftA: torch.FloatTensor,
+ scaleA: torch.FloatTensor,
+ gateA: torch.FloatTensor,
+ norm_eps: Optional[float] = None,
+ ) -> torch.FloatTensor:
if mode == "in":
- shiftB = comfy.ops.cast_to_input(shiftB, hid)
- scaleB = comfy.ops.cast_to_input(scaleB, hid)
+ shiftB = comfy.ops.cast_to_input(getattr(self, f"{layer}_shift"), hid)
+ scaleB = comfy.ops.cast_to_input(getattr(self, f"{layer}_scale"), hid)
+ if norm_eps is not None:
+ # rms_adaln applies rmsnorm(x) * (1 + scale) + shift, so hand it scale - 1.
+ return comfy.quant_ops.ck.rms_adaln(hid, scaleA + scaleB - 1.0, shiftA + shiftB, norm_eps)
return hid.mul_(scaleA + scaleB).add_(shiftA + shiftB)
if mode == "out":
+ gateB = getattr(self, f"{layer}_gate", None)
if gateB is not None:
- gateB = comfy.ops.cast_to_input(gateB, hid)
- return hid.mul_(gateA + gateB)
- else:
- return hid.mul_(gateA)
+ return hid.mul_(gateA + comfy.ops.cast_to_input(gateB, hid))
+ return hid.mul_(gateA)
raise ValueError(f"Unknown AdaSingle mode: {mode}")
diff --git a/comfy/ldm/seedvr/vae.py b/comfy/ldm/seedvr/vae.py
index 7a8070b65..6c8032855 100644
--- a/comfy/ldm/seedvr/vae.py
+++ b/comfy/ldm/seedvr/vae.py
@@ -1,17 +1,14 @@
-from typing import Literal, Optional, Tuple
+from typing import Optional
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch import Tensor
-from contextlib import contextmanager
from comfy.utils import ProgressBar
from comfy.ldm.seedvr.constants import (
BYTEDANCE_BLOCK_OUT_CHANNELS,
BYTEDANCE_GN_CHUNKS_FP16,
BYTEDANCE_GN_CHUNKS_FP32,
- BYTEDANCE_LOGVAR_CLAMP_MAX,
- BYTEDANCE_LOGVAR_CLAMP_MIN,
BYTEDANCE_SLICING_SAMPLE_MIN,
BYTEDANCE_VAE_CONV_MEM_GIB,
BYTEDANCE_VAE_NORM_MEM_GIB,
@@ -20,32 +17,63 @@ from comfy.ldm.seedvr.constants import (
BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE,
BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE,
SEEDVR2_LATENT_CHANNELS,
+ SEEDVR2_VAE_CACHE_QUANT_BYTES,
+ SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL,
+ SEEDVR2_DECODE_FIXED_BYTES,
+ SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL,
+ SEEDVR2_DECODE_LAB_BYTES_PER_OUTPUT_PIXEL,
+ SEEDVR2_ENCODE_BYTES_PER_PIXEL,
+ SEEDVR2_ENCODE_FIXED_BYTES,
+ SEEDVR2_EAGER_DECODE_FACTOR,
+ SEEDVR2_EAGER_ENCODE_FACTOR,
+ SEEDVR2_MAX_TILE_LATENT,
+ SEEDVR2_MIN_TILE_LATENT,
+ SEEDVR2_TILE_MEM_HEADROOM,
)
-from comfy.ldm.modules.attention import optimized_attention
from comfy.ldm.modules.diffusionmodules.model import vae_attention
+import collections
+import dataclasses
import math
from enum import Enum
-import logging
import comfy.model_management
import comfy.ops
-ops = comfy.ops.manual_cast
+import comfy.quant_ops
+import comfy_kitchen
+
+_CL = torch.channels_last_3d
+
+
+def _ck_eligible(x):
+ """fp16 on CUDA: the channels_last_3d path; the kitchen conv itself needs the fp16-accumulate opt-in."""
+ return x.is_cuda and x.dtype == torch.float16 and x.dim() == 5
+
+
+def _fp16_accumulate_wanted(x):
+ """The kitchen conv accumulates in fp16 (1.4-1.9x over cuDNN on this decoder, 73.9 -> 58 dB
+ against fp32); it is taken only when the user opted into fp16 accumulation, as the MiniMax H3
+ VAE and the fp16 GEMMs do."""
+ return comfy.ops._fp16_linear_wanted(x)
-def _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap, temporal_scale=1):
- if temporal_size is None:
- return None
+def _ndhwc_like(t):
+ """channels_last_3d, or a frame window of such a tensor (batch of one)."""
+ return t.is_contiguous(memory_format=_CL) or (t.dim() == 5 and t.stride(1) == 1 and t.stride(4) == t.size(1))
- temporal_size = int(temporal_size)
- if temporal_size <= 0:
- return None
- temporal_overlap = max(0, int(temporal_overlap or 0))
- temporal_overlap = min(temporal_overlap, temporal_size - 1)
- temporal_step = temporal_size - temporal_overlap
- temporal_scale = max(1, int(temporal_scale))
- return max(1, math.ceil(temporal_step / temporal_scale))
+def _match_layout(t, ref):
+ """Return ``t`` in ``ref``'s memory format, so a cat that lists it first keeps the layout."""
+ if ref.is_contiguous(memory_format=_CL) and not t.is_contiguous(memory_format=_CL):
+ return t.contiguous(memory_format=_CL)
+ return t
+
+
+def _ndhwc_filter(conv):
+ """Store a resident filter NDHWC once, so cast_bias_weight hands it back without a per-call copy."""
+ if conv.weight.is_cuda and not conv.weight.is_contiguous(memory_format=_CL):
+ conv.weight.data = conv.weight.data.contiguous(memory_format=_CL)
+ops = comfy.ops.manual_cast
def _seedvr2_clamped_spatial_overlap(overlap, tile_size):
@@ -54,78 +82,35 @@ def _seedvr2_clamped_spatial_overlap(overlap, tile_size):
return min(overlap, tile_size - 1)
-def tiled_vae(
- x,
- vae_model,
- tile_size=(512, 512),
- tile_overlap=(64, 64),
- temporal_size=16,
- temporal_overlap=0,
- encode=True,
-):
- if x.ndim != 5:
- x = x.unsqueeze(2)
-
+def tiled_vae(x, vae_model, tile_size=(512, 512), tile_overlap=(64, 64), encode=True):
+ """Spatial tiles, each through the model's own temporally sliced encode or decode, blended with
+ cosine ramps over the overlaps in fp32 on the intermediate device, as comfy.utils.tiled_scale does:
+ kept on the GPU, the full-size blend buffer was 12 bytes per output pixel per frame (6 GiB at 4K/61f)."""
_, _, d, h, w = x.shape
-
- sf_s = getattr(vae_model, "spatial_downsample_factor", BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE)
- sf_t = getattr(vae_model, "temporal_downsample_factor", BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE)
- if encode:
- slicing_attr = "slicing_sample_min_size"
- slicing_min_size = _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap)
- else:
- slicing_attr = "slicing_latent_min_size"
- slicing_min_size = _seedvr2_temporal_slicing_min_size(temporal_size, temporal_overlap, sf_t)
+ sf_s, sf_t = vae_model.spatial_downsample_factor, vae_model.temporal_downsample_factor
if encode:
ti_h, ti_w = tile_size
ov_h = _seedvr2_clamped_spatial_overlap(tile_overlap[0], ti_h)
ov_w = _seedvr2_clamped_spatial_overlap(tile_overlap[1], ti_w)
- blend_ov_h = max(0, ov_h // sf_s)
- blend_ov_w = max(0, ov_w // sf_s)
+ blend_ov_h, blend_ov_w = ov_h // sf_s, ov_w // sf_s
target_d = (d + sf_t - 1) // sf_t
- target_h = (h + sf_s - 1) // sf_s
- target_w = (w + sf_s - 1) // sf_s
+ target_h, target_w = (h + sf_s - 1) // sf_s, (w + sf_s - 1) // sf_s
else:
- ti_h = max(1, tile_size[0] // sf_s)
- ti_w = max(1, tile_size[1] // sf_s)
+ ti_h, ti_w = max(1, tile_size[0] // sf_s), max(1, tile_size[1] // sf_s)
ov_h = _seedvr2_clamped_spatial_overlap(tile_overlap[0] // sf_s, ti_h)
ov_w = _seedvr2_clamped_spatial_overlap(tile_overlap[1] // sf_s, ti_w)
- blend_ov_h = ov_h * sf_s
- blend_ov_w = ov_w * sf_s
-
+ blend_ov_h, blend_ov_w = ov_h * sf_s, ov_w * sf_s
target_d = max(1, d * sf_t - (sf_t - 1))
- target_h = h * sf_s
- target_w = w * sf_s
-
- stride_h = max(1, ti_h - ov_h)
- stride_w = max(1, ti_w - ov_w)
-
- storage_device = vae_model.device
- result = None
- count = None
- def run_temporal_chunks(spatial_tile, model=vae_model):
- t_chunk = spatial_tile.contiguous()
- old_device = getattr(model, "device", None)
- model.device = t_chunk.device
- old_slicing_min_size = getattr(model, slicing_attr, None)
- if old_slicing_min_size is not None and slicing_min_size is not None:
- if slicing_min_size <= 0:
- setattr(model, slicing_attr, t_chunk.shape[2])
- else:
- setattr(model, slicing_attr, slicing_min_size)
- try:
- if encode:
- out = model.encode(t_chunk)
- else:
- out = model.decode_(t_chunk)
- finally:
- if old_slicing_min_size is not None and slicing_min_size is not None:
- setattr(model, slicing_attr, old_slicing_min_size)
- if old_device is not None:
- model.device = old_device
- if out.ndim == 4:
+ target_h, target_w = h * sf_s, w * sf_s
+ stride_h, stride_w = max(1, ti_h - ov_h), max(1, ti_w - ov_w)
+ storage_device = comfy.model_management.intermediate_device()
+
+ def run(tile):
+ tile = tile.contiguous()
+ out = vae_model.encode(tile) if encode else vae_model.slicing_decode(tile)
+ if out.ndim == 4: # encode drops the time axis of a single-frame latent
out = out.unsqueeze(2)
- return out.to(storage_device)
+ return out.to(storage_device).float()
ramp_cache = {}
def get_ramp(steps):
@@ -145,87 +130,51 @@ def tiled_vae(
continue
tile_ranges.append((y_idx, y_end, x_idx, x_end))
- total_tiles = len(tile_ranges)
- bar = ProgressBar(total_tiles)
- single_spatial_tile = h <= ti_h and w <= ti_w
-
- def run_tile(tile_index, tile_range):
- y_idx, y_end, x_idx, x_end = tile_range
- tile_x = x[:, :, :, y_idx:y_end, x_idx:x_end]
- tile_out = run_temporal_chunks(tile_x)
- return tile_index, y_idx, y_end, x_idx, x_end, tile_out
-
- ordered_tile_outputs = (
- run_tile(tile_index, tile_range)
- for tile_index, tile_range in enumerate(tile_ranges)
- )
-
- for _, y_idx, y_end, x_idx, x_end, tile_out in ordered_tile_outputs:
-
- if single_spatial_tile:
- result = tile_out[:, :, :target_d, :target_h, :target_w]
- if result.device != x.device or result.dtype != x.dtype:
- result = result.to(device=x.device, dtype=x.dtype)
- if x.shape[2] == 1 and sf_t == 1:
- result = result.squeeze(2)
- bar.update(1)
- return result
+ bar = ProgressBar(len(tile_ranges))
+ if h <= ti_h and w <= ti_w:
+ result = run(x)[:, :, :target_d, :target_h, :target_w]
+ bar.update(1)
+ return result.to(dtype=x.dtype)
+ result = count = None
+ for y_idx, y_end, x_idx, x_end in tile_ranges:
+ tile_out = run(x[:, :, :, y_idx:y_end, x_idx:x_end])
if result is None:
- b_out, c_out = tile_out.shape[0], tile_out.shape[1]
- result = torch.zeros((b_out, c_out, target_d, target_h, target_w), device=storage_device, dtype=torch.float32)
+ result = torch.zeros((tile_out.shape[0], tile_out.shape[1], target_d, target_h, target_w), device=storage_device, dtype=torch.float32)
count = torch.zeros((1, 1, 1, target_h, target_w), device=storage_device, dtype=torch.float32)
- if encode:
- ys, ye = y_idx // sf_s, (y_idx // sf_s) + tile_out.shape[3]
- xs, xe = x_idx // sf_s, (x_idx // sf_s) + tile_out.shape[4]
- cur_ov_h = max(0, min(blend_ov_h, tile_out.shape[3] // 2))
- cur_ov_w = max(0, min(blend_ov_w, tile_out.shape[4] // 2))
- else:
- ys, ye = y_idx * sf_s, (y_idx * sf_s) + tile_out.shape[3]
- xs, xe = x_idx * sf_s, (x_idx * sf_s) + tile_out.shape[4]
- cur_ov_h = max(0, min(blend_ov_h, tile_out.shape[3] // 2))
- cur_ov_w = max(0, min(blend_ov_w, tile_out.shape[4] // 2))
+ ys = y_idx // sf_s if encode else y_idx * sf_s
+ xs = x_idx // sf_s if encode else x_idx * sf_s
+ ye, xe = ys + tile_out.shape[3], xs + tile_out.shape[4]
+ cur_ov_h = min(blend_ov_h, tile_out.shape[3] // 2)
+ cur_ov_w = min(blend_ov_w, tile_out.shape[4] // 2)
w_h = torch.ones((tile_out.shape[3],), device=storage_device)
w_w = torch.ones((tile_out.shape[4],), device=storage_device)
-
if cur_ov_h > 0:
r = get_ramp(cur_ov_h)
if y_idx > 0:
w_h[:cur_ov_h] = r
if y_end < h:
w_h[-cur_ov_h:] = 1.0 - r
-
if cur_ov_w > 0:
r = get_ramp(cur_ov_w)
if x_idx > 0:
w_w[:cur_ov_w] = r
if x_end < w:
w_w[-cur_ov_w:] = 1.0 - r
-
- final_weight = w_h.view(1,1,1,-1,1) * w_w.view(1,1,1,1,-1)
+ final_weight = w_h.view(1, 1, 1, -1, 1) * w_w.view(1, 1, 1, 1, -1)
valid_d = min(tile_out.shape[2], result.shape[2])
- tile_out = tile_out[:, :, :valid_d, :, :]
-
+ tile_out = tile_out[:, :, :valid_d]
tile_out.mul_(final_weight)
-
result[:, :, :valid_d, ys:ye, xs:xe] += tile_out
count[:, :, :, ys:ye, xs:xe] += final_weight
-
del tile_out, final_weight, w_h, w_w
bar.update(1)
result.div_(count.clamp(min=1e-6))
-
- if result.device != x.device or result.dtype != x.dtype:
- result = result.to(device=x.device, dtype=x.dtype)
-
- if x.shape[2] == 1 and sf_t == 1:
- result = result.squeeze(2)
-
- return result
+ return result.to(dtype=x.dtype)
_NORM_LIMIT = float("inf")
def get_norm_limit():
@@ -238,14 +187,235 @@ def set_norm_limit(value: Optional[float] = None):
value = float("inf")
_NORM_LIMIT = value
-@contextmanager
-def ignore_padding(model):
- orig_padding = model.padding
- model.padding = (0, 0, 0)
- try:
- yield
- finally:
- model.padding = orig_padding
+
+class CausalMemoryCache:
+ """Per-convolution temporal tails, kept between the slices of one encode or decode.
+
+ Large channels_last CUDA tails are int8-packed (ConvRot, per-token scales); with ``offload`` the
+ packed tails live in host memory and the next reader's tail is prefetched while this one runs.
+ A tail can be a ring of per-frame entries, so extending it packs one frame. free() ends the pass.
+ """
+
+ CONVROT_GROUPSIZE = 256
+
+ def __init__(self, offload=False):
+ self.offload = offload
+ self.plain = {}
+ self.packed = {} # key -> (qdata, params, shape), on the host when offloaded
+ self._order = [] # keys in packing order, which is the order the next slice reads them
+ self._staged = {} # key -> (qdata, params, event) on its way back to the GPU
+ self._spare = {} # (shape, dtype) -> host buffers of dropped entries
+ self._pinned = []
+ self._landing = None # (event, sources) of the copy to the host still in flight
+ self._stream = None
+ self._device = None
+ self._rings = {} # key -> deque of per-frame subkeys, oldest first
+ self._ring_serial = 0
+
+ def _transfer_stream(self, device):
+ """The one stream this cache copies on, so copies reusing a host buffer stay ordered;
+ it starts after the compute queued so far. None copies on the compute stream."""
+ if self._stream is None:
+ self._stream = comfy.model_management.get_offload_stream(device)
+ if self._stream is not None:
+ self._stream.wait_stream(comfy.model_management.current_stream(device))
+ return self._stream
+
+ def _host_like(self, t):
+ spare = self._spare.get((t.shape, t.dtype))
+ if spare:
+ return spare.pop()
+ host = torch.empty(t.shape, dtype=t.dtype, device="cpu")
+ if comfy.model_management.pin_memory(host, evict_active=False):
+ self._pinned.append(host)
+ return host
+
+ def _to_host(self, *tensors):
+ """Start copying ``tensors`` to host buffers; they stay alive until the copy lands.
+ One copy in flight at a time: sources held for copies the CPU ran ahead of are what grows
+ the peak (2x at 720p on either allocator); record_stream instead grows cudaMallocAsync's pool."""
+ if self._landing is not None:
+ self._landing[0].synchronize()
+ self._device = tensors[0].device
+ stream = self._transfer_stream(self._device)
+ host = [comfy.model_management.cast_to(t, None, torch.device("cpu"), non_blocking=True, stream=stream, r=self._host_like(t))
+ for t in tensors]
+ event = torch.cuda.Event()
+ event.record(stream)
+ self._landing = (event, tensors)
+ return host
+
+ def park(self, owner, frames):
+ """Hold channels_last ``frames`` unpacked in host memory until popped, the first already on
+ its way back; returns their keys."""
+ keys = [(owner, "parked", i) for i in range(len(frames))]
+ host = self._to_host(*(f.permute(0, 2, 3, 4, 1) for f in frames))
+ for key, h, f in zip(keys, host, frames):
+ self.packed[key] = (h, None, f.shape)
+ if key not in self._order:
+ self._order.append(key)
+ if keys:
+ self._staged[keys[0]] = self._bring_back(keys[0], self._device)
+ return keys
+
+ def _bring_back(self, key, device):
+ """Start the copy of an offloaded entry back to the GPU; returns (data, params, event)."""
+ hq, params, _ = self.packed[key]
+ # allocated on the compute stream, which the copy then waits on: queued work may still be releasing the block
+ qdata = torch.empty_like(hq, device=device)
+ stream = self._transfer_stream(device)
+ comfy.model_management.cast_to(hq, None, device, non_blocking=True, stream=stream, r=qdata)
+ if params is not None:
+ scale = torch.empty_like(params.scale, device=device)
+ comfy.model_management.cast_to(params.scale, None, device, non_blocking=True, stream=stream, r=scale)
+ params = dataclasses.replace(params, scale=scale)
+ event = torch.cuda.Event()
+ event.record(stream)
+ return qdata, params, event
+
+ def _top(self, key):
+ return key[0] if isinstance(key, tuple) and len(key) == 2 and key[0] in self._rings else key
+
+ def _prefetch_after(self, top, device):
+ """The reader after this one is next in line; have its tail on the way while this one runs."""
+ try:
+ nxt = self._order[self._order.index(top) + 1]
+ except (ValueError, IndexError):
+ return
+ for sub in self._rings.get(nxt, (nxt,)):
+ if sub in self.packed and sub not in self._staged:
+ self._staged[sub] = self._bring_back(sub, device)
+
+ def push_frames(self, key, frames, keep):
+ """Extend ``key``'s tail by ``frames``, keeping the last ``keep``; each frame is its own entry,
+ stored as given (the buffered convs pass frames of their padded buffer, border included)."""
+ ring = self._rings.setdefault(key, collections.deque())
+ for j in range(frames.size(2)):
+ self._ring_serial += 1
+ sub = (key, self._ring_serial)
+ frame = frames[:, :, j:j + 1]
+ if not self._packable(frame):
+ frame = frame.clone(memory_format=_CL) # held or sent to the host: not as a view of the caller's buffer
+ self[sub] = frame
+ ring.append(sub)
+ self.keep_last(key, keep)
+ if key not in self._order:
+ self._order.append(key)
+
+ def keep_last(self, key, keep):
+ """Drop all but the newest ``keep`` frames of ``key``'s ring."""
+ ring = self._rings.get(key, ())
+ while len(ring) > keep:
+ self.discard(ring.popleft())
+
+ def _packable(self, value):
+ return (
+ torch.is_tensor(value)
+ and value.dim() == 5
+ and value.is_cuda
+ and value.is_contiguous(memory_format=_CL)
+ and value.shape[1] % 4 == 0 # ConvRot rotates power-of-4 channel groups
+ and value.numel() * value.element_size() >= SEEDVR2_VAE_CACHE_QUANT_BYTES
+ )
+
+ def __setitem__(self, key, value):
+ self.discard(key) # a whole tail replaces a ring too
+ if self._packable(value):
+ c = value.shape[1]
+ g = self.CONVROT_GROUPSIZE
+ while g > 4 and c % g:
+ g //= 4
+ # rows of the NDHWC view are tokens: the rotation spreads each token's outliers over its channels
+ qdata, params = comfy.quant_ops.TensorWiseINT8Layout.quantize(
+ value.permute(0, 2, 3, 4, 1).reshape(-1, c), is_weight=True, per_channel=True, convrot=True, convrot_groupsize=g)
+ elif self.offload and value.is_cuda and value.dim() == 5 and value.is_contiguous(memory_format=_CL):
+ # too small to be worth packing, but off the GPU all the same: long-lived small blocks
+ # fragment the big activations' segments (720p needed 2 GiB more free VRAM)
+ qdata, params = value.permute(0, 2, 3, 4, 1), None
+ else:
+ self.plain[key] = value
+ return
+ if self.offload:
+ if params is None:
+ qdata, = self._to_host(qdata)
+ else:
+ qdata, scale = self._to_host(qdata, params.scale)
+ params = dataclasses.replace(params, scale=scale)
+ top = self._top(key)
+ if top not in self._order:
+ self._order.append(top)
+ self.packed[key] = (qdata, params, value.shape)
+
+ def get(self, key, default=None):
+ """A whole entry; a ring's frames are read with for_each_frame."""
+ if key not in self.plain and key not in self.packed:
+ return default
+ out = self._get_entry(key)
+ if self.offload and self._device is not None:
+ self._prefetch_after(key, self._device)
+ return out
+
+ def for_each_frame(self, key, fn, count=None):
+ """``fn(i, frame)`` over the first ``count`` frames of ``key``'s ring one at a time, never holding
+ the whole unpacked tail."""
+ for i, sub in enumerate(list(self._rings[key])[:count]):
+ fn(i, self._get_entry(sub))
+ if self.offload and self._device is not None:
+ self._prefetch_after(key, self._device)
+
+ def _get_entry(self, key):
+ if key in self.plain:
+ return self.plain[key]
+ qdata, params, (b, c, t, h, w) = self.packed[key]
+ if self.offload:
+ qdata, params, event = self._staged.pop(key, None) or self._bring_back(key, self._device)
+ comfy.model_management.current_stream(self._device).wait_event(event)
+ if params is None: # parked unpacked, NDHWC
+ return qdata.permute(0, 4, 1, 2, 3)
+ flat = comfy.quant_ops.TensorWiseINT8Layout.dequantize(qdata, params)
+ return flat.view(b, t, h, w, c).permute(0, 4, 1, 2, 3)
+
+ def pop(self, key, default=None):
+ value = self.get(key, default)
+ self.discard(key)
+ return value
+
+ def discard(self, key):
+ """Drop an entry, or a ring's frames, without bringing it back; its host buffers are reused."""
+ for sub in self._rings.pop(key, ()):
+ self.discard(sub)
+ self.plain.pop(key, None)
+ entry = self.packed.pop(key, None)
+ staged = self._staged.pop(key, None)
+ if staged is not None: # its GPU copy is freed on the compute stream
+ comfy.model_management.current_stream(self._device).wait_event(staged[2])
+ if entry is not None and self.offload:
+ for t in (entry[0],) if entry[1] is None else (entry[0], entry[1].scale):
+ self._spare.setdefault((t.shape, t.dtype), []).append(t)
+
+ def free(self):
+ """Unpin the host buffers at the end of the pass; the cache is empty afterwards."""
+ if self._pinned:
+ torch.cuda.synchronize(self._device) # no copy may still be using a buffer when it is unpinned
+ for t in self._pinned:
+ comfy.model_management.unpin_memory(t)
+ self._pinned, self._spare, self._staged, self._landing = [], {}, {}, None
+ self.plain, self.packed, self._rings, self._order = {}, {}, {}, []
+
+ def __contains__(self, key):
+ return key in self.plain or key in self.packed or key in self._rings
+
+
+def _eager_factor(dtype, factor):
+ """The memory figures are fitted on the fp16 channels-last path; other dtypes take the eager path."""
+ return 1 if dtype == torch.float16 else factor * dtype.itemsize / 2
+
+
+def _offload_caches_for(frame_pixels):
+ """Whether a pass over this frame area keeps its packed caches in host memory: when there is
+ room for them, reclaiming RAM from inactive models if needed."""
+ return comfy.model_management.ensure_pin_budget(frame_pixels * SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL, evict_active=False)
+
class MemoryState(Enum):
DISABLED = 0
@@ -253,6 +423,38 @@ class MemoryState(Enum):
ACTIVE = 2
UNSET = 3
+
+def _frame_chunked(owner, run, source, chunk, memory_state, memory_cache, frame_pixels):
+ """``run(source, memory_state, memory_cache)`` over a causal, spatial-only stage ``chunk`` frames at
+ a time, the cache carrying state between chunks: exact, and only a chunk's activations are live.
+ ``source`` is a 1-list the stage pops, so no caller keeps its input alive (the input of a module
+ call is held until that module returns)."""
+ if not chunk or source[0].size(2) <= chunk:
+ return run(source, memory_state, memory_cache)
+ local_cache = memory_state == MemoryState.DISABLED or memory_cache is None
+ if local_cache:
+ # under the same host-memory policy as a sliced pass
+ memory_cache = CausalMemoryCache(offload=_offload_caches_for(frame_pixels))
+ memory_state = MemoryState.INITIALIZING
+ outs = []
+ try:
+ pieces = list(source.pop().split(chunk, dim=2))
+ if memory_cache.offload and pieces[0].is_contiguous(memory_format=_CL):
+ # the rest waits in host memory, fetched a chunk ahead: resident, the whole input sat
+ # under every chunk's peak (2.5 GiB for the decoder's tail at 1080p)
+ pieces[0] = pieces[0].clone(memory_format=_CL)
+ pieces[1:] = memory_cache.park(owner, pieces[1:])
+
+ def take(i):
+ piece, pieces[i] = pieces[i], None
+ return [piece if torch.is_tensor(piece) else memory_cache.pop(piece)]
+ for i in range(len(pieces)):
+ outs.append(run(take(i), memory_state if i == 0 else MemoryState.ACTIVE, memory_cache))
+ finally:
+ if local_cache:
+ memory_cache.free()
+ return torch.cat(outs, dim=2)
+
def get_cache_size(conv_module, input_len, pad_len, dim=0):
dilated_kernel_size = conv_module.dilation[dim] * (conv_module.kernel_size[dim] - 1) + 1
output_len = (input_len + pad_len - dilated_kernel_size) // conv_module.stride[dim] + 1
@@ -268,225 +470,300 @@ def get_cache_size(conv_module, input_len, pad_len, dim=0):
)
return cache_len
-class DiagonalGaussianDistribution(object):
- def __init__(self, parameters: torch.Tensor):
- self.parameters = parameters
- self.mean, self.logvar = torch.chunk(parameters, 2, dim=1)
- self.logvar = torch.clamp(self.logvar, BYTEDANCE_LOGVAR_CLAMP_MIN, BYTEDANCE_LOGVAR_CLAMP_MAX)
-
- def mode(self):
- return self.mean
-
-class SpatialNorm(nn.Module):
- def __init__(
- self,
- f_channels: int,
- zq_channels: int,
- ):
- super().__init__()
- self.norm_layer = ops.GroupNorm(num_channels=f_channels, num_groups=32, eps=1e-6, affine=True)
- self.conv_y = ops.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0)
- self.conv_b = ops.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0)
-
- def forward(self, f: torch.Tensor, zq: torch.Tensor) -> torch.Tensor:
- f_size = f.shape[-2:]
- zq = F.interpolate(zq, size=f_size, mode="nearest")
- norm_f = self.norm_layer(f)
- new_f = norm_f * self.conv_y(zq) + self.conv_b(zq)
- return new_f
-
class Attention(nn.Module):
- def __init__(
- self,
- query_dim: int,
- heads: int = 8,
- dim_head: int = 64,
- bias: bool = False,
- norm_num_groups: Optional[int] = None,
- spatial_norm_dim: Optional[int] = None,
- out_bias: bool = True,
- eps: float = 1e-5,
- rescale_output_factor: float = 1.0,
- residual_connection: bool = False,
- ):
+ """The mid-block's single-head self-attention over each (c, h, w) frame, residual included."""
+ def __init__(self, query_dim, norm_num_groups, eps):
super().__init__()
-
- self.inner_dim = dim_head * heads
- self.rescale_output_factor = rescale_output_factor
- self.residual_connection = residual_connection
- self.out_dim = query_dim
- self.heads = heads
-
- if norm_num_groups is not None:
- self.group_norm = ops.GroupNorm(num_channels=query_dim, num_groups=norm_num_groups, eps=eps, affine=True)
- else:
- self.group_norm = None
-
- if spatial_norm_dim is not None:
- self.spatial_norm = SpatialNorm(f_channels=query_dim, zq_channels=spatial_norm_dim)
- else:
- self.spatial_norm = None
-
- self.to_q = ops.Linear(query_dim, self.inner_dim, bias=bias)
- self.to_k = ops.Linear(query_dim, self.inner_dim, bias=bias)
- self.to_v = ops.Linear(query_dim, self.inner_dim, bias=bias)
- self.to_out = nn.ModuleList([])
- self.to_out.append(ops.Linear(self.inner_dim, self.out_dim, bias=out_bias))
- self.to_out.append(nn.Identity())
-
+ self.group_norm = ops.GroupNorm(num_channels=query_dim, num_groups=norm_num_groups, eps=eps, affine=True)
+ self.to_q = ops.Linear(query_dim, query_dim)
+ self.to_k = ops.Linear(query_dim, query_dim)
+ self.to_v = ops.Linear(query_dim, query_dim)
+ self.to_out = nn.ModuleList([ops.Linear(query_dim, query_dim)])
self.optimized_vae_attention = vae_attention()
- def forward(
- self,
- hidden_states: torch.Tensor,
- temb: Optional[torch.Tensor] = None,
- ) -> torch.Tensor:
-
- residual = hidden_states
- if self.spatial_norm is not None:
- hidden_states = self.spatial_norm(hidden_states, temb)
-
- input_ndim = hidden_states.ndim
-
- if input_ndim == 4:
- batch_size, channel, height, width = hidden_states.shape
- hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)
-
- batch_size = hidden_states.shape[0]
-
- if self.group_norm is not None:
- hidden_states = self.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2)
-
- query = self.to_q(hidden_states)
- key = self.to_k(hidden_states)
- value = self.to_v(hidden_states)
-
- inner_dim = key.shape[-1]
- head_dim = inner_dim // self.heads
-
- query = query.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
-
- key = key.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
- value = value.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
-
- if input_ndim == 4 and self.heads == 1:
- query = query.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width)
- key = key.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width)
- value = value.squeeze(1).transpose(1, 2).reshape(batch_size, head_dim, height, width)
- hidden_states = self.optimized_vae_attention(query, key, value).reshape(batch_size, self.heads, head_dim, height * width).transpose(2, 3)
- else:
- hidden_states = optimized_attention(query, key, value, heads = self.heads, skip_reshape=True, skip_output_reshape=True)
-
- hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, self.heads * head_dim)
- hidden_states = hidden_states.to(query.dtype)
-
- hidden_states = self.to_out[0](hidden_states)
- hidden_states = self.to_out[1](hidden_states)
-
- if input_ndim == 4:
- hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)
-
- if self.residual_connection:
- hidden_states = hidden_states + residual
-
- hidden_states = hidden_states / self.rescale_output_factor
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ b, c, h, w = hidden_states.shape
+ x = self.group_norm(hidden_states.view(b, c, h * w)).transpose(1, 2)
+ q, k, v = (proj(x).transpose(1, 2).reshape(b, c, h, w) for proj in (self.to_q, self.to_k, self.to_v))
+ x = self.optimized_vae_attention(q, k, v).reshape(b, c, h * w).transpose(1, 2)
+ return self.to_out[0](x).transpose(1, 2).reshape(b, c, h, w) + hidden_states
- return hidden_states
-
-def causal_norm_wrapper(norm_layer: nn.Module, x: torch.Tensor) -> torch.Tensor:
- input_dtype = x.dtype
- if isinstance(norm_layer, (nn.LayerNorm, nn.RMSNorm)):
- if x.ndim == 4:
- x = x.permute(0, 2, 3, 1)
- x = norm_layer(x)
- x = x.permute(0, 3, 1, 2)
- return x.to(input_dtype)
- if x.ndim == 5:
- x = x.permute(0, 2, 3, 4, 1)
- x = norm_layer(x)
- x = x.permute(0, 4, 1, 2, 3)
- return x.to(input_dtype)
- if isinstance(norm_layer, (nn.GroupNorm, nn.BatchNorm2d, nn.SyncBatchNorm)):
- if x.ndim <= 4:
- return norm_layer(x).to(input_dtype)
- if x.ndim == 5:
- b, c, t, h, w = x.shape
- x = x.transpose(1, 2).reshape(b * t, c, h, w)
- memory_occupy = x.numel() * x.element_size() / 1024**3
- if isinstance(norm_layer, nn.GroupNorm) and memory_occupy > get_norm_limit():
- num_chunks = min(BYTEDANCE_GN_CHUNKS_FP16 if x.element_size() == 2 else BYTEDANCE_GN_CHUNKS_FP32, norm_layer.num_groups)
- if norm_layer.num_groups % num_chunks != 0:
- raise ValueError(
- f"SeedVR2 VAE GroupNorm groups must divide chunks: groups={norm_layer.num_groups}, chunks={num_chunks}."
- )
- num_groups_per_chunk = norm_layer.num_groups // num_chunks
-
- weights = comfy.ops.cast_to_input(norm_layer.weight, x).chunk(num_chunks, dim=0)
- biases = comfy.ops.cast_to_input(norm_layer.bias, x).chunk(num_chunks, dim=0)
- x = list(x.chunk(num_chunks, dim=1))
- for i, (w, bias) in enumerate(zip(weights, biases)):
- x[i] = F.group_norm(x[i], num_groups_per_chunk, w, bias, norm_layer.eps)
- x[i] = x[i].to(input_dtype)
- x = torch.cat(x, dim=1)
- else:
- x = norm_layer(x)
- x = x.reshape((b, t, x.size(1), x.size(2), x.size(3))).transpose(1, 2)
- return x.to(input_dtype)
- raise TypeError(f"SeedVR2 VAE unsupported norm layer type: {type(norm_layer).__name__}")
-
-_receptive_field_t = Literal["half", "full"]
-
-def extend_head(tensor, times: int = 2, memory = None):
- if memory is not None:
- return torch.cat((memory.to(tensor), tensor), dim=2)
- if times < 0:
- raise ValueError(f"SeedVR2 VAE extend_head expected times >= 0, got {times}.")
- if times == 0:
- return tensor
+def causal_norm_wrapper(norm_layer: nn.GroupNorm, x: torch.Tensor, silu: bool = False) -> torch.Tensor:
+ """Per-frame GroupNorm of a (b, c, t, h, w) tensor, with the SiLU folded in where the kernel does both."""
+ if _ck_eligible(x) and norm_layer.affine:
+ # accepts either layout and emits channels_last_3d, which is where the convolution wants its input
+ weight, bias, offload_stream = comfy.ops.cast_bias_weight(norm_layer, x, offloadable=True)
+ try:
+ return comfy_kitchen.group_norm_silu_pad3d(
+ x, weight, bias, norm_layer.num_groups, norm_layer.eps, (0, 0, 0, 0, 0), silu,
+ )
+ finally:
+ comfy.ops.uncast_bias_weight(norm_layer, weight, bias, offload_stream)
+ b, c, t, h, w = x.shape
+ x = x.transpose(1, 2).reshape(b * t, c, h, w)
+ if x.numel() * x.element_size() / 1024**3 > get_norm_limit():
+ num_chunks = min(BYTEDANCE_GN_CHUNKS_FP16 if x.element_size() == 2 else BYTEDANCE_GN_CHUNKS_FP32, norm_layer.num_groups)
+ if norm_layer.num_groups % num_chunks != 0:
+ raise ValueError(
+ f"SeedVR2 VAE GroupNorm groups must divide chunks: groups={norm_layer.num_groups}, chunks={num_chunks}."
+ )
+ num_groups_per_chunk = norm_layer.num_groups // num_chunks
+
+ weights = comfy.ops.cast_to_input(norm_layer.weight, x).chunk(num_chunks, dim=0)
+ biases = comfy.ops.cast_to_input(norm_layer.bias, x).chunk(num_chunks, dim=0)
+ x = list(x.chunk(num_chunks, dim=1))
+ for i, (w, bias) in enumerate(zip(weights, biases)):
+ x[i] = F.group_norm(x[i], num_groups_per_chunk, w, bias, norm_layer.eps)
+ x = torch.cat(x, dim=1)
else:
- tile_repeat = [1] * tensor.ndim
- tile_repeat[2] = times
- return torch.cat(tensors=(torch.tile(tensor[:, :, :1], tile_repeat), tensor), dim=2)
-
-def cache_send_recv(tensor, cache_size, times, memory=None):
- recv_buffer = None
+ x = norm_layer(x)
+ x = x.reshape((b, t, x.size(1), x.size(2), x.size(3))).transpose(1, 2)
+ if silu:
+ x = F.silu(x)
+ return x
- if memory is not None:
- recv_buffer = memory.to(tensor[0])
- elif times > 0:
- tile_repeat = [1] * tensor[0].ndim
- tile_repeat[2] = times
- recv_buffer = torch.tile(tensor[0][:, :, :1], tile_repeat)
-
- return recv_buffer
class InflatedCausalConv3d(ops.Conv3d):
- def __init__(
- self,
- *args,
- inflation_mode,
- **kwargs,
- ):
- self.inflation_mode = inflation_mode
+ def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.temporal_padding = self.padding[0]
self.padding = (0, *self.padding[1:])
- self.memory_limit = float("inf")
- self.logged_once = False
+ self.memory_limit = float("inf") # GiB of padded input above which slicing_forward splits
- def set_memory_limit(self, value: float):
- self.memory_limit = value
+ def _conv_padded(self, x, padding):
+ """The plain conv with ``padding`` in place of the module's own, through comfy's _conv_forward."""
+ orig_padding, self.padding = self.padding, padding
+ try:
+ return torch.nn.Conv3d.forward(self, x)
+ finally:
+ self.padding = orig_padding
+
+ def _conv_without_padding_copy(self, x, prev_cache, padding):
+ """Convolve with native symmetric padding and slice off the surplus; None with a halo or stride."""
+ if prev_cache is not None or any(st != 1 for st in self.stride):
+ return None
+
+ pads = ((padding[4], padding[5]), (padding[2], padding[3]), (padding[0], padding[1]))
+ native = tuple(max(lo, hi) for lo, hi in pads)
+ out = self._conv_padded(x, native)
+
+ for i, (dim, (lo, hi)) in enumerate(zip((2, 3, 4), pads)):
+ width = self.dilation[i] * (self.kernel_size[i] - 1)
+ length = x.size(dim) + lo + hi - width
+ start = native[i] - lo
+ if start or out.size(dim) != length:
+ out = out.narrow(dim, start, length)
+ return out
+
+ @staticmethod
+ def _padded_input(x, prev_cache, padding, concat_dim, channels_last=None):
+ """The conv's padded input (halo included) in one buffer, written once; ``copy_`` also casts the halo."""
+ pads = ((padding[4], padding[5]), (padding[2], padding[3]), (padding[0], padding[1]))
+ cache_len = 0 if prev_cache is None else prev_cache.size(concat_dim)
+ if cache_len == 0 and not any(lo or hi for lo, hi in pads):
+ return x.contiguous(memory_format=_CL) if channels_last else x
+
+ shape = list(x.shape)
+ for dim, (lo, hi) in zip((2, 3, 4), pads):
+ shape[dim] += lo + hi
+ shape[concat_dim] += cache_len
+
+ if channels_last is None:
+ channels_last = x.is_contiguous(memory_format=_CL)
+ buf = torch.empty(shape, dtype=x.dtype, device=x.device,
+ memory_format=_CL if channels_last else torch.contiguous_format)
+ for dim, (lo, hi) in zip((2, 3, 4), pads):
+ if lo:
+ buf.narrow(dim, 0, lo).zero_()
+ if hi:
+ buf.narrow(dim, shape[dim] - hi, hi).zero_()
+
+ core = buf
+ for dim, (lo, hi) in zip((2, 3, 4), pads):
+ core = core.narrow(dim, lo, shape[dim] - lo - hi)
+ if cache_len:
+ core.narrow(concat_dim, 0, cache_len).copy_(prev_cache)
+ core.narrow(concat_dim, cache_len, x.size(concat_dim)).copy_(x)
+ else:
+ core.copy_(x)
+ return buf
+
+ def _ck_applies(self, x):
+ """Whether this conv runs on the channels_last_3d path; any input layout, the pad copy lands it there."""
+ return (_ck_eligible(x) and self.padding_mode == "zeros" and self.groups == 1
+ and all(d == 1 for d in self.dilation))
+
+ def _ck_conv(self, padded):
+ """Run an already-padded channels_last_3d input through the conv, kitchen or cuDNN."""
+ _ndhwc_filter(self)
+ weight, bias, offload_stream = comfy.ops.cast_bias_weight(self, padded, offloadable=True)
+ try:
+ weight_cl = weight.contiguous(memory_format=_CL)
+ if _fp16_accumulate_wanted(padded):
+ return comfy_kitchen.fp16_conv3d(padded, weight_cl, bias, stride=tuple(self.stride))
+ # cuDNN keeps channels_last_3d in and out, so the layout still saves its transposes
+ return F.conv3d(padded, weight_cl, bias, stride=tuple(self.stride))
+ finally:
+ comfy.ops.uncast_bias_weight(self, weight, bias, offload_stream)
+
+ def _symmetric_padding(self):
+ return tuple(p for p in reversed(self.padding) for _ in range(2))
+
+ # The ck kernels index a buffer with 32-bit byte offsets; a view handed to one call stays
+ # under this many elements (2 GiB of fp16), which is one call at 1080p and two at 1440p.
+ _CK_MAX_VIEW_ELEMENTS = 2 ** 30
+
+ # _ck_taps runs each halo tap in this many bands of output rows: on cuDNN, which cannot accumulate
+ # into the conv's output, a tap's own output is then that fraction of a frame; the kitchen conv
+ # accumulates in place, and the bands keep its views under the 32-bit offsets
+ _TAP_BANDS = 4
+
+ def _ck_buffered(self, like, x_shape, memory_state, memory_cache, write_body, residual=None, pad=None):
+ """Run this conv from one padded buffer: halo in front, ``write_body`` writes the padded frames
+ behind it, and the conv reads and writes frame-window views. ``residual`` goes into the kitchen
+ conv's epilogue (added explicitly on the cuDNN path). ``pad`` is the buffer's spatial border
+ (top, bottom, left, right), the conv's own padding by default. Replicates forward()'s cache handling.
+ """
+ b, c, t, h, w = x_shape
+ _, ph, pw = self.padding
+ pad = pad or (ph, ph, pw, pw)
+ kt = self.kernel_size[0]
+ cache_size = kt - self.stride[0]
+ if memory_cache is None:
+ memory_cache = CausalMemoryCache() # DISABLED: nothing is kept anyway
+ elif memory_state != MemoryState.ACTIVE:
+ memory_cache.pop(self, None)
+ has_memory = self in memory_cache
+ halo = cache_size if has_memory else self.temporal_padding * 2
+ if t == 1 and halo == 2 and kt == 3 and tuple(self.stride) == (1, 1, 1) and pad == (ph, ph, pw, pw):
+ return self._ck_taps(like, x_shape, memory_state, memory_cache, write_body, residual, has_memory)
+ hp, wp = h + pad[0] + pad[1], w + pad[2] + pad[3]
+
+ buf = torch.empty((b, c, halo + t, hp, wp), dtype=like.dtype, device=like.device, memory_format=_CL)
+ write_body(buf[:, :, halo:])
+ del write_body # and with it the source it captured: nothing but the buffer feeds the conv
+ if halo:
+ front = buf[:, :, :halo]
+ if has_memory:
+ # after the body, whose source is freed by then; the frames come with their zero border
+ memory_cache.for_each_frame(self, lambda i, frame: front[:, :, i:i + 1].copy_(frame))
+ else:
+ # the causal head: the first frame repeated, already padded
+ front.copy_(buf[:, :, halo:halo + 1].expand(-1, -1, halo, -1, -1))
+ if cache_size and memory_state in (MemoryState.INITIALIZING, MemoryState.ACTIVE):
+ if has_memory:
+ # the halo frames are already the cache's newest entries: push only the new ones
+ memory_cache.push_frames(self, buf[:, :, halo:][:, :, -min(t, cache_size):], cache_size)
+ else:
+ memory_cache.push_frames(self, buf[:, :, -cache_size:], cache_size)
- def _conv_forward(self, input, weight, bias, *args, **kwargs):
+ _ndhwc_filter(self)
+ weight, bias, offload_stream = comfy.ops.cast_bias_weight(self, buf, offloadable=True)
+ try:
+ weight_cl = weight.contiguous(memory_format=_CL)
+ stride = sd, sh, sw = tuple(self.stride)
+ out_hw = ((hp - self.kernel_size[1]) // sh + 1) * ((wp - self.kernel_size[2]) // sw + 1)
+ fused_residual = None
+ # the kitchen's views must stay under its 32-bit offsets; when even one output frame's window
+ # does not, cuDNN takes the whole buffer (the kitchen would decline and copy a second output)
+ if (not _fp16_accumulate_wanted(buf) or kt * c * hp * wp > self._CK_MAX_VIEW_ELEMENTS
+ or weight.size(0) * out_hw > self._CK_MAX_VIEW_ELEMENTS):
+ out = F.conv3d(buf, weight_cl, bias, stride=stride) # cuDNN, NDHWC in and out
+ else:
+ zt = (buf.size(2) - kt) // sd + 1
+ out = torch.empty((b, weight.size(0), zt, (hp - self.kernel_size[1]) // sh + 1, (wp - self.kernel_size[2]) // sw + 1),
+ dtype=buf.dtype, device=buf.device, memory_format=_CL)
+ chunk = min(zt, (self._CK_MAX_VIEW_ELEMENTS // (c * hp * wp) - kt) // sd + 1,
+ self._CK_MAX_VIEW_ELEMENTS // (weight.size(0) * out_hw))
+ if residual is not None and residual.shape == out.shape and residual.dtype == out.dtype and _ndhwc_like(residual):
+ fused_residual = residual # otherwise added after: the epilogue wants the output's shape and layout
+ for z0 in range(0, zt, chunk):
+ z1 = min(zt, z0 + chunk)
+ comfy_kitchen.fp16_conv3d(buf[:, :, z0 * sd:(z1 - 1) * sd + kt], weight_cl, bias,
+ residual=None if fused_residual is None else fused_residual[:, :, z0:z1],
+ stride=stride, out=out[:, :, z0:z1])
+ finally:
+ comfy.ops.uncast_bias_weight(self, weight, bias, offload_stream)
+ if residual is not None and fused_residual is None:
+ out += residual
+ return out
+
+ def _ck_taps(self, like, x_shape, memory_state, memory_cache, write_body, residual, has_memory):
+ """A one-frame chunk against its two-frame halo as three single-frame convs summed into the
+ output: the 3-frame buffer was the tail's peak (1.5 GiB lower floor at 1080p). The halo taps
+ run straight from the unpacked frames, their padding in place of a buffer; with fp16
+ accumulation the kitchen conv adds each tap, and the residual, into the output in its
+ epilogue (1.7x over cuDNN plus an add). Without a cache the halo is the frame itself, so the
+ taps sum first."""
+ b, c, _, h, w = x_shape
+ _, ph, pw = self.padding
+ buf = torch.empty((b, c, 1, h + 2 * ph, w + 2 * pw), dtype=like.dtype, device=like.device, memory_format=_CL)
+ write_body(buf)
+ del write_body
+ _ndhwc_filter(self)
+ weight, bias, offload_stream = comfy.ops.cast_bias_weight(self, buf, offloadable=True)
try:
- return super()._conv_forward(input, weight, bias, *args, **kwargs)
- except NotImplementedError:
- # for: Could not run 'aten::cudnn_convolution' with arguments from the 'CPU' backend
- if not self.logged_once:
- logging.warning("VAE is on CPU for decoding. This is most likely due to not enough memory")
- self.logged_once = True
- return F.conv3d(input, weight, bias, *args, **kwargs)
+ tap = weight[:, :, 2:] if has_memory else weight.sum(dim=2, keepdim=True)
+ tap = tap.contiguous(memory_format=_CL)
+ fp16_acc = _fp16_accumulate_wanted(buf)
+ if fp16_acc:
+ fused = (residual is not None and residual.shape == (b, weight.size(0), 1, h, w)
+ and residual.dtype == buf.dtype and _ndhwc_like(residual))
+ out = comfy_kitchen.fp16_conv3d(buf, tap, bias, residual=residual if fused else None)
+ if fused:
+ residual = None
+ else:
+ out = F.conv3d(buf, tap, bias)
+ if memory_state in (MemoryState.INITIALIZING, MemoryState.ACTIVE):
+ # pushed now so the buffer can go; three kept, as the oldest is still this call's first tap
+ memory_cache.push_frames(self, buf if has_memory else buf.expand(-1, -1, 2, -1, -1), 3)
+ del buf
+ if has_memory:
+ rows = -(-out.size(3) // self._TAP_BANDS)
+
+ def halo_tap(i, frame):
+ tap_i = weight[:, :, i:i + 1].contiguous(memory_format=_CL)
+ for r0 in range(0, out.size(3), rows):
+ r1 = min(out.size(3), r0 + rows)
+ band, x = out[:, :, :, r0:r1], frame[:, :, :, r0:r1 + 2 * ph]
+ if fp16_acc:
+ comfy_kitchen.fp16_conv3d(x, tap_i, residual=band, out=band)
+ else:
+ band.add_(F.conv3d(x, tap_i))
+ memory_cache.for_each_frame(self, halo_tap, count=2)
+ memory_cache.keep_last(self, 2)
+ finally:
+ comfy.ops.uncast_bias_weight(self, weight, bias, offload_stream)
+ if residual is not None:
+ out += residual
+ return out
+
+ def _ck_buffered_applies(self, x, memory_state):
+ """Whether _ck_buffered can take this call: the frame-offset write needs a batch of one."""
+ return self._ck_applies(x) and memory_state != MemoryState.UNSET and x.size(0) == 1
+
+ def fused_norm_conv(self, source, norm, memory_state, memory_cache, residual=None):
+ """GroupNorm + SiLU written straight into the conv's buffer; ``source`` (a 1-list) is emptied
+ once read so the activation is freed early. None when the fused path does not apply."""
+ x = source[0]
+ if not (self._ck_buffered_applies(x, memory_state) and isinstance(norm, nn.GroupNorm)
+ and norm.affine):
+ return None
+ _, ph, pw = self.padding
+ like, shape = x.new_empty(0), x.shape
+ del x
+ gw, gb, offload_stream = comfy.ops.cast_bias_weight(norm, like, offloadable=True)
+
+ def write_body(dst):
+ comfy_kitchen.group_norm_silu_pad3d(
+ source.pop(), gw, gb, norm.num_groups, norm.eps, (pw, pw, ph, ph, 0), True,
+ zero_pad=True, out=dst,
+ )
+ try:
+ return self._ck_buffered(like, shape, memory_state, memory_cache, write_body, residual=residual)
+ finally:
+ comfy.ops.uncast_bias_weight(norm, gw, gb, offload_stream)
def memory_limit_conv(
self,
@@ -496,11 +773,6 @@ class InflatedCausalConv3d(ops.Conv3d):
padding=(0, 0, 0, 0, 0, 0),
prev_cache=None,
):
- if math.isinf(self.memory_limit):
- if prev_cache is not None:
- x = torch.cat([prev_cache, x], dim=split_dim - 1)
- return super().forward(x)
-
shape = list(x.size())
if prev_cache is not None:
shape[split_dim - 1] += prev_cache.size(split_dim - 1)
@@ -508,18 +780,13 @@ class InflatedCausalConv3d(ops.Conv3d):
shape[-3 + i] += pad_sum
memory_occupy = math.prod(shape) * x.element_size() / 1024**3 # GiB
if memory_occupy < self.memory_limit or split_dim == x.ndim:
- x_concat = x
- if prev_cache is not None:
- x_concat = torch.cat([prev_cache, x], dim=split_dim - 1)
-
- def pad_and_forward():
- padded = F.pad(x_concat, padding, mode='constant', value=0.0)
- if not padded.is_contiguous():
- padded = padded.contiguous()
- with ignore_padding(self):
- return torch.nn.Conv3d.forward(self, padded)
-
- return pad_and_forward()
+ if self._ck_applies(x):
+ return self._ck_conv(self._padded_input(x, prev_cache, padding, split_dim - 1, channels_last=True))
+ out = self._conv_without_padding_copy(x, prev_cache, padding)
+ if out is not None:
+ return out
+ padded = self._padded_input(x, prev_cache, padding, split_dim - 1)
+ return self._conv_padded(padded, (0, 0, 0))
num_splits = math.ceil(memory_occupy / self.memory_limit)
size_per_split = x.size(split_dim) // num_splits
@@ -583,8 +850,9 @@ class InflatedCausalConv3d(ops.Conv3d):
memory_cache = {}
if memory_state != MemoryState.ACTIVE:
memory_cache.pop(self, None)
+ # a 1x1x1 conv has no halo to save by splitting: split and concatenated, its output is held twice
if (
- math.isinf(self.memory_limit)
+ (math.isinf(self.memory_limit) or tuple(self.kernel_size) == (1, 1, 1))
and torch.is_tensor(input)
):
return self.basic_forward(input, memory_state, memory_cache)
@@ -594,9 +862,26 @@ class InflatedCausalConv3d(ops.Conv3d):
mem_size = self.stride[0] - self.kernel_size[0]
memory = memory_cache.get(self) if memory_cache is not None else None
if (memory is not None) and (memory_state == MemoryState.ACTIVE):
- input = extend_head(input, memory=memory, times=-1)
+ head = memory.to(input)
+ elif self.temporal_padding:
+ head = torch.tile(input[:, :, :1], [1, 1, self.temporal_padding * 2, 1, 1])
else:
- input = extend_head(input, times=self.temporal_padding * 2)
+ head = None
+ if self._ck_applies(input):
+ # The halo and the zero border share one buffer: the kernel needs a padded copy
+ # anyway, so the cat with the head is folded into it. On the largest
+ # activations that is one full pass and one activation's worth of peak.
+ padded = self._padded_input(input, head, self._symmetric_padding(), 2, channels_last=True)
+ if mem_size != 0 and memory_state != MemoryState.DISABLED and memory_cache is not None:
+ # From the halo'd buffer, head included: a one-frame slice
+ # still leaves a two-frame cache. The spatial border is cropped back off.
+ _, ph, pw = self.padding
+ tail = padded[:, :, mem_size:, ph:padded.size(3) - ph or None, pw:padded.size(4) - pw or None]
+ memory_cache[self] = tail.detach().clone(memory_format=_CL)
+ elif memory_cache is not None and memory_state != MemoryState.DISABLED:
+ memory_cache.pop(self, None)
+ return self._ck_conv(padded)
+ input = input if head is None else torch.cat((_match_layout(head, input), input), dim=2)
next_memory = (
input[:, :, mem_size:].detach()
if (mem_size != 0 and memory_state != MemoryState.DISABLED)
@@ -609,967 +894,516 @@ class InflatedCausalConv3d(ops.Conv3d):
memory_cache[self] = next_memory
return super().forward(input)
- def slicing_forward(
- self,
- input,
- memory_state: MemoryState = MemoryState.UNSET,
- memory_cache = None,
- ) -> Tensor:
- if memory_cache is None:
- memory_cache = {}
- squeeze_out = False
- if torch.is_tensor(input):
- input = [input]
- squeeze_out = True
-
+ def slicing_forward(self, input, memory_state, memory_cache):
+ """forward() for a finite memory_limit: the halo is handed to memory_limit_conv, which splits
+ the conv along H, then W, while its padded input exceeds the limit."""
cache_size = self.kernel_size[0] - self.stride[0]
- memory = memory_cache.get(self) if memory_cache is not None else None
- cache = cache_send_recv(
- input, cache_size=cache_size, memory=memory, times=self.temporal_padding * 2
- )
+ memory = memory_cache.get(self)
+ if memory is not None:
+ cache = _match_layout(memory.to(input), input)
+ elif self.temporal_padding:
+ cache = _match_layout(torch.tile(input[:, :, :1], [1, 1, self.temporal_padding * 2, 1, 1]), input)
+ else:
+ cache = None
- if (
- memory_state in [MemoryState.INITIALIZING, MemoryState.ACTIVE]
- and cache_size != 0
- ):
- if cache_size > input[-1].size(2) and cache is not None and len(input) == 1:
- input[0] = torch.cat([cache, input[0]], dim=2)
+ if memory_state in (MemoryState.INITIALIZING, MemoryState.ACTIVE) and cache_size != 0:
+ if cache_size > input.size(2) and cache is not None:
+ input = torch.cat([cache, input], dim=2)
cache = None
- if cache_size <= input[-1].size(2):
- memory_cache[self] = input[-1][:, :, -cache_size:].detach().contiguous()
-
- padding = tuple(x for x in reversed(self.padding) for _ in range(2))
- for i in range(len(input)):
- next_cache = None
- cache_size = 0
- if i < len(input) - 1:
- cache_len = cache.size(2) if cache is not None else 0
- cache_size = get_cache_size(self, input[i].size(2) + cache_len, pad_len=0)
- if cache_size != 0:
- if cache_size > input[i].size(2) and cache is not None:
- input[i] = torch.cat([cache, input[i]], dim=2)
- cache = None
- if cache_size > input[i].size(2):
- raise ValueError(f"SeedVR2 VAE cache size {cache_size} exceeds input length {input[i].size(2)}.")
- next_cache = input[i][:, :, -cache_size:]
-
- input[i] = self.memory_limit_conv(
- input[i],
- padding=padding,
- prev_cache=cache
- )
-
- cache = next_cache
-
- return input[0] if squeeze_out else input
+ if cache_size <= input.size(2):
+ # clone, not contiguous(): for a batch of one a frame slice of an NDHWC tensor
+ # already satisfies the channels_last_3d check, and contiguous() would return the
+ # view, pinning the whole activation for the next slice instead of two frames.
+ tail = input[:, :, -cache_size:].detach()
+ memory_cache[self] = tail.clone(
+ memory_format=_CL if input.is_contiguous(memory_format=_CL) else torch.contiguous_format)
-def remove_head(tensor: Tensor, times: int = 1) -> Tensor:
- if times == 0:
- return tensor
- return torch.cat(tensors=(tensor[:, :, :1], tensor[:, :, times + 1 :]), dim=2)
+ return self.memory_limit_conv(input, padding=self._symmetric_padding(), prev_cache=cache)
class Upsample3D(nn.Module):
+ """2x spatial (and 2x temporal) upsampling: a 1x1 expansion, its pixel shuffle, a causal 3x3x3 conv."""
- def __init__(
- self,
- channels,
- out_channels = None,
- inflation_mode = "tail",
- temporal_up: bool = False,
- spatial_up: bool = True,
- ):
+ def __init__(self, channels, temporal_up: bool = False):
super().__init__()
- self.channels = channels
- self.out_channels = out_channels or channels
-
- conv = InflatedCausalConv3d(
- self.channels,
- self.out_channels,
- 3,
- padding=1,
- inflation_mode=inflation_mode,
- )
-
self.temporal_up = temporal_up
- self.spatial_up = spatial_up
self.temporal_ratio = 2 if temporal_up else 1
- self.spatial_ratio = 2 if spatial_up else 1
-
- upscale_ratio = (self.spatial_ratio**2) * self.temporal_ratio
- self.upscale_conv = ops.Conv3d(
- self.channels, self.channels * upscale_ratio, kernel_size=1, padding=0
- )
-
- self.conv = conv
-
- def forward(
- self,
- hidden_states: torch.FloatTensor,
- memory_state=None,
- memory_cache=None,
- ) -> torch.FloatTensor:
- if hidden_states.shape[1] != self.channels:
- raise ValueError(f"SeedVR2 upsample expected {self.channels} channels, got {hidden_states.shape[1]}.")
-
- hidden_states = self.upscale_conv(hidden_states)
+ self.spatial_ratio = 2
+ self.upscale_conv = ops.Conv3d(channels, channels * self.spatial_ratio ** 2 * self.temporal_ratio, kernel_size=1, padding=0)
+ self.conv = InflatedCausalConv3d(channels, channels, 3, padding=1)
+
+ def _expand(self, x, weight, bias, i, j, t):
+ """One shuffle phase of the 1x1 expansion; its output channels are (sr, sr, tr, c). A helper
+ so the weight slice's copy is freed before the phase is written out."""
+ c = x.size(1)
+ k = ((i * self.spatial_ratio + j) * self.temporal_ratio + t) * c
+ wk = weight[k:k + c].contiguous(memory_format=_CL)
+ bk = None if bias is None else bias[k:k + c]
+ if _fp16_accumulate_wanted(x):
+ return comfy_kitchen.fp16_conv3d(x, wk, bk)
+ return F.conv3d(x, wk, bk)
+
+ def _expand_into(self, dst, x, phases, drop_head):
+ """Write the 1x1 expansion of ``x`` for temporal ``phases`` into the conv's padded buffer ``dst``,
+ one shuffle phase at a time at its strided place: a phase's output is a quarter (an eighth) of
+ the expanded tensor. With ``drop_head`` the first input frame keeps only its first phase, as
+ the video's first frame is not doubled."""
+ sr, tr = self.spatial_ratio, self.temporal_ratio
+ _, ph, pw = self.conv.padding
+ h_out, w_out = x.size(3) * sr, x.size(4) * sr
+ for dim, lo, hi in ((3, ph, ph + h_out), (4, pw, pw + w_out)):
+ if lo:
+ dst.narrow(dim, 0, lo).zero_()
+ dst.narrow(dim, hi, dst.size(dim) - hi).zero_()
+ core = dst[:, :, :, ph:ph + h_out, pw:pw + w_out]
+ _ndhwc_filter(self.upscale_conv)
+ weight, bias, offload_stream = comfy.ops.cast_bias_weight(self.upscale_conv, x, offloadable=True)
+ try:
+ for i in range(sr):
+ for j in range(sr):
+ for t in phases:
+ y = self._expand(x, weight, bias, i, j, t)
+ frames = core[:, :, :, i::sr, j::sr]
+ if len(phases) == 1:
+ frames.copy_(y)
+ elif not drop_head:
+ frames[:, :, t::tr].copy_(y)
+ elif t == 0:
+ frames[:, :, 0].copy_(y[:, :, 0])
+ frames[:, :, 1::tr].copy_(y[:, :, 1:])
+ else:
+ frames[:, :, t + 1::tr].copy_(y[:, :, 1:])
+ del y
+ finally:
+ comfy.ops.uncast_bias_weight(self.upscale_conv, weight, bias, offload_stream)
+
+ def frames_apply(self, x, memory_state):
+ return x.size(2) == 1 and x.is_contiguous(memory_format=_CL) and self.conv._ck_buffered_applies(x, memory_state)
+
+ def frames(self, source, memory_state, memory_cache):
+ """This upsampler over a one-frame ``source`` (a 1-list), an output frame at a time: each
+ temporal phase's expansion goes straight into the conv's buffer and the conv runs on that frame
+ alone. Each output is yielded boxed, so nothing here holds it while the caller uses it."""
+ x = source.pop()
+ b, c, _, h, w = x.shape
+ sr = self.spatial_ratio
+ phases = [0] if self.temporal_up and memory_state != MemoryState.ACTIVE else range(self.temporal_ratio)
+ for n, t in enumerate(phases):
+ state = memory_state if n == 0 else MemoryState.ACTIVE
+ box = [self.conv._ck_buffered(x.new_empty(0), (b, c, 1, h * sr, w * sr), state, memory_cache,
+ lambda dst, t=t: self._expand_into(dst, x, [t], False))]
+ yield box
+ del box
+
+ def run(self, source, memory_state, memory_cache):
+ """The upsampler on a 1-list ``source`` it empties: a module call would hold its input until it returns."""
+ sr, tr = self.spatial_ratio, self.temporal_ratio
+ drop_head = self.temporal_up and memory_state != MemoryState.ACTIVE
+ x = source[0]
+ if x.is_contiguous(memory_format=_CL) and self.conv._ck_buffered_applies(x, memory_state):
+ b, c, f, h, w = x.shape
+ like = x.new_empty(0) # dtype/device only, shares nothing
+ del x
+ return self.conv._ck_buffered(like, (b, c, f * tr - int(drop_head), h * sr, w * sr), memory_state, memory_cache,
+ lambda dst: self._expand_into(dst, source.pop(), range(tr), drop_head))
+ del x
+
+ hidden_states = self.upscale_conv(source.pop())
b, channels, f, h, w = hidden_states.shape
- c = channels // (self.spatial_ratio * self.spatial_ratio * self.temporal_ratio)
- hidden_states = hidden_states.view(b, self.spatial_ratio, self.spatial_ratio, self.temporal_ratio, c, f, h, w)
- hidden_states = hidden_states.permute(0, 4, 5, 3, 6, 1, 7, 2).reshape(
- b,
- c,
- f * self.temporal_ratio,
- h * self.spatial_ratio,
- w * self.spatial_ratio,
- )
-
- if self.temporal_up and memory_state != MemoryState.ACTIVE:
- hidden_states = remove_head(hidden_states)
-
- hidden_states = self.conv(hidden_states, memory_state=memory_state, memory_cache=memory_cache)
-
- return hidden_states
+ c = channels // (sr * sr * tr)
+ hidden_states = hidden_states.view(b, sr, sr, tr, c, f, h, w).permute(0, 4, 5, 3, 6, 1, 7, 2).reshape(
+ b, c, f * tr, h * sr, w * sr)
+ if drop_head:
+ hidden_states = torch.cat((hidden_states[:, :, :1], hidden_states[:, :, 2:]), dim=2)
+ return self.conv(hidden_states, memory_state=memory_state, memory_cache=memory_cache)
class Downsample3D(nn.Module):
- def __init__(
- self,
- channels,
- out_channels = None,
- inflation_mode = "tail",
- spatial_down: bool = False,
- temporal_down: bool = False,
- ):
- super().__init__()
- self.channels = channels
- self.out_channels = out_channels or channels
- self.temporal_down = temporal_down
- self.spatial_down = spatial_down
-
- self.temporal_ratio = 2 if temporal_down else 1
- self.spatial_ratio = 2 if spatial_down else 1
-
- self.temporal_kernel = 3 if temporal_down else 1
- self.spatial_kernel = 3 if spatial_down else 1
+ """2x spatial (and 2x temporal) downsampling: a strided conv over a right/bottom zero border."""
+ def __init__(self, channels, temporal_down: bool = False):
+ super().__init__()
self.conv = InflatedCausalConv3d(
- self.channels,
- self.out_channels,
- kernel_size=(self.temporal_kernel, self.spatial_kernel, self.spatial_kernel),
- stride=(self.temporal_ratio, self.spatial_ratio, self.spatial_ratio),
- padding=(1 if self.temporal_down else 0, 0, 0),
- inflation_mode=inflation_mode,
+ channels,
+ channels,
+ kernel_size=(3 if temporal_down else 1, 3, 3),
+ stride=(2 if temporal_down else 1, 2, 2),
+ padding=(1 if temporal_down else 0, 0, 0),
)
+ def run(self, source, memory_state, memory_cache):
+ """The downsampler on a 1-list ``source`` it empties: a module call would hold its input until it returns."""
+ x = source[0]
+ if x.is_contiguous(memory_format=_CL) and self.conv._ck_buffered_applies(x, memory_state):
+ # straight into the conv's buffer, its right and bottom zero border in place of the pad copy
+ b, c, t, h, w = x.shape
+ like = x.new_empty(0)
+ del x
- def forward(
- self,
- hidden_states: torch.FloatTensor,
- memory_state = None,
- memory_cache = None,
- ) -> torch.FloatTensor:
-
- if hidden_states.shape[1] != self.channels:
- raise ValueError(f"SeedVR2 downsample expected {self.channels} channels, got {hidden_states.shape[1]}.")
-
- if self.spatial_down:
- pad = (0, 1, 0, 1)
- hidden_states = F.pad(hidden_states, pad, mode="constant", value=0)
-
- if hidden_states.shape[1] != self.channels:
- raise ValueError(f"SeedVR2 downsample expected {self.channels} channels after padding, got {hidden_states.shape[1]}.")
-
- hidden_states = self.conv(hidden_states, memory_state=memory_state, memory_cache=memory_cache)
+ def write_body(dst):
+ dst[:, :, :, h:].zero_()
+ dst[:, :, :, :h, w:].zero_()
+ dst[:, :, :, :h, :w].copy_(source.pop())
+ return self.conv._ck_buffered(like, (b, c, t, h, w), memory_state, memory_cache, write_body, pad=(0, 1, 0, 1))
+ del x
- return hidden_states
+ hidden_states = F.pad(source.pop(), (0, 1, 0, 1), mode="constant", value=0)
+ return self.conv(hidden_states, memory_state=memory_state, memory_cache=memory_cache)
class ResnetBlock3D(nn.Module):
- def __init__(
- self,
- in_channels: int,
- out_channels: Optional[int] = None,
- temb_channels: int = 512,
- groups: int = 32,
- groups_out: Optional[int] = None,
- eps: float = 1e-6,
- output_scale_factor: float = 1.0,
- skip_time_act: bool = False,
- inflation_mode = "tail",
- time_receptive_field: _receptive_field_t = "half",
- ):
+ def __init__(self, in_channels: int, out_channels: int, groups: int = 32, eps: float = 1e-6):
super().__init__()
- self.in_channels = in_channels
- self.out_channels = in_channels if out_channels is None else out_channels
- self.output_scale_factor = output_scale_factor
- self.skip_time_act = skip_time_act
- self.nonlinearity = nn.SiLU()
- if temb_channels is not None:
- self.time_emb_proj = ops.Linear(temb_channels, self.out_channels)
- else:
- self.time_emb_proj = None
self.norm1 = ops.GroupNorm(num_groups=groups, num_channels=in_channels, eps=eps, affine=True)
- if groups_out is None:
- groups_out = groups
- self.norm2 = ops.GroupNorm(num_groups=groups_out, num_channels=self.out_channels, eps=eps, affine=True)
- self.use_in_shortcut = self.in_channels != self.out_channels
- self.conv1 = InflatedCausalConv3d(
- self.in_channels,
- self.out_channels,
- kernel_size=(1, 3, 3) if time_receptive_field == "half" else (3, 3, 3),
- stride=1,
- padding=(0, 1, 1) if time_receptive_field == "half" else (1, 1, 1),
- inflation_mode=inflation_mode,
- )
-
- self.conv2 = InflatedCausalConv3d(
- self.out_channels,
- self.out_channels,
- kernel_size=3,
- stride=1,
- padding=1,
- inflation_mode=inflation_mode,
- )
-
+ self.norm2 = ops.GroupNorm(num_groups=groups, num_channels=out_channels, eps=eps, affine=True)
+ self.conv1 = InflatedCausalConv3d(in_channels, out_channels, kernel_size=3, stride=1, padding=1)
+ self.conv2 = InflatedCausalConv3d(out_channels, out_channels, kernel_size=3, stride=1, padding=1)
self.conv_shortcut = None
- if self.use_in_shortcut:
- self.conv_shortcut = InflatedCausalConv3d(
- self.in_channels,
- self.out_channels,
- kernel_size=1,
- stride=1,
- padding=0,
- bias=True,
- inflation_mode=inflation_mode,
- )
-
- def forward(self, input_tensor, temb, memory_state = None, memory_cache = None):
- hidden_states = input_tensor
-
- hidden_states = causal_norm_wrapper(self.norm1, hidden_states)
-
- hidden_states = self.nonlinearity(hidden_states)
-
- hidden_states = self.conv1(hidden_states, memory_state=memory_state, memory_cache=memory_cache)
-
- if self.time_emb_proj is not None:
- if not self.skip_time_act:
- temb = self.nonlinearity(temb)
- temb = self.time_emb_proj(temb)[:, :, None, None]
-
- if temb is not None:
- hidden_states = hidden_states + temb
-
- hidden_states = causal_norm_wrapper(self.norm2, hidden_states)
-
- hidden_states = self.nonlinearity(hidden_states)
-
- hidden_states = self.conv2(hidden_states, memory_state=memory_state, memory_cache=memory_cache)
-
+ if in_channels != out_channels:
+ self.conv_shortcut = InflatedCausalConv3d(in_channels, out_channels, kernel_size=1, stride=1, padding=0, bias=True)
+
+ def forward(self, input_tensor, memory_state, memory_cache):
+ hidden_states = self.conv1.fused_norm_conv([input_tensor], self.norm1, memory_state, memory_cache)
+ if hidden_states is None:
+ hidden_states = causal_norm_wrapper(self.norm1, input_tensor, silu=True)
+ hidden_states = self.conv1(hidden_states, memory_state=memory_state, memory_cache=memory_cache)
+
+ # An identity skip rides in conv2's epilogue, sparing a pass over the output. A 1x1 shortcut
+ # runs after conv2 instead: computed first, its output would sit under conv2's buffer and
+ # output (the input is held either way).
+ residual = input_tensor if self.conv_shortcut is None else None
+ source = [hidden_states]
+ del hidden_states
+ hidden_states = self.conv2.fused_norm_conv(source, self.norm2, memory_state, memory_cache, residual=residual)
+ if hidden_states is None:
+ hidden_states = causal_norm_wrapper(self.norm2, source.pop(), silu=True)
+ hidden_states = self.conv2(hidden_states, memory_state=memory_state, memory_cache=memory_cache)
+ elif residual is not None:
+ return hidden_states
if self.conv_shortcut is not None:
input_tensor = self.conv_shortcut(input_tensor, memory_state=memory_state, memory_cache=memory_cache)
+ return hidden_states.add_(input_tensor)
- output_tensor = (input_tensor + hidden_states) / self.output_scale_factor
- return output_tensor
+def _run_block(resnets, samplers, source, memory_state, memory_cache):
+ """A down/up block's layers one by one on a 1-list ``source`` it empties, each sampler handed its
+ input boxed: a module call holds its input until it returns, so nothing may hold a layer's input
+ past the layer that consumes it."""
+ sample = source.pop()
+ for resnet in resnets:
+ sample = resnet(sample, memory_state, memory_cache)
+ for sampler in samplers or ():
+ box = [sample]
+ del sample
+ sample = sampler.run(box, memory_state, memory_cache)
+ return sample
class DownEncoderBlock3D(nn.Module):
- def __init__(
- self,
- in_channels: int,
- out_channels: int,
- num_layers: int = 1,
- resnet_eps: float = 1e-6,
- resnet_groups: int = 32,
- output_scale_factor: float = 1.0,
- add_downsample: bool = True,
- inflation_mode = "tail",
- time_receptive_field: _receptive_field_t = "half",
- temporal_down: bool = True,
- spatial_down: bool = True,
- ):
+ def __init__(self, in_channels: int, out_channels: int, num_layers: int, resnet_groups: int,
+ add_downsample: bool, temporal_down: bool):
super().__init__()
- resnets = []
-
- for i in range(num_layers):
- in_channels = in_channels if i == 0 else out_channels
- resnets.append(
- ResnetBlock3D(
- in_channels=in_channels,
- out_channels=out_channels,
- temb_channels=None,
- eps=resnet_eps,
- groups=resnet_groups,
- output_scale_factor=output_scale_factor,
- inflation_mode=inflation_mode,
- time_receptive_field=time_receptive_field,
- )
- )
-
- self.resnets = nn.ModuleList(resnets)
-
- if add_downsample:
- self.downsamplers = nn.ModuleList(
- [
- Downsample3D(
- out_channels,
- out_channels=out_channels,
- temporal_down=temporal_down,
- spatial_down=spatial_down,
- inflation_mode=inflation_mode,
- )
- ]
- )
- else:
- self.downsamplers = None
-
- def forward(
- self,
- hidden_states: torch.FloatTensor,
- memory_state = None,
- memory_cache = None,
- ) -> torch.FloatTensor:
- for resnet in self.resnets:
- hidden_states = resnet(hidden_states, temb=None, memory_state=memory_state, memory_cache=memory_cache)
-
- if self.downsamplers is not None:
- for downsampler in self.downsamplers:
- hidden_states = downsampler(hidden_states, memory_state=memory_state, memory_cache=memory_cache)
+ self.resnets = nn.ModuleList([
+ ResnetBlock3D(in_channels if i == 0 else out_channels, out_channels, groups=resnet_groups)
+ for i in range(num_layers)
+ ])
+ self.downsamplers = nn.ModuleList([Downsample3D(out_channels, temporal_down=temporal_down)]) if add_downsample else None
- return hidden_states
+ def run(self, source, memory_state, memory_cache):
+ return _run_block(self.resnets, self.downsamplers, source, memory_state, memory_cache)
class UpDecoderBlock3D(nn.Module):
- def __init__(
- self,
- in_channels: int,
- out_channels: int,
- num_layers: int = 1,
- resnet_eps: float = 1e-6,
- resnet_groups: int = 32,
- output_scale_factor: float = 1.0,
- add_upsample: bool = True,
- temb_channels: Optional[int] = None,
- inflation_mode = "tail",
- time_receptive_field: _receptive_field_t = "half",
- temporal_up: bool = True,
- spatial_up: bool = True,
- ):
+ def __init__(self, in_channels: int, out_channels: int, num_layers: int, resnet_groups: int,
+ add_upsample: bool, temporal_up: bool):
super().__init__()
- resnets = []
-
- for i in range(num_layers):
- input_channels = in_channels if i == 0 else out_channels
-
- resnets.append(
- ResnetBlock3D(
- in_channels=input_channels,
- out_channels=out_channels,
- temb_channels=temb_channels,
- eps=resnet_eps,
- groups=resnet_groups,
- output_scale_factor=output_scale_factor,
- inflation_mode=inflation_mode,
- time_receptive_field=time_receptive_field,
- )
- )
-
- self.resnets = nn.ModuleList(resnets)
-
- if add_upsample:
- self.upsamplers = nn.ModuleList(
- [
- Upsample3D(
- out_channels,
- out_channels=out_channels,
- temporal_up=temporal_up,
- spatial_up=spatial_up,
- inflation_mode=inflation_mode,
- )
- ]
- )
- else:
- self.upsamplers = None
+ self.resnets = nn.ModuleList([
+ ResnetBlock3D(in_channels if i == 0 else out_channels, out_channels, groups=resnet_groups)
+ for i in range(num_layers)
+ ])
+ self.upsamplers = nn.ModuleList([Upsample3D(out_channels, temporal_up=temporal_up)]) if add_upsample else None
- def forward(
- self,
- hidden_states: torch.FloatTensor,
- temb: Optional[torch.FloatTensor] = None,
- memory_state=None,
- memory_cache=None,
- ) -> torch.FloatTensor:
- for resnet in self.resnets:
- hidden_states = resnet(hidden_states, temb=None, memory_state=memory_state, memory_cache=memory_cache)
-
- if self.upsamplers is not None:
- for upsampler in self.upsamplers:
- hidden_states = upsampler(hidden_states, memory_state=memory_state, memory_cache=memory_cache)
-
- return hidden_states
+ def run(self, source, memory_state, memory_cache):
+ return _run_block(self.resnets, self.upsamplers, source, memory_state, memory_cache)
class UNetMidBlock3D(nn.Module):
- def __init__(
- self,
- in_channels: int,
- temb_channels: int,
- num_layers: int = 1,
- resnet_eps: float = 1e-6,
- resnet_time_scale_shift: str = "default", # default, spatial
- resnet_groups: int = 32,
- add_attention: bool = True,
- attention_head_dim: int = 1,
- output_scale_factor: float = 1.0,
- inflation_mode = "tail",
- time_receptive_field: _receptive_field_t = "half",
- ):
- super().__init__()
- resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)
- self.add_attention = add_attention
-
- resnets = [
- ResnetBlock3D(
- in_channels=in_channels,
- out_channels=in_channels,
- temb_channels=temb_channels,
- eps=resnet_eps,
- groups=resnet_groups,
- output_scale_factor=output_scale_factor,
- inflation_mode=inflation_mode,
- time_receptive_field=time_receptive_field,
- )
- ]
- attentions = []
-
- if attention_head_dim is None:
- attention_head_dim = in_channels
-
- for _ in range(num_layers):
- if self.add_attention:
- attentions.append(
- Attention(
- in_channels,
- heads=in_channels // attention_head_dim,
- dim_head=attention_head_dim,
- rescale_output_factor=output_scale_factor,
- eps=resnet_eps,
- norm_num_groups=(
- resnet_groups if resnet_time_scale_shift == "default" else None
- ),
- spatial_norm_dim=(
- temb_channels if resnet_time_scale_shift == "spatial" else None
- ),
- residual_connection=True,
- bias=True,
- )
- )
- else:
- attentions.append(None)
-
- resnets.append(
- ResnetBlock3D(
- in_channels=in_channels,
- out_channels=in_channels,
- temb_channels=temb_channels,
- eps=resnet_eps,
- groups=resnet_groups,
- output_scale_factor=output_scale_factor,
- inflation_mode=inflation_mode,
- time_receptive_field=time_receptive_field,
- )
- )
+ """A resnet, single-head self-attention within each frame, a resnet."""
- self.attentions = nn.ModuleList(attentions)
- self.resnets = nn.ModuleList(resnets)
-
- def forward(self, hidden_states, temb=None, memory_state=None, memory_cache=None):
- video_length = hidden_states.size(2)
- hidden_states = self.resnets[0](hidden_states, temb, memory_state=memory_state, memory_cache=memory_cache)
- for attn, resnet in zip(self.attentions, self.resnets[1:]):
- if attn is not None:
- b, c, f, h, w = hidden_states.shape
- hidden_states = hidden_states.transpose(1, 2).reshape(b * f, c, h, w)
- hidden_states = attn(hidden_states, temb=temb)
- hidden_states = hidden_states.reshape(b, video_length, c, h, w).transpose(1, 2)
- hidden_states = resnet(hidden_states, temb, memory_state=memory_state, memory_cache=memory_cache)
+ def __init__(self, channels: int, resnet_groups: int):
+ super().__init__()
+ self.attentions = nn.ModuleList([Attention(channels, norm_num_groups=resnet_groups, eps=1e-6)])
+ self.resnets = nn.ModuleList([ResnetBlock3D(channels, channels, groups=resnet_groups) for _ in range(2)])
- return hidden_states
+ def forward(self, hidden_states, memory_state, memory_cache):
+ hidden_states = self.resnets[0](hidden_states, memory_state, memory_cache)
+ b, c, f, h, w = hidden_states.shape
+ hidden_states = self.attentions[0](hidden_states.transpose(1, 2).reshape(b * f, c, h, w))
+ hidden_states = hidden_states.reshape(b, f, c, h, w).transpose(1, 2)
+ return self.resnets[1](hidden_states, memory_state, memory_cache)
class Encoder3D(nn.Module):
- def __init__(
- self,
- in_channels: int = 3,
- out_channels: int = 3,
- down_block_types: Tuple[str, ...] = ("DownEncoderBlock3D",),
- block_out_channels: Tuple[int, ...] = (64,),
- layers_per_block: int = 2,
- norm_num_groups: int = 32,
- mid_block_add_attention=True,
- temporal_down_num: int = 2,
- inflation_mode = "tail",
- time_receptive_field: _receptive_field_t = "half",
- ):
+ def __init__(self, in_channels: int, out_channels: int, block_out_channels, layers_per_block: int,
+ norm_num_groups: int, temporal_down_num: int):
super().__init__()
- self.layers_per_block = layers_per_block
- self.temporal_down_num = temporal_down_num
-
- self.conv_in = InflatedCausalConv3d(
- in_channels,
- block_out_channels[0],
- kernel_size=3,
- stride=1,
- padding=1,
- inflation_mode=inflation_mode,
- )
-
- self.mid_block = None
+ self.head_blocks = max(0, len(block_out_channels) - temporal_down_num - 1) # the spatial-only down blocks
+ self.conv_in = InflatedCausalConv3d(in_channels, block_out_channels[0], kernel_size=3, stride=1, padding=1)
self.down_blocks = nn.ModuleList([])
-
output_channel = block_out_channels[0]
- for i, down_block_type in enumerate(down_block_types):
+ for i in range(len(block_out_channels)):
input_channel = output_channel
output_channel = block_out_channels[i]
- is_final_block = i == len(block_out_channels) - 1
- is_temporal_down_block = i >= len(block_out_channels) - self.temporal_down_num - 1
-
- if down_block_type != "DownEncoderBlock3D":
- raise ValueError(f"SeedVR2 encoder only supports DownEncoderBlock3D, got {down_block_type}.")
-
- down_block = DownEncoderBlock3D(
- num_layers=self.layers_per_block,
- in_channels=input_channel,
- out_channels=output_channel,
- add_downsample=not is_final_block,
- resnet_eps=1e-6,
+ self.down_blocks.append(DownEncoderBlock3D(
+ input_channel,
+ output_channel,
+ num_layers=layers_per_block,
resnet_groups=norm_num_groups,
- temporal_down=is_temporal_down_block,
- spatial_down=True,
- inflation_mode=inflation_mode,
- time_receptive_field=time_receptive_field,
- )
- self.down_blocks.append(down_block)
-
- self.mid_block = UNetMidBlock3D(
- in_channels=block_out_channels[-1],
- resnet_eps=1e-6,
- output_scale_factor=1,
- resnet_time_scale_shift="default",
- attention_head_dim=block_out_channels[-1],
- resnet_groups=norm_num_groups,
- temb_channels=None,
- add_attention=mid_block_add_attention,
- inflation_mode=inflation_mode,
- time_receptive_field=time_receptive_field,
- )
-
- self.conv_norm_out = ops.GroupNorm(
- num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6
- )
- self.conv_act = nn.SiLU()
-
- conv_out_channels = 2 * out_channels
- self.conv_out = InflatedCausalConv3d(
- block_out_channels[-1], conv_out_channels, 3, padding=1, inflation_mode=inflation_mode
- )
-
-
- def forward(
- self,
- sample: torch.FloatTensor,
- memory_state = None,
- memory_cache = None,
- ) -> torch.FloatTensor:
- sample = sample.to(next(self.parameters()).device)
- sample = self.conv_in(sample, memory_state=memory_state, memory_cache=memory_cache)
- for down_block in self.down_blocks:
- sample = down_block(sample, memory_state=memory_state, memory_cache=memory_cache)
-
- sample = self.mid_block(sample, memory_state=memory_state, memory_cache=memory_cache)
-
- sample = causal_norm_wrapper(self.conv_norm_out, sample)
- sample = self.conv_act(sample)
- sample = self.conv_out(sample, memory_state=memory_state, memory_cache=memory_cache)
-
- return sample
+ add_downsample=i < len(block_out_channels) - 1,
+ temporal_down=i >= len(block_out_channels) - temporal_down_num - 1,
+ ))
+ self.mid_block = UNetMidBlock3D(block_out_channels[-1], resnet_groups=norm_num_groups)
+ self.conv_norm_out = ops.GroupNorm(num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6)
+ self.conv_out = InflatedCausalConv3d(block_out_channels[-1], 2 * out_channels, 3, padding=1)
+
+ # Frames per pass through the full-resolution, spatial-only head; None runs a slice's frames at once.
+ head_frames = 1
+
+ def _head(self, source, memory_state, memory_cache):
+ box = [self.conv_in(source.pop(), memory_state=memory_state, memory_cache=memory_cache)]
+ for down_block in self.down_blocks[:self.head_blocks]:
+ box = [down_block.run(box, memory_state, memory_cache)]
+ return box.pop()
+
+ def forward(self, sample, memory_state, memory_cache):
+ frame_pixels = sample.size(-2) * sample.size(-1)
+ source = [sample]
+ del sample
+ box = [_frame_chunked(self, self._head, source, self.head_frames, memory_state, memory_cache, frame_pixels)]
+ for down_block in self.down_blocks[self.head_blocks:]:
+ box = [down_block.run(box, memory_state, memory_cache)]
+ sample = self.mid_block(box.pop(), memory_state, memory_cache)
+ sample = causal_norm_wrapper(self.conv_norm_out, sample, silu=True)
+ return self.conv_out(sample, memory_state=memory_state, memory_cache=memory_cache)
class Decoder3D(nn.Module):
-
- def __init__(
- self,
- in_channels: int = 3,
- out_channels: int = 3,
- up_block_types: Tuple[str, ...] = ("UpDecoderBlock3D",),
- block_out_channels: Tuple[int, ...] = (64,),
- layers_per_block: int = 2,
- norm_num_groups: int = 32,
- mid_block_add_attention=True,
- inflation_mode = "tail",
- time_receptive_field: _receptive_field_t = "half",
- temporal_up_num: int = 2,
- ):
+ def __init__(self, in_channels: int, out_channels: int, block_out_channels, layers_per_block: int,
+ norm_num_groups: int, temporal_up_num: int):
super().__init__()
- self.layers_per_block = layers_per_block
self.temporal_up_num = temporal_up_num
-
- self.conv_in = InflatedCausalConv3d(
- in_channels,
- block_out_channels[-1],
- kernel_size=3,
- stride=1,
- padding=1,
- inflation_mode=inflation_mode,
- )
-
- self.mid_block = None
+ self.conv_in = InflatedCausalConv3d(in_channels, block_out_channels[-1], kernel_size=3, stride=1, padding=1)
+ self.mid_block = UNetMidBlock3D(block_out_channels[-1], resnet_groups=norm_num_groups)
self.up_blocks = nn.ModuleList([])
-
- temb_channels = None
-
- self.mid_block = UNetMidBlock3D(
- in_channels=block_out_channels[-1],
- resnet_eps=1e-6,
- output_scale_factor=1,
- resnet_time_scale_shift="default",
- attention_head_dim=block_out_channels[-1],
- resnet_groups=norm_num_groups,
- temb_channels=temb_channels,
- add_attention=mid_block_add_attention,
- inflation_mode=inflation_mode,
- time_receptive_field=time_receptive_field,
- )
-
reversed_block_out_channels = list(reversed(block_out_channels))
output_channel = reversed_block_out_channels[0]
- for i, up_block_type in enumerate(up_block_types):
+ for i in range(len(block_out_channels)):
prev_output_channel = output_channel
output_channel = reversed_block_out_channels[i]
-
- is_final_block = i == len(block_out_channels) - 1
- is_temporal_up_block = i < self.temporal_up_num
- if up_block_type != "UpDecoderBlock3D":
- raise ValueError(f"SeedVR2 decoder only supports UpDecoderBlock3D, got {up_block_type}.")
- up_block = UpDecoderBlock3D(
- num_layers=self.layers_per_block + 1,
- in_channels=prev_output_channel,
- out_channels=output_channel,
- add_upsample=not is_final_block,
- resnet_eps=1e-6,
+ self.up_blocks.append(UpDecoderBlock3D(
+ prev_output_channel,
+ output_channel,
+ num_layers=layers_per_block + 1,
resnet_groups=norm_num_groups,
- temb_channels=temb_channels,
- temporal_up=is_temporal_up_block,
- inflation_mode=inflation_mode,
- time_receptive_field=time_receptive_field,
- )
- self.up_blocks.append(up_block)
- prev_output_channel = output_channel
-
- self.conv_norm_out = ops.GroupNorm(
- num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6
- )
- self.conv_act = nn.SiLU()
- self.conv_out = InflatedCausalConv3d(
- block_out_channels[0], out_channels, 3, padding=1, inflation_mode=inflation_mode
- )
-
-
- def forward(
- self,
- sample: torch.FloatTensor,
- latent_embeds: Optional[torch.FloatTensor] = None,
- memory_state = None,
- memory_cache = None,
- ) -> torch.FloatTensor:
-
- sample = sample.to(next(self.parameters()).device)
+ add_upsample=i < len(block_out_channels) - 1,
+ temporal_up=i < temporal_up_num,
+ ))
+ self.conv_norm_out = ops.GroupNorm(num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6)
+ self.conv_out = InflatedCausalConv3d(block_out_channels[0], out_channels, 3, padding=1)
+
+ # Frames per pass through the spatial-only tail; None runs a slice's frames at once.
+ tail_frames = 1
+
+ def _tail(self, source, memory_state, memory_cache):
+ for up_block in self.up_blocks[self.temporal_up_num:]:
+ source = [up_block.run(source, memory_state, memory_cache)]
+ out = self.conv_out.fused_norm_conv(source, self.conv_norm_out, memory_state, memory_cache)
+ if out is None:
+ sample = causal_norm_wrapper(self.conv_norm_out, source.pop(), silu=True)
+ out = self.conv_out(sample, memory_state=memory_state, memory_cache=memory_cache)
+ return out
+
+ def forward(self, sample, memory_state, memory_cache):
sample = self.conv_in(sample, memory_state=memory_state, memory_cache=memory_cache)
-
- upscale_dtype = next(iter(self.up_blocks.parameters())).dtype
- sample = self.mid_block(sample, latent_embeds, memory_state=memory_state, memory_cache=memory_cache)
- sample = sample.to(upscale_dtype)
-
- for up_block in self.up_blocks:
- sample = up_block(sample, latent_embeds, memory_state=memory_state, memory_cache=memory_cache)
-
- sample = causal_norm_wrapper(self.conv_norm_out, sample)
- sample = self.conv_act(sample)
- sample = self.conv_out(sample, memory_state=memory_state, memory_cache=memory_cache)
-
- return sample
-
-class VideoAutoencoderKL(nn.Module):
- def __init__(
- self,
- in_channels: int = 3,
- out_channels: int = 3,
- layers_per_block: int = 2,
- latent_channels: int = SEEDVR2_LATENT_CHANNELS,
- norm_num_groups: int = 32,
- temporal_scale_num: int = 2,
- inflation_mode = "pad",
- time_receptive_field: _receptive_field_t = "full",
- slicing_sample_min_size = BYTEDANCE_SLICING_SAMPLE_MIN,
- ):
- self.slicing_sample_min_size = slicing_sample_min_size
- self.slicing_latent_min_size = slicing_sample_min_size // (2**temporal_scale_num)
- block_out_channels = BYTEDANCE_BLOCK_OUT_CHANNELS
- down_block_types = ("DownEncoderBlock3D",) * 4
- up_block_types = ("UpDecoderBlock3D",) * 4
+ sample = self.mid_block(sample, memory_state, memory_cache)
+
+ *head, last = self.up_blocks[:self.temporal_up_num]
+ box = [sample]
+ del sample
+ for up_block in head:
+ box = [up_block.run(box, memory_state, memory_cache)]
+ sample = box.pop()
+ for resnet in last.resnets:
+ sample = resnet(sample, memory_state, memory_cache)
+
+ # From the last temporal upsampler on, everything is causal and per frame: it runs an input
+ # frame at a time, each of its output frames going on through the spatial-only tail as it is made.
+ ups = sum(1 for blk in self.up_blocks[self.temporal_up_num - 1:] if blk.upsamplers is not None)
+ frame_pixels = sample.size(-2) * sample.size(-1) * 4 ** ups
+ up = last.upsamplers[0]
+
+ def upsample_and_tail(source, state, cache):
+ if up.frames_apply(source[0], state):
+ outs = [self._tail(box, state if n == 0 else MemoryState.ACTIVE, cache)
+ for n, box in enumerate(up.frames(source, state, cache))]
+ return torch.cat(outs, dim=2)
+ box = [up.run(source, state, cache)]
+ return _frame_chunked(self, self._tail, box, self.tail_frames, state, cache, frame_pixels)
+ source = [sample]
+ del sample
+ return _frame_chunked(last, upsample_and_tail, source, self.tail_frames and max(1, self.tail_frames // up.temporal_ratio),
+ memory_state, memory_cache, frame_pixels)
+
+
+class VideoAutoencoderKLWrapper(nn.Module):
+ """The SeedVR2 causal video VAE, run a temporal slice at a time with the causal caches carried
+ across slices."""
+ spatial_downsample_factor = BYTEDANCE_VAE_SPATIAL_DOWNSAMPLE
+ temporal_downsample_factor = BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE
+ slicing_sample_min_size = BYTEDANCE_SLICING_SAMPLE_MIN
+ slicing_latent_min_size = BYTEDANCE_SLICING_SAMPLE_MIN // BYTEDANCE_VAE_TEMPORAL_DOWNSAMPLE
+
+ def __init__(self):
super().__init__()
+ self.encoder = Encoder3D(3, SEEDVR2_LATENT_CHANNELS, BYTEDANCE_BLOCK_OUT_CHANNELS, layers_per_block=2,
+ norm_num_groups=32, temporal_down_num=2)
+ self.decoder = Decoder3D(SEEDVR2_LATENT_CHANNELS, 3, BYTEDANCE_BLOCK_OUT_CHANNELS, layers_per_block=2,
+ norm_num_groups=32, temporal_up_num=2)
+ set_norm_limit(BYTEDANCE_VAE_NORM_MEM_GIB)
+ for m in self.modules():
+ if isinstance(m, InflatedCausalConv3d):
+ m.memory_limit = BYTEDANCE_VAE_CONV_MEM_GIB
- self.encoder = Encoder3D(
- in_channels=in_channels,
- out_channels=latent_channels,
- down_block_types=down_block_types,
- block_out_channels=block_out_channels,
- layers_per_block=layers_per_block,
- norm_num_groups=norm_num_groups,
- temporal_down_num=temporal_scale_num,
- inflation_mode=inflation_mode,
- time_receptive_field=time_receptive_field,
- )
-
- self.decoder = Decoder3D(
- in_channels=latent_channels,
- out_channels=out_channels,
- up_block_types=up_block_types,
- block_out_channels=block_out_channels,
- layers_per_block=layers_per_block,
- norm_num_groups=norm_num_groups,
- temporal_up_num=temporal_scale_num,
- inflation_mode=inflation_mode,
- time_receptive_field=time_receptive_field,
- )
-
- self.use_slicing = True
-
- def encode(self, x: torch.FloatTensor, return_dict: bool = True):
- h = self.slicing_encode(x)
- posterior = DiagonalGaussianDistribution(h).mode()
-
- if not return_dict:
- return (posterior,)
-
- return posterior
-
- def decode_(
- self, z: torch.Tensor, return_dict: bool = True
- ):
- decoded = self.slicing_decode(z)
-
- if not return_dict:
- return (decoded,)
-
- return decoded
-
- def _encode(
- self, x, memory_state = MemoryState.DISABLED, memory_cache = None
- ) -> torch.Tensor:
- _x = x.to(self.device)
- h = self.encoder(_x, memory_state=memory_state, memory_cache=memory_cache)
+ def _encode(self, x, memory_state=MemoryState.DISABLED, memory_cache=None):
+ h = self.encoder(x.to(self.device), memory_state=memory_state, memory_cache=memory_cache)
return h.to(x.device)
- def _decode(
- self, z, memory_state = MemoryState.DISABLED, memory_cache = None
- ) -> torch.Tensor:
- _z = z.to(self.device)
- output = self.decoder(_z, memory_state=memory_state, memory_cache=memory_cache)
+ def _decode(self, z, memory_state=MemoryState.DISABLED, memory_cache=None):
+ output = self.decoder(z.to(self.device), memory_state=memory_state, memory_cache=memory_cache)
return output.to(z.device)
def slicing_encode(self, x: torch.Tensor) -> torch.Tensor:
- if self.use_slicing and (x.shape[2] - 1) > self.slicing_sample_min_size:
- memory_cache = {}
- split_size = max(
- self.slicing_sample_min_size,
- getattr(self, "temporal_downsample_factor", 1),
- )
- x_slices = list(x[:, :, 1:].split(split_size=split_size, dim=2))
- min_active_len = getattr(self, "temporal_downsample_factor", 1)
- if len(x_slices) > 1 and x_slices[-1].shape[2] < min_active_len:
- x_slices[-2] = torch.cat((x_slices[-2], x_slices[-1]), dim=2)
- x_slices.pop()
- encoded_slices = [
- self._encode(
- torch.cat((x[:, :, :1], x_slices[0]), dim=2),
- memory_state=MemoryState.INITIALIZING,
- memory_cache=memory_cache,
- )
- ]
- for x_idx in range(1, len(x_slices)):
- encoded_slices.append(
- self._encode(x_slices[x_idx], memory_state=MemoryState.ACTIVE, memory_cache=memory_cache)
- )
- out = torch.cat(encoded_slices, dim=2)
- return out
- else:
- return self._encode(x)
-
- def slicing_decode(self, z: torch.Tensor) -> torch.Tensor:
- if self.use_slicing and (z.shape[2] - 1) > self.slicing_latent_min_size:
- memory_cache = {}
- z_slices = z[:, :, 1:].split(split_size=self.slicing_latent_min_size, dim=2)
- decoded_slices = [
- self._decode(
- torch.cat((z[:, :, :1], z_slices[0]), dim=2),
- memory_state=MemoryState.INITIALIZING,
- memory_cache=memory_cache,
- )
- ]
- for z_idx in range(1, len(z_slices)):
- decoded_slices.append(
- self._decode(z_slices[z_idx], memory_state=MemoryState.ACTIVE, memory_cache=memory_cache)
- )
- out = torch.cat(decoded_slices, dim=2)
- return out
- else:
- return self._decode(z)
-
- def forward(self, x: torch.FloatTensor, mode: Literal["encode", "decode", "all"] = "all"):
- def _unwrap(value):
- return value[0] if isinstance(value, tuple) else value
-
- if mode == "encode":
- return _unwrap(self.encode(x))
- if mode == "decode":
- return _unwrap(self.decode_(x))
- if mode == "all":
- latent = _unwrap(self.encode(x))
- return _unwrap(self.decode_(latent))
- raise ValueError(f"Unknown SeedVR2 VAE forward mode: {mode}")
-
-class VideoAutoencoderKLWrapper(VideoAutoencoderKL):
- def __init__(
- self,
- spatial_downsample_factor = 8,
- temporal_downsample_factor = 4,
- ):
- self.spatial_downsample_factor = spatial_downsample_factor
- self.temporal_downsample_factor = temporal_downsample_factor
- super().__init__()
- self.set_memory_limit(BYTEDANCE_VAE_CONV_MEM_GIB, BYTEDANCE_VAE_NORM_MEM_GIB)
+ if x.size(0) > 1: # a video at a time: the buffered path and the memory estimate are per video
+ return torch.cat([self.slicing_encode(x[i:i + 1]) for i in range(x.size(0))])
+ if (x.shape[2] - 1) <= self.slicing_sample_min_size:
+ return self._encode(x.to(self.device))
+ memory_cache = CausalMemoryCache(offload=_offload_caches_for(x.shape[-2] * x.shape[-1]))
+ split_size = max(self.slicing_sample_min_size, self.temporal_downsample_factor)
+ x_slices = list(x[:, :, 1:].split(split_size=split_size, dim=2)) # _encode moves each to the device
+ if len(x_slices) > 1 and x_slices[-1].shape[2] < self.temporal_downsample_factor:
+ x_slices[-2] = torch.cat((x_slices[-2], x_slices[-1]), dim=2)
+ x_slices.pop()
+ try:
+ encoded_slices = [self._encode(torch.cat((x[:, :, :1], x_slices[0]), dim=2),
+ memory_state=MemoryState.INITIALIZING, memory_cache=memory_cache)]
+ for x_slice in x_slices[1:]:
+ encoded_slices.append(self._encode(x_slice, memory_state=MemoryState.ACTIVE, memory_cache=memory_cache))
+ finally:
+ memory_cache.free()
+ return torch.cat(encoded_slices, dim=2)
+
+ def slicing_decode(self, z: torch.Tensor, output_buffer=None) -> torch.Tensor:
+ """Decode slice by slice; with ``output_buffer`` each slice is written into it (no cat, no accumulation)."""
+ if z.size(0) > 1: # a video at a time: the buffered path and the memory estimate are per video
+ outs = [self.slicing_decode(z[i:i + 1], None if output_buffer is None else output_buffer[i:i + 1])
+ for i in range(z.size(0))]
+ return output_buffer if output_buffer is not None else torch.cat(outs)
+ if (z.shape[2] - 1) <= self.slicing_latent_min_size:
+ decoded = self._decode(z)
+ return decoded if output_buffer is None else output_buffer.copy_(decoded)
+ memory_cache = CausalMemoryCache(offload=_offload_caches_for(z.shape[-2] * z.shape[-1] * self.spatial_downsample_factor ** 2))
+ z_slices = z[:, :, 1:].split(split_size=self.slicing_latent_min_size, dim=2)
+ decoded_slices = []
+ write_pos = 0
+
+ def emit(decoded):
+ nonlocal write_pos
+ if output_buffer is None:
+ decoded_slices.append(decoded)
+ return
+ output_buffer[:, :, write_pos:write_pos + decoded.size(2)].copy_(decoded)
+ write_pos += decoded.size(2)
- def forward(self, x: torch.FloatTensor):
- z, p = self._encode_with_raw_latent(x)
- x = self.decode(z)
- return x, z, p
+ try:
+ emit(self._decode(torch.cat((z[:, :, :1], z_slices[0]), dim=2),
+ memory_state=MemoryState.INITIALIZING, memory_cache=memory_cache))
+ for z_slice in z_slices[1:]:
+ emit(self._decode(z_slice, memory_state=MemoryState.ACTIVE, memory_cache=memory_cache))
+ finally:
+ memory_cache.free()
+ return output_buffer if output_buffer is not None else torch.cat(decoded_slices, dim=2)
- def _encode_with_raw_latent(self, x):
+ def encode(self, x, device=None):
+ """The latent mean of (b, c, t, h, w) pixels, or of an image batch; a single latent frame loses
+ its time axis. sd.py leaves the pixels wherever they were when it uses the chunked-io protocol:
+ the slicing moves each slice to ``device`` as it encodes, so the whole clip never sits on the GPU."""
if x.ndim == 4:
x = x.unsqueeze(2)
- self.device = x.device
- p = super().encode(x)
- z = p.squeeze(2)
- return z, p
-
- def encode(self, x):
- z, _ = self._encode_with_raw_latent(x)
- return z
-
- def decode(self, z, seedvr2_tiling=None):
- seedvr2_tiling = {} if seedvr2_tiling is None else seedvr2_tiling
- if not isinstance(seedvr2_tiling, dict):
- raise RuntimeError(
- "SeedVR2 VideoAutoencoderKLWrapper.decode: `seedvr2_tiling` must be a dict; "
- f"got {type(seedvr2_tiling).__name__} with value {seedvr2_tiling!r}."
- )
+ self.device = x.device if device is None else device
+ return self.slicing_encode(x).chunk(2, dim=1)[0].squeeze(2)
- if z.ndim == 5:
- _, c, _, _, _ = z.shape
- if c != SEEDVR2_LATENT_CHANNELS:
- raise RuntimeError(
- "SeedVR2 VideoAutoencoderKLWrapper.decode: 5-D latent input must "
- f"have {SEEDVR2_LATENT_CHANNELS} channels; got shape {tuple(z.shape)}."
- )
- latent = z
- elif z.ndim == 4:
- b, tc, h, w = z.shape
- if tc % SEEDVR2_LATENT_CHANNELS != 0:
- raise RuntimeError(
- "SeedVR2 VideoAutoencoderKLWrapper.decode: 4-D latent input must "
- f"use collapsed channel layout (B, {SEEDVR2_LATENT_CHANNELS}*T, H, W); "
- f"got shape {tuple(z.shape)}."
- )
- latent = z.reshape(b, SEEDVR2_LATENT_CHANNELS, -1, h, w)
- else:
- raise RuntimeError(
- "SeedVR2 VideoAutoencoderKLWrapper.decode: latent input must be "
- f"4-D collapsed (B, {SEEDVR2_LATENT_CHANNELS}*T, H, W) or "
- f"5-D (B, {SEEDVR2_LATENT_CHANNELS}, T, H, W); "
- f"got shape {tuple(z.shape)}."
- )
- scale = BYTEDANCE_VAE_SCALING_FACTOR
- shift = BYTEDANCE_VAE_SHIFTING_FACTOR
- latent = latent / scale + shift
+ # sd.py preallocates the output on its device/dtype and hands slices of it to decode(); the
+ # decoded frames then never accumulate on the GPU (see slicing_decode).
+ comfy_has_chunked_io = True
- self.device = latent.device
- enable_tiling = seedvr2_tiling.get("enable_tiling", False)
-
- if enable_tiling:
- decode_seedvr2_args = dict(seedvr2_tiling)
- decode_seedvr2_args.pop("enable_tiling", None)
- tile_h, tile_w = decode_seedvr2_args.get("tile_size", (512, 512))
- ov_h, ov_w = decode_seedvr2_args.get("tile_overlap", (64, 64))
- decode_seedvr2_args["tile_overlap"] = (
- min(ov_h, max(0, tile_h - 8)),
- min(ov_w, max(0, tile_w - 8)),
- )
- x = tiled_vae(latent, self, **decode_seedvr2_args, encode=False)
- if x.ndim == 4:
- # tiled_vae squeezes the temporal axis when
- # temporal_downsample_factor == 1 AND latent T == 1
- # (see tiled_vae line 179-180); re-add it so the post-decode
- # pipeline can keep batch and time distinct on the tiled path.
- x = x.unsqueeze(2)
- else:
- x = super().decode_(latent)
+ def decode_output_shape(self, input_shape):
+ b, c, t, h, w = self._latent_dims(input_shape)
+ frames = max(1, (t - 1) * self.temporal_downsample_factor + 1)
+ return (b, 3, frames, h * self.spatial_downsample_factor, w * self.spatial_downsample_factor)
- h, w = x.shape[-2:]
- w2 = w - (w % 2)
- h2 = h - (h % 2)
- x = x[..., :h2, :w2]
+ @staticmethod
+ def _latent_dims(shape):
+ """(b, c, t, h, w) of a 5-D latent or the collapsed 4-D (B, 16*T, H, W) form."""
+ if len(shape) == 5:
+ return tuple(shape)
+ b, tc, h, w = shape
+ return b, SEEDVR2_LATENT_CHANNELS, tc // SEEDVR2_LATENT_CHANNELS, h, w
- return x
+ def _unscaled(self, z):
+ """A scaled latent, in either form, as the decoder's (b, c, t, h, w) input."""
+ return z.reshape(self._latent_dims(z.shape)) / BYTEDANCE_VAE_SCALING_FACTOR + BYTEDANCE_VAE_SHIFTING_FACTOR
+
+ def decode(self, z, output_buffer=None):
+ latent = self._unscaled(z)
+ self.device = latent.device
+ return self.slicing_decode(latent, output_buffer=output_buffer)
+
+ # comfy_memory_used_decode is fitted against measured peaks, so a decode it says will not fit
+ # can be tiled without trying first. See preferred_decode_tile for the sizing that follows.
+ comfy_decode_estimate_is_reliable = True
+
+ def preferred_decode_tile(self, free_memory, dtype):
+ """Largest square tile, in latent units, whose decode fits ``free_memory``, from the same per-pixel
+ figures as the estimate: the temporal caches count too when there is no room to pin them."""
+ factor = _eager_factor(dtype, SEEDVR2_EAGER_DECODE_FACTOR)
+ budget = free_memory * SEEDVR2_TILE_MEM_HEADROOM - SEEDVR2_DECODE_FIXED_BYTES * factor
+ if budget <= 0:
+ return SEEDVR2_MIN_TILE_LATENT
+ area = budget / (SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL * factor)
+ if not _offload_caches_for(area):
+ area = budget / ((SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL + SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL) * factor)
+ side = int(math.sqrt(max(area, 1.0))) // self.spatial_downsample_factor
+ side = (side // 8) * 8 # keep tiles a whole number of latent blocks
+ return max(SEEDVR2_MIN_TILE_LATENT, min(SEEDVR2_MAX_TILE_LATENT, side))
def decode_tiled(self, z, tile_x=32, tile_y=32, overlap=8, tile_t=None, overlap_t=None):
# SeedVR2's causal VAE owns temporal via the MemoryState cache; external
# temporal tiling breaks that continuity, so only spatial tiling is applied.
+ latent = self._unscaled(z)
+ self.device = latent.device
sf = self.spatial_downsample_factor
- seedvr2_tiling = {
- "enable_tiling": True,
- "tile_size": (tile_y * sf, tile_x * sf),
- "tile_overlap": (overlap * sf, overlap * sf),
- "temporal_size": None,
- "temporal_overlap": None,
- }
- return self.decode(z, seedvr2_tiling=seedvr2_tiling)
+ tile_h, tile_w, ov = tile_y * sf, tile_x * sf, overlap * sf
+ return tiled_vae(latent, self, tile_size=(tile_h, tile_w),
+ tile_overlap=(min(ov, max(0, tile_h - 8)), min(ov, max(0, tile_w - 8))), encode=False)
def encode_tiled(self, x, tile_x=None, tile_y=None, overlap=None, tile_t=None, overlap_t=None):
# External temporal tiling knobs are discarded; the causal VAE keeps its
# own internal MemoryState slicing.
- if tile_y is None:
- tile_y = 512
- if tile_x is None:
- tile_x = 512
- if overlap is None:
- overlap_y = 64
- overlap_x = 64
- else:
- overlap_y = overlap
- overlap_x = overlap
- overlap_y = min(overlap_y, max(0, tile_y - 8))
- overlap_x = min(overlap_x, max(0, tile_x - 8))
+ tile_y = 512 if tile_y is None else tile_y
+ tile_x = 512 if tile_x is None else tile_x
+ overlap = 64 if overlap is None else overlap
self.device = x.device
- return tiled_vae(
- x,
- self,
- tile_size=(tile_y, tile_x),
- tile_overlap=(overlap_y, overlap_x),
- temporal_size=None,
- temporal_overlap=None,
- encode=True,
- )
+ return tiled_vae(x, self, tile_size=(tile_y, tile_x),
+ tile_overlap=(min(overlap, max(0, tile_y - 8)), min(overlap, max(0, tile_x - 8))), encode=True)
def comfy_format_encoded(self, samples):
if samples.ndim == 4:
@@ -1578,33 +1412,22 @@ class VideoAutoencoderKLWrapper(VideoAutoencoderKL):
samples = samples * BYTEDANCE_VAE_SCALING_FACTOR
return samples
- def comfy_memory_used_decode(self, shape):
- bytes_per_output_pixel = 160
-
- def output_pixels(latent_t, latent_h, latent_w):
- output_t = max(1, (latent_t - 1) * 4 + 1)
- return output_t * latent_h * 8 * latent_w * 8
-
- # SeedVR2 decode performs full-frame LAB histogram matching: fp32 channels
- # plus int64 sort indices dominate peak memory, not the VAE weight dtype.
- if len(shape) == 5:
- candidates = []
- if shape[1] == SEEDVR2_LATENT_CHANNELS:
- candidates.append((shape[2], shape[3], shape[4]))
- if shape[-1] == SEEDVR2_LATENT_CHANNELS:
- candidates.append((shape[1], shape[2], shape[3]))
- if len(candidates) == 0:
- candidates.append((shape[2], shape[3], shape[4]))
- pixels = max(output_pixels(*candidate) for candidate in candidates)
- elif len(shape) == 4:
- latent_t = max(1, (shape[1] + SEEDVR2_LATENT_CHANNELS - 1) // SEEDVR2_LATENT_CHANNELS)
- pixels = output_pixels(latent_t, shape[2], shape[3])
- else:
- pixels = output_pixels(1, shape[-2], shape[-1])
- return pixels * bytes_per_output_pixel
-
- def set_memory_limit(self, conv_max_mem: Optional[float], norm_max_mem: Optional[float]):
- set_norm_limit(norm_max_mem)
- for m in self.modules():
- if isinstance(m, InflatedCausalConv3d):
- m.set_memory_limit(conv_max_mem if conv_max_mem is not None else float("inf"))
+ def comfy_memory_used_encode(self, shape, dtype):
+ """Peak device memory of an encode of (b, c, t, h, w) pixels, in bytes: one frame's area sets it
+ (the videos of a batch are encoded one at a time)."""
+ peak = shape[-2] * shape[-1] * SEEDVR2_ENCODE_BYTES_PER_PIXEL + SEEDVR2_ENCODE_FIXED_BYTES
+ return int(peak * _eager_factor(dtype, SEEDVR2_EAGER_ENCODE_FACTOR))
+
+ def comfy_memory_used_decode(self, shape, dtype):
+ """Peak device memory of a decode, in bytes: one frame's area sets it, not the clip length
+ (the videos of a batch are decoded one at a time)."""
+ _, _, _, latent_h, latent_w = self._latent_dims(shape)
+ area = latent_h * self.spatial_downsample_factor * latent_w * self.spatial_downsample_factor
+ decode_peak = area * SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL + SEEDVR2_DECODE_FIXED_BYTES
+ if not _offload_caches_for(area):
+ decode_peak += area * SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL # the temporal caches stay resident
+ # The node's colour correction runs after the decode, one frame (LAB) or a free-memory
+ # sized chunk (wavelet, adain) at a time with its own OOM back-off, so it never coincides
+ # with the decode's working set and only a frame of it is worth reserving.
+ colour_transfer = area * SEEDVR2_DECODE_LAB_BYTES_PER_OUTPUT_PIXEL
+ return int(max(decode_peak * _eager_factor(dtype, SEEDVR2_EAGER_DECODE_FACTOR), colour_transfer))
diff --git a/comfy/sd.py b/comfy/sd.py
index 1023ad9ea..73a35aaa4 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -604,8 +604,8 @@ class VAE:
self.latent_channels = comfy.ldm.seedvr.vae.SEEDVR2_LATENT_CHANNELS
self.latent_dim = 3
self.disable_offload = True
- self.memory_used_decode = lambda shape, dtype: self.first_stage_model.comfy_memory_used_decode(shape)
- self.memory_used_encode = lambda shape, dtype: (max(shape[2], 5) * shape[3] * shape[4] * 64) * model_management.dtype_size(dtype)
+ self.memory_used_decode = self.first_stage_model.comfy_memory_used_decode
+ self.memory_used_encode = self.first_stage_model.comfy_memory_used_encode
self.working_dtypes = [torch.float16, torch.bfloat16, torch.float32]
self.handles_tiling = True
self.format_encoded = self.first_stage_model.comfy_format_encoded
@@ -1270,6 +1270,14 @@ class VAE:
batch_number = int(free_memory / memory_used)
batch_number = max(1, batch_number)
+ # A VAE with a reliable estimate skips the doomed untiled attempt.
+ if getattr(self.first_stage_model, "comfy_decode_estimate_is_reliable", False):
+ if memory_used > free_memory:
+ logging.info("VAE decode needs more than the free VRAM; tiling directly.")
+ raise model_management.OOM_EXCEPTION(
+ "decode estimate exceeds free memory; going straight to tiled"
+ )
+
# Pre-allocate output for VAEs that support direct buffer writes
preallocated = False
if getattr(self.first_stage_model, 'comfy_has_chunked_io', False):
@@ -1313,6 +1321,10 @@ class VAE:
pixel_samples = self.decode_tiled_(samples_in)
elif dims == 3:
tile = 256 // self.spacial_compression_decode()
+ # A VAE that can size its own tiles raises the 256-pixel floor.
+ prefers_tile = getattr(self.first_stage_model, "preferred_decode_tile", None)
+ if prefers_tile is not None:
+ tile = max(tile, prefers_tile(self.patcher.get_free_memory(self.device), self.vae_dtype))
overlap = tile // 4
if self.handles_tiling:
memory_used = self.memory_used_decode(self._tile_bounded_shape(samples_in.shape, tile, tile, None), self.vae_dtype)
diff --git a/tests-unit/comfy_test/seedvr_vae_forward_test.py b/tests-unit/comfy_test/seedvr_vae_forward_test.py
deleted file mode 100644
index 7ea7a143e..000000000
--- a/tests-unit/comfy_test/seedvr_vae_forward_test.py
+++ /dev/null
@@ -1,74 +0,0 @@
-"""Regression tests for the SeedVR2 VAE forward return contract."""
-
-import pytest
-import torch
-import torch.nn as nn
-
-from comfy.cli_args import args as cli_args
-
-if not torch.cuda.is_available():
- cli_args.cpu = True
-
-from comfy.ldm.seedvr.vae import SEEDVR2_LATENT_CHANNELS, VideoAutoencoderKL # noqa: E402
-
-
-_LATENT_SHAPE = (1, SEEDVR2_LATENT_CHANNELS, 2, 2, 2)
-_DECODED_SHAPE = (1, 3, 5, 16, 16)
-_INPUT_ENCODE_SHAPE = (1, 3, 5, 16, 16)
-_INPUT_DECODE_SHAPE = _LATENT_SHAPE
-
-
-class _StubVAE(VideoAutoencoderKL):
- def __init__(self):
- nn.Module.__init__(self)
- self._encode_out = torch.zeros(*_LATENT_SHAPE)
- self._decode_out = torch.zeros(*_DECODED_SHAPE)
-
- def encode(self, x, return_dict=True):
- return self._encode_out
-
- def decode_(self, z, return_dict=True):
- return self._decode_out
-
-
-def test_forward_encode_returns_tensor():
- vae = _StubVAE()
- x = torch.zeros(*_INPUT_ENCODE_SHAPE)
- result = vae.forward(x, mode="encode")
- assert type(result) is torch.Tensor
- assert result.shape == torch.Size(_LATENT_SHAPE)
-
-
-def test_forward_decode_returns_tensor():
- vae = _StubVAE()
- z = torch.zeros(*_INPUT_DECODE_SHAPE)
- result = vae.forward(z, mode="decode")
- assert type(result) is torch.Tensor
- assert result.shape == torch.Size(_DECODED_SHAPE)
-
-
-class _TupleReturningStubVAE(VideoAutoencoderKL):
- def __init__(self):
- nn.Module.__init__(self)
- self._encode_tensor = torch.zeros(*_LATENT_SHAPE)
- self._decode_tensor = torch.zeros(*_DECODED_SHAPE)
-
- def encode(self, x, return_dict=True):
- return (self._encode_tensor,)
-
- def decode_(self, z, return_dict=True):
- return (self._decode_tensor,)
-
-
-def test_forward_all_unwraps_one_tuple_at_each_step():
- vae = _TupleReturningStubVAE()
- x = torch.zeros(*_INPUT_ENCODE_SHAPE)
- result = vae.forward(x, mode="all")
- assert type(result) is torch.Tensor
- assert result.shape == torch.Size(_DECODED_SHAPE)
-
-
-def test_forward_rejects_unknown_mode():
- vae = _StubVAE()
- with pytest.raises(ValueError, match="Unknown SeedVR2 VAE forward mode"):
- vae.forward(torch.zeros(*_INPUT_ENCODE_SHAPE), mode="bogus")
diff --git a/tests-unit/comfy_test/test_seedvr2_dtype.py b/tests-unit/comfy_test/test_seedvr2_dtype.py
index d743cc848..4dd6939f9 100644
--- a/tests-unit/comfy_test/test_seedvr2_dtype.py
+++ b/tests-unit/comfy_test/test_seedvr2_dtype.py
@@ -40,39 +40,28 @@ def test_seedvr2_text_conditioning_accepts_cfg1_single_branch():
torch.testing.assert_close(txt_shape, torch.tensor([[3]], device=context.device))
-def test_seedvr2_vae_decode_memory_covers_full_frame_lab_transfer():
- wrapper = seedvr_vae.VideoAutoencoderKLWrapper.__new__(seedvr_vae.VideoAutoencoderKLWrapper)
- latent_channels = seedvr_vae.SEEDVR2_LATENT_CHANNELS
- estimate = wrapper.comfy_memory_used_decode((1, latent_channels, 26, 120, 160))
- old_estimate = latent_channels * 120 * 160 * (4 * 8 * 8) * 2
-
- assert estimate == 101 * 960 * 1280 * 160
- assert estimate > 15 * 1024 ** 3
- assert estimate > old_estimate * 100
-
-
def test_seedvr2_vae_encode_preserves_compute_dtype(monkeypatch):
wrapper = seedvr_vae.VideoAutoencoderKLWrapper.__new__(seedvr_vae.VideoAutoencoderKLWrapper)
nn.Module.__init__(wrapper)
wrapper._dummy = nn.Parameter(torch.empty(1, dtype=torch.float16))
input_dtype = None
- def encode(self, x):
+ def slicing_encode(self, x):
nonlocal input_dtype
input_dtype = x.dtype
return x
- monkeypatch.setattr(seedvr_vae.VideoAutoencoderKL, "encode", encode)
+ monkeypatch.setattr(seedvr_vae.VideoAutoencoderKLWrapper, "slicing_encode", slicing_encode)
x = torch.zeros((1, 3, 1, 8, 8), dtype=torch.float32)
- wrapper._encode_with_raw_latent(x)
+ wrapper.encode(x)
assert input_dtype == torch.float32
def test_seedvr2_vae_ops_cast_weights_to_compute_dtype():
- attention = seedvr_vae.Attention(query_dim=4, heads=1, dim_head=4).to(torch.float16)
- hidden_states = torch.zeros((1, 2, 4), dtype=torch.float32)
+ attention = seedvr_vae.Attention(query_dim=4, norm_num_groups=2, eps=1e-6).to(torch.float16)
+ hidden_states = torch.zeros((1, 4, 2, 2), dtype=torch.float32)
output = attention(hidden_states)
diff --git a/tests-unit/comfy_test/test_seedvr2_internals.py b/tests-unit/comfy_test/test_seedvr2_internals.py
index fe4bde1c4..9e620a215 100644
--- a/tests-unit/comfy_test/test_seedvr2_internals.py
+++ b/tests-unit/comfy_test/test_seedvr2_internals.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import inspect
from unittest.mock import patch
import pytest
@@ -27,6 +28,27 @@ _NUM_CHANNELS = 8
_NUM_GROUPS = 4
_TENSOR_SHAPE = (1, 8, 2, 4, 4)
+
[email protected](autouse=True)
+def _isolate_weight_patches():
+ """Keep these tests independent of leaked weight patches.
+
+ ``comfy.ops.CastWeightBiasOp`` holds ``weight_function``/``bias_function`` as *class* level
+ lists, so any code that appends to one without first giving the module its own list mutates the
+ shared default and every comfy layer created afterwards silently inherits the patch. That is
+ what happens when these tests run after tests-unit/comfy_quant in the same process.
+ """
+ cls = comfy_ops.CastWeightBiasOp
+ saved = (list(cls.weight_function), list(cls.bias_function))
+ cls.weight_function.clear()
+ cls.bias_function.clear()
+ try:
+ yield
+ finally:
+ cls.weight_function[:] = saved[0]
+ cls.bias_function[:] = saved[1]
+
+
_GROUPNORM_SUBCLASSES = [
pytest.param(comfy_ops.disable_weight_init.GroupNorm, id="disable_weight_init"),
pytest.param(comfy_ops.manual_cast.GroupNorm, id="manual_cast"),
@@ -124,6 +146,78 @@ def test_seedvr2_7b_swin_attention_forward_uses_optimized_var_attention(monkeypa
assert call["cu_seqlens_k"] == [0, 7, 14]
+def _make_swin_attention(rope_type, dim=16, heads=2, head_dim=8):
+ torch.manual_seed(0)
+ attn = seedvr_model.NaSwinAttention(
+ vid_dim=dim,
+ txt_dim=dim,
+ heads=heads,
+ head_dim=head_dim,
+ qk_bias=False,
+ qk_norm=comfy_ops.disable_weight_init.RMSNorm,
+ qk_norm_eps=1e-6,
+ rope_type=rope_type,
+ rope_dim=head_dim,
+ shared_weights=False,
+ window=(2, 2, 2),
+ window_method="720pwin_by_size_bysize",
+ version=(rope_type == "rope3d"),
+ device="cpu",
+ dtype=torch.float32,
+ operations=comfy_ops.disable_weight_init,
+ )
+ for param in attn.parameters():
+ torch.nn.init.normal_(param, std=0.5)
+ return attn
+
+
[email protected]("rope_type", [None, "rope3d", "mmrope3d"])
+def test_seedvr2_swin_attention_batched_samples_match_one_at_a_time(rope_type):
+ """A batched cond+uncond forward must give each sample its own text tokens."""
+ attn = _make_swin_attention(rope_type)
+ generator = torch.Generator(device="cpu").manual_seed(3)
+ vids = [torch.randn(2, 6, 6, 16, generator=generator), torch.randn(3, 6, 8, 16, generator=generator)]
+ txts = [torch.randn(5, 16, generator=generator), torch.randn(7, 16, generator=generator)]
+
+ vid, vid_shape = seedvr_model.flatten(vids)
+ txt, txt_shape = seedvr_model.flatten(txts)
+ batched_vid, batched_txt = attn(vid, txt, vid_shape, txt_shape, seedvr_model.Cache())
+
+ single_vid, single_txt = [], []
+ for one_vid, one_txt in zip(vids, txts):
+ vid_i, vid_shape_i = seedvr_model.flatten([one_vid])
+ txt_i, txt_shape_i = seedvr_model.flatten([one_txt])
+ out_vid, out_txt = attn(vid_i, txt_i, vid_shape_i, txt_shape_i, seedvr_model.Cache())
+ single_vid.append(out_vid)
+ single_txt.append(out_txt)
+
+ torch.testing.assert_close(batched_vid, torch.cat(single_vid), rtol=1e-4, atol=1e-5)
+ torch.testing.assert_close(batched_txt, torch.cat(single_txt), rtol=1e-4, atol=1e-5)
+
+
+def test_var_attention_optimized_split_batches_equal_length_windows(monkeypatch):
+ heads = 2
+ head_dim = 3
+ q = torch.arange(36, dtype=torch.float32).reshape(6, heads, head_dim)
+ k = q + 100
+ v = q + 200
+ cu = [0, 2, 4, 6]
+ calls = []
+
+ def fake_optimized_attention(q_arg, k_arg, v_arg, heads_arg, **kwargs):
+ calls.append(tuple(q_arg.shape))
+ return q_arg + v_arg
+
+ monkeypatch.setattr(attention, "optimized_attention", fake_optimized_attention)
+
+ out = var_attention_optimized_split(q, k, v, heads, cu, cu, skip_reshape=True, skip_output_reshape=True)
+
+ assert calls == [(3, heads, 2, head_dim)], (
+ f"equal-length windows must share one batched attention call; got {calls}"
+ )
+ torch.testing.assert_close(out, q + v, rtol=0, atol=0)
+
+
def test_var_attention_optimized_split_calls_dense_backend_per_window(monkeypatch):
heads = 2
head_dim = 3
@@ -167,3 +261,462 @@ def test_var_attention_optimized_split_calls_dense_backend_per_window(monkeypatc
assert all(call["kwargs"]["skip_output_reshape"] is True for call in calls)
torch.testing.assert_close(out, q + v, rtol=0, atol=0)
+
+class _AlwaysPackedCache(vae_mod.CausalMemoryCache):
+ """Drops the size gate so small tails exercise the packing."""
+
+ def _packable(self, value):
+ return torch.is_tensor(value) and value.is_cuda and value.is_contiguous(memory_format=torch.channels_last_3d)
+
+
+def _cache_tail(channels, frames=2):
+ torch.manual_seed(0)
+ tail = torch.randn(1, channels, frames, 48, 64, device="cuda") * 2
+ tail[:, ::5] *= 20.0 # the per-channel outliers the rotation exists to spread
+ return tail.half().contiguous(memory_format=torch.channels_last_3d)
+
+
+def _rel_err(a, b):
+ return ((a.float() - b.float()).norm() / b.float().norm()).item()
+
+
+def test_causal_memory_cache_holds_unpackable_tails_as_they_are():
+ cache = vae_mod.CausalMemoryCache()
+ assert not cache._packable(torch.zeros(1, 128, 2, 4, 4)), "cpu tails stay unpacked"
+ assert not cache._packable(torch.zeros(4, 8)) and not cache._packable("not a tensor")
+ tail = torch.randn(1, 96, 2, 4, 4)
+ cache["conv"] = tail
+ assert cache.get("conv") is tail and "conv" in cache
+ assert cache.pop("conv") is tail and "conv" not in cache
+ assert cache.pop("conv", "fallback") == "fallback"
+ assert cache.get("missing") is None
+
+
[email protected](not torch.cuda.is_available(), reason="tails pack on CUDA")
[email protected]("channels", [128, 256, 512])
+def test_causal_memory_cache_packing_round_trips(channels):
+ tail = _cache_tail(channels)
+ cache = _AlwaysPackedCache()
+ cache["conv"] = tail
+ assert "conv" in cache.packed
+ restored = cache.get("conv")
+ assert restored.shape == tail.shape and restored.dtype == tail.dtype
+ assert restored.is_contiguous(memory_format=torch.channels_last_3d)
+ assert _rel_err(restored, tail) < 1e-2
+ plain = torch.randn(1, 96, 2, 4, 4)
+ cache["conv"] = plain # replaces the packed entry
+ assert cache.get("conv") is plain and "conv" not in cache.packed
+
+
[email protected](not torch.cuda.is_available(), reason="offload moves tails between CUDA and host memory")
+def test_causal_memory_cache_offload_round_trips_across_slices():
+ """Offloaded tails come back exactly as the resident packing returns them, in the order the
+ next slice reads them (which the one-ahead prefetch assumes), and free() unpins the host buffers."""
+ keys = [f"conv{i}" for i in range(6)]
+ tails = {k: _cache_tail(128) for k in keys}
+ pinned_before = vae_mod.comfy.model_management.TOTAL_PINNED_MEMORY
+ resident = _AlwaysPackedCache()
+ offloaded = _AlwaysPackedCache(offload=True)
+ for slice_idx in range(3):
+ for k in keys:
+ resident[k] = tails[k] * (slice_idx + 1)
+ offloaded[k] = tails[k] * (slice_idx + 1)
+ assert not any(q.is_cuda for q, *_ in offloaded.packed.values()), "offloaded tails stay off the GPU between slices"
+ for k in keys:
+ a, b = resident.get(k), offloaded.get(k)
+ assert b.is_contiguous(memory_format=torch.channels_last_3d)
+ assert torch.equal(a, b), f"{k} slice {slice_idx}: offloaded tail differs"
+ assert offloaded.pop("conv0") is not None and "conv0" not in offloaded
+ offloaded.free()
+ assert vae_mod.comfy.model_management.TOTAL_PINNED_MEMORY == pinned_before
+
+
[email protected](not torch.cuda.is_available(), reason="offload moves tails between CUDA and host memory")
+def test_causal_memory_cache_offloads_small_tails_and_parked_frames_exactly():
+ """Below the packing threshold an offloaded tail goes to the host as it is, like parked frames:
+ both come back bit-exact."""
+ cache = vae_mod.CausalMemoryCache(offload=True)
+ tail = _cache_tail(64)
+ cache["conv"] = tail
+ assert not cache.packed["conv"][0].is_cuda
+ assert torch.equal(cache.pop("conv"), tail)
+ frames = list(_cache_tail(128, frames=3).split(1, dim=2))
+ keys = cache.park("tail", frames)
+ assert all(torch.equal(cache.pop(k), f) for k, f in zip(keys, frames))
+ cache.free()
+
+
+def _ring(cache, key):
+ frames = []
+ cache.for_each_frame(key, lambda i, frame: frames.append(frame.clone()))
+ return torch.cat(frames, dim=2)
+
+
[email protected](not torch.cuda.is_available(), reason="ring entries pack and offload on CUDA")
+def test_causal_memory_cache_frame_ring_keeps_the_last_frames():
+ """push_frames extends a key's tail one frame at a time; for_each_frame reads the last `keep`
+ frames in order, retired frames are dropped, and a plain assignment replaces the ring."""
+ cache = _AlwaysPackedCache(offload=True)
+ frames = [f.contiguous(memory_format=torch.channels_last_3d) for f in _cache_tail(128, frames=4).split(1, dim=2)]
+ cache.push_frames("conv", torch.cat(frames[:2], dim=2), keep=2)
+ assert "conv" in cache
+ assert _rel_err(_ring(cache, "conv"), torch.cat(frames[:2], dim=2)) < 1e-2
+ cache.push_frames("conv", frames[2], keep=2) # one new frame: one packed entry
+ assert len(cache._rings["conv"]) == 2
+ assert _rel_err(_ring(cache, "conv"), torch.cat(frames[1:3], dim=2)) < 1e-2
+ cache.push_frames("conv", frames[3], keep=2)
+ assert len(cache.plain) + len(cache.packed) == 2, "retired frames must be dropped, not accumulated"
+ cache["conv"] = frames[0] # plain assignment replaces the ring
+ assert "conv" not in cache._rings and cache.get("conv").shape == frames[0].shape
+ assert len(cache.plain) + len(cache.packed) == 1
+ cache.free()
+
+
[email protected](not torch.cuda.is_available(), reason="ring entries are CUDA frames")
[email protected]("offload", [False, True])
+def test_causal_memory_cache_push_frames_copies_out_of_the_callers_buffer(offload):
+ """Frames too small to pack are pushed as views of the conv's padded buffer; held as views they
+ would keep the whole buffer alive, and the caller reuses it once the push returns."""
+ cache = vae_mod.CausalMemoryCache(offload=offload)
+ buf = _cache_tail(64, frames=4)
+ expected = buf[:, :, -2:].clone()
+ cache.push_frames("conv", buf[:, :, -2:], keep=2)
+ buf.zero_()
+ if not offload:
+ assert all(t.untyped_storage().data_ptr() != buf.untyped_storage().data_ptr() for t in cache.plain.values())
+ assert torch.equal(_ring(cache, "conv"), expected)
+ cache.free()
+
+
+def _half_cuda(module):
+ torch.manual_seed(0)
+ for p in module.parameters():
+ torch.nn.init.normal_(p, std=0.1)
+ return module.to("cuda", torch.float16)
+
+
+def _slices(channels, frames, h, w):
+ """fp16 activations for consecutive slices; the reference runs them NCDHW, which takes the eager
+ path. Sized so the fp16-accumulate runs reach the kitchen conv, which leaves small launches to cuDNN."""
+ torch.manual_seed(1)
+ return [torch.randn(1, channels, t, h, w, device="cuda", dtype=torch.float16) for t in frames]
+
+
+def _states(n):
+ return [vae_mod.MemoryState.INITIALIZING] + [vae_mod.MemoryState.ACTIVE] * (n - 1)
+
+
[email protected](not torch.cuda.is_available(), reason="the buffered convs run fp16 on CUDA")
[email protected]("fp16acc", [False, True])
[email protected]("temporal_down", [False, True])
+def test_seedvr2_downsample_buffered_matches_padded_conv(temporal_down, fp16acc, monkeypatch):
+ """The buffered downsampler writes into the conv's buffer with its own right/bottom border in
+ place of F.pad; across slices it must match the pad-then-conv path and its cache."""
+ down = _half_cuda(vae_mod.Downsample3D(16, temporal_down=temporal_down))
+ slices = _slices(16, (9, 8) if temporal_down else (5, 4), h=160, w=160)
+ ref_cache, cache = vae_mod.CausalMemoryCache(), vae_mod.CausalMemoryCache()
+ for x, state in zip(slices, _states(len(slices))):
+ ref = down.run([x], state, ref_cache)
+ with monkeypatch.context() as m:
+ m.setattr(torch.backends.cuda.matmul, "allow_fp16_accumulation", fp16acc, raising=False)
+ got = down.run([x.contiguous(memory_format=torch.channels_last_3d)], state, cache)
+ assert got.shape == ref.shape
+ assert _rel_err(got, ref) < (2e-3 if fp16acc else 1e-3)
+
+
[email protected](not torch.cuda.is_available(), reason="the buffered convs run fp16 on CUDA")
[email protected]("fp16acc", [False, True])
+def test_seedvr2_upsample_per_frame_matches_whole_slice(fp16acc, monkeypatch):
+ """The decoder's last temporal upsampler runs a latent frame at a time through frames(); its
+ output frames, and the whole-slice buffered run, must match the eager shuffle-then-conv path."""
+ up = _half_cuda(vae_mod.Upsample3D(16, temporal_up=True))
+ slices = _slices(16, (1, 1, 1), h=64, w=64)
+ caches = [vae_mod.CausalMemoryCache() for _ in range(3)]
+ for x, state in zip(slices, _states(len(slices))):
+ ref = up.run([x], state, caches[0])
+ x = x.contiguous(memory_format=torch.channels_last_3d)
+ with monkeypatch.context() as m:
+ m.setattr(torch.backends.cuda.matmul, "allow_fp16_accumulation", fp16acc, raising=False)
+ whole = up.run([x], state, caches[1])
+ per_frame = torch.cat([box.pop() for box in up.frames([x], state, caches[2])], dim=2)
+ for got in (whole, per_frame):
+ assert got.shape == ref.shape
+ assert _rel_err(got, ref) < (2e-3 if fp16acc else 1e-3)
+
+
+def _make_block(is_last_layer, dim=16, heads=2, head_dim=8):
+ torch.manual_seed(0)
+ block = seedvr_model.NaMMSRTransformerBlock(
+ vid_dim=dim, txt_dim=dim, emb_dim=dim * 6, heads=heads, head_dim=head_dim,
+ expand_ratio=2, norm=comfy_ops.disable_weight_init.RMSNorm, norm_eps=1e-6,
+ ada=seedvr_model.AdaSingle, qk_bias=False, qk_norm=comfy_ops.disable_weight_init.RMSNorm,
+ mlp_type="normal", shared_weights=False, rope_type="mmrope3d", rope_dim=head_dim,
+ is_last_layer=is_last_layer, window=(2, 2, 2), window_method="720pwin_by_size_bysize",
+ version=False, device="cpu", dtype=torch.float32,
+ operations=comfy_ops.disable_weight_init,
+ )
+ for param in block.parameters():
+ torch.nn.init.normal_(param, std=0.5)
+ return block
+
+
[email protected]("is_last_layer", [False, True])
+def test_seedvr2_norm_ada_in_matches_unfused_norm_then_modulate(is_last_layer):
+ """The fused norm+modulate must agree with norm() then ada() on BOTH branches.
+
+ The last block's attn_norm normalizes txt while its ada modulates vid alone, so a fused path
+ that lets ada carry the norm hands attention an unnormalized txt to read as K/V.
+ """
+ block = _make_block(is_last_layer)
+ generator = torch.Generator(device="cpu").manual_seed(5)
+ vid = torch.randn(12, 16, generator=generator) * 30.0
+ txt = torch.randn(7, 16, generator=generator) * 30.0
+ emb = torch.randn(1, 16 * 6, generator=generator)
+
+ for norm, layer in ((block.attn_norm, "attn"), (block.mlp_norm, "mlp")):
+ ada_kwargs = {
+ "emb": emb,
+ "hid_len": seedvr_model.MMArg(torch.tensor([12]), torch.tensor([7])),
+ "cache": seedvr_model.Cache(),
+ "branch_tag": seedvr_model.MMArg("vid", "txt"),
+ }
+ fused_vid, fused_txt = block._norm_ada_in(norm, vid.clone(), txt.clone(), layer, ada_kwargs)
+
+ ada_kwargs["cache"] = seedvr_model.Cache()
+ ref_vid, ref_txt = norm(vid.clone(), txt.clone())
+ ref_vid, ref_txt = block.ada(ref_vid, ref_txt, layer=layer, mode="in", **ada_kwargs)
+
+ torch.testing.assert_close(fused_vid, ref_vid, rtol=1e-4, atol=1e-4)
+ torch.testing.assert_close(
+ fused_txt, ref_txt, rtol=1e-4, atol=1e-4,
+ msg=lambda m: f"{layer} txt branch diverged (is_last_layer={is_last_layer}):\n{m}",
+ )
+
+
+def test_seedvr2_vid_out_ada_reuses_the_block_attn_modulation():
+ """``vid_out_ada`` has layers=["out"], so its own slice of a 6*dim embedding is twice as wide
+ as hid. Upstream survives that because its cache key collides with the blocks' "attn" entry and
+ it silently takes theirs; the released 3B weights were exported against that aliasing."""
+ dim = 16
+ torch.manual_seed(0)
+ block_ada = seedvr_model.AdaSingle(dim=dim, emb_dim=dim * 6, layers=["attn", "mlp"])
+ out_ada = seedvr_model.AdaSingle(dim=dim, emb_dim=dim * 6, layers=["out"], modes=["in"])
+ for module in (block_ada, out_ada):
+ for param in module.parameters():
+ torch.nn.init.normal_(param, std=0.1)
+
+ generator = torch.Generator(device="cpu").manual_seed(1)
+ emb = torch.randn(1, dim * 6, generator=generator)
+ hid_len = torch.tensor([9])
+ cache = seedvr_model.Cache()
+
+ block_ada(torch.randn(9, dim, generator=generator), emb=emb, layer="attn", mode="in",
+ cache=cache, branch_tag="vid", hid_len=hid_len)
+ assert "emb_repeat_0_vid" in cache.cache
+
+ hid = torch.randn(9, dim, generator=generator)
+ out = out_ada(hid.clone(), emb=emb, layer="out", mode="in",
+ cache=cache, branch_tag="vid", hid_len=hid_len)
+
+ shiftA, scaleA, _ = cache.cache["emb_repeat_0_vid"].unbind(-1)
+ expected = hid * (scaleA + out_ada.out_scale) + (shiftA + out_ada.out_shift)
+ torch.testing.assert_close(out, expected, rtol=1e-5, atol=1e-6)
+
+
[email protected]("padding", [(0, 0, 0, 0, 0, 0), (1, 1, 1, 1, 0, 0), (1, 2, 0, 1, 3, 0)])
[email protected]("concat_dim", [2, 3])
[email protected]("with_cache", [False, True])
+def test_seedvr2_padded_input_matches_cat_then_pad(padding, concat_dim, with_cache):
+ """``_padded_input`` replaces cat-then-F.pad with one buffer; it must be value-identical, a
+ no-op when there is nothing to add, and cast a cache of another dtype where cat would raise."""
+ generator = torch.Generator(device="cpu").manual_seed(7)
+ x = torch.randn(1, 8, 4, 5, 6, generator=generator)
+ cache = None
+ if with_cache:
+ shape = list(x.shape)
+ shape[concat_dim] = 2
+ cache = torch.randn(*shape, generator=generator).to(torch.float16)
+
+ got = vae_mod.InflatedCausalConv3d._padded_input(x, cache, padding, concat_dim)
+
+ reference = x if cache is None else torch.cat([cache.to(x.dtype), x], dim=concat_dim)
+ reference = torch.nn.functional.pad(reference, padding, mode="constant", value=0.0)
+ assert got.dtype == x.dtype and got.shape == reference.shape
+ torch.testing.assert_close(got, reference, rtol=0, atol=0)
+ if cache is None and not any(padding):
+ assert got is x
+
+
+def _reference_group_norm(norm, x, silu):
+ """What causal_norm_wrapper computes: per-frame statistics via the (b*t, c, h, w) view."""
+ b, c, t, h, w = x.shape
+ out = torch.nn.functional.group_norm(
+ x.transpose(1, 2).reshape(b * t, c, h, w), norm.num_groups, norm.weight, norm.bias, norm.eps,
+ )
+ if silu:
+ out = torch.nn.functional.silu(out)
+ return out.reshape(b, t, c, h, w).transpose(1, 2)
+
+
[email protected]("silu", [False, True])
+def test_seedvr2_causal_norm_wrapper_silu_matches_norm_then_silu(silu):
+ """The silu flag must fold in exactly, on whichever path this machine takes."""
+ torch.manual_seed(0)
+ norm = comfy_ops.disable_weight_init.GroupNorm(4, 16, eps=1e-6, affine=True)
+ torch.nn.init.normal_(norm.weight, mean=1.0, std=0.2)
+ torch.nn.init.normal_(norm.bias, std=0.2)
+ x = torch.randn(1, 16, 3, 5, 7)
+
+ got = causal_norm_wrapper(norm, x, silu=silu)
+
+ torch.testing.assert_close(got, _reference_group_norm(norm, x, silu), rtol=1e-5, atol=1e-6)
+
+
+def _decode_estimate(frames, height, width, dtype=torch.float16, batch=1, offload=True):
+ """Whether the caches can be pinned in host memory depends on the machine's RAM; the measured
+ floors are with them offloaded, so the test decides it instead of the machine."""
+ wrapper = vae_mod.VideoAutoencoderKLWrapper.__new__(vae_mod.VideoAutoencoderKLWrapper)
+ latent_t = (frames - 1) // 4 + 1
+ with patch.object(vae_mod, "_offload_caches_for", lambda frame_pixels: offload):
+ return wrapper.comfy_memory_used_decode((batch, 16, latent_t, height // 8, width // 8), dtype)
+
+
+def test_seedvr2_decode_estimate_charges_resident_caches_without_offload():
+ """Without room to pin the caches in host memory they stay on the GPU, and the estimate says so."""
+ resident = _decode_estimate(21, 1080, 1920, offload=False) - _decode_estimate(21, 1080, 1920)
+ assert resident == 1080 * 1920 * vae_mod.SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL
+
+
+def _encode_estimate(frames, height, width, dtype=torch.float16, batch=1):
+ wrapper = vae_mod.VideoAutoencoderKLWrapper.__new__(vae_mod.VideoAutoencoderKLWrapper)
+ return wrapper.comfy_memory_used_encode((batch, 3, frames, height, width), dtype)
+
+
+def test_seedvr2_decode_estimate_is_flat_in_clip_length():
+ """Slicing bounds the working set and the decoded frames go straight to sd.py's output buffer
+ off the GPU, so clip length must not move the estimate: charging it per frame sent long clips to
+ the reliable-estimate tiled path (about 1.75x slower) for memory they never use."""
+ assert _decode_estimate(241, 1080, 1920) == _decode_estimate(9, 1080, 1920)
+
+
[email protected]("frames, height, width, measured_gib", [
+ (9, 480, 864, 1.51),
+ (21, 720, 1280, 3.01),
+ (9, 864, 1536, 4.01),
+ (21, 1080, 1920, 6.26),
+ (9, 2160, 3840, 23.98),
+])
+def test_seedvr2_decode_estimate_tracks_measured_peak(frames, height, width, measured_gib):
+ """The least free VRAM (to 0.5 GiB) an untiled decode ran in under cudaMallocAsync, RTX 5090;
+ the estimate should sit just above it, never under it."""
+ estimate_gib = _decode_estimate(frames, height, width) / 1024 ** 3
+ assert estimate_gib >= measured_gib, (
+ f"estimate {estimate_gib:.2f} GiB under-reports the measured {measured_gib:.2f} GiB peak"
+ )
+ assert estimate_gib < measured_gib * 1.35, (
+ f"estimate {estimate_gib:.2f} GiB is far above the measured {measured_gib:.2f} GiB peak"
+ )
+
+
[email protected]("frames, height, width, measured_gib", [
+ (9, 480, 864, 1.51),
+ (21, 720, 1280, 2.51),
+ (9, 864, 1536, 3.01),
+ (21, 1088, 1920, 5.01),
+ (9, 2160, 3840, 19.73),
+])
+def test_seedvr2_encode_estimate_tracks_measured_floor(frames, height, width, measured_gib):
+ """Same measurement as the decode table; slicing and the frame-chunked head keep an encode flat
+ in clip length, so the estimate must not grow with it."""
+ estimate = _encode_estimate(frames, height, width)
+ assert measured_gib <= estimate / 1024 ** 3 < measured_gib * 1.35
+ assert _encode_estimate(frames * 10, height, width) == estimate
+
+
[email protected]("dtype, decode_factor, encode_factor", [(torch.bfloat16, 5, 3), (torch.float32, 10, 6)])
+def test_seedvr2_estimates_scale_for_the_eager_path(dtype, decode_factor, encode_factor):
+ """Only fp16 runs the channels-last path the figures are fitted on; bf16 and fp32 run the eager
+ path, measured at 2.4-5x (bf16) and 4-8.4x (fp32) the fp16 estimate. A batch is run a video at a
+ time, so the estimate is per video."""
+ assert _decode_estimate(21, 720, 1280, dtype) / _decode_estimate(21, 720, 1280) == pytest.approx(decode_factor)
+ assert _encode_estimate(21, 720, 1280, dtype) / _encode_estimate(21, 720, 1280) == pytest.approx(encode_factor)
+ assert _decode_estimate(21, 720, 1280, dtype, batch=3) == _decode_estimate(21, 720, 1280, dtype)
+ assert _encode_estimate(21, 720, 1280, dtype, batch=3) == _encode_estimate(21, 720, 1280, dtype)
+ assert _tile_side(8, dtype) < _tile_side(8)
+
+
+def test_seedvr2_batch_runs_a_video_at_a_time():
+ """The buffered convs and the estimates are per video: a batch is split before slicing, and the
+ slices land back in their video's place (in the preallocated output too)."""
+ vae = vae_mod.VideoAutoencoderKLWrapper.__new__(vae_mod.VideoAutoencoderKLWrapper)
+ vae.slicing_sample_min_size = vae.slicing_latent_min_size = 2
+ seen = []
+
+ def run(x, memory_state=vae_mod.MemoryState.DISABLED, memory_cache=None):
+ seen.append(x.size(0))
+ return x[:, :3].clone()
+ vae._decode = vae._encode = run
+ z = torch.randn(2, 16, 7, 4, 4)
+ torch.testing.assert_close(vae.slicing_decode(z), z[:, :3], rtol=0, atol=0)
+ out = torch.empty(2, 3, 7, 4, 4)
+ assert vae.slicing_decode(z, output_buffer=out) is out
+ torch.testing.assert_close(out, z[:, :3], rtol=0, atol=0)
+ torch.testing.assert_close(vae.slicing_encode(z), z[:, :3], rtol=0, atol=0)
+ assert set(seen) == {1}
+
+
+def test_seedvr2_encode_accepts_the_chunked_io_device_kwarg():
+ """``comfy_has_chunked_io`` is one flag for both directions: sd.py leaves the pixels where they
+ are and calls ``encode(x, device=...)``, so encode must take it and move the data itself."""
+ sig = inspect.signature(vae_mod.VideoAutoencoderKLWrapper.encode)
+ assert "device" in sig.parameters, "encode must accept the chunked-io device kwarg"
+ assert sig.parameters["device"].default is None, "device must be optional"
+ # the wrapper claims the protocol, so both sides of it have to exist
+ assert vae_mod.VideoAutoencoderKLWrapper.comfy_has_chunked_io is True
+ assert hasattr(vae_mod.VideoAutoencoderKLWrapper, "decode_output_shape")
+ assert "output_buffer" in inspect.signature(vae_mod.VideoAutoencoderKLWrapper.decode).parameters
+
+
+def test_seedvr2_decode_output_shape_matches_decode():
+ """sd.py preallocates from this; it must equal what decode() returns (frames from the 4n+1
+ rule, spatial 8x, cropped to even) for both latent layouts."""
+ wrapper = vae_mod.VideoAutoencoderKLWrapper.__new__(vae_mod.VideoAutoencoderKLWrapper)
+ wrapper.spatial_downsample_factor = 8
+ assert wrapper.decode_output_shape((1, 16, 6, 90, 160)) == (1, 3, 21, 720, 1280)
+ assert wrapper.decode_output_shape((1, 16, 1, 135, 240)) == (1, 3, 1, 1080, 1920)
+ assert wrapper.decode_output_shape((2, 16 * 3, 45, 81)) == (2, 3, 9, 360, 648)
+ assert wrapper.decode_output_shape((1, 16, 2, 13, 13)) == (1, 3, 5, 104, 104)
+ assert vae_mod.VideoAutoencoderKLWrapper.comfy_has_chunked_io is True
+
+
+def _tile_side(free_gib, dtype=torch.float16, offload=True):
+ wrapper = vae_mod.VideoAutoencoderKLWrapper.__new__(vae_mod.VideoAutoencoderKLWrapper)
+ wrapper.spatial_downsample_factor = 8
+ with patch.object(vae_mod, "_offload_caches_for", lambda frame_pixels: offload):
+ return wrapper.preferred_decode_tile(free_gib * 1024 ** 3, dtype)
+
+
+def test_seedvr2_tile_side_tracks_free_memory_within_bounds():
+ """Grows with free memory, stays inside [min, max] on whole latent blocks, and the tile it
+ picks is predicted to fit the memory it was sized against."""
+ assert _tile_side(0.1) == vae_mod.SEEDVR2_MIN_TILE_LATENT
+ assert _tile_side(10_000) == vae_mod.SEEDVR2_MAX_TILE_LATENT
+ assert _tile_side(3) < _tile_side(30)
+ assert _tile_side(30) * 8 >= 512
+ for free in (2, 4, 8, 16, 24, 32, 80):
+ side = _tile_side(free)
+ assert vae_mod.SEEDVR2_MIN_TILE_LATENT <= side <= vae_mod.SEEDVR2_MAX_TILE_LATENT and side % 8 == 0
+ if side != vae_mod.SEEDVR2_MIN_TILE_LATENT:
+ predicted = (side * 8) ** 2 * vae_mod.SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL + vae_mod.SEEDVR2_DECODE_FIXED_BYTES
+ assert predicted <= free * 1024 ** 3
+
+
+def test_seedvr2_tile_side_counts_resident_caches_without_offload():
+ """With no room to pin the caches they stay on the GPU, and the chosen tile must still fit with them."""
+ assert _tile_side(8, offload=False) < _tile_side(8), "the resident caches must shrink the tile"
+ per_pixel = vae_mod.SEEDVR2_DECODE_BYTES_PER_FRAME_PIXEL + vae_mod.SEEDVR2_CACHE_BYTES_PER_FRAME_PIXEL
+ for free in (8, 16, 32):
+ side = _tile_side(free, offload=False)
+ assert side <= _tile_side(free)
+ predicted = (side * 8) ** 2 * per_pixel + vae_mod.SEEDVR2_DECODE_FIXED_BYTES
+ assert predicted <= free * 1024 ** 3 * vae_mod.SEEDVR2_TILE_MEM_HEADROOM
diff --git a/tests-unit/comfy_test/test_seedvr2_model.py b/tests-unit/comfy_test/test_seedvr2_model.py
index 1d454aaf1..b5aad00ca 100644
--- a/tests-unit/comfy_test/test_seedvr2_model.py
+++ b/tests-unit/comfy_test/test_seedvr2_model.py
@@ -91,11 +91,9 @@ class _EncodeWrapper(seedvr_vae_mod.VideoAutoencoderKLWrapper):
def __init__(self, encoded):
nn.Module.__init__(self)
self.encoded = encoded
- self.spatial_downsample_factor = 8
- self.temporal_downsample_factor = 4
self.seen = []
- def encode(self, x):
+ def encode(self, x, device=None):
self.seen.append(tuple(x.shape))
return self.encoded.to(device=x.device, dtype=x.dtype)
@@ -103,60 +101,29 @@ class _EncodeWrapper(seedvr_vae_mod.VideoAutoencoderKLWrapper):
class _DecodeWrapper(seedvr_vae_mod.VideoAutoencoderKLWrapper):
def __init__(self):
nn.Module.__init__(self)
- self.spatial_downsample_factor = 8
- self.temporal_downsample_factor = 4
- self.calls = []
-
- def decode(self, z, seedvr2_tiling=None):
- self.calls.append({"shape": tuple(z.shape), "seedvr2_tiling": seedvr2_tiling})
- if z.ndim == 4:
- b, tc, h, w = z.shape
- t = tc // _LATENT_CHANNELS
- else:
- b, _, t, h, w = z.shape
- return torch.zeros(b, 3, t, h * 8, w * 8, dtype=z.dtype, device=z.device)
def test_seedvr2_wrapper_public_encode_returns_tensor(monkeypatch):
- raw_latent = torch.full((1, _LATENT_CHANNELS, 1, 4, 5), 2.0)
+ moments = torch.cat([torch.full((1, _LATENT_CHANNELS, 1, 4, 5), 2.0), torch.full((1, _LATENT_CHANNELS, 1, 4, 5), -1.0)], dim=1)
seen_shapes = []
- def base_encode(self, x):
+ def slicing_encode(self, x):
seen_shapes.append(tuple(x.shape))
- return raw_latent.to(device=x.device, dtype=x.dtype)
+ return moments.to(device=x.device, dtype=x.dtype)
- monkeypatch.setattr(seedvr_vae_mod.VideoAutoencoderKL, "encode", base_encode)
+ monkeypatch.setattr(seedvr_vae_mod.VideoAutoencoderKLWrapper, "slicing_encode", slicing_encode)
vae = seedvr_vae_mod.VideoAutoencoderKLWrapper.__new__(seedvr_vae_mod.VideoAutoencoderKLWrapper)
nn.Module.__init__(vae)
- vae._dummy = nn.Parameter(torch.zeros((), dtype=torch.float32))
latent = vae.encode(torch.zeros(1, 3, 32, 40))
assert type(latent) is torch.Tensor
assert tuple(latent.shape) == (1, _LATENT_CHANNELS, 4, 5)
+ assert torch.equal(latent, torch.full_like(latent, 2.0)), "the latent is the posterior mean"
assert seen_shapes == [(1, 3, 1, 32, 40)]
-def test_seedvr2_wrapper_private_encode_helper_keeps_raw_latent(monkeypatch):
- raw_latent = torch.full((1, _LATENT_CHANNELS, 1, 4, 5), 3.0)
-
- def base_encode(self, x):
- return raw_latent.to(device=x.device, dtype=x.dtype)
-
- monkeypatch.setattr(seedvr_vae_mod.VideoAutoencoderKL, "encode", base_encode)
-
- vae = seedvr_vae_mod.VideoAutoencoderKLWrapper.__new__(seedvr_vae_mod.VideoAutoencoderKLWrapper)
- nn.Module.__init__(vae)
- vae._dummy = nn.Parameter(torch.zeros((), dtype=torch.float32))
-
- latent, raw = vae._encode_with_raw_latent(torch.zeros(1, 3, 32, 40))
-
- assert tuple(latent.shape) == (1, _LATENT_CHANNELS, 4, 5)
- assert tuple(raw.shape) == (1, _LATENT_CHANNELS, 1, 4, 5)
- assert torch.equal(raw, raw_latent)
-
-
def _make_vae(wrapper):
vae = sd_mod.VAE.__new__(sd_mod.VAE)
vae.first_stage_model = wrapper
@@ -209,6 +176,20 @@ def test_seedvr2_7b_keeps_final_block_text_path(monkeypatch):
]
+def _bytedance_interleaved_rope(x, freqs):
+ """ByteDance SeedVR2 pixel-RoPE: adjacent feature pairs rotated by the interleaved angles,
+ leaving the head-dim tail past the rotary width untouched."""
+ rot = freqs.shape[-1]
+ angles = freqs[:, ::2].float().unsqueeze(1)
+ cos, sin = torch.cos(angles), torch.sin(angles)
+ out = x.float().clone()
+ even = out[..., 0:rot:2].clone()
+ odd = out[..., 1:rot:2].clone()
+ out[..., 0:rot:2] = even * cos - odd * sin
+ out[..., 1:rot:2] = odd * cos + even * sin
+ return out.to(x.dtype)
+
+
def test_seedvr2_7b_rope3d_matches_wrapper_oracle():
rope = seedvr_model.get_na_rope("rope3d", dim=64)
generator = torch.Generator(device="cpu").manual_seed(0)
@@ -217,19 +198,13 @@ def test_seedvr2_7b_rope3d_matches_wrapper_oracle():
shape = torch.tensor([[1, 2, 2]], dtype=torch.long)
freqs = rope.get_axial_freqs(1, 2, 2).reshape(4, -1)
- expected_q = seedvr_model._apply_seedvr2_rotary_emb(
- freqs,
- q.permute(1, 0, 2).float(),
- ).to(q.dtype).permute(1, 0, 2)
- expected_k = seedvr_model._apply_seedvr2_rotary_emb(
- freqs,
- k.permute(1, 0, 2).float(),
- ).to(k.dtype).permute(1, 0, 2)
+ expected_q = _bytedance_interleaved_rope(q, freqs)
+ expected_k = _bytedance_interleaved_rope(k, freqs)
actual_q, actual_k = rope(q.clone(), k.clone(), shape, seedvr_model.Cache(disable=True))
- torch.testing.assert_close(actual_q, expected_q, rtol=0, atol=0)
- torch.testing.assert_close(actual_k, expected_k, rtol=0, atol=0)
+ torch.testing.assert_close(actual_q, expected_q, rtol=1e-5, atol=1e-6)
+ torch.testing.assert_close(actual_k, expected_k, rtol=1e-5, atol=1e-6)
def test_seedvr2_forward_requires_conditioning_latents():
@@ -293,6 +268,8 @@ def test_seedvr2_encode_and_encode_tiled_preserve_native_latent_contract(monkeyp
def test_vaedecode_tiled_spatial_applies_temporal_discarded(monkeypatch):
monkeypatch.setattr(sd_mod.model_management, "load_models_gpu", lambda *a, **k: None)
+ tiled = MagicMock(return_value=torch.zeros(1, 3, 5, 32, 40))
+ monkeypatch.setattr(seedvr_vae_mod, "tiled_vae", tiled)
vae = _make_vae(_DecodeWrapper())
nodes_mod.VAEDecodeTiled().decode(
@@ -306,15 +283,7 @@ def test_vaedecode_tiled_spatial_applies_temporal_discarded(monkeypatch):
# Spatial inputs flow through; temporal inputs are discarded as public tiling
# knobs, but SeedVR2's internal MemoryState causal slicing is left intact.
- assert vae.first_stage_model.calls == [
- {
- "shape": (1, _LATENT_CHANNELS, 2, 4, 5),
- "seedvr2_tiling": {
- "enable_tiling": True,
- "tile_size": (512, 512),
- "tile_overlap": (64, 64),
- "temporal_size": None,
- "temporal_overlap": None,
- },
- }
- ]
+ assert tiled.call_count == 1
+ (latent, _), kwargs = tiled.call_args
+ assert tuple(latent.shape) == (1, _LATENT_CHANNELS, 2, 4, 5)
+ assert kwargs == {"tile_size": (512, 512), "tile_overlap": (64, 64), "encode": False}
diff --git a/tests-unit/comfy_test/test_seedvr2_vae_decode.py b/tests-unit/comfy_test/test_seedvr2_vae_decode.py
index c486b9195..10c195374 100644
--- a/tests-unit/comfy_test/test_seedvr2_vae_decode.py
+++ b/tests-unit/comfy_test/test_seedvr2_vae_decode.py
@@ -24,7 +24,7 @@ def _make_wrapper() -> vae_mod.VideoAutoencoderKLWrapper:
return wrapper
-def _fingerprint_decode_(self, z, return_dict=True):
+def _fingerprint_decode(self, z, output_buffer=None):
b = int(z.shape[0])
t = int(z.shape[2])
h = int(z.shape[3])
@@ -36,7 +36,7 @@ def _fingerprint_decode_(self, z, return_dict=True):
def _decode_with_patches(wrapper, z):
- with patch.object(vae_mod.VideoAutoencoderKL, "decode_", _fingerprint_decode_):
+ with patch.object(vae_mod.VideoAutoencoderKLWrapper, "slicing_decode", _fingerprint_decode):
return wrapper.decode(z)
@@ -53,10 +53,8 @@ class _Wrapper(vae_mod.VideoAutoencoderKLWrapper):
nn.Module.__init__(self)
self.calls = []
- def parameters(self):
- return iter([torch.nn.Parameter(torch.zeros(()))])
-def _decode_stub(self, latent):
+def _decode_stub(self, latent, output_buffer=None):
self.calls.append(tuple(latent.shape))
return torch.zeros(latent.shape[0], 3, latent.shape[2], latent.shape[3] * 8, latent.shape[4] * 8)
@@ -64,20 +62,13 @@ def _decode_stub(self, latent):
def test_seedvr2_wrapper_decode_accepts_5d_channel_first_latents_without_preprocessor_state():
wrapper = _Wrapper()
- with patch.object(vae_mod.VideoAutoencoderKL, "decode_", _decode_stub):
+ with patch.object(vae_mod.VideoAutoencoderKLWrapper, "slicing_decode", _decode_stub):
out = wrapper.decode(torch.zeros(1, _LATENT_CHANNELS, 2, 4, 5))
assert tuple(out.shape) == (1, 3, 2, 32, 40)
assert wrapper.calls == [(1, _LATENT_CHANNELS, 2, 4, 5)]
-def test_seedvr2_wrapper_decode_rejects_wrong_rank_latents():
- wrapper = _Wrapper()
-
- with pytest.raises(RuntimeError, match=r"latent input must be 4-D collapsed .* or 5-D"):
- wrapper.decode(torch.zeros(1, _LATENT_CHANNELS, 4))
-
-
def _t_padded(t_in: int) -> int:
if t_in == 1:
return 1
diff --git a/tests-unit/comfy_test/test_seedvr2_vae_tiled.py b/tests-unit/comfy_test/test_seedvr2_vae_tiled.py
index d64f51918..322f48e28 100644
--- a/tests-unit/comfy_test/test_seedvr2_vae_tiled.py
+++ b/tests-unit/comfy_test/test_seedvr2_vae_tiled.py
@@ -1,7 +1,6 @@
from contextlib import ExitStack
from unittest.mock import MagicMock, patch
-import pytest
import torch
import torch.nn as nn
@@ -10,85 +9,14 @@ from comfy.cli_args import args as cli_args
if not torch.cuda.is_available():
cli_args.cpu = True
-import comfy.ldm.seedvr.vae as vae_mod # noqa: E402
import comfy.ldm.seedvr.vae as seedvr_vae_mod # noqa: E402
import comfy.sd as sd_mod # noqa: E402
-from comfy.ldm.seedvr.vae import MemoryState, tiled_vae # noqa: E402
+from comfy.ldm.seedvr.vae import tiled_vae # noqa: E402
_LATENT_CHANNELS = seedvr_vae_mod.SEEDVR2_LATENT_CHANNELS
-def test_runtime_decode_zero_temporal_size_preserves_model_slicing():
- class StubVAEModel(torch.nn.Module):
- def __init__(self):
- super().__init__()
- self.slicing_latent_min_size = 2
- self.spatial_downsample_factor = 8
- self.temporal_downsample_factor = 4
- self.device = torch.device("cpu")
- self.use_slicing = True
- self._dummy = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32))
- self.decode_min_sizes = []
- self.memory_states = []
-
- def decode_(self, t_chunk):
- self.decode_min_sizes.append(self.slicing_latent_min_size)
- return vae_mod.VideoAutoencoderKL.slicing_decode(self, t_chunk)
-
- def _decode(self, z, memory_state=MemoryState.DISABLED, memory_cache=None):
- self.memory_states.append(memory_state)
- b, c, d, h, w = z.shape
- return torch.zeros((b, 3, d, h * 8, w * 8), dtype=z.dtype)
-
- vae = StubVAEModel()
- z = torch.zeros((1, _LATENT_CHANNELS, 5, 8, 8), dtype=torch.float32)
-
- tiled_vae(
- z,
- vae,
- tile_size=(64, 64),
- tile_overlap=(0, 0),
- temporal_size=0,
- temporal_overlap=0,
- encode=False,
- )
-
- assert vae.decode_min_sizes == [2]
- assert vae.memory_states == [MemoryState.INITIALIZING, MemoryState.ACTIVE]
- assert vae.slicing_latent_min_size == 2
-
-
-def test_zero_temporal_size_preserves_min_size_when_encode_raises():
- class RaisingVAEModel(torch.nn.Module):
- def __init__(self):
- super().__init__()
- self.slicing_sample_min_size = 4
- self.spatial_downsample_factor = 8
- self.temporal_downsample_factor = 4
- self.device = torch.device("cpu")
- self._dummy = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32))
-
- def encode(self, t_chunk):
- raise RuntimeError("simulated encode failure")
-
- vae = RaisingVAEModel()
- x = torch.zeros((1, 3, 12, 64, 64), dtype=torch.float32)
-
- with pytest.raises(RuntimeError, match="simulated encode failure"):
- tiled_vae(
- x,
- vae,
- tile_size=(64, 64),
- tile_overlap=(0, 0),
- temporal_size=0,
- temporal_overlap=0,
- encode=True,
- )
-
- assert vae.slicing_sample_min_size == 4
-
-
def test_tiled_vae_encode_uses_tensor_return_without_indexing():
class TensorEncodeVAEModel(torch.nn.Module):
def __init__(self):
@@ -113,8 +41,6 @@ def test_tiled_vae_encode_uses_tensor_return_without_indexing():
vae,
tile_size=(64, 64),
tile_overlap=(0, 0),
- temporal_size=0,
- temporal_overlap=0,
encode=True,
)
@@ -166,87 +92,12 @@ def test_tiled_vae_preserves_input_dtype_on_single_tile():
FloatOutputVAEModel(),
tile_size=(64, 64),
tile_overlap=(0, 0),
- temporal_size=0,
- temporal_overlap=0,
encode=True,
)
assert out.dtype == torch.float16
-class _SlicingDecodeVAE(nn.Module):
- def __init__(self, slicing_latent_min_size):
- super().__init__()
- self.slicing_latent_min_size = slicing_latent_min_size
- self.spatial_downsample_factor = 8
- self.temporal_downsample_factor = 4
- self.device = torch.device("cpu")
- self.use_slicing = True
- self._dummy = nn.Parameter(torch.zeros(1, dtype=torch.float32))
- self.decode_min_sizes = []
- self.memory_states = []
-
- def decode_(self, z):
- self.decode_min_sizes.append(self.slicing_latent_min_size)
- return vae_mod.VideoAutoencoderKL.slicing_decode(self, z)
-
- def _decode(self, z, memory_state=MemoryState.DISABLED, memory_cache=None):
- self.memory_states.append(memory_state)
- x = z[:, :1].repeat(
- 1,
- 3,
- 1,
- self.spatial_downsample_factor,
- self.spatial_downsample_factor,
- )
- return x
-
-
-def test_decode_tiled_vae_maps_temporal_args_to_latent_slicing_min_size():
- vae = _SlicingDecodeVAE(slicing_latent_min_size=2)
- z = torch.arange(
- _LATENT_CHANNELS * 5 * 8 * 8,
- dtype=torch.float32,
- ).reshape(1, _LATENT_CHANNELS, 5, 8, 8)
-
- tiled_vae(
- z,
- vae,
- tile_size=(64, 64),
- tile_overlap=(0, 0),
- temporal_size=12,
- temporal_overlap=4,
- encode=False,
- )
-
- assert vae.decode_min_sizes == [2]
- assert vae.memory_states == [MemoryState.INITIALIZING, MemoryState.ACTIVE]
- assert vae.slicing_latent_min_size == 2
-
- wrapper = vae_mod.VideoAutoencoderKLWrapper.__new__(
- vae_mod.VideoAutoencoderKLWrapper
- )
- nn.Module.__init__(wrapper)
- seedvr2_tiling = {
- "enable_tiling": True,
- "tile_size": (64, 64),
- "tile_overlap": (0, 0),
- "temporal_size": 8,
- "temporal_overlap": 7,
- }
-
- captured = {}
-
- def _fake_tiled_vae(latent, model, **kwargs):
- captured.update(kwargs)
- return torch.zeros(1, 3, 1, 16, 16)
-
- with patch.object(vae_mod, "tiled_vae", side_effect=_fake_tiled_vae):
- wrapper.decode(torch.zeros(1, _LATENT_CHANNELS, 2, 2), seedvr2_tiling=seedvr2_tiling)
-
- assert captured["temporal_overlap"] == 7
-
-
def _force_oom(*a, **k):
raise torch.cuda.OutOfMemoryError("forced OOM for dispatcher test")