The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
# Wan video super-resolution
# RelayVSR https://github.com/kopperx/RelayVSR (Apache-2.0): one-step Wan keyframes guiding a small per-frame transformer
import math
import comfy_kitchen
import torch
import torch.nn as nn
import torch.nn.functional as F
from comfy.ldm.flux.layers import EmbedND
from comfy.ldm.flux.math import apply_rope1
from comfy.ldm.wan.model import WanAttentionBlock, WanModel, WanSelfAttention, modulate, sinusoidal_embedding_1d
import comfy.ldm.common_dit
import comfy.model_management
import comfy.model_prefetch
import comfy.ops
import comfy.utils
ops = comfy.ops.disable_weight_init
RELAY_KEYFRAME_INTERVAL = 15
RELAY_WAN_CONTEXT = 4
RELAY_WAN_WINDOW = (22, 40)
RELAY_WAN_CHUNK_TOKENS = 32768
# the per-layer cache needs about as much VRAM as this many keyframes of a single pass
RELAY_WAN_CACHE_KEYFRAMES = 7
RELAY_FLASH_CONDITION_WINDOW = (48, 48)
RELAY_FLASH_VIDEO_WINDOW = (24, 24)
def relay_keyframes(frames):
return list(range(0, frames - 1, RELAY_KEYFRAME_INTERVAL)) + [frames - 1]
def relay_latent_size(height, width, latent_stride):
# whole 2x2 patches of the model's pixel stride
stride = latent_stride * 2
return math.ceil(height / stride) * 2, math.ceil(width / stride) * 2
class RelayLQProjection(nn.Module):
def __init__(self, upscale_factor, hidden_dim=768, hidden_dim2=3072, out_dim=3072, eps=1e-6, device=None, dtype=None, operations=None):
super().__init__()
self.upscale_factor = upscale_factor
self.conv1 = operations.Conv2d(3 * (4 * upscale_factor) ** 2, hidden_dim, 1, device=device, dtype=dtype)
self.norm1 = operations.RMSNorm(hidden_dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)
self.conv2 = operations.Conv2d(hidden_dim, hidden_dim2, 3, stride=2, padding=1, device=device, dtype=dtype)
self.norm2 = operations.RMSNorm(hidden_dim2, eps=eps, elementwise_affine=True, device=device, dtype=dtype)
self.linear = operations.Linear(hidden_dim2, out_dim, device=device, dtype=dtype)
def forward(self, video):
b, c, f, h, w = video.shape
x = video.movedim(2, 1).reshape(b * f, c, h, w)
if self.upscale_factor == 4:
x = F.interpolate(x, scale_factor=4, mode="bilinear", align_corners=False)
x = self.conv1(F.pixel_unshuffle(x, 4 * self.upscale_factor))
x = F.silu(self.norm1(x.movedim(1, -1))).movedim(-1, 1)
x = F.silu(self.norm2(self.conv2(x).flatten(2).transpose(1, 2)))
return self.linear(x).view(b, -1, self.linear.out_features)
class RelayVSRSelfAttention(WanSelfAttention):
def forward(self, x, freqs, cache, start, store, transformer_options={}):
# causal over the last keyframes, per-query windows slid to stay inside the frame
b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim
frames, height, width = transformer_options["grid_sizes"]
grid = (b, frames, height, width, n, d)
q = apply_rope1(self.norm_q(self.q(x)).view(b, s, n, d), freqs).view(grid)
k = apply_rope1(self.norm_k(self.k(x)).view(b, s, n, d), freqs).view(grid)
v = self.v(x).view(grid)
# keyframes before this chunk come from the cache, which holds the last ones in order as int8 rows with a scale each
past = min(start, RELAY_WAN_CONTEXT - 1)
keys, values = k, v
if past > 0:
cached, scales = cache
k_past, v_past = (cached[:, :, :past].to(k.dtype) * scales[:, :, :past]).unbind(0)
keys, values = torch.cat((k_past, k), dim=1), torch.cat((v_past, v), dim=1)
x = comfy_kitchen.window_attention(q, keys, values, RELAY_WAN_WINDOW, temporal_window=RELAY_WAN_CONTEXT)
if store:
# the cache moves on to the last keyframes for the next chunk, older cached ones first when this one is short
cached, scales = cache
new = min(frames, RELAY_WAN_CONTEXT - 1)
old = min(past, RELAY_WAN_CONTEXT - 1 - new)
if old > 0:
cached[:, :, :old] = cached[:, :, past - old:past].clone()
scales[:, :, :old] = scales[:, :, past - old:past].clone()
for i, rows in enumerate((k[:, frames - new:], v[:, frames - new:])):
scale = (rows.abs().amax(dim=-1, keepdim=True) / 127.0).clamp_(min=torch.finfo(rows.dtype).tiny)
scales[i, :, old:old + new] = scale
cached[i, :, old:old + new] = (rows.float() / scale.float()).round_().clamp_(-127.0, 127.0)
return self.o(x.view(b, s, n * d))
class RelayVSRAttentionBlock(WanAttentionBlock):
def forward(self, x, e, freqs, context, cache, start, store, transformer_options={}):
# residual updated in place, nothing allocated in the block outlives its memory compiler scope
e = (comfy.model_management.cast_to(self.modulation, dtype=x.dtype, device=x.device).unsqueeze(0) + e).unbind(2)
x.addcmul_(self.self_attn(modulate(x, self.norm1, e[0], e[1]), freqs, cache, start, store, transformer_options=transformer_options), e[2])
x.add_(self.cross_attn(self.norm3(x), context, transformer_options=transformer_options))
x.addcmul_(self.ffn(modulate(x, self.norm2, e[3], e[4])), e[5])
return x
class RelayVSRWanModel(WanModel):
def __init__(self, model_type=None, upscale_factor=4, operations=None, device=None, dtype=None, **kwargs):
super().__init__(model_type="t2v", wan_attn_block_class=RelayVSRAttentionBlock, operations=operations, device=device, dtype=dtype, **kwargs)
self.register_buffer("positive_conditioning", torch.empty((self.text_len, self.text_dim), device=device, dtype=dtype))
operation_settings = {"operations": operations, "device": device, "dtype": dtype}
for block in self.blocks:
block.self_attn = RelayVSRSelfAttention(self.dim, self.num_heads, eps=self.eps, operation_settings=operation_settings)
self.lq_proj_in = RelayLQProjection(upscale_factor, out_dim=self.dim, device=device, dtype=dtype, operations=operations)
self.latent_stride = 16 // upscale_factor
def forward_orig(self, x, t, context, freqs=None, lq_pixels=None, transformer_options={}, **kwargs):
b, frames = x.shape[0], x.shape[2]
height, width = x.shape[3] // self.patch_size[1], x.shape[4] // self.patch_size[2]
tokens = height * width
e = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, t.flatten()).to(dtype=x.dtype))
e = e.reshape(t.shape[0], -1, e.shape[-1])
e0 = self.time_projection(e).unflatten(2, (6, self.dim))
context = self.text_embedding(context)
# keyframes go through in chunks, every layer's cache carries the keys and values of the last ones across
chunk = max(1, RELAY_WAN_CHUNK_TOKENS // tokens)
if frames <= chunk + RELAY_WAN_CACHE_KEYFRAMES:
chunk = frames
shape = (2, b, RELAY_WAN_CONTEXT - 1, height, width, self.num_heads, self.dim // self.num_heads)
caches = [(torch.empty(shape, device=x.device, dtype=torch.int8), torch.empty((*shape[:-1], 1), device=x.device, dtype=x.dtype)) if frames > chunk else None for _ in self.blocks]
out = torch.empty((b, self.out_dim, *x.shape[2:]), device=x.device, dtype=x.dtype)
blocks_replace = transformer_options.get("patches_replace", {}).get("dit", {})
transformer_options["total_blocks"] = len(self.blocks)
transformer_options["block_type"] = "double"
starts = range(0, frames, chunk)
# the sampler's bars only count the one step
progress = comfy.utils.ProgressBar(len(starts) * len(self.blocks))
with comfy.utils.model_trange(len(starts) * len(self.blocks), desc="RelayVSR", unit="block") as pbar:
for start in starts:
end = min(start + chunk, frames)
# only a following chunk reads the cache
store = end < frames
grid_sizes = (end - start, height, width)
transformer_options["grid_sizes"] = grid_sizes
# the projected LQ frames go first so the sum is row-major, the blocks update it in place
hidden = self.lq_proj_in(lq_pixels[:, :, start:end]) + self.patch_embedding(x[:, :, start:end].float()).to(x.dtype).flatten(2).transpose(1, 2)
chunk_freqs = freqs[:, start * tokens:end * tokens]
prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.blocks), x.device, transformer_options)
comfy.model_prefetch.malloc_graph_begin(x.device)
for i, block in enumerate(self.blocks):
comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, x.device, block, x.dtype, malloc_scope="block")
transformer_options["block_index"] = i
if ("double_block", i) in blocks_replace:
def block_wrap(args):
return {"img": block(args["img"], args["vec"], args["pe"], args["txt"], caches[i], start, store, transformer_options=args["transformer_options"])}
hidden = blocks_replace[("double_block", i)]({"img": hidden, "txt": context, "vec": e0, "pe": chunk_freqs, "transformer_options": transformer_options}, {"original_block": block_wrap})["img"]
else:
hidden = block(hidden, e0, chunk_freqs, context, caches[i], start, store, transformer_options=transformer_options)
pbar.update(1)
progress.update(1)
comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, x.device, None, malloc_scope="block")
comfy.model_prefetch.malloc_graph_end()
out[:, :, start:end] = self.unpatchify(self.head(hidden, e), grid_sizes)
return out
def _forward(self, x, timestep, context, clip_fea=None, time_dim_concat=None, transformer_options={}, lq_pixels=None, **kwargs):
if lq_pixels is None:
raise ValueError("RelayVSR needs the low-quality video, connect the model through the RelayVSR Conditioning node.")
t, h, w = x.shape[2:]
x = comfy.ldm.common_dit.pad_to_patch_size(x, self.patch_size, padding_mode="replicate")
tp, hp, wp = x.shape[2:]
lq_height, lq_width = hp * self.latent_stride, wp * self.latent_stride
lq_pixels = lq_pixels[..., :lq_height, :lq_width]
lq_pixels = F.pad(lq_pixels, (0, lq_width - lq_pixels.shape[-1], 0, lq_height - lq_pixels.shape[-2], 0, 0), mode="replicate")
freqs = self.rope_encode(tp, hp, wp, device=x.device, dtype=x.dtype, transformer_options=transformer_options)
return self.forward_orig(x, timestep, context, freqs=freqs, lq_pixels=lq_pixels, transformer_options=transformer_options, **kwargs)[:, :, :t, :h, :w]
class RelayFlashAttention(nn.Module):
def __init__(self, dim, num_heads, kv_groups, eps, device=None, dtype=None):
super().__init__()
self.num_heads = num_heads
self.kv_groups = kv_groups
self.head_dim = dim // num_heads
self.query = ops.Linear(dim, dim, device=device, dtype=dtype)
self.key = ops.Linear(dim, kv_groups * self.head_dim, device=device, dtype=dtype)
self.value = ops.Linear(dim, kv_groups * self.head_dim, device=device, dtype=dtype)
self.condition_key = ops.Linear(dim, kv_groups * self.head_dim, device=device, dtype=dtype)
self.condition_value = ops.Linear(dim, kv_groups * self.head_dim, device=device, dtype=dtype)
self.key_norm = ops.RMSNorm(self.head_dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)
self.value_norm = ops.RMSNorm(self.head_dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)
self.output = ops.Linear(dim, dim, device=device, dtype=dtype)
def project(self, x, key, value, freqs):
b, s, _ = x.shape
q = apply_rope1(self.query(x).view(b, s, self.num_heads, self.head_dim), freqs)
k = self.key_norm(key(x).view(b, s, self.kv_groups, self.head_dim))
v = self.value_norm(value(x).view(b, s, self.kv_groups, self.head_dim))
return q, k, v
def condition(self, x, freqs, height, width):
# the rotated condition keys are reused as they are by every frame of the window
b, s, c = x.shape
q, k, v = self.project(x, self.condition_key, self.condition_value, freqs)
k = apply_rope1(k, freqs).view(b, 1, height, width, self.kv_groups, self.head_dim)
v = v.view(k.shape)
x = comfy_kitchen.window_attention(q.view(b, 1, height, width, self.num_heads, self.head_dim), k, v, RELAY_FLASH_CONDITION_WINDOW, clamp=False)
return self.output(x.view(b, s, c)), (k, v)
def video(self, x, freqs, freqs_past, height, width, condition, past):
b, s, c = x.shape
plane = (b, 1, height, width, self.kv_groups, self.head_dim)
q, k, v = self.project(x, self.key, self.value, freqs)
# both condition keyframes, then the previous frame and this one
(k_start, v_start), (k_end, v_end) = condition
keys, values = [k_start, k_end], [v_start, v_end]
if past is not None:
keys.append(apply_rope1(past[0], freqs_past).view(plane))
values.append(past[1].view(plane))
keys.append(apply_rope1(k, freqs).view(plane))
values.append(v.view(plane))
windows = [RELAY_FLASH_CONDITION_WINDOW] * 2 + [RELAY_FLASH_VIDEO_WINDOW] * (len(keys) - 2)
x = comfy_kitchen.window_attention(q.view(b, 1, height, width, self.num_heads, self.head_dim), torch.cat(keys, dim=1), torch.cat(values, dim=1), windows, clamp=False)
return self.output(x.view(b, s, c)), (k, v)
class RelayFlashFeedForward(nn.Module):
def __init__(self, dim, hidden_dim, device=None, dtype=None):
super().__init__()
self.gate_up_proj = ops.Linear(dim, 2 * hidden_dim, bias=False, device=device, dtype=dtype)
self.down = ops.Linear(hidden_dim, dim, bias=False, device=device, dtype=dtype)
def forward(self, x):
return comfy.ops.linear_input_act(self.down, self.gate_up_proj(x), "swiglu")
class RelayFlashBlock(nn.Module):
def __init__(self, dim, num_heads, kv_groups, hidden_dim, eps, device=None, dtype=None):
super().__init__()
self.attention_norm = ops.RMSNorm(dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)
self.attention = RelayFlashAttention(dim, num_heads, kv_groups, eps, device=device, dtype=dtype)
self.feed_forward_norm = ops.RMSNorm(dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)
self.feed_forward = RelayFlashFeedForward(dim, hidden_dim, device=device, dtype=dtype)
def condition(self, x, *args):
y, kv = self.attention.condition(self.attention_norm(x), *args)
x = x + y
return x + self.feed_forward(self.feed_forward_norm(x)), kv
def video(self, x, *args):
y, kv = self.attention.video(self.attention_norm(x), *args)
x = x + y
return x + self.feed_forward(self.feed_forward_norm(x)), kv
class RelayFlashEncoder(nn.Module):
def __init__(self, last_stride, dim, eps, device=None, dtype=None):
super().__init__()
self.layers = nn.Sequential(
ops.Conv2d(3, 64, 3, stride=2, padding=1, device=device, dtype=dtype), nn.SiLU(),
ops.Conv2d(64, 128, 3, stride=2, padding=1, device=device, dtype=dtype), nn.SiLU(),
ops.Conv2d(128, 128, 3, stride=last_stride, padding=1, device=device, dtype=dtype), nn.SiLU(),
ops.Conv2d(128, dim, 1, device=device, dtype=dtype))
self.norm = ops.RMSNorm(dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)
def forward(self, x):
return self.norm(self.layers(x).flatten(2).transpose(1, 2))
class RelayFlashConditionEncoder(nn.Module):
def __init__(self, in_channels, dim, eps, device=None, dtype=None):
super().__init__()
self.projection = ops.Conv2d(in_channels, dim, 1, device=device, dtype=dtype)
self.norm = ops.RMSNorm(dim, eps=eps, elementwise_affine=True, device=device, dtype=dtype)
def forward(self, x):
return self.norm(self.projection(x).flatten(2).transpose(1, 2))
class RelayFlashBackbone(nn.Module):
def __init__(self, depth, dim, num_heads, kv_groups, hidden_dim, eps, device=None, dtype=None):
super().__init__()
self.blocks = nn.ModuleList([RelayFlashBlock(dim, num_heads, kv_groups, hidden_dim, eps, device=device, dtype=dtype) for _ in range(depth)])
class RelayFlash(nn.Module):
"""Per-frame decoder conditioned on the keyframe latents on either side of the frame, VAE slot of RelayVSR."""
comfy_has_chunked_io = True
def __init__(self, upscale_factor, latent_channels=48, dim=512, depth=10, num_heads=8, kv_groups=2, hidden_dim=1408, head_dim=1024, eps=1e-5, device=None, dtype=None):
super().__init__()
self.upscale_factor = upscale_factor
self.latent_stride = 16 // upscale_factor
self.lr_encoder = RelayFlashEncoder(self.latent_stride // 4, dim, eps, device=device, dtype=dtype)
self.condition_encoder = RelayFlashConditionEncoder(latent_channels, dim, eps, device=device, dtype=dtype)
self.video_type_embedding = nn.Parameter(torch.empty(1, 1, dim, device=device, dtype=dtype))
self.condition_type_embedding = nn.Parameter(torch.empty(1, 1, dim, device=device, dtype=dtype))
self.backbone = RelayFlashBackbone(depth, dim, num_heads, kv_groups, hidden_dim, eps, device=device, dtype=dtype)
self.spatial_head = nn.Sequential(ops.Linear(dim, head_dim, device=device, dtype=dtype), nn.SiLU())
self.residual_projection = ops.Linear(head_dim, 768, device=device, dtype=dtype)
self.rope_embedder = EmbedND(dim=dim // num_heads, theta=10000.0, axes_dim=[16, 24, 24])
def grid_rope(self, t, height, width, device):
ids = torch.zeros((height, width, 3), device=device, dtype=torch.float32)
ids[..., 0] = t
ids[..., 1] = torch.arange(height, device=device, dtype=torch.float32)[:, None]
ids[..., 2] = torch.arange(width, device=device, dtype=torch.float32)[None]
return self.rope_embedder(ids.reshape(1, -1, 3)).movedim(1, 2)
def prepare_condition(self, latent, freqs):
height, width = latent.shape[-2:]
x = self.condition_encoder(latent) + comfy.ops.cast_to_input(self.condition_type_embedding, latent)
out = []
for block in self.backbone.blocks:
x, kv = block.condition(x, freqs, height, width)
out.append(kv)
return out
def decode_frame(self, lr, dtype, freqs, freqs_past, condition, history, detail):
height, width = lr.shape[-2] // self.latent_stride, lr.shape[-1] // self.latent_stride
x = self.lr_encoder(lr.to(dtype))
x = x + comfy.ops.cast_to_input(self.video_type_embedding, x)
next_history = []
for block, cond, past in zip(self.backbone.blocks, condition, history):
x, kv = block.video(x, freqs, freqs_past, height, width, cond, past)
next_history.append(kv)
residual = F.pixel_shuffle(self.residual_projection(self.spatial_head(x)).transpose(1, 2).reshape(lr.shape[0], 768, height, width), 16)
if detail != 1.0:
residual = residual * detail
return F.interpolate(lr, scale_factor=self.upscale_factor, mode="bicubic", align_corners=False) + residual, next_history
def decode_output_shape(self, input_shape, lq_images=None, detail=1.0):
if lq_images is None:
raise ValueError("The RelayVSR Flash decoder needs the low-quality frames, decode with the VSR Decode node.")
frames, height, width = lq_images.shape[:3]
# the x2 and x4 decoders load from the same weight shapes, a latent of the other factor would decode garbage or fail on the copy
if tuple(input_shape[2:]) != (len(relay_keyframes(frames)), *relay_latent_size(height, width, self.latent_stride)):
raise ValueError(f"The latent {tuple(input_shape[2:])} doesn't fit the x{self.upscale_factor} RelayVSR Flash decoder and these {frames} {width}x{height} frames: use the RelayVSR model and Flash decoder of the same factor, and the same video in RelayVSR Conditioning and VSR Decode.")
return (input_shape[0], 3, frames, height * self.upscale_factor, width * self.upscale_factor)
def decode(self, x, lq_images=None, output_buffer=None, detail=1.0):
# detail scales what the model adds to the bicubic upscale of each frame
out = output_buffer
if out is None:
out = torch.empty(self.decode_output_shape(x.shape, lq_images), device=comfy.model_management.intermediate_device())
b, _, _, height, width = x.shape
frames, lq_height, lq_width = lq_images.shape[:3]
depth = len(self.backbone.blocks)
keyframes = relay_keyframes(frames)
device = x.device
freqs, freqs_past = self.grid_rope(0, height, width, device), self.grid_rope(-1, height, width, device)
pbar = comfy.utils.ProgressBar(frames)
# finished frames go to the output without the GPU waiting on each copy
with comfy.model_management.HostWriter(device, out, (b, 3, *out.shape[-2:])) as writer, comfy.utils.model_trange(frames, desc="RelayVSR Flash decode", unit="frame") as console:
start = self.prepare_condition(x[:, :, 0], freqs)
for window, (first, last) in enumerate(zip(keyframes, keyframes[1:] or keyframes)):
comfy.model_management.throw_exception_if_processing_interrupted()
end = self.prepare_condition(x[:, :, min(window + 1, len(keyframes) - 1)], freqs)
condition = list(zip(start, end))
history = [None] * depth
# one upload per window, an upload from pageable memory waits for the whole stream
lr_window = lq_images[first:last + 1].to(device=device)[..., :3].movedim(-1, 1) * 2.0 - 1.0
lr_window = F.pad(lr_window, (0, width * self.latent_stride - lq_width, 0, height * self.latent_stride - lq_height), mode="replicate")
for i, frame in enumerate(range(first, last + 1)):
decoded, history = self.decode_frame(lr_window[i:i + 1].expand(b, -1, -1, -1), x.dtype, freqs, freqs_past, condition, history, detail)
# the shared keyframe was already output by the previous window, here it only seeds the history
if window == 0 or frame > first:
# finished [0, 1] pixels on the GPU, the VAE's process_output is the identity
writer.write(out[:, :, frame], decoded[..., :out.shape[-2], :out.shape[-1]].add_(1.0).div_(2.0).clamp_(0.0, 1.0).to(out.dtype))
pbar.update(1)
console.update(1)
start = end
return out