Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Make NAF neighborhood attention work per LR cell (#17155)

Jukka Seppänen446ef4e005fc
diff --git a/comfy/image_encoders/naf.py b/comfy/image_encoders/naf.py
index 9d18c87df..d942daa2e 100644
--- a/comfy/image_encoders/naf.py
+++ b/comfy/image_encoders/naf.py
@@ -18,112 +18,44 @@ import comfy.ops
# Pure-torch neighborhood attention (replaces natten.na2d / na2d_qk + na2d_av).
-def upsample_lr_slice(src_lr: torch.Tensor, lr_dh: int, lr_dw: int,
- hr_h_range: Tuple[int, int], hr_w_range: Tuple[int, int]) -> torch.Tensor:
- """Slice a LR-layout tensor [B, h_lr, w_lr, n, C], permute to BCHW, and
- nearest-exact upsample only the region covering [hr_h_range, hr_w_range].
- Returns BCHW at hr_h_end-hr_h_start x hr_w_end-hr_w_start (no padding for
- out-of-bounds regions)."""
- B = src_lr.shape[0]
- n = src_lr.shape[-2]
- C = src_lr.shape[-1]
- h_hr_start, h_hr_end = hr_h_range
- w_hr_start, w_hr_end = hr_w_range
- # LR positions covering [h_hr_start, h_hr_end). Nearest-exact maps HR p → p // D.
- lr_h_start = h_hr_start // lr_dh
- lr_h_end = (h_hr_end - 1) // lr_dh + 1
- lr_w_start = w_hr_start // lr_dw
- lr_w_end = (w_hr_end - 1) // lr_dw + 1
- lr_slice = src_lr[:, lr_h_start:lr_h_end, lr_w_start:lr_w_end]
- lh, lw = lr_slice.shape[1], lr_slice.shape[2]
- lr_bcd = lr_slice.permute(0, 3, 4, 1, 2).reshape(B * n, C, lh, lw).contiguous()
- up = F.interpolate(lr_bcd, scale_factor=(lr_dh, lr_dw), mode="nearest-exact")
- offset_h = h_hr_start - lr_h_start * lr_dh
- offset_w = w_hr_start - lr_w_start * lr_dw
- return up[:, :, offset_h:offset_h + (h_hr_end - h_hr_start),
- offset_w:offset_w + (w_hr_end - w_hr_start)]
-
+_CHUNK_BYTES = 2 ** 29
def na2d_pure(
- q: torch.Tensor, # [B, H, W, n_heads, d_qk] at HR.
- k_lr: torch.Tensor, # [B, h_lr, w_lr, n_heads, d_qk] at LR
- v_lr: torch.Tensor, # [B, h_lr, w_lr, n_heads, d_v] at LR
- kernel_size: Tuple[int, int], # (Kh, Kw) attention window.
- dilation: Tuple[int, int], # (Dh, Dw) stride within the unrolled K/V grid; also the LR→HR upsample factor.
+ q: torch.Tensor, # [B, n_heads, d_qk, H, W] at HR.
+ k: torch.Tensor, # [B, n_heads * d_qk, h, w] at LR; H and W are multiples of h and w.
+ v: torch.Tensor, # [B, n_heads * d_v, h, w] at LR.
+ kernel_size: Tuple[int, int], # (Kh, Kw) attention window over the LR grid.
scale: float, # 1 / sqrt(d_qk) scaling for the Q·K scores.
- tile: int = 128, # Spatial tile size (output positions per tile)
- v_chunk: int = 64, # Sub-divide d_v into chunks of this size when computing attn·V. None disables chunking.
output: torch.Tensor = None, # Pre-allocated [B, n_heads, d_v, H, W] buffer (may be on CPU).
) -> torch.Tensor: # [B, n_heads, d_v, H, W] (caller views as BCHW).
- """Neighborhood attention in pure torch via F.unfold + per-tile slicing.
-
- K and V are passed at LR resolution and upsampled (nearest-exact) per-tile only
- for the slice the unfold needs. Avoids the [B, n*d, H, W] HR allocations for K
- (512 MB) and V (2 GB) at tex_1024 fp16. Spatial tiling bounds the per-tile
- F.unfold blob; `v_chunk` further slices d_v so attn·V is computed in C-sized
- chunks (attn is reused, computed once from Q/K).
+ """natten.na2d on K/V nearest-upsampled to the HR grid, with the upsample factor as dilation.
+ All HR queries of one LR cell see the same Kh x Kw window of the zero-padded LR grid, so each cell is one small
+ attention of its queries against that window instead of one unfolded window per HR position.
"""
- B, H, W, n, d_qk = q.shape
- d_v = v_lr.shape[-1]
+ B, n, d_qk, H, W = q.shape
+ h, w = k.shape[-2:]
+ d_v = v.shape[1] // n
Kh, Kw = kernel_size
- Dh, Dw = dilation
- pad_h, pad_w = (Kh // 2) * Dh, (Kw // 2) * Dw
-
+ Dh, Dw = H // h, W // w
+ KK, DD = Kh * Kw, Dh * Dw
out = output if output is not None else torch.empty((B, n, d_v, H, W), device=q.device, dtype=q.dtype)
-
- th = min(tile, H) if tile else H
- tw = min(tile, W) if tile else W
- chunk = v_chunk if (v_chunk and v_chunk < d_v) else d_v
-
- for h0 in range(0, H, th):
- for w0 in range(0, W, tw):
- h1, w1 = min(h0 + th, H), min(w0 + tw, W)
- t_h, t_w = h1 - h0, w1 - w0
-
- # Padded HR region the unfold needs (kernel span = (K-1)*D + 1).
- h_src_start = max(0, h0 - pad_h)
- h_src_end = min(H, h1 + pad_h)
- w_src_start = max(0, w0 - pad_w)
- w_src_end = min(W, w1 + pad_w)
- pad_top = max(0, pad_h - h0)
- pad_bot = max(0, (h1 + pad_h) - H)
- pad_lft = max(0, pad_w - w0)
- pad_rgt = max(0, (w1 + pad_w) - W)
-
- # Upsample only the tile region from k_lr / v_lr.
- k_tile = upsample_lr_slice(k_lr, Dh, Dw,
- (h_src_start, h_src_end),
- (w_src_start, w_src_end))
- v_tile = upsample_lr_slice(v_lr, Dh, Dw,
- (h_src_start, h_src_end),
- (w_src_start, w_src_end))
- if pad_top or pad_bot or pad_lft or pad_rgt:
- k_tile = F.pad(k_tile, [pad_lft, pad_rgt, pad_top, pad_bot])
- v_tile = F.pad(v_tile, [pad_lft, pad_rgt, pad_top, pad_bot])
-
- # Q·K → attention weights (small: KK=81 per output position).
- KK = Kh * Kw
- k_w = F.unfold(k_tile, kernel_size=(Kh, Kw), dilation=(Dh, Dw), padding=0)
- k_w = k_w.view(B, n, d_qk, KK, t_h * t_w).permute(0, 1, 4, 3, 2) # [B, n, t, KK, d_qk]
- # q is [B, H, W, n, d_qk]; per-tile slice + permute -> [B, n, t_h*t_w, 1, d_qk].
- q_tile = q[:, h0:h1, w0:w1].permute(0, 3, 1, 2, 4).reshape(B, n, t_h * t_w, 1, d_qk)
- scores = torch.matmul(q_tile, k_w.transpose(-1, -2)) * scale
- attn = scores.softmax(dim=-1)
- del k_w, scores, q_tile, k_tile
-
- # attn · V, chunked over d_v.
- for c0 in range(0, d_v, chunk):
- c1 = min(c0 + chunk, d_v)
- v_w = F.unfold(v_tile[:, c0:c1], kernel_size=(Kh, Kw),dilation=(Dh, Dw), padding=0) # [B*n, (c1-c0)*KK, t]
- v_w = v_w.view(B, n, c1 - c0, KK, t_h * t_w).permute(0, 1, 4, 3, 2)
- out_chunk = torch.matmul(attn, v_w).squeeze(-2) # [B, n, t, c1-c0]
- out_chunk = out_chunk.view(B, n, t_h, t_w, c1 - c0).permute(0, 1, 4, 2, 3)
- out[:, :, c0:c1, h0:h1, w0:w1] = out_chunk
- del v_w, out_chunk
- del attn, v_tile
-
- return out # [B, n, d_v, H, W] — sole caller (CrossAttention) views it as BCHW directly.
+ pad = (Kw // 2, Kw // 2, Kh // 2, Kh // 2)
+ k, v = F.pad(k, pad), F.pad(v, pad)
+ # LR rows per chunk, from one row's live temporaries: K/V windows, Q, fp32 copies and scores, outputs
+ es = q.element_size()
+ row_bytes = B * n * w * (KK * (d_qk * (es + 4) + d_v * es) + DD * (d_qk * (es + 4) + KK * (8 + es) + 2 * d_v * es))
+ rows = max(1, _CHUNK_BYTES // row_bytes)
+ for r0 in range(0, h, rows):
+ r1 = min(r0 + rows, h)
+ cells = (r1 - r0) * w
+ k_w = F.unfold(k[:, :, r0:r1 + Kh - 1], (Kh, Kw)).view(B, n, d_qk, KK, cells).transpose(2, 4) # [B, n, cells, KK, d_qk]
+ v_w = F.unfold(v[:, :, r0:r1 + Kh - 1], (Kh, Kw)).view(B, n, d_v, KK, cells).transpose(2, 4) # [B, n, cells, KK, d_v]
+ q_c = q[..., r0 * Dh:r1 * Dh, :].reshape(B, n, d_qk, r1 - r0, Dh, w, Dw).permute(0, 1, 3, 5, 4, 6, 2).reshape(B, n, cells, DD, d_qk)
+ attn = (torch.matmul(q_c.float(), k_w.float().transpose(-1, -2)) * scale).softmax(dim=-1).to(v_w.dtype)
+ o = torch.matmul(attn, v_w).view(B, n, r1 - r0, w, Dh, Dw, d_v)
+ out[..., r0 * Dh:r1 * Dh, :] = o.permute(0, 1, 6, 2, 4, 3, 5).reshape(B, n, d_v, (r1 - r0) * Dh, W)
+ return out
class CrossAttention(nn.Module):
@@ -137,23 +69,12 @@ class CrossAttention(nn.Module):
self.kernel_size = kernel_size
self.scale = (dim // num_heads) ** -0.5
- @staticmethod
- def _split_heads_lr(x: torch.Tensor, num_heads: int) -> torch.Tensor:
- """[B, n*d, h, w] -> [B, h, w, n, d] at the input resolution (no upsample)."""
- B, C, H, W = x.shape
- return x.view(B, num_heads, C // num_heads, H, W).permute(0, 3, 4, 1, 2).contiguous()
-
def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
output=None) -> torch.Tensor:
- hq, wq = q.shape[-2:]
- hk, wk = k.shape[-2:]
- dilation = (hq // hk, wq // wk)
- B, C, _, _ = q.shape
- q = q.view(B, self.num_heads, C // self.num_heads, hq, wq).permute(0, 3, 4, 1, 2).contiguous()
- k_lr = self._split_heads_lr(k, self.num_heads).to(q.dtype)
- v_lr = self._split_heads_lr(v, self.num_heads).to(q.dtype)
- out_buf = output.view(B, self.num_heads, v.shape[1] // self.num_heads, hq, wq) if output is not None else None
- out = na2d_pure(q, k_lr, v_lr, self.kernel_size, dilation, self.scale, output=out_buf)
+ B, C, hq, wq = q.shape
+ n = self.num_heads
+ out_buf = output.view(B, n, v.shape[1] // n, hq, wq) if output is not None else None
+ out = na2d_pure(q.view(B, n, C // n, hq, wq), k.to(q.dtype), v.to(q.dtype), self.kernel_size, self.scale, output=out_buf)
return out.view(B, -1, hq, wq)