FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Static window / compressed top-k index builders."""
from __future__ import annotations
import torch
import torch.nn.functional as F
def get_window_topk_idxs(window_size, bsz, seqlen, start_pos):
if start_pos >= window_size - 1:
start_pos %= window_size
matrix = torch.cat([torch.arange(start_pos + 1, window_size), torch.arange(0, start_pos + 1)], dim=0)
elif start_pos > 0:
matrix = F.pad(torch.arange(start_pos + 1), (0, window_size - start_pos - 1), value=-1)
else:
base = torch.arange(seqlen).unsqueeze(1)
matrix = (base - window_size + 1).clamp(0) + torch.arange(min(seqlen, window_size))
matrix = torch.where(matrix > base, -1, matrix)
return matrix.unsqueeze(0).expand(bsz, -1, -1)
def get_compress_topk_idxs(ratio, bsz, seqlen, start_pos, offset):
if start_pos > 0:
matrix = torch.arange(0, (start_pos + 1) // ratio) + offset
else:
matrix = torch.arange(seqlen // ratio).repeat(seqlen, 1)
mask = matrix >= torch.arange(1, seqlen + 1).unsqueeze(1) // ratio
matrix = torch.where(mask, -1, matrix + offset)
return matrix.unsqueeze(0).expand(bsz, -1, -1)