FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
perf(attention): copy triton metadata without blocking the host (#726)
Xiaoze Fanbc0e2feb7cf0
diff --git a/python/freetoken/attention/triton.py b/python/freetoken/attention/triton.pyindex 8bf9792d..30bedf4f 100644--- a/python/freetoken/attention/triton.py+++ b/python/freetoken/attention/triton.py@@ -230,17 +230,16 @@ class TritonAttentionBackend(BaseAttnBackend):cached_lens = [req.cached_len for req in reqs]num_query_tokens = sum(seqlens_q)is_decode = max(seqlens_q) == 1- prefix_lens = torch.tensor(cached_lens, dtype=torch.int32, device=device)+ CPU_KWARGS = {"device": "cpu", "dtype": torch.int32, "pin_memory": device.type == "cuda"}+ prefix_lens = torch.tensor(cached_lens, **CPU_KWARGS).to(device, non_blocking=True)- indptr = torch.tensor([0] + seqlens_k, dtype=torch.int32, device=device).cumsum_(0)+ indptr = torch.tensor([0] + seqlens_k, **CPU_KWARGS).cumsum_(0).to(device, non_blocking=True)if is_decode:cu_seqlens_q_gpu = torch.arange(0, padded_size + 1, device=device, dtype=torch.int32)elif all(l == 0 for l in cached_lens):cu_seqlens_q_gpu = indptrelse:- cu_seqlens_q_gpu = torch.tensor(- [0] + seqlens_q, dtype=torch.int32, device=device- ).cumsum_(0)+ cu_seqlens_q_gpu = torch.tensor([0] + seqlens_q, **CPU_KWARGS).cumsum_(0).to(device, non_blocking=True)indices = torch.cat([page_table[req.table_idx, : req.device_len] for req in reqs])swa_indices = Noneif getattr(self.kvcache, "swa_paged", False):