FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(rocm): clamp QSA KV tile to the device LDS budget on RDNA (#587)
Zhang Jason7d6490c5d6ba
diff --git a/python/freetoken/kernel/triton/qsa/attend.py b/python/freetoken/kernel/triton/qsa/attend.pyindex 541e27c6..4c55f820 100644--- a/python/freetoken/kernel/triton/qsa/attend.py+++ b/python/freetoken/kernel/triton/qsa/attend.py@@ -277,6 +277,18 @@ def qsa_sparse_paged_attention(else:block_n, target_splits, partial_warps = 64, 1, 2+ # These tiles were tuned for the GB300's 228KB LDS; AMD parts have less and it varies by arch+ # (e.g. 64KB/CU on RDNA gfx11/gfx12), so k+v staging can overflow. Query the device's actual+ # shared-memory limit rather than assuming a fixed size, and shrink the KV tile only when it+ # doesn't fit -- so a larger-LDS arch keeps the wide tile. ROCm-gated so the NVIDIA tiles stay+ # byte-identical (a small NV part could report a 48KB default and would otherwise clamp).+ if torch.version.hip is not None:+ _props = torch.cuda.get_device_properties(q.device)+ _lds_budget = getattr(_props, "shared_memory_per_block", 64 * 1024) or 64 * 1024+ _kv_bytes = k_cache.element_size()+ while block_n > 16 and 2 * block_n * q.shape[2] * _kv_bytes + 2048 > _lds_budget:+ block_n //= 2+num_tiles = triton.cdiv(logical_indices.shape[1], block_n)# Avoid empty splits when the selection width is smaller than the profile.max_useful_splits = 1 << (num_tiles.bit_length() - 1)