FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(rotary): preserve fp32 inputs in mrope torch fallbacks (#641)
cherry77-cloud507a8bfa81d5
diff --git a/python/freetoken/kernel/triton/rope.py b/python/freetoken/kernel/triton/rope.pyindex 4cd25c91..7e67a4c3 100644--- a/python/freetoken/kernel/triton/rope.py+++ b/python/freetoken/kernel/triton/rope.py@@ -263,8 +263,11 @@ def apply_mrope_torch_fallback(for t, heads in ((query, query.shape[1] // head_size), (key, key.shape[1] // head_size)):v = t.view(nnz, heads, head_size)lo, hi = v[..., :half].float(), v[..., half:rotary_dim].float()- v[..., :half] = (lo * cos - hi * sin).to(v.dtype)- v[..., half:rotary_dim] = (hi * cos + lo * sin).to(v.dtype)+ # Compute both halves before writing: float() aliases FP32 inputs.+ lo_new = lo * cos - hi * sin+ hi_new = hi * cos + lo * sin+ v[..., :half] = lo_new.to(v.dtype)+ v[..., half:rotary_dim] = hi_new.to(v.dtype)__all__ = ["apply_rope_with_cos_sin_cache_inplace", "apply_mrope_with_cos_sin_cache_inplace"]diff --git a/python/freetoken/layers/rotary.py b/python/freetoken/layers/rotary.pyindex d0178f3b..f2792ac1 100644--- a/python/freetoken/layers/rotary.py+++ b/python/freetoken/layers/rotary.py@@ -86,8 +86,11 @@ def _mrope_torch(for t, heads in ((query, query.shape[1] // head_size), (key, key.shape[1] // head_size)):v = t.view(nnz, heads, head_size)lo, hi = v[..., :half].float(), v[..., half:rotary_dim].float()- v[..., :half] = (lo * cos - hi * sin).to(v.dtype)- v[..., half:rotary_dim] = (hi * cos + lo * sin).to(v.dtype)+ # Compute both halves before writing: float() aliases FP32 inputs.+ lo_new = lo * cos - hi * sin+ hi_new = hi * cos + lo * sin+ v[..., :half] = lo_new.to(v.dtype)+ v[..., half:rotary_dim] = hi_new.to(v.dtype)MROPE_LAYOUTS = ("contiguous", "interleaved", "interleaved_glm")