Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(rotary): preserve fp32 inputs in mrope torch fallbacks (#641)

cherry77-cloud507a8bfa81d5
diff --git a/python/freetoken/kernel/triton/rope.py b/python/freetoken/kernel/triton/rope.py
index 4cd25c91..7e67a4c3 100644
--- a/python/freetoken/kernel/triton/rope.py
+++ b/python/freetoken/kernel/triton/rope.py
@@ -263,8 +263,11 @@ def apply_mrope_torch_fallback(
for t, heads in ((query, query.shape[1] // head_size), (key, key.shape[1] // head_size)):
v = t.view(nnz, heads, head_size)
lo, hi = v[..., :half].float(), v[..., half:rotary_dim].float()
- v[..., :half] = (lo * cos - hi * sin).to(v.dtype)
- v[..., half:rotary_dim] = (hi * cos + lo * sin).to(v.dtype)
+ # Compute both halves before writing: float() aliases FP32 inputs.
+ lo_new = lo * cos - hi * sin
+ hi_new = hi * cos + lo * sin
+ v[..., :half] = lo_new.to(v.dtype)
+ v[..., half:rotary_dim] = hi_new.to(v.dtype)
__all__ = ["apply_rope_with_cos_sin_cache_inplace", "apply_mrope_with_cos_sin_cache_inplace"]
diff --git a/python/freetoken/layers/rotary.py b/python/freetoken/layers/rotary.py
index d0178f3b..f2792ac1 100644
--- a/python/freetoken/layers/rotary.py
+++ b/python/freetoken/layers/rotary.py
@@ -86,8 +86,11 @@ def _mrope_torch(
for t, heads in ((query, query.shape[1] // head_size), (key, key.shape[1] // head_size)):
v = t.view(nnz, heads, head_size)
lo, hi = v[..., :half].float(), v[..., half:rotary_dim].float()
- v[..., :half] = (lo * cos - hi * sin).to(v.dtype)
- v[..., half:rotary_dim] = (hi * cos + lo * sin).to(v.dtype)
+ # Compute both halves before writing: float() aliases FP32 inputs.
+ lo_new = lo * cos - hi * sin
+ hi_new = hi * cos + lo * sin
+ v[..., :half] = lo_new.to(v.dtype)
+ v[..., half:rotary_dim] = hi_new.to(v.dtype)
MROPE_LAYOUTS = ("contiguous", "interleaved", "interleaved_glm")