Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Two-GPU decode RoPE test: check the copy bitwise and the rounding at 2 ulp (#14752)

Daniel Han6394a86ca804
diff --git a/tests/test_decode_fused_rope.py b/tests/test_decode_fused_rope.py
index fff5ea78ab1..26cee4576ad 100644
--- a/tests/test_decode_fused_rope.py
+++ b/tests/test_decode_fused_rope.py
@@ -239,9 +239,18 @@ def test_decode_rope_on_another_device_orders_with_the_copy():
for _ in range(3):
q = 3 * torch.randn(4, 32, 1, 128, device = "cuda:1", dtype = torch.bfloat16)
k = 3 * torch.randn(4, 8, 1, 128, device = "cuda:1", dtype = torch.bfloat16)
+ # Same kernel, same inputs, rotated with its own device current: bitwise what the copy must
+ # see (a racing copy sees unrotated values). Fused rounds once and the unfused ops several
+ # times, so versus unfused only the 2 ulp bound of test_decode_rope_inplace_matches_unfused
+ # holds (0.0625 differences at |x| >= 8 fail a fixed atol = 0.05).
+ qr, kr = q.clone(), k.clone()
+ with torch.cuda.device(1):
+ assert RE.decode_rope_inplace(qr, kr, cos, sin, p)
qu, ku = _unfused_rope(q.clone(), k.clone(), cos, sin, p)
with torch.cuda.device(0):
assert RE.decode_rope_inplace(q, k, cos, sin, p)
q0, k0 = q.to("cuda:0"), k.to("cuda:0")
- assert torch.allclose(q0.cpu().float(), qu.cpu().float(), atol = 0.05)
- assert torch.allclose(k0.cpu().float(), ku.cpu().float(), atol = 0.05)
+ assert torch.equal(q0.cpu(), qr.cpu()) and torch.equal(k0.cpu(), kr.cpu())
+ ulp = torch.finfo(q.dtype).eps * max(qu.abs().max().item(), ku.abs().max().item())
+ assert (q0.cpu().float() - qu.cpu().float()).abs().max().item() <= 2 * ulp
+ assert (k0.cpu().float() - ku.cpu().float()).abs().max().item() <= 2 * ulp
diff --git a/tests/test_fp8_dequant_save_policy.py b/tests/test_fp8_dequant_save_policy.py
index 6ffa659f1ad..f4d071824da 100644
--- a/tests/test_fp8_dequant_save_policy.py
+++ b/tests/test_fp8_dequant_save_policy.py
@@ -35,7 +35,9 @@ def _restore_fp8_integration(monkeypatch):
if not name.startswith("__"):
monkeypatch.setattr(cls, name, value)
if hasattr(fp8_integration, "replace_with_fp8_linear"):
- monkeypatch.setattr(fp8_integration, "replace_with_fp8_linear", fp8_integration.replace_with_fp8_linear)
+ monkeypatch.setattr(
+ fp8_integration, "replace_with_fp8_linear", fp8_integration.replace_with_fp8_linear
+ )
def _helper():