PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: handle mantissa round overflow (#2302)
Xin He9478fcf8c8b2
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex f72c875ea..fbe025843 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -125,34 +125,20 @@ FLOAT8_UE5M3_MAX = 114688def float_to_e5m3_frexp(x: torch.Tensor) -> torch.Tensor:- x = torch.clamp(x, min=0.0)- e5m3 = torch.zeros_like(x, dtype=torch.uint8)-- mask = x > 0- x_masked = x[mask]-- # normal number: x >= 2^-14- normal_mask = x_masked >= 2**-14- x_normal = x_masked[normal_mask]- mantissa, exponent = torch.frexp(x_normal)-- m3 = torch.clamp(torch.round((mantissa - 0.5) * 16), 0, 7).to(torch.uint8)- e5 = torch.clamp(exponent + 14, 0, 31).to(torch.uint8) # 0 reserved for subnormal, 31 reserved for NaN-- e5m3_vals = ((e5 << 3) | m3).to(torch.uint8)-- # sumnorm:0 < x < 2^-14- subnormal_mask = ~normal_mask- x_subnormal = x_masked[subnormal_mask]- m_sub = torch.clamp(torch.round(x_subnormal / (2**-14) * 8), 1, 7).to(torch.uint8) # exponent = 0- e5m3_sub = m_sub # top 5 bits = 0-- out_vals = torch.zeros_like(x_masked, dtype=torch.uint8)- out_vals[normal_mask] = e5m3_vals- out_vals[subnormal_mask] = e5m3_sub-- e5m3[mask] = out_vals- return e5m3+ input_fp32 = x.to(torch.float32)+ finite = torch.nan_to_num(input_fp32, nan=0.0, posinf=FLOAT8_UE5M3_MAX, neginf=0.0).clamp_(0.0, FLOAT8_UE5M3_MAX)++ mantissa, exponent = torch.frexp(finite.clamp_min(2**-14))+ m3 = torch.round((mantissa - 0.5) * 16).to(torch.int32)+ carry = m3 == 8+ m3 = torch.where(carry, 0, m3)+ e5 = exponent + 14 + carry.to(exponent.dtype)+ normal = (e5 << 3) | m3++ # RNE may underflow to zero or carry from the largest subnormal to 0x08.+ subnormal = torch.round(finite * 2**17).to(torch.int32)+ encoded = torch.where(finite < 2**-14, subnormal, normal).clamp_(0, 0xFE).to(torch.uint8)+ return torch.where(torch.isnan(input_fp32), 0xFF, encoded)def e5m3_to_float_tensor(e5m3: torch.Tensor) -> torch.Tensor:diff --git a/auto_round_extension/cuda/cute_nvfp4_e5m3.py b/auto_round_extension/cuda/cute_nvfp4_e5m3.pyindex b71ae4614..4be4769e2 100644--- a/auto_round_extension/cuda/cute_nvfp4_e5m3.py+++ b/auto_round_extension/cuda/cute_nvfp4_e5m3.py@@ -70,6 +70,9 @@ def _make_qdq_kernel():exponent = math.floor(math.log2(scale)) + cutlass.Float32(1.0)mantissa = scale / math.exp2(exponent)mantissa_bits = math.roundeven((mantissa - cutlass.Float32(0.5)) * cutlass.Float32(16.0))+ if mantissa_bits == cutlass.Float32(8.0):+ mantissa_bits = cutlass.Float32(0.0)+ exponent = exponent + cutlass.Float32(1.0)mantissa_bits = cute.arch.fmin(cute.arch.fmax(mantissa_bits, 0.0), 7.0)scale = (cutlass.Float32(1.0) + mantissa_bits / cutlass.Float32(8.0)) * math.exp2(exponent - cutlass.Float32(1.0)diff --git a/test/unit/common/data_type/test_nvfp.py b/test/unit/common/data_type/test_nvfp.pyindex 930dd0a28..a70d1e3ab 100644--- a/test/unit/common/data_type/test_nvfp.py+++ b/test/unit/common/data_type/test_nvfp.py@@ -454,6 +454,35 @@ class TestFloatToE5m3Frexp:out = float_to_e5m3_frexp(x)assert (out == 0).all()+ def test_mantissa_round_overflow_carries_to_exponent(self):+ # mantissa=0.96875 yields ((m-0.5)*16)=7.5 -> RNE to 8, which must carry.+ x = torch.tensor([1.9375], dtype=torch.float32) # frexp -> (0.96875, 1)+ out = float_to_e5m3_frexp(x)+ decoded = e5m3_to_float_tensor(out)+ assert out.item() == 0x80 # (exp=16, mantissa=0)+ assert decoded.item() == pytest.approx(2.0, rel=1e-6)++ def test_rne_boundaries_and_saturation(self):+ min_subnormal = 2**-17+ max_subnormal = 7 * min_subnormal+ min_normal = 2**-14+ x = torch.tensor(+ [+ min_subnormal / 2,+ torch.nextafter(torch.tensor((max_subnormal + min_normal) / 2), torch.tensor(float("inf"))).item(),+ min_normal,+ FLOAT8_UE5M3_MAX,+ float("inf"),+ float("nan"),+ ],+ dtype=torch.float32,+ )++ assert torch.equal(+ float_to_e5m3_frexp(x),+ torch.tensor([0x00, 0x08, 0x08, 0xFE, 0xFE, 0xFF], dtype=torch.uint8),+ )+class TestE5m3ToFloatTensor:"""Test e5m3_to_float_tensor."""diff --git a/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.py b/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.pyindex e6a9e2b89..ced441ab1 100644--- a/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.py+++ b/test/unit/test_cuda/quantization/test_nvfp4_e5m3_cute.py@@ -49,11 +49,25 @@ def test_cute_nvfp4_e5m3_weight_dq_matches_reference_for_multiple_groups(dtype):def test_nvfp4_v2_uses_direct_scale_at_rounding_boundary():- activation = torch.tensor([[1.25] + [1.0] * 15], dtype=torch.float32)+ activation = torch.tensor([[11.625] + [1.0] * 15], dtype=torch.float32)actual, scale, _ = nvfp4_v2(activation, bits=4, group_size=16)expected = cast_to_fp4(activation / scale) * scale+ assert scale.item() == 2.0+ torch.testing.assert_close(actual, expected)++[email protected](not torch.cuda.is_available(), reason="CUDA is required")+def test_cute_nvfp4_v2_qdq_carries_mantissa_round_overflow():+ activation = torch.tensor([[11.625] + [1.0] * 15], device="cuda", dtype=torch.float32)+ if not can_use_cute_nvfp4_v2_qdq(activation, 16):+ pytest.skip("CuTe DSL is not available")++ expected, _, _ = nvfp4_v2(activation, bits=4, group_size=16)+ actual = try_cute_nvfp4_v2_qdq(activation, 16)++ assert actual is not Nonetorch.testing.assert_close(actual, expected)