PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Add MXFP8 MOE/Linear and MXFP4 Linear (#1034)
Yi Liu61bcfdb68513
diff --git a/auto_round_extension/vllm_ext/README.md b/auto_round_extension/vllm_ext/README.mdnew file mode 100644index 000000000..1f0cb832f--- /dev/null+++ b/auto_round_extension/vllm_ext/README.md@@ -0,0 +1,12 @@+- Build and Install vLLM++```+git clone --branch fused-moe-ar https://github.com/yiliu30/vllm-fork.git+VLLM_USE_PRECOMPILED=1 pip install --editable . -vvv+```+++- Enable vLLM-Ext at Runtime+```bash+VLLM_ENABLE_AR_EXT=1 vllm serve ...+```\ No newline at end of filediff --git a/auto_round_extension/vllm_ext/__init__.py b/auto_round_extension/vllm_ext/__init__.pyindex 3e145334f..748e917e2 100644--- a/auto_round_extension/vllm_ext/__init__.py+++ b/auto_round_extension/vllm_ext/__init__.py@@ -18,9 +18,9 @@def apply():- import vllm.model_executor.layers.quantization.auto_round as auto_round_module+ import auto_round_extension.vllm_ext.auto_round_ext+ import auto_round_extension.vllm_ext.envs_ext- from auto_round_extension.vllm_ext.auto_round_ext import AutoRoundExtensionConfig-- auto_round_module.AutoRoundConfig = AutoRoundExtensionConfig- from auto_round_extension.vllm_ext.envs_ext import extra_environment_variables+ print("*****************************************************************************")+ print("* !!! VLLM_ENABLE_AR_EXT is set to 1, applying auto_round_vllm_extension *")+ print("*****************************************************************************")diff --git a/auto_round_extension/vllm_ext/apply_ext.sh b/auto_round_extension/vllm_ext/apply_ext.shdeleted file mode 100644index b327ec22c..000000000--- a/auto_round_extension/vllm_ext/apply_ext.sh+++ /dev/null@@ -1,46 +0,0 @@-#!/bin/bash--# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--# Define the relative path for the `auto-round` installation-AUTO_ROUND_PATH="auto_round/../auto_round_extension/vllm_ext/sitecustomize.py"--# Try to find the pip installation location-PIP_LOCATION=$(pip show auto-round 2>/dev/null | grep "Location:" | awk '{print $2}')--if [ -n "$PIP_LOCATION" ]; then- SITE_CUSTOMIZE_PATH="$PIP_LOCATION/$AUTO_ROUND_PATH"- echo "Checking for sitecustomize.py at: $SITE_CUSTOMIZE_PATH"-- if [ -f "$SITE_CUSTOMIZE_PATH" ]; then- echo "Found sitecustomize.py at: $SITE_CUSTOMIZE_PATH"- export PYTHONPATH=$(dirname "$SITE_CUSTOMIZE_PATH"):$PYTHONPATH- echo "PYTHONPATH set to: $PYTHONPATH"- return 0 2>/dev/null || true- fi-fi--# Fallback: check current directory-LOCAL_SITE_CUSTOMIZE="./sitecustomize.py"-if [ -f "$LOCAL_SITE_CUSTOMIZE" ]; then- echo "Found sitecustomize.py at current directory."- export PYTHONPATH=$(pwd):$PYTHONPATH- echo "PYTHONPATH set to: $PYTHONPATH"- return 0 2>/dev/null || true-fi--echo "Warning: sitecustomize.py not found in pip installation or current directory."-# Do not exit the shell-return 1 2>/dev/null || true\ No newline at end of filediff --git a/auto_round_extension/vllm_ext/auto_round_ext.py b/auto_round_extension/vllm_ext/auto_round_ext.pyindex cce2c912d..d665fd568 100644--- a/auto_round_extension/vllm_ext/auto_round_ext.py+++ b/auto_round_extension/vllm_ext/auto_round_ext.py@@ -18,17 +18,18 @@ import torchfrom vllm.logger import init_loggerfrom vllm.model_executor.layers.fused_moe import FusedMoEfrom vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod-from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig+from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig as _BaseAutoRoundConfigfrom auto_round.schemes import QuantizationScheme+from auto_round_extension.vllm_ext.quant_method_linear import AutoRoundQuantLinearMethodfrom auto_round_extension.vllm_ext.quant_method_moe import AutoRoundMoEMethodlogger = init_logger(__name__)-class AutoRoundExtensionConfig(AutoRoundConfig):- SUPPORTED_DTYPES = AutoRoundConfig.SUPPORTED_DTYPES.union({"mx_fp"})- SUPPORTED_FORMATS = AutoRoundConfig.SUPPORTED_FORMATS.union({"auto_round:llm_compressor"})+class AutoRoundExtensionConfig(_BaseAutoRoundConfig):+ SUPPORTED_DTYPES = _BaseAutoRoundConfig.SUPPORTED_DTYPES.union({"mx_fp"})+ SUPPORTED_FORMATS = _BaseAutoRoundConfig.SUPPORTED_FORMATS.union({"auto_round:llm_compressor"})def get_quant_method(self, layer: torch.nn.Module, prefix: str):# FIXME: (yi) make it compatible with `AutoRoundConfig`@@ -36,7 +37,7 @@ class AutoRoundExtensionConfig(AutoRoundConfig):quant_method = AutoRoundMoEMethod.get_moe_method(self, layer, prefix)return quant_methodelif isinstance(layer, LinearBase):- return UnquantizedLinearMethod()+ return AutoRoundQuantLinearMethod.get_method(self, layer, prefix)else:return None@@ -48,7 +49,7 @@ class AutoRoundExtensionConfig(AutoRoundConfig):return quant_scheme@classmethod- def from_config(cls, config: dict[str, Any]) -> AutoRoundConfig:+ def from_config(cls, config: dict[str, Any]) -> _BaseAutoRoundConfig:ar_config = super().from_config(config)# TODO: (yi) refine below implementationquant_scheme = AutoRoundExtensionConfig._parse_quant_scheme(config)@@ -61,3 +62,9 @@ class AutoRoundExtensionConfig(AutoRoundConfig):ar_config.quant_scheme = quant_schemear_config.layer_schemes = layer_schemesreturn ar_config+++# Patch vLLM’s AutoRoundConfig at import time+import vllm.model_executor.layers.quantization.auto_round as _auto_round_module++_auto_round_module.AutoRoundConfig = AutoRoundExtensionConfigdiff --git a/auto_round_extension/vllm_ext/envs_ext.py b/auto_round_extension/vllm_ext/envs_ext.pyindex 325b9de7c..70fbb3238 100644--- a/auto_round_extension/vllm_ext/envs_ext.py+++ b/auto_round_extension/vllm_ext/envs_ext.py@@ -21,9 +21,11 @@ logger = init_logger(__name__)# Define extra environment variablesextra_environment_variables: dict[str, Callable[[], Any]] = {- "VLLM_MXFP4_PRE_UNPACK_WEIGHTS": lambda: os.getenv("VLLM_MXFP4_PRE_UNPACK_WEIGHTS", "1") in ("1", "true", "True"),- "VLLM_ENABLE_STATIC_MOE": lambda: os.getenv("VLLM_ENABLE_STATIC_MOE", "1") in ("1", "true", "True"),- "VLLM_AR_MXFP4_MODULAR_MOE": lambda: os.getenv("VLLM_AR_MXFP4_MODULAR_MOE", "0") in ("1", "true", "True"),+ "VLLM_MXFP4_PRE_UNPACK_WEIGHTS": lambda: os.getenv("VLLM_MXFP4_PRE_UNPACK_WEIGHTS", "0") in ("1", "true", "True"),+ "VLLM_MXFP4_PRE_UNPACK_TO_FP8": lambda: os.getenv("VLLM_MXFP4_PRE_UNPACK_TO_FP8", "1") in ("1", "true", "True"),+ "VLLM_ENABLE_STATIC_MOE": lambda: os.getenv("VLLM_ENABLE_STATIC_MOE", "0") in ("1", "true", "True"),+ "VLLM_AR_MXFP4_MODULAR_MOE": lambda: os.getenv("VLLM_AR_MXFP4_MODULAR_MOE", "1") in ("1", "true", "True"),+ "VLLM_AR_POST_PROCESS_GPTOSS": lambda: os.getenv("VLLM_AR_POST_PROCESS_GPTOSS", "0") in ("1", "true", "True"),}# Add the extra environment variables to vllm.envsimport vllm.envs as envsdiff --git a/auto_round_extension/vllm_ext/fp4_utils.py b/auto_round_extension/vllm_ext/fp4_utils.pyindex adb67d190..fb7b6e935 100644--- a/auto_round_extension/vllm_ext/fp4_utils.py+++ b/auto_round_extension/vllm_ext/fp4_utils.py@@ -51,7 +51,7 @@ def pack_fp4_to_uint8(x: torch.Tensor) -> torch.Tensor:indices = indices.reshape(-1)# Handle odd length by padding if necessary- assert indices.numel() % 2 != 0, f"Expected even number of elements, got {indices.numel()}"+ # assert indices.numel() % 2 != 0, f"Expected even number of elements, got {indices.numel()}"# Reshape to pair consecutive elementsindices = indices.reshape(-1, 2)diff --git a/auto_round_extension/vllm_ext/linear_impl_mxfp4.py b/auto_round_extension/vllm_ext/linear_impl_mxfp4.pynew file mode 100644index 000000000..04d5e20f8--- /dev/null+++ b/auto_round_extension/vllm_ext/linear_impl_mxfp4.py@@ -0,0 +1,129 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# SPDX-License-Identifier: Apache-2.0+from typing import Callable, Optional++import torch+import vllm.envs as envs+from torch.nn.parameter import Parameter+from vllm.logger import init_logger+from vllm.model_executor.parameter import GroupQuantScaleParameter, ModelWeightParameter, PerTensorScaleParameter+from vllm.platforms import current_platform++from auto_round_extension.vllm_ext.mxfp4_qdq_utils import (+ dequant_mxfp4_to_fp8,+ mxfp4_gemm_with_unpacked_weight,+ run_mxfp4_emulations,+)++logger = init_logger(__name__)++__all__ = ["AutoRoundMXFP4LinearImpl"]++from auto_round_extension.vllm_ext.quant_impl import AutoRoundQuantImpl+++class AutoRoundMXFP4LinearImpl(AutoRoundQuantImpl):+ def __init__(self, quant_scheme):+ self.quant_scheme = quant_scheme+ self.group_size = 32++ @classmethod+ def get_min_capability(cls) -> int:+ if envs.VLLM_USE_MXFP4_CT_EMULATIONS:+ return 80+ return 100++ def create_weights(+ self,+ layer: torch.nn.Module,+ output_partition_sizes: list[int],+ input_size_per_partition: int,+ params_dtype: torch.dtype,+ weight_loader: Callable,+ **kwargs,+ ):+ output_size_per_partition = sum(output_partition_sizes)+ layer.logical_widths = output_partition_sizes+ layer.input_size_per_partition = input_size_per_partition+ layer.output_size_per_partition = output_size_per_partition++ # Weight+ weight = ModelWeightParameter(+ data=torch.empty(sum(output_partition_sizes), input_size_per_partition // 2, dtype=torch.uint8),+ input_dim=1,+ output_dim=0,+ weight_loader=weight_loader,+ )+ layer.register_parameter("weight_packed", weight)++ # Per Group Weight Scale+ weight_scale = GroupQuantScaleParameter(+ data=torch.empty(+ sum(output_partition_sizes),+ input_size_per_partition // self.group_size,+ # dtype=torch.uint8,+ dtype=torch.uint8,+ ),+ input_dim=1,+ output_dim=0,+ weight_loader=weight_loader,+ )++ layer.register_parameter("weight_scale", weight_scale)++ def process_weights_after_loading(self, layer) -> None:+ # FIXME: may dequant to bf16+ if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:++ weight_fp8, scale_bf16 = dequant_mxfp4_to_fp8(+ data_lp=layer.weight_packed,+ scale_e8m0=layer.weight_scale,+ )+ del layer.weight_packed+ del layer.weight_scale+ layer.weight_packed = None+ layer.weight_scale = None+ layer.register_parameter(+ "weight_unpacked_fp8",+ torch.nn.Parameter(+ weight_fp8,+ requires_grad=False,+ ),+ )+ layer.register_parameter(+ "weight_scale_bf16",+ torch.nn.Parameter(+ scale_bf16,+ requires_grad=False,+ ),+ )++ def apply_weights(+ self, layer: torch.nn.Module, x: torch.Tensor, bias: Optional[torch.Tensor] = None+ ) -> torch.Tensor:+ if not envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:+ out = run_mxfp4_emulations(x=x, weight=layer.weight_packed, weight_scale=layer.weight_scale)+ if bias is not None:+ out = out + bias+ return out+ else:+ out = mxfp4_gemm_with_unpacked_weight(+ x=x,+ weight_fp8=layer.weight_unpacked_fp8,+ weight_scale_bf16=layer.weight_scale_bf16,+ bias=bias,+ )+ return outdiff --git a/auto_round_extension/vllm_ext/linear_impl_mxfp8.py b/auto_round_extension/vllm_ext/linear_impl_mxfp8.pynew file mode 100644index 000000000..cf1382054--- /dev/null+++ b/auto_round_extension/vllm_ext/linear_impl_mxfp8.py@@ -0,0 +1,112 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from typing import Callable, Optional++import torch+import vllm.envs as envs+from vllm.model_executor.parameter import (+ GroupQuantScaleParameter,+ ModelWeightParameter,+ PerTensorScaleParameter,+)++from auto_round_extension.vllm_ext.mxfp8_qdq_utils import dequant_mx_fp8, quant_mx_fp8+from auto_round_extension.vllm_ext.quant_impl import AutoRoundQuantImpl+++class AutoRoundMXFP8LinearImpl(AutoRoundQuantImpl):+ def __init__(self, quant_scheme):+ self.quant_scheme = quant_scheme+ self.strategy = "TENSOR_GROUP"+ self.out_dtype = torch.get_default_dtype()+ self.group_size = 32++ @classmethod+ def get_min_capability(cls) -> int:+ return 80++ def process_weights_after_loading(self, layer) -> None:+ return++ def create_weights(+ self,+ layer: torch.nn.Module,+ output_partition_sizes: list[int],+ input_size_per_partition: int,+ params_dtype: torch.dtype,+ weight_loader: Callable,+ **kwargs,+ ):+ # maybe_create_device_identity()++ output_size_per_partition = sum(output_partition_sizes)+ layer.logical_widths = output_partition_sizes++ # WEIGHT+ weight = ModelWeightParameter(+ data=torch.empty(+ output_size_per_partition,+ input_size_per_partition,+ dtype=torch.float8_e4m3fn,+ ),+ input_dim=1,+ output_dim=0,+ weight_loader=weight_loader,+ )+ layer.register_parameter("weight", weight)++ # WEIGHT SCALE+ # Per Group Weight Scale+ weight_scale = GroupQuantScaleParameter(+ data=torch.empty(+ sum(output_partition_sizes),+ input_size_per_partition // self.group_size,+ dtype=torch.uint8, # E8M0 for MXFP8 scale+ ),+ input_dim=1,+ output_dim=0,+ weight_loader=weight_loader,+ )+ layer.register_parameter("weight_scale", weight_scale)++ def apply_weights(+ self,+ layer: torch.nn.Module,+ x: torch.Tensor,+ bias: Optional[torch.Tensor] = None,+ ) -> torch.Tensor:+ # dequant weight+ weight = layer.weight+ weight_scale = layer.weight_scale+ dequnat_weight = dequant_mx_fp8(+ weight_fp8=weight.data,+ scale_e8m0=weight_scale.data,+ block_size=self.group_size,+ target_dtype=x.dtype,+ )+ dequnat_weight = dequnat_weight.to(x.dtype)+ # if not envs.VLLM_AR_MXFP8_DISABLE_INPUT_QDQ:+ # q-dq input+ x_scale, x_quant = quant_mx_fp8(x)+ dequant_x = dequant_mx_fp8(+ weight_fp8=x_quant,+ scale_e8m0=x_scale,+ block_size=self.group_size,+ target_dtype=x.dtype,+ )+ x = dequant_x.to(x.dtype)++ out = x @ dequnat_weight.t()+ return out.to(x.dtype) + (bias if bias is not None else 0)diff --git a/auto_round_extension/vllm_ext/moe_impl_mxfp4.py b/auto_round_extension/vllm_ext/moe_impl_mxfp4.pyindex 77ec6570d..0ff5241ab 100644--- a/auto_round_extension/vllm_ext/moe_impl_mxfp4.py+++ b/auto_round_extension/vllm_ext/moe_impl_mxfp4.py@@ -161,6 +161,33 @@ class AutoRoundMoEMethodMXFp4Impl(AutoRoundMoEMethod):ocp_mx_moe_quant_config,)+ if envs.VLLM_MXFP4_PRE_UNPACK_TO_FP8:+ self.input_dtype = "mxfp8_e4m3"+ self.weight_dtype = "mxfp8_e4m3"+ ocp_config = ocp_mx_moe_quant_config(+ quant_dtype=self.input_dtype,+ weight_dtype=self.weight_dtype,+ w1_scale=layer.w13_weight_scale,+ w2_scale=layer.w2_weight_scale,+ a1_scale=None,+ a2_scale=None,+ w1_bias=layer.w13_bias if self.has_bias else None,+ w2_bias=layer.w2_bias if self.has_bias else None,+ block_shape=None,+ )+ from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (+ OCP_MX_DTYPES,+ OCP_MX_Scheme,+ )++ ocp_config.ocp_mx_scheme = OCP_MX_Scheme.from_quant_dtype(+ input_dtype="mxfp8_e4m3",+ weight_dtype="mxfp8_e4m3",+ original_dtype="mxfp4",+ )+ logger.warning_once(f"Set OCP_MX_Scheme to {ocp_config.ocp_mx_scheme}")+ return ocp_config+self.input_dtype = "mxfp4"self.weight_dtype = "mxfp4"return ocp_mx_moe_quant_config(@@ -176,52 +203,59 @@ class AutoRoundMoEMethodMXFp4Impl(AutoRoundMoEMethod):)return None+ def _dequant_fp4_to_fp8(self, layer):+ weight_name_lst = ["w13_weight", "w2_weight"]++ for weight_name_prefix in weight_name_lst:+ weight_name = f"{weight_name_prefix}_packed"+ weight = getattr(layer, weight_name)+ weight_scale_name = f"{weight_name_prefix}_scale"+ weight_scale = getattr(layer, weight_scale_name)+ new_weight_name = f"{weight_name_prefix}_unpacked"+ new_scale_name = weight_scale_name+ num_experts, _, _ = weight.shape+ unpacked_weight_lst = []+ scale_list = []+ for expert_index in range(num_experts):+ weight_fp8, scale_bf16 = dequant_mxfp4_to_fp8(+ data_lp=weight[expert_index],+ scale_e8m0=weight_scale[expert_index],+ )++ unpacked_weight_lst.append(weight_fp8)+ scale_list.append(scale_bf16)+ unpacked_weight_fp8 = torch.stack(unpacked_weight_lst, dim=0)+ scale_bf16 = torch.stack(scale_list, dim=0)+ assert unpacked_weight_fp8.shape[0] == num_experts, (+ f"Expected {num_experts} unpacked weights, got " f"{unpacked_weight_fp8.shape[0]}"+ )+ delattr(layer, weight_name)+ delattr(layer, weight_scale_name)+ layer.register_parameter(+ new_weight_name,+ torch.nn.Parameter(+ unpacked_weight_fp8,+ requires_grad=False,+ ),+ )+ layer.register_parameter(+ new_scale_name,+ torch.nn.Parameter(+ scale_bf16,+ requires_grad=False,+ ),+ )+def process_weights_after_loading(self, layer: torch.nn.Module) -> None:+ logger.debug(f"Processing weights after loading for layer: {layer._prefix}")if envs.VLLM_ENABLE_STATIC_MOE:if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:- weight_name_lst = ["w13_weight", "w2_weight"]-- for weight_name_prefix in weight_name_lst:- weight_name = f"{weight_name_prefix}_packed"- weight = getattr(layer, weight_name)- weight_scale_name = f"{weight_name_prefix}_scale"- weight_scale = getattr(layer, weight_scale_name)- new_weight_name = f"{weight_name_prefix}_unpacked"- new_scale_name = weight_scale_name- num_experts, _, _ = weight.shape- unpacked_weight_lst = []- scale_list = []- for expert_index in range(num_experts):- weight_fp8, scale_bf16 = dequant_mxfp4_to_fp8(- data_lp=weight[expert_index],- scale_e8m0=weight_scale[expert_index],- )-- unpacked_weight_lst.append(weight_fp8)- scale_list.append(scale_bf16)- unpacked_weight_fp8 = torch.stack(unpacked_weight_lst, dim=0)- scale_bf16 = torch.stack(scale_list, dim=0)- assert unpacked_weight_fp8.shape[0] == num_experts, (- f"Expected {num_experts} unpacked weights, got " f"{unpacked_weight_fp8.shape[0]}"- )- delattr(layer, weight_name)- delattr(layer, weight_scale_name)- layer.register_parameter(- new_weight_name,- torch.nn.Parameter(- unpacked_weight_fp8,- requires_grad=False,- ),- )- layer.register_parameter(- new_scale_name,- torch.nn.Parameter(- scale_bf16,- requires_grad=False,- ),- )-+ self._dequant_fp4_to_fp8(layer)+ returnelif envs.VLLM_AR_MXFP4_MODULAR_MOE:+ if envs.VLLM_MXFP4_PRE_UNPACK_TO_FP8:+ self._dequant_fp4_to_fp8(layer)+ returndef revert_interleaved_bias(bias):"""@@ -250,8 +284,10 @@ class AutoRoundMoEMethodMXFp4Impl(AutoRoundMoEMethod):return revert_bias# breakpoint()- w13_bias_swapped = revert_interleaved_bias(layer.w13_bias)- layer.w13_bias.data.copy_(w13_bias_swapped)+ if self.has_bias:+ if envs.VLLM_AR_POST_PROCESS_GPTOSS:+ w13_bias_swapped = revert_interleaved_bias(layer.w13_bias)+ layer.w13_bias.data.copy_(w13_bias_swapped)if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:@@ -272,7 +308,8 @@ class AutoRoundMoEMethodMXFp4Impl(AutoRoundMoEMethod):new_w1[:, 1::2, :] = w1[:, N // 2 :, :]return new_w1- w1 = revert_interleaved_w1(w1)+ if envs.VLLM_AR_POST_PROCESS_GPTOSS:+ w1 = revert_interleaved_w1(w1)w1_scale = Nonew2 = layer.w2_weight_packed@@ -352,6 +389,23 @@ class AutoRoundMoEMethodMXFp4Impl(AutoRoundMoEMethod):if envs.VLLM_AR_MXFP4_MODULAR_MOE:from vllm.model_executor.layers.fused_moe import fused_experts+ if envs.VLLM_MXFP4_PRE_UNPACK_TO_FP8:+ w1 = layer.w13_weight_unpacked+ w2 = layer.w2_weight_unpacked+ out = fused_experts(+ x,+ w1,+ w2,+ topk_weights=topk_weights,+ topk_ids=topk_ids,+ inplace=True,+ activation=activation,+ global_num_experts=global_num_experts,+ apply_router_weight_on_input=apply_router_weight_on_input,+ expert_map=expert_map,+ quant_config=self.moe_quant_config,+ )+ return outif envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:w1 = layer.w13_weight_unpackedw2 = layer.w2_weight_unpackeddiff --git a/auto_round_extension/vllm_ext/moe_impl_mxfp8.py b/auto_round_extension/vllm_ext/moe_impl_mxfp8.pynew file mode 100644index 000000000..ecd4183ff--- /dev/null+++ b/auto_round_extension/vllm_ext/moe_impl_mxfp8.py@@ -0,0 +1,208 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from typing import Callable, Optional, Union++import torch+import torch.nn.functional as F+from vllm.logger import init_logger+from vllm.model_executor.layers.fused_moe import (+ FusedMoE,+ FusedMoEConfig,+ FusedMoeWeightScaleSupported,+)+from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig+from vllm.model_executor.utils import set_weight_attrs++from auto_round_extension.vllm_ext.quant_method_moe import AutoRoundMoEMethod++logger = init_logger(__name__)+++def apply_act(local_w1_out: torch.Tensor, local_w3_out: torch.Tensor, activation: str) -> torch.Tensor:+ if activation == "silu":+ act_fn = F.silu+ w13_out = act_fn(local_w1_out) * local_w3_out+ elif activation == "swigluoai":+ limit = 7.0+ alpha = 1.702+ local_w1_out = local_w1_out.clamp(min=None, max=limit)+ local_w3_out = local_w3_out.clamp(min=-limit, max=limit)+ glu = (local_w1_out) * F.sigmoid(local_w1_out * alpha)+ w13_out = (local_w3_out + 1) * glu+ else:+ raise NotImplementedError(f"Activation {activation} is not implemented.")+ return w13_out+++class AutoRoundMoEMethodMXFp8Impl(AutoRoundMoEMethod):+ def __init__(+ self,+ quant_config: "AutoRoundConfig", # type: ignore # noqa E501+ moe: FusedMoEConfig,+ ):+ super().__init__(moe)+ self.use_marlin = False+ self.group_size = 32+ self.quant_config = quant_config+ self.has_bias = self.moe.has_bias++ def create_weights(+ self,+ layer: torch.nn.Module,+ num_experts: int,+ hidden_size: int,+ intermediate_size_per_partition: int,+ params_dtype: torch.dtype,+ **extra_weight_attrs,+ ):+ layer.intermediate_size_per_partition = intermediate_size_per_partition+ layer.hidden_size = hidden_size+ layer.num_experts = num_experts+ layer.orig_dtype = params_dtype+ layer.weight_block_size = None++ params_dtype = torch.float8_e4m3fn++ # WEIGHTS+ w13_weight = torch.nn.Parameter(+ torch.empty(+ num_experts,+ 2 * intermediate_size_per_partition,+ hidden_size,+ dtype=params_dtype,+ ),+ requires_grad=False,+ )+ layer.register_parameter("w13_weight", w13_weight)+ set_weight_attrs(w13_weight, extra_weight_attrs)++ w2_weight = torch.nn.Parameter(+ torch.empty(+ num_experts,+ hidden_size,+ intermediate_size_per_partition,+ dtype=params_dtype,+ ),+ requires_grad=False,+ )+ layer.register_parameter("w2_weight", w2_weight)+ set_weight_attrs(w2_weight, extra_weight_attrs)++ # WEIGHT_SCALES+ w13_weight_scale = torch.nn.Parameter(+ data=torch.empty(+ num_experts,+ 2 * intermediate_size_per_partition,+ hidden_size // self.group_size,+ dtype=torch.uint8, # E8M0 for MXFP8 scale+ ),+ requires_grad=False,+ )+ layer.register_parameter("w13_weight_scale", w13_weight_scale)+ # w2+ w2_weight_scale = torch.nn.Parameter(+ data=torch.empty(+ num_experts,+ hidden_size,+ intermediate_size_per_partition // self.group_size,+ dtype=torch.uint8, # E8M0 for MXFP8 scale+ ),+ requires_grad=False,+ )+ layer.register_parameter("w2_weight_scale", w2_weight_scale)+ # Add PER-TENSORGROUP quantization for FusedMoE.weight_loader.+ extra_weight_attrs.update({"quant_method": FusedMoeWeightScaleSupported.GROUP.value})+ set_weight_attrs(w13_weight_scale, extra_weight_attrs)+ set_weight_attrs(w2_weight_scale, extra_weight_attrs)++ def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> Optional[FusedMoEQuantConfig]:+ from vllm.model_executor.layers.fused_moe.config import (+ ocp_mx_moe_quant_config,+ )++ self.input_dtype = "mxfp8_e4m3"+ self.weight_dtype = "mxfp8_e4m3"+ return ocp_mx_moe_quant_config(+ quant_dtype=self.input_dtype,+ weight_dtype=self.weight_dtype,+ w1_scale=layer.w13_weight_scale,+ w2_scale=layer.w2_weight_scale,+ a1_scale=None,+ a2_scale=None,+ w1_bias=layer.w13_bias if self.has_bias else None,+ w2_bias=layer.w2_bias if self.has_bias else None,+ block_shape=None,+ )+ return None++ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:+ return++ @torch.inference_mode()+ def apply(+ self,+ layer: torch.nn.Module,+ x: torch.Tensor,+ router_logits: torch.Tensor,+ top_k: int,+ renormalize: bool,+ use_grouped_topk: bool = False,+ topk_group: Optional[int] = None,+ num_expert_group: Optional[int] = None,+ global_num_experts: int = -1,+ expert_map: Optional[torch.Tensor] = None,+ custom_routing_function: Optional[Callable] = None,+ scoring_func: str = "softmax",+ routed_scaling_factor: float = 1.0,+ e_score_correction_bias: Optional[torch.Tensor] = None,+ apply_router_weight_on_input: bool = False,+ activation: str = "silu",+ enable_eplb: bool = False,+ expert_load_view: Optional[torch.Tensor] = None,+ logical_to_physical_map: Optional[torch.Tensor] = None,+ logical_replica_count: Optional[torch.Tensor] = None,+ ) -> Union[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:+ topk_weights, topk_ids, _ = FusedMoE.select_experts(+ hidden_states=x,+ router_logits=router_logits,+ use_grouped_topk=use_grouped_topk,+ top_k=top_k,+ renormalize=renormalize,+ topk_group=topk_group,+ num_expert_group=num_expert_group,+ custom_routing_function=custom_routing_function,+ scoring_func=scoring_func,+ e_score_correction_bias=e_score_correction_bias,+ )+ assert self.fused_experts is None++ from vllm.model_executor.layers.fused_moe import fused_experts++ w1 = layer.w13_weight+ w2 = layer.w2_weight+ out = fused_experts(+ x,+ w1,+ w2,+ topk_weights=topk_weights,+ topk_ids=topk_ids,+ inplace=True,+ activation=activation,+ global_num_experts=global_num_experts,+ apply_router_weight_on_input=apply_router_weight_on_input,+ expert_map=expert_map,+ quant_config=self.moe_quant_config,+ )+ return outdiff --git a/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py b/auto_round_extension/vllm_ext/mxfp4_qdq_utils.pyindex 197d757fa..1916fbc62 100644--- a/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py+++ b/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py@@ -159,8 +159,10 @@ def dequant_mxfp4_to_fp8(data_lp, scale_e8m0):def mxfp4_fp8_weight_to_bf16(weight_fp8, scale_bf16):+origin_shape = weight_fp8.shapeweight_fp8 = weight_fp8.reshape(-1, 32)+ scale_bf16 = scale_bf16.reshape(-1, 1)assert weight_fp8.shape[0] == scale_bf16.shape[0], f"shape mismatch: {weight_fp8.shape} vs {scale_bf16.shape}"dequant_weight_bf16 = weight_fp8.to(torch.bfloat16) * scale_bf16dequant_weight_bf16 = dequant_weight_bf16.reshape(origin_shape)diff --git a/auto_round_extension/vllm_ext/mxfp8_qdq_utils.py b/auto_round_extension/vllm_ext/mxfp8_qdq_utils.pynew file mode 100644index 000000000..482d77f85--- /dev/null+++ b/auto_round_extension/vllm_ext/mxfp8_qdq_utils.py@@ -0,0 +1,63 @@+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project+import torch++__all__ = ["get_fp_scale", "dequant_mx_fp8", "quant_mx_fp8"]+++# def get_fp_scale(scale_e8m0):+# # https://github.com/pytorch/ao/blob/994a4ba6c869854fcaa6ca7e118fcbd75e6c28cc/torchao/prototype/mx_formats/mx_tensor.py#L337+# assert scale_e8m0.dtype == torch.uint8, f"Expected uint8, got {scale_e8m0.dtype}"+# E8M0_EXPONENT_BIAS = 127+# scale_e8m0 = scale_e8m0.view(torch.uint8)+# s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS+# # TODO(later): it would be nice if there was a way to do the 2^x operation+# # in PyTorch without creating a tensor of twos+# two = torch.full(s_offset.size(), 2.0, device=scale_e8m0.device)+# # pow(two, s_offset) can be out of range of floating point formats.+# # TODO(later): handle this for float16 if we decide to support float16+# # scales.+# s_fp = torch.pow(two, s_offset)++# return s_fp+++def get_fp_scale(scale_e8m0):+ # https://github.com/pytorch/ao/blob/994a4ba6c869854fcaa6ca7e118fcbd75e6c28cc/torchao/prototype/mx_formats/mx_tensor.py#L337+ E8M0_EXPONENT_BIAS = 127++ scale_e8m0 = scale_e8m0.view(torch.uint8)+ s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS+ # TODO(later): it would be nice if there was a way to do the 2^x operation+ # in PyTorch without creating a tensor of twos+ # two = torch.full(s_offset.size(), 2.0, device=scale_e8m0.device)+ # pow(two, s_offset) can be out of range of floating point formats.+ # TODO(later): handle this for float16 if we decide to support float16+ # scales.+ # s_fp = torch.pow(two, s_offset)+ # !!!!NOTE Critical: fixed the OoM issue when using HPU graph+ s_fp = torch.pow(2.0, s_offset.to(torch.float))++ return s_fp+++def dequant_mx_fp8(weight_fp8, scale_e8m0, block_size, target_dtype):+ scale_float = get_fp_scale(scale_e8m0)+ weight_bf16 = weight_fp8.to(torch.bfloat16)+ weight_original_shape = weight_bf16.shape+ weight_bf16 = weight_bf16.reshape(-1, block_size)+ scale_float = scale_float.reshape(-1, 1)+ dequant_weight = weight_bf16 * scale_float+ dequant_weight = dequant_weight.reshape(weight_original_shape)+ return dequant_weight.to(target_dtype)+++def quant_mx_fp8(tensor):+ from auto_round_extension.vllm_ext.utils import to_mx_fp8e4m3++ scale_e8m0_biased, data_lp = to_mx_fp8e4m3(+ data_hp=tensor,+ elem_dtype=torch.float8_e4m3fn,+ block_size=32,+ )+ return scale_e8m0_biased, data_lpdiff --git a/auto_round_extension/vllm_ext/quant_impl.py b/auto_round_extension/vllm_ext/quant_impl.pynew file mode 100644index 000000000..b88843462--- /dev/null+++ b/auto_round_extension/vllm_ext/quant_impl.py@@ -0,0 +1,45 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from abc import ABC, abstractmethod+from typing import Optional++import torch+++class AutoRoundQuantImpl(ABC):+ @classmethod+ @abstractmethod+ def get_min_capability(cls) -> int:+ """+ Get minimum device capability.+ """+ raise NotImplementedError++ @abstractmethod+ def create_weights(self, *args, **kwargs):+ raise NotImplementedError++ @abstractmethod+ def apply_weights(+ self,+ layer: torch.nn.Module,+ x: torch.Tensor,+ bias: Optional[torch.Tensor],+ ):+ raise NotImplementedError++ @abstractmethod+ def process_weights_after_loading(self, layer: torch.nn.Module):+ raise NotImplementedErrordiff --git a/auto_round_extension/vllm_ext/quant_method_linear.py b/auto_round_extension/vllm_ext/quant_method_linear.pynew file mode 100644index 000000000..eecc97902--- /dev/null+++ b/auto_round_extension/vllm_ext/quant_method_linear.py@@ -0,0 +1,112 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from typing import Optional++import torch+from vllm.logger import init_logger+from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase, UnquantizedLinearMethod+from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig++from auto_round.schemes import QuantizationScheme+from auto_round_extension.vllm_ext.utils import _is_mxfp4_w4a4, _is_mxfp8_w8a8, get_scheme, need_quantize++logger = init_logger(__name__)+++QLINEAR_METHODS_DISPATCH_TABLE = {}+++class AutoRoundQuantLinearMethod(LinearMethodBase):++ def __init__(self, impl, config=None, scheme=None):+ self.config = config+ self.impl = impl+ self.scheme = scheme++ @staticmethod+ def get_method(+ quant_config: AutoRoundConfig,+ layer: torch.nn.Module,+ prefix: str,+ ) -> "AutoRoundQuantLinearMethod":++ def get_impl(scheme: QuantizationScheme):+ if _is_mxfp8_w8a8(scheme):+ from auto_round_extension.vllm_ext.linear_impl_mxfp8 import AutoRoundMXFP8LinearImpl++ return AutoRoundMXFP8LinearImpl(quant_config)++ elif _is_mxfp4_w4a4(scheme):+ from auto_round_extension.vllm_ext.linear_impl_mxfp4 import AutoRoundMXFP4LinearImpl++ return AutoRoundMXFP4LinearImpl(quant_config)++ raise ValueError(f"Unsupported Linear scheme: {scheme}, layer: {prefix}")++ packed_modules_mapping = quant_config.packed_modules_mapping+ for packed_name, child_names in packed_modules_mapping.items():+ if prefix.endswith(packed_name):+ prefix = prefix.replace(packed_name, child_names[0])+ break++ # TODO: use a more robust way to map layer names+ if prefix.endswith("gate_up_proj"):+ # update gate_up_proj to gate_proj, assume both gate and up share the same quantization scheme+ prefix = prefix.replace("gate_up_proj", "up_proj")+ if prefix.endswith("qkv_proj"):+ # update qkv_proj to q_proj, assume all qkv share the same quantization scheme+ prefix = prefix.replace("qkv_proj", "q_proj")+ layer_scheme = get_scheme(quant_config, prefix)+ if not need_quantize(layer_scheme.bits):+ return UnquantizedLinearMethod()+ impl = get_impl(layer_scheme)+ logger.info("Apply %s to %s", impl.__class__.__name__, prefix)+ return AutoRoundQuantLinearMethod(impl=impl)++ @classmethod+ def get_min_capability(cls) -> int:+ return cls.impl.get_min_capability()++ def create_weights(+ self,+ layer: torch.nn.Module,+ input_size_per_partition: int,+ output_partition_sizes: list[int],+ input_size: int,+ output_size: int,+ params_dtype: torch.dtype,+ **extra_weight_attrs,+ ):+ weight_loader = extra_weight_attrs.get("weight_loader")+ return self.impl.create_weights(+ layer=layer,+ input_size=input_size,+ input_size_per_partition=input_size_per_partition,+ output_partition_sizes=output_partition_sizes,+ output_size=output_size,+ params_dtype=params_dtype,+ weight_loader=weight_loader,+ )++ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:+ return self.impl.process_weights_after_loading(layer)++ def apply(+ self,+ layer: torch.nn.Module,+ x: torch.Tensor,+ bias: Optional[torch.Tensor] = None,+ ):+ return self.impl.apply_weights(layer, x, bias=bias)diff --git a/auto_round_extension/vllm_ext/quant_method_moe.py b/auto_round_extension/vllm_ext/quant_method_moe.pyindex ae7b81775..ef5c75291 100644--- a/auto_round_extension/vllm_ext/quant_method_moe.py+++ b/auto_round_extension/vllm_ext/quant_method_moe.py@@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfigfrom vllm.model_executor.layers.quantization.auto_round import AutoRoundConfigfrom auto_round.schemes import QuantizationScheme+from auto_round_extension.vllm_ext.utils import _is_mxfp4_w4a4, _is_mxfp8_w8a8, get_scheme, need_quantizelogger = init_logger(__name__)@@ -31,11 +32,6 @@ logger = init_logger(__name__)QMOE_METHODS_DISPATCH_TABLE = {}-def _is_mxfp4_w4a4(scheme: QuantizationScheme):- # FIXME: below impl is incomplete- return scheme.bits == 4 and scheme.group_size == 32--class AutoRoundMoEMethod(FusedMoEMethodBase):def __init__(self, moe: FusedMoEConfig):super().__init__(moe)@@ -47,21 +43,8 @@ class AutoRoundMoEMethod(FusedMoEMethodBase):prefix: str,) -> "AutoRoundMoEMethod":- def get_scheme(quant_config: AutoRoundConfig, prefix: str):- # Check extra_config first- layer_schemes = quant_config.layer_schemes- # FIXME: make more robust- for name, scheme in layer_schemes.items():- if prefix.startswith(name):- return scheme- # If not found, use default- return quant_config.quant_scheme-- def check_quantized(weight_bits: int) -> bool:- return weight_bits < 16-def get_impl(scheme: QuantizationScheme):- if not check_quantized(scheme.bits):+ if not need_quantize(scheme.bits):from vllm.model_executor.layers.fused_moe.layer import (UnquantizedFusedMoEMethod,)@@ -73,10 +56,16 @@ class AutoRoundMoEMethod(FusedMoEMethodBase):return AutoRoundMoEMethodMXFp4Impl(quant_config, layer.moe_config)+ elif _is_mxfp8_w8a8(scheme):+ from auto_round_extension.vllm_ext.moe_impl_mxfp8 import AutoRoundMoEMethodMXFp8Impl++ return AutoRoundMoEMethodMXFp8Impl(quant_config, layer.moe_config)+raise ValueError(f"Unsupported FusedMoe scheme: {scheme}")layer_scheme = get_scheme(quant_config, prefix)impl = get_impl(layer_scheme)+ layer._prefix = prefixlogger.debug("Apply %s to %s", impl.__class__.__name__, prefix)return impldiff --git a/auto_round_extension/vllm_ext/tests/test_mxfp4_moe.py b/auto_round_extension/vllm_ext/tests/test_models.pysimilarity index 63%rename from auto_round_extension/vllm_ext/tests/test_mxfp4_moe.pyrename to auto_round_extension/vllm_ext/tests/test_models.pyindex a2de32832..faa783cde 100644--- a/auto_round_extension/vllm_ext/tests/test_mxfp4_moe.py+++ b/auto_round_extension/vllm_ext/tests/test_models.py@@ -18,10 +18,24 @@ from vllm.platforms import current_platformMODELS = [# "/data5/yliu7/HF_HOME/unsloth-gpt-oss-20b-BF16-ar-MXFP4/"# "/data5/yliu7/HF_HOME/Qwen2.5-0.5B-Instruct-test-FP8_STATIC-fp8kv/"- "/data6/yiliu4/Qwen3-15B-A2B-Base-MXFP4"+ # "/data6/yiliu4/Qwen3-15B-A2B-Base-MXFP4",+ # "/data6/yiliu4/Llama-3.2-1B-Instruct-MXFP4-fp8attention",+ # "/data6/yiliu4/Llama-3.2-1B-Instruct-MXFP8"+ "/storage/yiliu7/ar_vllm_ext/quantized_model_qwen_mxfp4",+ "/storage/yiliu7/ar_vllm_ext/quantized_model_qwen_mxfp8",][email protected](autouse=True)+def set_vllm_ar_env(monkeypatch):+ monkeypatch.setenv("VLLM_AR_MXFP4_MODULAR_MOE", "1")+ monkeypatch.setenv("VLLM_MXFP4_PRE_UNPACK_TO_FP8", "1")+ monkeypatch.setenv("VLLM_MXFP4_PRE_UNPACK_WEIGHTS", "0")+ monkeypatch.setenv("VLLM_ENABLE_STATIC_MOE", "0")+ monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "0")+ monkeypatch.setenv("VLLM_ENABLE_AR_EXT", "1")++@pytest.mark.skipif(not current_platform.is_cuda(),reason="only supports CUDA backend.",diff --git a/auto_round_extension/vllm_ext/utils.py b/auto_round_extension/vllm_ext/utils.pyindex f98eb2325..db5a0dff1 100644--- a/auto_round_extension/vllm_ext/utils.py+++ b/auto_round_extension/vllm_ext/utils.py@@ -12,12 +12,41 @@# See the License for the specific language governing permissions and# limitations under the License.+from typing import Union+import torch+from auto_round.schemes import QuantizationScheme+E8M0_EXPONENT_BIAS = 127E8M0_EXPONENT_NAN_VAL = 255+def get_scheme(quant_config, prefix: str):+ # Check extra_config first+ layer_schemes = quant_config.layer_schemes+ # FIXME: make more robust+ for name, scheme in layer_schemes.items():+ if prefix.startswith(name):+ return scheme+ # If not found, use default+ return quant_config.quant_scheme+++def need_quantize(weight_bits: int) -> bool:+ return weight_bits < 16+++def _is_mxfp4_w4a4(scheme: QuantizationScheme):+ # FIXME: below impl is incomplete+ return scheme.bits == 4 and scheme.group_size == 32+++def _is_mxfp8_w8a8(scheme: QuantizationScheme):+ # FIXME: below impl is incomplete+ return scheme.bits == 8 and scheme.group_size == 32++def get_fp_scale(scale_e8m0):scale_e8m0 = scale_e8m0.view(torch.uint8)s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS@@ -79,3 +108,60 @@ def _to_mx_rceil(# scale and saturated cast the data elements to max of target dtypedata_lp = torch.clamp(data_hp * descale_fp.unsqueeze(1), min=-1 * max_pos, max=max_pos)return exponent, data_lp+++def to_mx_fp8e4m3(+ data_hp: torch.Tensor,+ elem_dtype: Union[torch.dtype, str],+ block_size: int,+):+ """+ Takes a high precision tensor and converts to MX scale and raw data, in+ naive layout (scale and raw data are separate tensors).+ """++ assert data_hp.dtype in (+ torch.bfloat16,+ torch.float,+ ), f"{data_hp.dtype} is not supported yet"+ # TODO(future PR): consider supporting padding+ data_hp = data_hp.contiguous()++ # calculate the scale in e8m0 format+ orig_shape = data_hp.shape+ data_hp = data_hp.reshape(-1, block_size)++ # find max value of the data+ # Note: this only implements the `minimally supported` version of+ # https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf+ # section 6.3.+ max_abs = torch.amax(torch.abs(data_hp), 1)++ # Set X to be the largest power-of-two less than or equal to+ # max_abs(v), divided by the largest power of two representable+ # in the element data type, and get the mbits at the same time+ assert elem_dtype == torch.float8_e4m3fn, f"only float8_e4m3fn is supported now, got {elem_dtype}"++ max_pos = torch.finfo(torch.float8_e4m3fn).max # 448.0+ # TODO: add support for other rounding modes+ scale_e8m0_biased, data_lp = _to_mx_rceil(data_hp, max_abs, max_pos)++ data_lp = data_lp.to(elem_dtype)+ # need to reshape at the end to help inductor fuse things+ data_lp = data_lp.reshape(orig_shape)++ # scale_e8m0_biased = scale_e8m0_biased.view(torch.float8_e8m0fnu)+ return scale_e8m0_biased, data_lp+++def down_size(size):+ assert size[-1] % 2 == 0, f"{size} last dim not divisible by two"+ return (*size[:-1], size[-1] // 2)+++def pack_uint4(uint8_data: torch.Tensor) -> torch.Tensor:+ # converting to uint8 for operations+ shape = uint8_data.shape+ assert shape[-1] % 2 == 0+ uint8_data = uint8_data.contiguous().view(-1)+ return (uint8_data[::2] << 4 | uint8_data[1::2]).view(down_size(shape))