Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Add MXFP8 MOE/Linear and MXFP4 Linear (#1034)

Yi Liu61bcfdb68513
diff --git a/auto_round_extension/vllm_ext/README.md b/auto_round_extension/vllm_ext/README.md
new file mode 100644
index 000000000..1f0cb832f
--- /dev/null
+++ b/auto_round_extension/vllm_ext/README.md
@@ -0,0 +1,12 @@
+- Build and Install vLLM
+
+```
+git clone --branch fused-moe-ar https://github.com/yiliu30/vllm-fork.git
+VLLM_USE_PRECOMPILED=1 pip install --editable . -vvv
+```
+
+
+- Enable vLLM-Ext at Runtime
+```bash
+VLLM_ENABLE_AR_EXT=1 vllm serve ...
+```
\ No newline at end of file
diff --git a/auto_round_extension/vllm_ext/__init__.py b/auto_round_extension/vllm_ext/__init__.py
index 3e145334f..748e917e2 100644
--- a/auto_round_extension/vllm_ext/__init__.py
+++ b/auto_round_extension/vllm_ext/__init__.py
@@ -18,9 +18,9 @@
def apply():
- import vllm.model_executor.layers.quantization.auto_round as auto_round_module
+ import auto_round_extension.vllm_ext.auto_round_ext
+ import auto_round_extension.vllm_ext.envs_ext
- from auto_round_extension.vllm_ext.auto_round_ext import AutoRoundExtensionConfig
-
- auto_round_module.AutoRoundConfig = AutoRoundExtensionConfig
- from auto_round_extension.vllm_ext.envs_ext import extra_environment_variables
+ print("*****************************************************************************")
+ print("* !!! VLLM_ENABLE_AR_EXT is set to 1, applying auto_round_vllm_extension *")
+ print("*****************************************************************************")
diff --git a/auto_round_extension/vllm_ext/apply_ext.sh b/auto_round_extension/vllm_ext/apply_ext.sh
deleted file mode 100644
index b327ec22c..000000000
--- a/auto_round_extension/vllm_ext/apply_ext.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define the relative path for the `auto-round` installation
-AUTO_ROUND_PATH="auto_round/../auto_round_extension/vllm_ext/sitecustomize.py"
-
-# Try to find the pip installation location
-PIP_LOCATION=$(pip show auto-round 2>/dev/null | grep "Location:" | awk '{print $2}')
-
-if [ -n "$PIP_LOCATION" ]; then
- SITE_CUSTOMIZE_PATH="$PIP_LOCATION/$AUTO_ROUND_PATH"
- echo "Checking for sitecustomize.py at: $SITE_CUSTOMIZE_PATH"
-
- if [ -f "$SITE_CUSTOMIZE_PATH" ]; then
- echo "Found sitecustomize.py at: $SITE_CUSTOMIZE_PATH"
- export PYTHONPATH=$(dirname "$SITE_CUSTOMIZE_PATH"):$PYTHONPATH
- echo "PYTHONPATH set to: $PYTHONPATH"
- return 0 2>/dev/null || true
- fi
-fi
-
-# Fallback: check current directory
-LOCAL_SITE_CUSTOMIZE="./sitecustomize.py"
-if [ -f "$LOCAL_SITE_CUSTOMIZE" ]; then
- echo "Found sitecustomize.py at current directory."
- export PYTHONPATH=$(pwd):$PYTHONPATH
- echo "PYTHONPATH set to: $PYTHONPATH"
- return 0 2>/dev/null || true
-fi
-
-echo "Warning: sitecustomize.py not found in pip installation or current directory."
-# Do not exit the shell
-return 1 2>/dev/null || true
\ No newline at end of file
diff --git a/auto_round_extension/vllm_ext/auto_round_ext.py b/auto_round_extension/vllm_ext/auto_round_ext.py
index cce2c912d..d665fd568 100644
--- a/auto_round_extension/vllm_ext/auto_round_ext.py
+++ b/auto_round_extension/vllm_ext/auto_round_ext.py
@@ -18,17 +18,18 @@ import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
-from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig
+from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig as _BaseAutoRoundConfig
from auto_round.schemes import QuantizationScheme
+from auto_round_extension.vllm_ext.quant_method_linear import AutoRoundQuantLinearMethod
from auto_round_extension.vllm_ext.quant_method_moe import AutoRoundMoEMethod
logger = init_logger(__name__)
-class AutoRoundExtensionConfig(AutoRoundConfig):
- SUPPORTED_DTYPES = AutoRoundConfig.SUPPORTED_DTYPES.union({"mx_fp"})
- SUPPORTED_FORMATS = AutoRoundConfig.SUPPORTED_FORMATS.union({"auto_round:llm_compressor"})
+class AutoRoundExtensionConfig(_BaseAutoRoundConfig):
+ SUPPORTED_DTYPES = _BaseAutoRoundConfig.SUPPORTED_DTYPES.union({"mx_fp"})
+ SUPPORTED_FORMATS = _BaseAutoRoundConfig.SUPPORTED_FORMATS.union({"auto_round:llm_compressor"})
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
# FIXME: (yi) make it compatible with `AutoRoundConfig`
@@ -36,7 +37,7 @@ class AutoRoundExtensionConfig(AutoRoundConfig):
quant_method = AutoRoundMoEMethod.get_moe_method(self, layer, prefix)
return quant_method
elif isinstance(layer, LinearBase):
- return UnquantizedLinearMethod()
+ return AutoRoundQuantLinearMethod.get_method(self, layer, prefix)
else:
return None
@@ -48,7 +49,7 @@ class AutoRoundExtensionConfig(AutoRoundConfig):
return quant_scheme
@classmethod
- def from_config(cls, config: dict[str, Any]) -> AutoRoundConfig:
+ def from_config(cls, config: dict[str, Any]) -> _BaseAutoRoundConfig:
ar_config = super().from_config(config)
# TODO: (yi) refine below implementation
quant_scheme = AutoRoundExtensionConfig._parse_quant_scheme(config)
@@ -61,3 +62,9 @@ class AutoRoundExtensionConfig(AutoRoundConfig):
ar_config.quant_scheme = quant_scheme
ar_config.layer_schemes = layer_schemes
return ar_config
+
+
+# Patch vLLM’s AutoRoundConfig at import time
+import vllm.model_executor.layers.quantization.auto_round as _auto_round_module
+
+_auto_round_module.AutoRoundConfig = AutoRoundExtensionConfig
diff --git a/auto_round_extension/vllm_ext/envs_ext.py b/auto_round_extension/vllm_ext/envs_ext.py
index 325b9de7c..70fbb3238 100644
--- a/auto_round_extension/vllm_ext/envs_ext.py
+++ b/auto_round_extension/vllm_ext/envs_ext.py
@@ -21,9 +21,11 @@ logger = init_logger(__name__)
# Define extra environment variables
extra_environment_variables: dict[str, Callable[[], Any]] = {
- "VLLM_MXFP4_PRE_UNPACK_WEIGHTS": lambda: os.getenv("VLLM_MXFP4_PRE_UNPACK_WEIGHTS", "1") in ("1", "true", "True"),
- "VLLM_ENABLE_STATIC_MOE": lambda: os.getenv("VLLM_ENABLE_STATIC_MOE", "1") in ("1", "true", "True"),
- "VLLM_AR_MXFP4_MODULAR_MOE": lambda: os.getenv("VLLM_AR_MXFP4_MODULAR_MOE", "0") in ("1", "true", "True"),
+ "VLLM_MXFP4_PRE_UNPACK_WEIGHTS": lambda: os.getenv("VLLM_MXFP4_PRE_UNPACK_WEIGHTS", "0") in ("1", "true", "True"),
+ "VLLM_MXFP4_PRE_UNPACK_TO_FP8": lambda: os.getenv("VLLM_MXFP4_PRE_UNPACK_TO_FP8", "1") in ("1", "true", "True"),
+ "VLLM_ENABLE_STATIC_MOE": lambda: os.getenv("VLLM_ENABLE_STATIC_MOE", "0") in ("1", "true", "True"),
+ "VLLM_AR_MXFP4_MODULAR_MOE": lambda: os.getenv("VLLM_AR_MXFP4_MODULAR_MOE", "1") in ("1", "true", "True"),
+ "VLLM_AR_POST_PROCESS_GPTOSS": lambda: os.getenv("VLLM_AR_POST_PROCESS_GPTOSS", "0") in ("1", "true", "True"),
}
# Add the extra environment variables to vllm.envs
import vllm.envs as envs
diff --git a/auto_round_extension/vllm_ext/fp4_utils.py b/auto_round_extension/vllm_ext/fp4_utils.py
index adb67d190..fb7b6e935 100644
--- a/auto_round_extension/vllm_ext/fp4_utils.py
+++ b/auto_round_extension/vllm_ext/fp4_utils.py
@@ -51,7 +51,7 @@ def pack_fp4_to_uint8(x: torch.Tensor) -> torch.Tensor:
indices = indices.reshape(-1)
# Handle odd length by padding if necessary
- assert indices.numel() % 2 != 0, f"Expected even number of elements, got {indices.numel()}"
+ # assert indices.numel() % 2 != 0, f"Expected even number of elements, got {indices.numel()}"
# Reshape to pair consecutive elements
indices = indices.reshape(-1, 2)
diff --git a/auto_round_extension/vllm_ext/linear_impl_mxfp4.py b/auto_round_extension/vllm_ext/linear_impl_mxfp4.py
new file mode 100644
index 000000000..04d5e20f8
--- /dev/null
+++ b/auto_round_extension/vllm_ext/linear_impl_mxfp4.py
@@ -0,0 +1,129 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# SPDX-License-Identifier: Apache-2.0
+from typing import Callable, Optional
+
+import torch
+import vllm.envs as envs
+from torch.nn.parameter import Parameter
+from vllm.logger import init_logger
+from vllm.model_executor.parameter import GroupQuantScaleParameter, ModelWeightParameter, PerTensorScaleParameter
+from vllm.platforms import current_platform
+
+from auto_round_extension.vllm_ext.mxfp4_qdq_utils import (
+ dequant_mxfp4_to_fp8,
+ mxfp4_gemm_with_unpacked_weight,
+ run_mxfp4_emulations,
+)
+
+logger = init_logger(__name__)
+
+__all__ = ["AutoRoundMXFP4LinearImpl"]
+
+from auto_round_extension.vllm_ext.quant_impl import AutoRoundQuantImpl
+
+
+class AutoRoundMXFP4LinearImpl(AutoRoundQuantImpl):
+ def __init__(self, quant_scheme):
+ self.quant_scheme = quant_scheme
+ self.group_size = 32
+
+ @classmethod
+ def get_min_capability(cls) -> int:
+ if envs.VLLM_USE_MXFP4_CT_EMULATIONS:
+ return 80
+ return 100
+
+ def create_weights(
+ self,
+ layer: torch.nn.Module,
+ output_partition_sizes: list[int],
+ input_size_per_partition: int,
+ params_dtype: torch.dtype,
+ weight_loader: Callable,
+ **kwargs,
+ ):
+ output_size_per_partition = sum(output_partition_sizes)
+ layer.logical_widths = output_partition_sizes
+ layer.input_size_per_partition = input_size_per_partition
+ layer.output_size_per_partition = output_size_per_partition
+
+ # Weight
+ weight = ModelWeightParameter(
+ data=torch.empty(sum(output_partition_sizes), input_size_per_partition // 2, dtype=torch.uint8),
+ input_dim=1,
+ output_dim=0,
+ weight_loader=weight_loader,
+ )
+ layer.register_parameter("weight_packed", weight)
+
+ # Per Group Weight Scale
+ weight_scale = GroupQuantScaleParameter(
+ data=torch.empty(
+ sum(output_partition_sizes),
+ input_size_per_partition // self.group_size,
+ # dtype=torch.uint8,
+ dtype=torch.uint8,
+ ),
+ input_dim=1,
+ output_dim=0,
+ weight_loader=weight_loader,
+ )
+
+ layer.register_parameter("weight_scale", weight_scale)
+
+ def process_weights_after_loading(self, layer) -> None:
+ # FIXME: may dequant to bf16
+ if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:
+
+ weight_fp8, scale_bf16 = dequant_mxfp4_to_fp8(
+ data_lp=layer.weight_packed,
+ scale_e8m0=layer.weight_scale,
+ )
+ del layer.weight_packed
+ del layer.weight_scale
+ layer.weight_packed = None
+ layer.weight_scale = None
+ layer.register_parameter(
+ "weight_unpacked_fp8",
+ torch.nn.Parameter(
+ weight_fp8,
+ requires_grad=False,
+ ),
+ )
+ layer.register_parameter(
+ "weight_scale_bf16",
+ torch.nn.Parameter(
+ scale_bf16,
+ requires_grad=False,
+ ),
+ )
+
+ def apply_weights(
+ self, layer: torch.nn.Module, x: torch.Tensor, bias: Optional[torch.Tensor] = None
+ ) -> torch.Tensor:
+ if not envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:
+ out = run_mxfp4_emulations(x=x, weight=layer.weight_packed, weight_scale=layer.weight_scale)
+ if bias is not None:
+ out = out + bias
+ return out
+ else:
+ out = mxfp4_gemm_with_unpacked_weight(
+ x=x,
+ weight_fp8=layer.weight_unpacked_fp8,
+ weight_scale_bf16=layer.weight_scale_bf16,
+ bias=bias,
+ )
+ return out
diff --git a/auto_round_extension/vllm_ext/linear_impl_mxfp8.py b/auto_round_extension/vllm_ext/linear_impl_mxfp8.py
new file mode 100644
index 000000000..cf1382054
--- /dev/null
+++ b/auto_round_extension/vllm_ext/linear_impl_mxfp8.py
@@ -0,0 +1,112 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Callable, Optional
+
+import torch
+import vllm.envs as envs
+from vllm.model_executor.parameter import (
+ GroupQuantScaleParameter,
+ ModelWeightParameter,
+ PerTensorScaleParameter,
+)
+
+from auto_round_extension.vllm_ext.mxfp8_qdq_utils import dequant_mx_fp8, quant_mx_fp8
+from auto_round_extension.vllm_ext.quant_impl import AutoRoundQuantImpl
+
+
+class AutoRoundMXFP8LinearImpl(AutoRoundQuantImpl):
+ def __init__(self, quant_scheme):
+ self.quant_scheme = quant_scheme
+ self.strategy = "TENSOR_GROUP"
+ self.out_dtype = torch.get_default_dtype()
+ self.group_size = 32
+
+ @classmethod
+ def get_min_capability(cls) -> int:
+ return 80
+
+ def process_weights_after_loading(self, layer) -> None:
+ return
+
+ def create_weights(
+ self,
+ layer: torch.nn.Module,
+ output_partition_sizes: list[int],
+ input_size_per_partition: int,
+ params_dtype: torch.dtype,
+ weight_loader: Callable,
+ **kwargs,
+ ):
+ # maybe_create_device_identity()
+
+ output_size_per_partition = sum(output_partition_sizes)
+ layer.logical_widths = output_partition_sizes
+
+ # WEIGHT
+ weight = ModelWeightParameter(
+ data=torch.empty(
+ output_size_per_partition,
+ input_size_per_partition,
+ dtype=torch.float8_e4m3fn,
+ ),
+ input_dim=1,
+ output_dim=0,
+ weight_loader=weight_loader,
+ )
+ layer.register_parameter("weight", weight)
+
+ # WEIGHT SCALE
+ # Per Group Weight Scale
+ weight_scale = GroupQuantScaleParameter(
+ data=torch.empty(
+ sum(output_partition_sizes),
+ input_size_per_partition // self.group_size,
+ dtype=torch.uint8, # E8M0 for MXFP8 scale
+ ),
+ input_dim=1,
+ output_dim=0,
+ weight_loader=weight_loader,
+ )
+ layer.register_parameter("weight_scale", weight_scale)
+
+ def apply_weights(
+ self,
+ layer: torch.nn.Module,
+ x: torch.Tensor,
+ bias: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
+ # dequant weight
+ weight = layer.weight
+ weight_scale = layer.weight_scale
+ dequnat_weight = dequant_mx_fp8(
+ weight_fp8=weight.data,
+ scale_e8m0=weight_scale.data,
+ block_size=self.group_size,
+ target_dtype=x.dtype,
+ )
+ dequnat_weight = dequnat_weight.to(x.dtype)
+ # if not envs.VLLM_AR_MXFP8_DISABLE_INPUT_QDQ:
+ # q-dq input
+ x_scale, x_quant = quant_mx_fp8(x)
+ dequant_x = dequant_mx_fp8(
+ weight_fp8=x_quant,
+ scale_e8m0=x_scale,
+ block_size=self.group_size,
+ target_dtype=x.dtype,
+ )
+ x = dequant_x.to(x.dtype)
+
+ out = x @ dequnat_weight.t()
+ return out.to(x.dtype) + (bias if bias is not None else 0)
diff --git a/auto_round_extension/vllm_ext/moe_impl_mxfp4.py b/auto_round_extension/vllm_ext/moe_impl_mxfp4.py
index 77ec6570d..0ff5241ab 100644
--- a/auto_round_extension/vllm_ext/moe_impl_mxfp4.py
+++ b/auto_round_extension/vllm_ext/moe_impl_mxfp4.py
@@ -161,6 +161,33 @@ class AutoRoundMoEMethodMXFp4Impl(AutoRoundMoEMethod):
ocp_mx_moe_quant_config,
)
+ if envs.VLLM_MXFP4_PRE_UNPACK_TO_FP8:
+ self.input_dtype = "mxfp8_e4m3"
+ self.weight_dtype = "mxfp8_e4m3"
+ ocp_config = ocp_mx_moe_quant_config(
+ quant_dtype=self.input_dtype,
+ weight_dtype=self.weight_dtype,
+ w1_scale=layer.w13_weight_scale,
+ w2_scale=layer.w2_weight_scale,
+ a1_scale=None,
+ a2_scale=None,
+ w1_bias=layer.w13_bias if self.has_bias else None,
+ w2_bias=layer.w2_bias if self.has_bias else None,
+ block_shape=None,
+ )
+ from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
+ OCP_MX_DTYPES,
+ OCP_MX_Scheme,
+ )
+
+ ocp_config.ocp_mx_scheme = OCP_MX_Scheme.from_quant_dtype(
+ input_dtype="mxfp8_e4m3",
+ weight_dtype="mxfp8_e4m3",
+ original_dtype="mxfp4",
+ )
+ logger.warning_once(f"Set OCP_MX_Scheme to {ocp_config.ocp_mx_scheme}")
+ return ocp_config
+
self.input_dtype = "mxfp4"
self.weight_dtype = "mxfp4"
return ocp_mx_moe_quant_config(
@@ -176,52 +203,59 @@ class AutoRoundMoEMethodMXFp4Impl(AutoRoundMoEMethod):
)
return None
+ def _dequant_fp4_to_fp8(self, layer):
+ weight_name_lst = ["w13_weight", "w2_weight"]
+
+ for weight_name_prefix in weight_name_lst:
+ weight_name = f"{weight_name_prefix}_packed"
+ weight = getattr(layer, weight_name)
+ weight_scale_name = f"{weight_name_prefix}_scale"
+ weight_scale = getattr(layer, weight_scale_name)
+ new_weight_name = f"{weight_name_prefix}_unpacked"
+ new_scale_name = weight_scale_name
+ num_experts, _, _ = weight.shape
+ unpacked_weight_lst = []
+ scale_list = []
+ for expert_index in range(num_experts):
+ weight_fp8, scale_bf16 = dequant_mxfp4_to_fp8(
+ data_lp=weight[expert_index],
+ scale_e8m0=weight_scale[expert_index],
+ )
+
+ unpacked_weight_lst.append(weight_fp8)
+ scale_list.append(scale_bf16)
+ unpacked_weight_fp8 = torch.stack(unpacked_weight_lst, dim=0)
+ scale_bf16 = torch.stack(scale_list, dim=0)
+ assert unpacked_weight_fp8.shape[0] == num_experts, (
+ f"Expected {num_experts} unpacked weights, got " f"{unpacked_weight_fp8.shape[0]}"
+ )
+ delattr(layer, weight_name)
+ delattr(layer, weight_scale_name)
+ layer.register_parameter(
+ new_weight_name,
+ torch.nn.Parameter(
+ unpacked_weight_fp8,
+ requires_grad=False,
+ ),
+ )
+ layer.register_parameter(
+ new_scale_name,
+ torch.nn.Parameter(
+ scale_bf16,
+ requires_grad=False,
+ ),
+ )
+
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
+ logger.debug(f"Processing weights after loading for layer: {layer._prefix}")
if envs.VLLM_ENABLE_STATIC_MOE:
if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:
- weight_name_lst = ["w13_weight", "w2_weight"]
-
- for weight_name_prefix in weight_name_lst:
- weight_name = f"{weight_name_prefix}_packed"
- weight = getattr(layer, weight_name)
- weight_scale_name = f"{weight_name_prefix}_scale"
- weight_scale = getattr(layer, weight_scale_name)
- new_weight_name = f"{weight_name_prefix}_unpacked"
- new_scale_name = weight_scale_name
- num_experts, _, _ = weight.shape
- unpacked_weight_lst = []
- scale_list = []
- for expert_index in range(num_experts):
- weight_fp8, scale_bf16 = dequant_mxfp4_to_fp8(
- data_lp=weight[expert_index],
- scale_e8m0=weight_scale[expert_index],
- )
-
- unpacked_weight_lst.append(weight_fp8)
- scale_list.append(scale_bf16)
- unpacked_weight_fp8 = torch.stack(unpacked_weight_lst, dim=0)
- scale_bf16 = torch.stack(scale_list, dim=0)
- assert unpacked_weight_fp8.shape[0] == num_experts, (
- f"Expected {num_experts} unpacked weights, got " f"{unpacked_weight_fp8.shape[0]}"
- )
- delattr(layer, weight_name)
- delattr(layer, weight_scale_name)
- layer.register_parameter(
- new_weight_name,
- torch.nn.Parameter(
- unpacked_weight_fp8,
- requires_grad=False,
- ),
- )
- layer.register_parameter(
- new_scale_name,
- torch.nn.Parameter(
- scale_bf16,
- requires_grad=False,
- ),
- )
-
+ self._dequant_fp4_to_fp8(layer)
+ return
elif envs.VLLM_AR_MXFP4_MODULAR_MOE:
+ if envs.VLLM_MXFP4_PRE_UNPACK_TO_FP8:
+ self._dequant_fp4_to_fp8(layer)
+ return
def revert_interleaved_bias(bias):
"""
@@ -250,8 +284,10 @@ class AutoRoundMoEMethodMXFp4Impl(AutoRoundMoEMethod):
return revert_bias
# breakpoint()
- w13_bias_swapped = revert_interleaved_bias(layer.w13_bias)
- layer.w13_bias.data.copy_(w13_bias_swapped)
+ if self.has_bias:
+ if envs.VLLM_AR_POST_PROCESS_GPTOSS:
+ w13_bias_swapped = revert_interleaved_bias(layer.w13_bias)
+ layer.w13_bias.data.copy_(w13_bias_swapped)
if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:
@@ -272,7 +308,8 @@ class AutoRoundMoEMethodMXFp4Impl(AutoRoundMoEMethod):
new_w1[:, 1::2, :] = w1[:, N // 2 :, :]
return new_w1
- w1 = revert_interleaved_w1(w1)
+ if envs.VLLM_AR_POST_PROCESS_GPTOSS:
+ w1 = revert_interleaved_w1(w1)
w1_scale = None
w2 = layer.w2_weight_packed
@@ -352,6 +389,23 @@ class AutoRoundMoEMethodMXFp4Impl(AutoRoundMoEMethod):
if envs.VLLM_AR_MXFP4_MODULAR_MOE:
from vllm.model_executor.layers.fused_moe import fused_experts
+ if envs.VLLM_MXFP4_PRE_UNPACK_TO_FP8:
+ w1 = layer.w13_weight_unpacked
+ w2 = layer.w2_weight_unpacked
+ out = fused_experts(
+ x,
+ w1,
+ w2,
+ topk_weights=topk_weights,
+ topk_ids=topk_ids,
+ inplace=True,
+ activation=activation,
+ global_num_experts=global_num_experts,
+ apply_router_weight_on_input=apply_router_weight_on_input,
+ expert_map=expert_map,
+ quant_config=self.moe_quant_config,
+ )
+ return out
if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:
w1 = layer.w13_weight_unpacked
w2 = layer.w2_weight_unpacked
diff --git a/auto_round_extension/vllm_ext/moe_impl_mxfp8.py b/auto_round_extension/vllm_ext/moe_impl_mxfp8.py
new file mode 100644
index 000000000..ecd4183ff
--- /dev/null
+++ b/auto_round_extension/vllm_ext/moe_impl_mxfp8.py
@@ -0,0 +1,208 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Callable, Optional, Union
+
+import torch
+import torch.nn.functional as F
+from vllm.logger import init_logger
+from vllm.model_executor.layers.fused_moe import (
+ FusedMoE,
+ FusedMoEConfig,
+ FusedMoeWeightScaleSupported,
+)
+from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
+from vllm.model_executor.utils import set_weight_attrs
+
+from auto_round_extension.vllm_ext.quant_method_moe import AutoRoundMoEMethod
+
+logger = init_logger(__name__)
+
+
+def apply_act(local_w1_out: torch.Tensor, local_w3_out: torch.Tensor, activation: str) -> torch.Tensor:
+ if activation == "silu":
+ act_fn = F.silu
+ w13_out = act_fn(local_w1_out) * local_w3_out
+ elif activation == "swigluoai":
+ limit = 7.0
+ alpha = 1.702
+ local_w1_out = local_w1_out.clamp(min=None, max=limit)
+ local_w3_out = local_w3_out.clamp(min=-limit, max=limit)
+ glu = (local_w1_out) * F.sigmoid(local_w1_out * alpha)
+ w13_out = (local_w3_out + 1) * glu
+ else:
+ raise NotImplementedError(f"Activation {activation} is not implemented.")
+ return w13_out
+
+
+class AutoRoundMoEMethodMXFp8Impl(AutoRoundMoEMethod):
+ def __init__(
+ self,
+ quant_config: "AutoRoundConfig", # type: ignore # noqa E501
+ moe: FusedMoEConfig,
+ ):
+ super().__init__(moe)
+ self.use_marlin = False
+ self.group_size = 32
+ self.quant_config = quant_config
+ self.has_bias = self.moe.has_bias
+
+ def create_weights(
+ self,
+ layer: torch.nn.Module,
+ num_experts: int,
+ hidden_size: int,
+ intermediate_size_per_partition: int,
+ params_dtype: torch.dtype,
+ **extra_weight_attrs,
+ ):
+ layer.intermediate_size_per_partition = intermediate_size_per_partition
+ layer.hidden_size = hidden_size
+ layer.num_experts = num_experts
+ layer.orig_dtype = params_dtype
+ layer.weight_block_size = None
+
+ params_dtype = torch.float8_e4m3fn
+
+ # WEIGHTS
+ w13_weight = torch.nn.Parameter(
+ torch.empty(
+ num_experts,
+ 2 * intermediate_size_per_partition,
+ hidden_size,
+ dtype=params_dtype,
+ ),
+ requires_grad=False,
+ )
+ layer.register_parameter("w13_weight", w13_weight)
+ set_weight_attrs(w13_weight, extra_weight_attrs)
+
+ w2_weight = torch.nn.Parameter(
+ torch.empty(
+ num_experts,
+ hidden_size,
+ intermediate_size_per_partition,
+ dtype=params_dtype,
+ ),
+ requires_grad=False,
+ )
+ layer.register_parameter("w2_weight", w2_weight)
+ set_weight_attrs(w2_weight, extra_weight_attrs)
+
+ # WEIGHT_SCALES
+ w13_weight_scale = torch.nn.Parameter(
+ data=torch.empty(
+ num_experts,
+ 2 * intermediate_size_per_partition,
+ hidden_size // self.group_size,
+ dtype=torch.uint8, # E8M0 for MXFP8 scale
+ ),
+ requires_grad=False,
+ )
+ layer.register_parameter("w13_weight_scale", w13_weight_scale)
+ # w2
+ w2_weight_scale = torch.nn.Parameter(
+ data=torch.empty(
+ num_experts,
+ hidden_size,
+ intermediate_size_per_partition // self.group_size,
+ dtype=torch.uint8, # E8M0 for MXFP8 scale
+ ),
+ requires_grad=False,
+ )
+ layer.register_parameter("w2_weight_scale", w2_weight_scale)
+ # Add PER-TENSORGROUP quantization for FusedMoE.weight_loader.
+ extra_weight_attrs.update({"quant_method": FusedMoeWeightScaleSupported.GROUP.value})
+ set_weight_attrs(w13_weight_scale, extra_weight_attrs)
+ set_weight_attrs(w2_weight_scale, extra_weight_attrs)
+
+ def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> Optional[FusedMoEQuantConfig]:
+ from vllm.model_executor.layers.fused_moe.config import (
+ ocp_mx_moe_quant_config,
+ )
+
+ self.input_dtype = "mxfp8_e4m3"
+ self.weight_dtype = "mxfp8_e4m3"
+ return ocp_mx_moe_quant_config(
+ quant_dtype=self.input_dtype,
+ weight_dtype=self.weight_dtype,
+ w1_scale=layer.w13_weight_scale,
+ w2_scale=layer.w2_weight_scale,
+ a1_scale=None,
+ a2_scale=None,
+ w1_bias=layer.w13_bias if self.has_bias else None,
+ w2_bias=layer.w2_bias if self.has_bias else None,
+ block_shape=None,
+ )
+ return None
+
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
+ return
+
+ @torch.inference_mode()
+ def apply(
+ self,
+ layer: torch.nn.Module,
+ x: torch.Tensor,
+ router_logits: torch.Tensor,
+ top_k: int,
+ renormalize: bool,
+ use_grouped_topk: bool = False,
+ topk_group: Optional[int] = None,
+ num_expert_group: Optional[int] = None,
+ global_num_experts: int = -1,
+ expert_map: Optional[torch.Tensor] = None,
+ custom_routing_function: Optional[Callable] = None,
+ scoring_func: str = "softmax",
+ routed_scaling_factor: float = 1.0,
+ e_score_correction_bias: Optional[torch.Tensor] = None,
+ apply_router_weight_on_input: bool = False,
+ activation: str = "silu",
+ enable_eplb: bool = False,
+ expert_load_view: Optional[torch.Tensor] = None,
+ logical_to_physical_map: Optional[torch.Tensor] = None,
+ logical_replica_count: Optional[torch.Tensor] = None,
+ ) -> Union[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
+ topk_weights, topk_ids, _ = FusedMoE.select_experts(
+ hidden_states=x,
+ router_logits=router_logits,
+ use_grouped_topk=use_grouped_topk,
+ top_k=top_k,
+ renormalize=renormalize,
+ topk_group=topk_group,
+ num_expert_group=num_expert_group,
+ custom_routing_function=custom_routing_function,
+ scoring_func=scoring_func,
+ e_score_correction_bias=e_score_correction_bias,
+ )
+ assert self.fused_experts is None
+
+ from vllm.model_executor.layers.fused_moe import fused_experts
+
+ w1 = layer.w13_weight
+ w2 = layer.w2_weight
+ out = fused_experts(
+ x,
+ w1,
+ w2,
+ topk_weights=topk_weights,
+ topk_ids=topk_ids,
+ inplace=True,
+ activation=activation,
+ global_num_experts=global_num_experts,
+ apply_router_weight_on_input=apply_router_weight_on_input,
+ expert_map=expert_map,
+ quant_config=self.moe_quant_config,
+ )
+ return out
diff --git a/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py b/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py
index 197d757fa..1916fbc62 100644
--- a/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py
+++ b/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py
@@ -159,8 +159,10 @@ def dequant_mxfp4_to_fp8(data_lp, scale_e8m0):
def mxfp4_fp8_weight_to_bf16(weight_fp8, scale_bf16):
+
origin_shape = weight_fp8.shape
weight_fp8 = weight_fp8.reshape(-1, 32)
+ scale_bf16 = scale_bf16.reshape(-1, 1)
assert weight_fp8.shape[0] == scale_bf16.shape[0], f"shape mismatch: {weight_fp8.shape} vs {scale_bf16.shape}"
dequant_weight_bf16 = weight_fp8.to(torch.bfloat16) * scale_bf16
dequant_weight_bf16 = dequant_weight_bf16.reshape(origin_shape)
diff --git a/auto_round_extension/vllm_ext/mxfp8_qdq_utils.py b/auto_round_extension/vllm_ext/mxfp8_qdq_utils.py
new file mode 100644
index 000000000..482d77f85
--- /dev/null
+++ b/auto_round_extension/vllm_ext/mxfp8_qdq_utils.py
@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import torch
+
+__all__ = ["get_fp_scale", "dequant_mx_fp8", "quant_mx_fp8"]
+
+
+# def get_fp_scale(scale_e8m0):
+# # https://github.com/pytorch/ao/blob/994a4ba6c869854fcaa6ca7e118fcbd75e6c28cc/torchao/prototype/mx_formats/mx_tensor.py#L337
+# assert scale_e8m0.dtype == torch.uint8, f"Expected uint8, got {scale_e8m0.dtype}"
+# E8M0_EXPONENT_BIAS = 127
+# scale_e8m0 = scale_e8m0.view(torch.uint8)
+# s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS
+# # TODO(later): it would be nice if there was a way to do the 2^x operation
+# # in PyTorch without creating a tensor of twos
+# two = torch.full(s_offset.size(), 2.0, device=scale_e8m0.device)
+# # pow(two, s_offset) can be out of range of floating point formats.
+# # TODO(later): handle this for float16 if we decide to support float16
+# # scales.
+# s_fp = torch.pow(two, s_offset)
+
+# return s_fp
+
+
+def get_fp_scale(scale_e8m0):
+ # https://github.com/pytorch/ao/blob/994a4ba6c869854fcaa6ca7e118fcbd75e6c28cc/torchao/prototype/mx_formats/mx_tensor.py#L337
+ E8M0_EXPONENT_BIAS = 127
+
+ scale_e8m0 = scale_e8m0.view(torch.uint8)
+ s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS
+ # TODO(later): it would be nice if there was a way to do the 2^x operation
+ # in PyTorch without creating a tensor of twos
+ # two = torch.full(s_offset.size(), 2.0, device=scale_e8m0.device)
+ # pow(two, s_offset) can be out of range of floating point formats.
+ # TODO(later): handle this for float16 if we decide to support float16
+ # scales.
+ # s_fp = torch.pow(two, s_offset)
+ # !!!!NOTE Critical: fixed the OoM issue when using HPU graph
+ s_fp = torch.pow(2.0, s_offset.to(torch.float))
+
+ return s_fp
+
+
+def dequant_mx_fp8(weight_fp8, scale_e8m0, block_size, target_dtype):
+ scale_float = get_fp_scale(scale_e8m0)
+ weight_bf16 = weight_fp8.to(torch.bfloat16)
+ weight_original_shape = weight_bf16.shape
+ weight_bf16 = weight_bf16.reshape(-1, block_size)
+ scale_float = scale_float.reshape(-1, 1)
+ dequant_weight = weight_bf16 * scale_float
+ dequant_weight = dequant_weight.reshape(weight_original_shape)
+ return dequant_weight.to(target_dtype)
+
+
+def quant_mx_fp8(tensor):
+ from auto_round_extension.vllm_ext.utils import to_mx_fp8e4m3
+
+ scale_e8m0_biased, data_lp = to_mx_fp8e4m3(
+ data_hp=tensor,
+ elem_dtype=torch.float8_e4m3fn,
+ block_size=32,
+ )
+ return scale_e8m0_biased, data_lp
diff --git a/auto_round_extension/vllm_ext/quant_impl.py b/auto_round_extension/vllm_ext/quant_impl.py
new file mode 100644
index 000000000..b88843462
--- /dev/null
+++ b/auto_round_extension/vllm_ext/quant_impl.py
@@ -0,0 +1,45 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from typing import Optional
+
+import torch
+
+
+class AutoRoundQuantImpl(ABC):
+ @classmethod
+ @abstractmethod
+ def get_min_capability(cls) -> int:
+ """
+ Get minimum device capability.
+ """
+ raise NotImplementedError
+
+ @abstractmethod
+ def create_weights(self, *args, **kwargs):
+ raise NotImplementedError
+
+ @abstractmethod
+ def apply_weights(
+ self,
+ layer: torch.nn.Module,
+ x: torch.Tensor,
+ bias: Optional[torch.Tensor],
+ ):
+ raise NotImplementedError
+
+ @abstractmethod
+ def process_weights_after_loading(self, layer: torch.nn.Module):
+ raise NotImplementedError
diff --git a/auto_round_extension/vllm_ext/quant_method_linear.py b/auto_round_extension/vllm_ext/quant_method_linear.py
new file mode 100644
index 000000000..eecc97902
--- /dev/null
+++ b/auto_round_extension/vllm_ext/quant_method_linear.py
@@ -0,0 +1,112 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+
+import torch
+from vllm.logger import init_logger
+from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase, UnquantizedLinearMethod
+from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig
+
+from auto_round.schemes import QuantizationScheme
+from auto_round_extension.vllm_ext.utils import _is_mxfp4_w4a4, _is_mxfp8_w8a8, get_scheme, need_quantize
+
+logger = init_logger(__name__)
+
+
+QLINEAR_METHODS_DISPATCH_TABLE = {}
+
+
+class AutoRoundQuantLinearMethod(LinearMethodBase):
+
+ def __init__(self, impl, config=None, scheme=None):
+ self.config = config
+ self.impl = impl
+ self.scheme = scheme
+
+ @staticmethod
+ def get_method(
+ quant_config: AutoRoundConfig,
+ layer: torch.nn.Module,
+ prefix: str,
+ ) -> "AutoRoundQuantLinearMethod":
+
+ def get_impl(scheme: QuantizationScheme):
+ if _is_mxfp8_w8a8(scheme):
+ from auto_round_extension.vllm_ext.linear_impl_mxfp8 import AutoRoundMXFP8LinearImpl
+
+ return AutoRoundMXFP8LinearImpl(quant_config)
+
+ elif _is_mxfp4_w4a4(scheme):
+ from auto_round_extension.vllm_ext.linear_impl_mxfp4 import AutoRoundMXFP4LinearImpl
+
+ return AutoRoundMXFP4LinearImpl(quant_config)
+
+ raise ValueError(f"Unsupported Linear scheme: {scheme}, layer: {prefix}")
+
+ packed_modules_mapping = quant_config.packed_modules_mapping
+ for packed_name, child_names in packed_modules_mapping.items():
+ if prefix.endswith(packed_name):
+ prefix = prefix.replace(packed_name, child_names[0])
+ break
+
+ # TODO: use a more robust way to map layer names
+ if prefix.endswith("gate_up_proj"):
+ # update gate_up_proj to gate_proj, assume both gate and up share the same quantization scheme
+ prefix = prefix.replace("gate_up_proj", "up_proj")
+ if prefix.endswith("qkv_proj"):
+ # update qkv_proj to q_proj, assume all qkv share the same quantization scheme
+ prefix = prefix.replace("qkv_proj", "q_proj")
+ layer_scheme = get_scheme(quant_config, prefix)
+ if not need_quantize(layer_scheme.bits):
+ return UnquantizedLinearMethod()
+ impl = get_impl(layer_scheme)
+ logger.info("Apply %s to %s", impl.__class__.__name__, prefix)
+ return AutoRoundQuantLinearMethod(impl=impl)
+
+ @classmethod
+ def get_min_capability(cls) -> int:
+ return cls.impl.get_min_capability()
+
+ def create_weights(
+ self,
+ layer: torch.nn.Module,
+ input_size_per_partition: int,
+ output_partition_sizes: list[int],
+ input_size: int,
+ output_size: int,
+ params_dtype: torch.dtype,
+ **extra_weight_attrs,
+ ):
+ weight_loader = extra_weight_attrs.get("weight_loader")
+ return self.impl.create_weights(
+ layer=layer,
+ input_size=input_size,
+ input_size_per_partition=input_size_per_partition,
+ output_partition_sizes=output_partition_sizes,
+ output_size=output_size,
+ params_dtype=params_dtype,
+ weight_loader=weight_loader,
+ )
+
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
+ return self.impl.process_weights_after_loading(layer)
+
+ def apply(
+ self,
+ layer: torch.nn.Module,
+ x: torch.Tensor,
+ bias: Optional[torch.Tensor] = None,
+ ):
+ return self.impl.apply_weights(layer, x, bias=bias)
diff --git a/auto_round_extension/vllm_ext/quant_method_moe.py b/auto_round_extension/vllm_ext/quant_method_moe.py
index ae7b81775..ef5c75291 100644
--- a/auto_round_extension/vllm_ext/quant_method_moe.py
+++ b/auto_round_extension/vllm_ext/quant_method_moe.py
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig
from auto_round.schemes import QuantizationScheme
+from auto_round_extension.vllm_ext.utils import _is_mxfp4_w4a4, _is_mxfp8_w8a8, get_scheme, need_quantize
logger = init_logger(__name__)
@@ -31,11 +32,6 @@ logger = init_logger(__name__)
QMOE_METHODS_DISPATCH_TABLE = {}
-def _is_mxfp4_w4a4(scheme: QuantizationScheme):
- # FIXME: below impl is incomplete
- return scheme.bits == 4 and scheme.group_size == 32
-
-
class AutoRoundMoEMethod(FusedMoEMethodBase):
def __init__(self, moe: FusedMoEConfig):
super().__init__(moe)
@@ -47,21 +43,8 @@ class AutoRoundMoEMethod(FusedMoEMethodBase):
prefix: str,
) -> "AutoRoundMoEMethod":
- def get_scheme(quant_config: AutoRoundConfig, prefix: str):
- # Check extra_config first
- layer_schemes = quant_config.layer_schemes
- # FIXME: make more robust
- for name, scheme in layer_schemes.items():
- if prefix.startswith(name):
- return scheme
- # If not found, use default
- return quant_config.quant_scheme
-
- def check_quantized(weight_bits: int) -> bool:
- return weight_bits < 16
-
def get_impl(scheme: QuantizationScheme):
- if not check_quantized(scheme.bits):
+ if not need_quantize(scheme.bits):
from vllm.model_executor.layers.fused_moe.layer import (
UnquantizedFusedMoEMethod,
)
@@ -73,10 +56,16 @@ class AutoRoundMoEMethod(FusedMoEMethodBase):
return AutoRoundMoEMethodMXFp4Impl(quant_config, layer.moe_config)
+ elif _is_mxfp8_w8a8(scheme):
+ from auto_round_extension.vllm_ext.moe_impl_mxfp8 import AutoRoundMoEMethodMXFp8Impl
+
+ return AutoRoundMoEMethodMXFp8Impl(quant_config, layer.moe_config)
+
raise ValueError(f"Unsupported FusedMoe scheme: {scheme}")
layer_scheme = get_scheme(quant_config, prefix)
impl = get_impl(layer_scheme)
+ layer._prefix = prefix
logger.debug("Apply %s to %s", impl.__class__.__name__, prefix)
return impl
diff --git a/auto_round_extension/vllm_ext/tests/test_mxfp4_moe.py b/auto_round_extension/vllm_ext/tests/test_models.py
similarity index 63%
rename from auto_round_extension/vllm_ext/tests/test_mxfp4_moe.py
rename to auto_round_extension/vllm_ext/tests/test_models.py
index a2de32832..faa783cde 100644
--- a/auto_round_extension/vllm_ext/tests/test_mxfp4_moe.py
+++ b/auto_round_extension/vllm_ext/tests/test_models.py
@@ -18,10 +18,24 @@ from vllm.platforms import current_platform
MODELS = [
# "/data5/yliu7/HF_HOME/unsloth-gpt-oss-20b-BF16-ar-MXFP4/"
# "/data5/yliu7/HF_HOME/Qwen2.5-0.5B-Instruct-test-FP8_STATIC-fp8kv/"
- "/data6/yiliu4/Qwen3-15B-A2B-Base-MXFP4"
+ # "/data6/yiliu4/Qwen3-15B-A2B-Base-MXFP4",
+ # "/data6/yiliu4/Llama-3.2-1B-Instruct-MXFP4-fp8attention",
+ # "/data6/yiliu4/Llama-3.2-1B-Instruct-MXFP8"
+ "/storage/yiliu7/ar_vllm_ext/quantized_model_qwen_mxfp4",
+ "/storage/yiliu7/ar_vllm_ext/quantized_model_qwen_mxfp8",
]
[email protected](autouse=True)
+def set_vllm_ar_env(monkeypatch):
+ monkeypatch.setenv("VLLM_AR_MXFP4_MODULAR_MOE", "1")
+ monkeypatch.setenv("VLLM_MXFP4_PRE_UNPACK_TO_FP8", "1")
+ monkeypatch.setenv("VLLM_MXFP4_PRE_UNPACK_WEIGHTS", "0")
+ monkeypatch.setenv("VLLM_ENABLE_STATIC_MOE", "0")
+ monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "0")
+ monkeypatch.setenv("VLLM_ENABLE_AR_EXT", "1")
+
+
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="only supports CUDA backend.",
diff --git a/auto_round_extension/vllm_ext/utils.py b/auto_round_extension/vllm_ext/utils.py
index f98eb2325..db5a0dff1 100644
--- a/auto_round_extension/vllm_ext/utils.py
+++ b/auto_round_extension/vllm_ext/utils.py
@@ -12,12 +12,41 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from typing import Union
+
import torch
+from auto_round.schemes import QuantizationScheme
+
E8M0_EXPONENT_BIAS = 127
E8M0_EXPONENT_NAN_VAL = 255
+def get_scheme(quant_config, prefix: str):
+ # Check extra_config first
+ layer_schemes = quant_config.layer_schemes
+ # FIXME: make more robust
+ for name, scheme in layer_schemes.items():
+ if prefix.startswith(name):
+ return scheme
+ # If not found, use default
+ return quant_config.quant_scheme
+
+
+def need_quantize(weight_bits: int) -> bool:
+ return weight_bits < 16
+
+
+def _is_mxfp4_w4a4(scheme: QuantizationScheme):
+ # FIXME: below impl is incomplete
+ return scheme.bits == 4 and scheme.group_size == 32
+
+
+def _is_mxfp8_w8a8(scheme: QuantizationScheme):
+ # FIXME: below impl is incomplete
+ return scheme.bits == 8 and scheme.group_size == 32
+
+
def get_fp_scale(scale_e8m0):
scale_e8m0 = scale_e8m0.view(torch.uint8)
s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS
@@ -79,3 +108,60 @@ def _to_mx_rceil(
# scale and saturated cast the data elements to max of target dtype
data_lp = torch.clamp(data_hp * descale_fp.unsqueeze(1), min=-1 * max_pos, max=max_pos)
return exponent, data_lp
+
+
+def to_mx_fp8e4m3(
+ data_hp: torch.Tensor,
+ elem_dtype: Union[torch.dtype, str],
+ block_size: int,
+):
+ """
+ Takes a high precision tensor and converts to MX scale and raw data, in
+ naive layout (scale and raw data are separate tensors).
+ """
+
+ assert data_hp.dtype in (
+ torch.bfloat16,
+ torch.float,
+ ), f"{data_hp.dtype} is not supported yet"
+ # TODO(future PR): consider supporting padding
+ data_hp = data_hp.contiguous()
+
+ # calculate the scale in e8m0 format
+ orig_shape = data_hp.shape
+ data_hp = data_hp.reshape(-1, block_size)
+
+ # find max value of the data
+ # Note: this only implements the `minimally supported` version of
+ # https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf
+ # section 6.3.
+ max_abs = torch.amax(torch.abs(data_hp), 1)
+
+ # Set X to be the largest power-of-two less than or equal to
+ # max_abs(v), divided by the largest power of two representable
+ # in the element data type, and get the mbits at the same time
+ assert elem_dtype == torch.float8_e4m3fn, f"only float8_e4m3fn is supported now, got {elem_dtype}"
+
+ max_pos = torch.finfo(torch.float8_e4m3fn).max # 448.0
+ # TODO: add support for other rounding modes
+ scale_e8m0_biased, data_lp = _to_mx_rceil(data_hp, max_abs, max_pos)
+
+ data_lp = data_lp.to(elem_dtype)
+ # need to reshape at the end to help inductor fuse things
+ data_lp = data_lp.reshape(orig_shape)
+
+ # scale_e8m0_biased = scale_e8m0_biased.view(torch.float8_e8m0fnu)
+ return scale_e8m0_biased, data_lp
+
+
+def down_size(size):
+ assert size[-1] % 2 == 0, f"{size} last dim not divisible by two"
+ return (*size[:-1], size[-1] // 2)
+
+
+def pack_uint4(uint8_data: torch.Tensor) -> torch.Tensor:
+ # converting to uint8 for operations
+ shape = uint8_data.shape
+ assert shape[-1] % 2 == 0
+ uint8_data = uint8_data.contiguous().view(-1)
+ return (uint8_data[::2] << 4 | uint8_data[1::2]).view(down_size(shape))