Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[1/N] Initial vllm-ext evaluation support (MXFP4 MOE) (#935)

Yi Liue8bc3536d22e
diff --git a/auto_round_extension/vllm_ext/__init__.py b/auto_round_extension/vllm_ext/__init__.py
new file mode 100644
index 000000000..3e145334f
--- /dev/null
+++ b/auto_round_extension/vllm_ext/__init__.py
@@ -0,0 +1,26 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ==---------------------------------------------------------------------------==
+# Apply the extension
+# ==---------------------------------------------------------------------------==
+
+
+def apply():
+ import vllm.model_executor.layers.quantization.auto_round as auto_round_module
+
+ from auto_round_extension.vllm_ext.auto_round_ext import AutoRoundExtensionConfig
+
+ auto_round_module.AutoRoundConfig = AutoRoundExtensionConfig
+ from auto_round_extension.vllm_ext.envs_ext import extra_environment_variables
diff --git a/auto_round_extension/vllm_ext/apply_ext.sh b/auto_round_extension/vllm_ext/apply_ext.sh
new file mode 100644
index 000000000..b327ec22c
--- /dev/null
+++ b/auto_round_extension/vllm_ext/apply_ext.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Define the relative path for the `auto-round` installation
+AUTO_ROUND_PATH="auto_round/../auto_round_extension/vllm_ext/sitecustomize.py"
+
+# Try to find the pip installation location
+PIP_LOCATION=$(pip show auto-round 2>/dev/null | grep "Location:" | awk '{print $2}')
+
+if [ -n "$PIP_LOCATION" ]; then
+ SITE_CUSTOMIZE_PATH="$PIP_LOCATION/$AUTO_ROUND_PATH"
+ echo "Checking for sitecustomize.py at: $SITE_CUSTOMIZE_PATH"
+
+ if [ -f "$SITE_CUSTOMIZE_PATH" ]; then
+ echo "Found sitecustomize.py at: $SITE_CUSTOMIZE_PATH"
+ export PYTHONPATH=$(dirname "$SITE_CUSTOMIZE_PATH"):$PYTHONPATH
+ echo "PYTHONPATH set to: $PYTHONPATH"
+ return 0 2>/dev/null || true
+ fi
+fi
+
+# Fallback: check current directory
+LOCAL_SITE_CUSTOMIZE="./sitecustomize.py"
+if [ -f "$LOCAL_SITE_CUSTOMIZE" ]; then
+ echo "Found sitecustomize.py at current directory."
+ export PYTHONPATH=$(pwd):$PYTHONPATH
+ echo "PYTHONPATH set to: $PYTHONPATH"
+ return 0 2>/dev/null || true
+fi
+
+echo "Warning: sitecustomize.py not found in pip installation or current directory."
+# Do not exit the shell
+return 1 2>/dev/null || true
\ No newline at end of file
diff --git a/auto_round_extension/vllm_ext/auto_round_ext.py b/auto_round_extension/vllm_ext/auto_round_ext.py
new file mode 100644
index 000000000..cce2c912d
--- /dev/null
+++ b/auto_round_extension/vllm_ext/auto_round_ext.py
@@ -0,0 +1,63 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any
+
+import torch
+from vllm.logger import init_logger
+from vllm.model_executor.layers.fused_moe import FusedMoE
+from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
+from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig
+
+from auto_round.schemes import QuantizationScheme
+from auto_round_extension.vllm_ext.quant_method_moe import AutoRoundMoEMethod
+
+logger = init_logger(__name__)
+
+
+class AutoRoundExtensionConfig(AutoRoundConfig):
+ SUPPORTED_DTYPES = AutoRoundConfig.SUPPORTED_DTYPES.union({"mx_fp"})
+ SUPPORTED_FORMATS = AutoRoundConfig.SUPPORTED_FORMATS.union({"auto_round:llm_compressor"})
+
+ def get_quant_method(self, layer: torch.nn.Module, prefix: str):
+ # FIXME: (yi) make it compatible with `AutoRoundConfig`
+ if isinstance(layer, FusedMoE):
+ quant_method = AutoRoundMoEMethod.get_moe_method(self, layer, prefix)
+ return quant_method
+ elif isinstance(layer, LinearBase):
+ return UnquantizedLinearMethod()
+ else:
+ return None
+
+ @staticmethod
+ def _parse_quant_scheme(config: dict):
+ quant_scheme_attrs = QuantizationScheme.get_attributes()
+ filter_config = {key: value for key, value in config.items() if key in quant_scheme_attrs}
+ quant_scheme = QuantizationScheme.from_dict(filter_config)
+ return quant_scheme
+
+ @classmethod
+ def from_config(cls, config: dict[str, Any]) -> AutoRoundConfig:
+ ar_config = super().from_config(config)
+ # TODO: (yi) refine below implementation
+ quant_scheme = AutoRoundExtensionConfig._parse_quant_scheme(config)
+ layer_schemes = {}
+ layer_schemes = {} # ensure dict
+ extra_config = getattr(ar_config, "extra_config", None)
+ if extra_config is not None:
+ for layer_name, layer_config in extra_config.items():
+ layer_schemes[layer_name] = AutoRoundExtensionConfig._parse_quant_scheme(layer_config)
+ ar_config.quant_scheme = quant_scheme
+ ar_config.layer_schemes = layer_schemes
+ return ar_config
diff --git a/auto_round_extension/vllm_ext/envs_ext.py b/auto_round_extension/vllm_ext/envs_ext.py
new file mode 100644
index 000000000..325b9de7c
--- /dev/null
+++ b/auto_round_extension/vllm_ext/envs_ext.py
@@ -0,0 +1,39 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from typing import Any, Callable
+
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+# Define extra environment variables
+extra_environment_variables: dict[str, Callable[[], Any]] = {
+ "VLLM_MXFP4_PRE_UNPACK_WEIGHTS": lambda: os.getenv("VLLM_MXFP4_PRE_UNPACK_WEIGHTS", "1") in ("1", "true", "True"),
+ "VLLM_ENABLE_STATIC_MOE": lambda: os.getenv("VLLM_ENABLE_STATIC_MOE", "1") in ("1", "true", "True"),
+ "VLLM_AR_MXFP4_MODULAR_MOE": lambda: os.getenv("VLLM_AR_MXFP4_MODULAR_MOE", "0") in ("1", "true", "True"),
+}
+# Add the extra environment variables to vllm.envs
+import vllm.envs as envs
+from vllm.envs import environment_variables
+
+# Merge the environment variables
+all_environment_variables = {**environment_variables, **extra_environment_variables}
+
+
+for name, value_fn in extra_environment_variables.items():
+ setattr(envs, name, value_fn())
+
+logger.warning_once(f"Added extra environment variables: {list(extra_environment_variables.keys())}")
diff --git a/auto_round_extension/vllm_ext/fp4_utils.py b/auto_round_extension/vllm_ext/fp4_utils.py
new file mode 100644
index 000000000..adb67d190
--- /dev/null
+++ b/auto_round_extension/vllm_ext/fp4_utils.py
@@ -0,0 +1,113 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+
+import torch
+
+# Module-level device tensor cache to fix cuda graph issue
+_DEVICE_E2M1_TENSORS = {}
+
+# Constants for FP4 values (E2M1 format)
+_E2M1_VALUES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
+
+
+def get_e2m1_tensor(device):
+ """Get device-specific E2M1 lookup tensor, creating it if needed."""
+ device_str = str(device)
+ if device_str not in _DEVICE_E2M1_TENSORS:
+ _DEVICE_E2M1_TENSORS[device_str] = torch.tensor(_E2M1_VALUES, dtype=torch.float32, device=device)
+ return _DEVICE_E2M1_TENSORS[device_str]
+
+
+def pack_fp4_to_uint8(x: torch.Tensor) -> torch.Tensor:
+ m, n = x.shape
+ device = x.device
+
+ # Create lookup table for FP4 values to indices
+ # Map the absolute values to 0-7 indices
+ kE2M1 = get_e2m1_tensor(x.device)
+
+ # Find closest valid FP4 value index for each element
+ abs_x = torch.abs(x)
+ abs_diff_x = torch.abs(abs_x.unsqueeze(-1) - kE2M1) # [m, n, 8]
+ abs_indices = torch.argmin(abs_diff_x, dim=-1) # [m, n]
+
+ # Apply sign bit (bit 3) to get final 4-bit representation
+ indices = abs_indices + (torch.signbit(x).to(torch.long) << 3)
+
+ # Reshape to prepare for packing pairs of values
+ indices = indices.reshape(-1)
+
+ # Handle odd length by padding if necessary
+ assert indices.numel() % 2 != 0, f"Expected even number of elements, got {indices.numel()}"
+
+ # Reshape to pair consecutive elements
+ indices = indices.reshape(-1, 2)
+
+ # Pack pairs of 4-bit values into 8-bit values
+ packed = (indices[:, 0] | (indices[:, 1] << 4)).to(torch.uint8)
+
+ return packed.reshape(m, n // 2)
+
+
+def unpack_fp4_from_uint8(
+ a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16
+) -> torch.Tensor:
+ """
+ Unpacks uint8 values into fp4. Each uint8 consists of two fp4 values
+ (i.e. first four bits correspond to one fp4 value, last four correspond to a
+ consecutive fp4 value). The bits represent an index, which are mapped to an fp4
+ value.
+
+ :param a: tensor to unpack
+ :param m: original dim 0 size of the unpacked tensor
+ :param n: original dim 1 size of the unpacked tensor
+ :param dtype: dense dtype to cast the unpacked tensor to
+ """
+ assert a.dtype == torch.uint8, f"expected uint8, got {a.dtype}"
+
+ # Vectorized nibble processing
+ a_flat = a.flatten()
+ high = (a_flat & 0xF0) >> 4 # Upper nibbles
+ low = a_flat & 0x0F # Lower nibbles
+
+ # Combine nibbles for batch processing
+ combined = torch.stack((low, high), dim=1).flatten()
+
+ # Vectorized sign and magnitude extraction
+ signs = (combined & 0x08).to(torch.bool) # Sign bits
+ abs_vals = (combined & 0x07).to(torch.long) # Magnitude indices
+
+ # Device-aware lookup and sign application
+ kE2M1 = get_e2m1_tensor(a.device)
+
+ values = kE2M1[abs_vals] * torch.where(signs, -1.0, 1.0)
+
+ # Reshape to final form
+ return values.reshape(m, n).to(dtype=dtype)
+
+
+def cast_to_fp4(x):
+ sign = torch.sign(x)
+ x = torch.abs(x)
+ x[(x >= 0.0) & (x <= 0.25)] = 0.0
+ x[(x > 0.25) & (x < 0.75)] = 0.5
+ x[(x >= 0.75) & (x <= 1.25)] = 1.0
+ x[(x > 1.25) & (x < 1.75)] = 1.5
+ x[(x >= 1.75) & (x <= 2.5)] = 2.0
+ x[(x > 2.5) & (x < 3.5)] = 3.0
+ x[(x >= 3.5) & (x <= 5.0)] = 4.0
+ x[x > 5.0] = 6.0
+ return x * sign
diff --git a/auto_round_extension/vllm_ext/moe_impl_mxfp4.py b/auto_round_extension/vllm_ext/moe_impl_mxfp4.py
new file mode 100644
index 000000000..77ec6570d
--- /dev/null
+++ b/auto_round_extension/vllm_ext/moe_impl_mxfp4.py
@@ -0,0 +1,539 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Callable, Optional, Union
+
+import torch
+import torch.nn.functional as F
+import vllm.envs as envs
+from vllm.distributed import get_tensor_model_parallel_rank
+from vllm.logger import init_logger
+from vllm.model_executor.layers.fused_moe import (
+ FusedMoE,
+ FusedMoEConfig,
+ FusedMoeWeightScaleSupported,
+)
+from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
+from vllm.model_executor.utils import set_weight_attrs
+
+import auto_round_extension.vllm_ext.mxfp4_qdq_utils as mxfp4_utils
+from auto_round_extension.vllm_ext.mxfp4_qdq_utils import (
+ dequant_mxfp4_to_fp8,
+ mxfp4_gemm_with_unpacked_weight,
+ run_mxfp4_emulations,
+)
+from auto_round_extension.vllm_ext.quant_method_moe import AutoRoundMoEMethod
+
+logger = init_logger(__name__)
+
+
+def apply_act(local_w1_out: torch.Tensor, local_w3_out: torch.Tensor, activation: str) -> torch.Tensor:
+ if activation == "silu":
+ act_fn = F.silu
+ w13_out = act_fn(local_w1_out) * local_w3_out
+ elif activation == "swigluoai":
+ limit = 7.0
+ alpha = 1.702
+ local_w1_out = local_w1_out.clamp(min=None, max=limit)
+ local_w3_out = local_w3_out.clamp(min=-limit, max=limit)
+ glu = (local_w1_out) * F.sigmoid(local_w1_out * alpha)
+ w13_out = (local_w3_out + 1) * glu
+ else:
+ raise NotImplementedError(f"Activation {activation} is not implemented.")
+ return w13_out
+
+
+class AutoRoundMoEMethodMXFp4Impl(AutoRoundMoEMethod):
+ def __init__(
+ self,
+ quant_config: "AutoRoundConfig", # type: ignore # noqa E501
+ moe: FusedMoEConfig,
+ ):
+ super().__init__(moe)
+ self.use_marlin = False
+ self.group_size = 32
+ self.quant_config = quant_config
+ self.has_bias = self.moe.has_bias
+ self.mask_weights_buffer = None
+ self.experts_mask_buffer = None
+ self.num_all_tokens_threshold = 16 * 1024
+ # self.num_all_tokens_threshold = 0
+
+ def create_weights(
+ self,
+ layer: torch.nn.Module,
+ num_experts: int,
+ hidden_size: int,
+ intermediate_size_per_partition: int,
+ params_dtype: torch.dtype,
+ **extra_weight_attrs,
+ ):
+ layer.num_experts = num_experts
+ layer.params_dtype = params_dtype
+
+ w13_weight = torch.nn.Parameter(
+ torch.empty(
+ num_experts,
+ 2 * intermediate_size_per_partition,
+ # 2 fp4 items are packed in the input dimension
+ hidden_size // 2,
+ requires_grad=False,
+ dtype=torch.uint8,
+ ),
+ requires_grad=False,
+ )
+ layer.register_parameter("w13_weight_packed", w13_weight)
+ set_weight_attrs(w13_weight, extra_weight_attrs)
+
+ w2_weight = torch.nn.Parameter(
+ torch.empty(
+ num_experts,
+ hidden_size,
+ # 2 fp4 items are packed in the input dimension
+ intermediate_size_per_partition // 2,
+ dtype=torch.uint8,
+ ),
+ requires_grad=False,
+ )
+ layer.register_parameter("w2_weight_packed", w2_weight)
+ set_weight_attrs(w2_weight, extra_weight_attrs)
+
+ # Weight Scales
+ w13_weight_scale = torch.nn.Parameter(
+ torch.empty(
+ num_experts,
+ 2 * intermediate_size_per_partition,
+ # 2 fp4 items are packed in the input dimension
+ hidden_size // self.group_size,
+ dtype=torch.uint8,
+ ),
+ requires_grad=False,
+ )
+ layer.register_parameter("w13_weight_scale", w13_weight_scale)
+ extra_weight_attrs.update({"quant_method": FusedMoeWeightScaleSupported.GROUP.value})
+ set_weight_attrs(w13_weight_scale, extra_weight_attrs)
+
+ w2_weight_scale = torch.nn.Parameter(
+ torch.empty(
+ num_experts,
+ hidden_size,
+ # 2 fp4 items are packed in the input dimension
+ intermediate_size_per_partition // self.group_size,
+ dtype=torch.uint8,
+ ),
+ requires_grad=False,
+ )
+ layer.register_parameter("w2_weight_scale", w2_weight_scale)
+ extra_weight_attrs.update({"quant_method": FusedMoeWeightScaleSupported.GROUP.value})
+ set_weight_attrs(w2_weight_scale, extra_weight_attrs)
+
+ E = num_experts
+ H = hidden_size
+ IN = intermediate_size_per_partition
+ if self.has_bias:
+ # TODO: yiliu30 use the dtype in CK
+ bias_dtype = torch.bfloat16
+ w13_bias = torch.nn.Parameter(torch.zeros(E, 2 * IN, dtype=bias_dtype), requires_grad=False)
+ layer.register_parameter("w13_bias", w13_bias)
+ set_weight_attrs(w13_bias, extra_weight_attrs)
+
+ w2_bias = torch.nn.Parameter(
+ torch.zeros(E, H, dtype=bias_dtype),
+ requires_grad=False,
+ )
+ layer.register_parameter("w2_bias", w2_bias)
+ set_weight_attrs(w2_bias, extra_weight_attrs)
+
+ def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> Optional[FusedMoEQuantConfig]:
+ if envs.VLLM_AR_MXFP4_MODULAR_MOE:
+ from vllm.model_executor.layers.fused_moe.config import (
+ ocp_mx_moe_quant_config,
+ )
+
+ self.input_dtype = "mxfp4"
+ self.weight_dtype = "mxfp4"
+ return ocp_mx_moe_quant_config(
+ quant_dtype=self.input_dtype,
+ weight_dtype=self.weight_dtype,
+ w1_scale=layer.w13_weight_scale,
+ w2_scale=layer.w2_weight_scale,
+ a1_scale=None,
+ a2_scale=None,
+ w1_bias=layer.w13_bias if self.has_bias else None,
+ w2_bias=layer.w2_bias if self.has_bias else None,
+ block_shape=None,
+ )
+ return None
+
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
+ if envs.VLLM_ENABLE_STATIC_MOE:
+ if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:
+ weight_name_lst = ["w13_weight", "w2_weight"]
+
+ for weight_name_prefix in weight_name_lst:
+ weight_name = f"{weight_name_prefix}_packed"
+ weight = getattr(layer, weight_name)
+ weight_scale_name = f"{weight_name_prefix}_scale"
+ weight_scale = getattr(layer, weight_scale_name)
+ new_weight_name = f"{weight_name_prefix}_unpacked"
+ new_scale_name = weight_scale_name
+ num_experts, _, _ = weight.shape
+ unpacked_weight_lst = []
+ scale_list = []
+ for expert_index in range(num_experts):
+ weight_fp8, scale_bf16 = dequant_mxfp4_to_fp8(
+ data_lp=weight[expert_index],
+ scale_e8m0=weight_scale[expert_index],
+ )
+
+ unpacked_weight_lst.append(weight_fp8)
+ scale_list.append(scale_bf16)
+ unpacked_weight_fp8 = torch.stack(unpacked_weight_lst, dim=0)
+ scale_bf16 = torch.stack(scale_list, dim=0)
+ assert unpacked_weight_fp8.shape[0] == num_experts, (
+ f"Expected {num_experts} unpacked weights, got " f"{unpacked_weight_fp8.shape[0]}"
+ )
+ delattr(layer, weight_name)
+ delattr(layer, weight_scale_name)
+ layer.register_parameter(
+ new_weight_name,
+ torch.nn.Parameter(
+ unpacked_weight_fp8,
+ requires_grad=False,
+ ),
+ )
+ layer.register_parameter(
+ new_scale_name,
+ torch.nn.Parameter(
+ scale_bf16,
+ requires_grad=False,
+ ),
+ )
+
+ elif envs.VLLM_AR_MXFP4_MODULAR_MOE:
+
+ def revert_interleaved_bias(bias):
+ """
+ Convert from blocked bias format to interleaved format.
+
+ Args:
+ bias: Tensor of shape [E, intermediate_size*2] where the first half contains
+ w1 biases and second half contains w3 biases for each expert
+
+ Returns:
+ Tensor with interleaved w1 and w3 biases
+ """
+ # loaded bias[0]: [e0_w1_bias_0, e0_w1_bias_1, ..., e0_w3_bias_0, e0_w3_bias_1...]
+ # Expected bias[0]: [e0_w1_bias_0, e0_w3_bias_0, e0_w1_bias_1, e0_w3_bias_1...]
+ # The expected bias order is used in triton kernel https://github.com/vllm-project/vllm/pull/22508
+ revert_bias = torch.zeros_like(bias, device=bias.device)
+ E, two_IN = bias.shape
+
+ # Verify the shape is as expected
+ if two_IN % 2 != 0:
+ raise ValueError(f"Expected even number of bias elements, got {two_IN}")
+
+ revert_bias[..., ::2] = bias[..., : two_IN // 2]
+ revert_bias[..., 1::2] = bias[..., two_IN // 2 :]
+
+ return revert_bias
+
+ # breakpoint()
+ w13_bias_swapped = revert_interleaved_bias(layer.w13_bias)
+ layer.w13_bias.data.copy_(w13_bias_swapped)
+
+ if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:
+
+ w1 = layer.w13_weight_packed
+ w1_scale = layer.w13_weight_scale
+ w1 = mxfp4_utils.to_dtype(
+ data_lp=w1,
+ scale_e8m0=w1_scale,
+ elem_dtype="fp4_e2m1",
+ block_size=32,
+ target_dtype=torch.bfloat16,
+ )
+
+ def revert_interleaved_w1(w1):
+ new_w1 = torch.zeros_like(w1)
+ E, N, H = w1.shape
+ new_w1[:, ::2, :] = w1[:, : N // 2, :]
+ new_w1[:, 1::2, :] = w1[:, N // 2 :, :]
+ return new_w1
+
+ w1 = revert_interleaved_w1(w1)
+
+ w1_scale = None
+ w2 = layer.w2_weight_packed
+ w2_scale = layer.w2_weight_scale
+ w2 = mxfp4_utils.to_dtype(
+ data_lp=w2,
+ scale_e8m0=w2_scale,
+ elem_dtype="fp4_e2m1",
+ block_size=32,
+ target_dtype=torch.bfloat16,
+ )
+ w2_scale = None
+ del layer.w13_weight_packed
+ del layer.w13_weight_scale
+ del layer.w2_weight_packed
+ del layer.w2_weight_scale
+ layer.w13_weight_scale = None
+ layer.w2_weight_scale = None
+ layer.register_parameter(
+ "w13_weight_unpacked",
+ torch.nn.Parameter(
+ w1,
+ requires_grad=False,
+ ),
+ )
+ layer.register_parameter(
+ "w2_weight_unpacked",
+ torch.nn.Parameter(
+ w2,
+ requires_grad=False,
+ ),
+ )
+
+ else:
+ raise NotImplementedError("process_weights_after_loading is not implemented for now.")
+
+ @torch.inference_mode()
+ def apply(
+ self,
+ layer: torch.nn.Module,
+ x: torch.Tensor,
+ router_logits: torch.Tensor,
+ top_k: int,
+ renormalize: bool,
+ use_grouped_topk: bool = False,
+ topk_group: Optional[int] = None,
+ num_expert_group: Optional[int] = None,
+ global_num_experts: int = -1,
+ expert_map: Optional[torch.Tensor] = None,
+ custom_routing_function: Optional[Callable] = None,
+ scoring_func: str = "softmax",
+ routed_scaling_factor: float = 1.0,
+ e_score_correction_bias: Optional[torch.Tensor] = None,
+ apply_router_weight_on_input: bool = False,
+ activation: str = "silu",
+ enable_eplb: bool = False,
+ expert_load_view: Optional[torch.Tensor] = None,
+ logical_to_physical_map: Optional[torch.Tensor] = None,
+ logical_replica_count: Optional[torch.Tensor] = None,
+ ) -> Union[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
+ topk_weights, topk_ids, _ = FusedMoE.select_experts(
+ hidden_states=x,
+ router_logits=router_logits,
+ use_grouped_topk=use_grouped_topk,
+ top_k=top_k,
+ renormalize=renormalize,
+ topk_group=topk_group,
+ num_expert_group=num_expert_group,
+ custom_routing_function=custom_routing_function,
+ scoring_func=scoring_func,
+ e_score_correction_bias=e_score_correction_bias,
+ )
+ assert self.fused_experts is None
+
+ # There are three implementations:
+
+ if envs.VLLM_AR_MXFP4_MODULAR_MOE:
+ from vllm.model_executor.layers.fused_moe import fused_experts
+
+ if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:
+ w1 = layer.w13_weight_unpacked
+ w2 = layer.w2_weight_unpacked
+ else:
+ w1 = layer.w13_weight_packed
+ w2 = layer.w2_weight_packed
+ out = fused_experts(
+ x,
+ w1,
+ w2,
+ topk_weights=topk_weights,
+ topk_ids=topk_ids,
+ inplace=True,
+ activation=activation,
+ global_num_experts=global_num_experts,
+ apply_router_weight_on_input=apply_router_weight_on_input,
+ expert_map=expert_map,
+ quant_config=self.moe_quant_config,
+ )
+ return out
+
+ num_all_tokens, hidden_dim = x.shape
+ num_experts = layer.local_num_experts
+ total_num_experts = router_logits.size(-1)
+ if (
+ self.mask_weights_buffer is None
+ or self.mask_weights_buffer.dtype != x.dtype
+ or self.mask_weights_buffer.device != x.device
+ or self.mask_weights_buffer.shape[0] < num_all_tokens
+ or self.mask_weights_buffer.shape[1] < total_num_experts
+ ):
+ if num_all_tokens > self.num_all_tokens_threshold:
+ mask_weights = torch.zeros((num_all_tokens, total_num_experts), dtype=x.dtype, device=x.device)
+ if self.mask_weights_buffer is None and self.num_all_tokens_threshold != 0:
+ self.mask_weights_buffer = torch.zeros(
+ (self.num_all_tokens_threshold, total_num_experts),
+ dtype=x.dtype,
+ device=x.device,
+ )
+ self.experts_mask_buffer = torch.zeros(
+ (self.num_all_tokens_threshold, total_num_experts),
+ dtype=x.dtype,
+ device=x.device,
+ )
+ else:
+ self.mask_weights_buffer = torch.zeros(
+ (num_all_tokens, total_num_experts), dtype=x.dtype, device=x.device
+ )
+ self.experts_mask_buffer = torch.zeros(
+ (num_all_tokens, total_num_experts), dtype=x.dtype, device=x.device
+ )
+ mask_weights = self.mask_weights_buffer
+ experts_mask = self.experts_mask_buffer
+ else:
+ self.mask_weights_buffer.zero_()
+ mask_weights = self.mask_weights_buffer
+ self.experts_mask_buffer.zero_()
+ experts_mask = self.experts_mask_buffer
+
+ topk_ids = topk_ids.to(torch.int64)
+ topk_weights = topk_weights.to(x.dtype)
+ experts_mask.scatter_(-1, topk_ids, topk_weights)
+ mask_weights.scatter_(-1, topk_ids, 1)
+ mask_weights = mask_weights[:num_all_tokens, :total_num_experts]
+ mask_weights = mask_weights.transpose(0, 1)
+ experts_mask = experts_mask[:num_all_tokens, :total_num_experts]
+ experts_mask = experts_mask.transpose(0, 1)
+ # Note: ep_size equal tp_size
+ ep_rank = get_tensor_model_parallel_rank() if expert_map is not None else 0
+ ep_shift = ep_rank * num_experts
+
+ if envs.VLLM_ENABLE_STATIC_MOE and not envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:
+ num_experts, intermediate_size_per_partition_x2, _ = layer.w13_weight_packed.shape
+ intermediate_size_per_partition = intermediate_size_per_partition_x2 // 2
+ for expert_index in range(num_experts):
+ mask_weight = mask_weights[expert_index + ep_shift].unsqueeze(1)
+ current_state_static = x * mask_weight
+
+ local_w13_packed = layer.w13_weight_packed[expert_index]
+ local_w13_scale = layer.w13_weight_scale[expert_index]
+ local_w2_packed = layer.w2_weight_packed[expert_index]
+ local_w2_scale = layer.w2_weight_scale[expert_index]
+
+ local_w1_packed = local_w13_packed[:intermediate_size_per_partition, ...]
+ local_w1_scale = local_w13_scale[:intermediate_size_per_partition, ...]
+
+ local_w3_packed = local_w13_packed[intermediate_size_per_partition:, ...]
+ local_w3_scale = local_w13_scale[intermediate_size_per_partition:, ...]
+
+ local_w1_bias = None
+ local_w2_bias = None
+ local_w3_bias = None
+ if self.has_bias:
+ local_w13_bias = layer.w13_bias[expert_index]
+ local_w1_bias = local_w13_bias[:intermediate_size_per_partition]
+ local_w3_bias = local_w13_bias[intermediate_size_per_partition:]
+ local_w2_bias = layer.w2_bias[expert_index]
+
+ local_w1_out = run_mxfp4_emulations(
+ x=current_state_static,
+ weight=local_w1_packed,
+ weight_scale=local_w1_scale,
+ bias=local_w1_bias,
+ )
+ local_w3_out = run_mxfp4_emulations(
+ x=current_state_static,
+ weight=local_w3_packed,
+ weight_scale=local_w3_scale,
+ bias=local_w3_bias,
+ )
+
+ w13_out = apply_act(local_w1_out, local_w3_out, activation)
+
+ local_w2_out = run_mxfp4_emulations(
+ x=w13_out,
+ weight=local_w2_packed,
+ weight_scale=local_w2_scale,
+ bias=local_w2_bias,
+ )
+ padded_weight = experts_mask[expert_index + ep_shift].unsqueeze(1)
+ local_w2_out = local_w2_out * padded_weight
+ if expert_index == 0:
+ final_hidden_states = local_w2_out
+ else:
+ final_hidden_states += local_w2_out
+ return final_hidden_states
+ if envs.VLLM_ENABLE_STATIC_MOE and envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:
+ num_experts, intermediate_size_per_partition_x2, _ = layer.w13_weight_unpacked.shape
+ intermediate_size_per_partition = intermediate_size_per_partition_x2 // 2
+
+ for expert_index in range(num_experts):
+ mask_weight = mask_weights[expert_index + ep_shift].unsqueeze(1)
+ current_state_static = x * mask_weight
+
+ local_unpacked_w13 = layer.w13_weight_unpacked[expert_index]
+ local_w13_scale = layer.w13_weight_scale[expert_index]
+
+ local_unpacked_w2 = layer.w2_weight_unpacked[expert_index]
+ local_w2_scale = layer.w2_weight_scale[expert_index]
+
+ local_unpacked_w1 = local_unpacked_w13[:intermediate_size_per_partition, ...]
+ half_scale = local_w13_scale.shape[0] // 2
+ local_w1_scale = local_w13_scale[:half_scale, ...]
+ local_unpacked_w3 = local_unpacked_w13[intermediate_size_per_partition:, ...]
+ local_w3_scale = local_w13_scale[half_scale:, ...]
+
+ local_w1_bias = None
+ local_w2_bias = None
+ local_w3_bias = None
+ if self.has_bias:
+ local_w13_bias = layer.w13_bias[expert_index]
+ local_w1_bias = local_w13_bias[:intermediate_size_per_partition]
+ local_w3_bias = local_w13_bias[intermediate_size_per_partition:]
+ local_w2_bias = layer.w2_bias[expert_index]
+
+ local_w1_out = mxfp4_gemm_with_unpacked_weight(
+ x=current_state_static,
+ weight_fp8=local_unpacked_w1,
+ weight_scale_bf16=local_w1_scale,
+ bias=local_w1_bias,
+ )
+ local_w3_out = mxfp4_gemm_with_unpacked_weight(
+ x=current_state_static,
+ weight_fp8=local_unpacked_w3,
+ weight_scale_bf16=local_w3_scale,
+ bias=local_w3_bias,
+ )
+
+ w13_out = apply_act(local_w1_out, local_w3_out, activation)
+
+ local_w2_out = mxfp4_gemm_with_unpacked_weight(
+ x=w13_out,
+ weight_fp8=local_unpacked_w2,
+ weight_scale_bf16=local_w2_scale,
+ bias=local_w2_bias,
+ )
+
+ padded_weight = experts_mask[expert_index + ep_shift].unsqueeze(1)
+ local_w2_out = local_w2_out * padded_weight
+ if expert_index == 0:
+ final_hidden_states = local_w2_out
+ else:
+ final_hidden_states += local_w2_out
+ return final_hidden_states
+ raise NotImplementedError("Not implemented for now.")
diff --git a/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py b/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py
new file mode 100644
index 000000000..197d757fa
--- /dev/null
+++ b/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py
@@ -0,0 +1,220 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Union
+
+import torch
+
+from auto_round_extension.vllm_ext.fp4_utils import cast_to_fp4, pack_fp4_to_uint8, unpack_fp4_from_uint8
+from auto_round_extension.vllm_ext.utils import _to_mx_rceil, get_fp_scale
+
+F4_E2M1_MAX = 6.0
+F32_EXP_BIAS = 127
+
+F32_MIN_NORMAL = 2 ** (-F32_EXP_BIAS + 1)
+
+
+def to_mxfp4_rceil(
+ data_hp: torch.Tensor,
+ elem_dtype: Union[torch.dtype, str],
+ block_size: int,
+):
+
+ assert data_hp.dtype in (
+ torch.bfloat16,
+ torch.float,
+ ), f"{data_hp.dtype} is not supported yet"
+ # TODO(future PR): consider supporting padding
+ assert data_hp.numel() % block_size == 0, f"data size must be multiple of block_size={block_size}"
+ assert data_hp.is_contiguous(), f"data must be contiguous, got {data_hp.stride()}"
+
+ # calculate the scale in e8m0 format
+
+ orig_shape = data_hp.shape
+ data_hp = data_hp.reshape(-1, block_size)
+
+ # find max value of the data
+ # Note: this only implements the `minimally supported` version of
+ # https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf
+ # section 6.3.
+ max_abs = torch.amax(torch.abs(data_hp), 1)
+
+ # Add an epsilon to prevent the log2 function call for returning -inf
+ # where the values are zero.
+ eps = F32_MIN_NORMAL * (max_abs == 0).type(max_abs.dtype)
+
+ # Set X to be the largest power-of-two less than or equal to
+ # max_abs(v), divided by the largest power of two representable
+ # in the element data type, and get the mbits at the same time
+
+ max_pos = F4_E2M1_MAX
+ scale_e8m0_biased, data_lp = _to_mx_rceil(data_hp, max_abs, max_pos)
+
+ data_lp = data_lp.reshape(orig_shape)
+ orig_shape = [*orig_shape[:-1], orig_shape[-1] // 2]
+ data_lp = cast_to_fp4(data_lp)
+ data_lp = pack_fp4_to_uint8(data_lp)
+
+ scale_e8m0_biased = scale_e8m0_biased.view(torch.uint8)
+ return scale_e8m0_biased, data_lp
+
+
+def to_dtype(
+ data_lp,
+ scale_e8m0,
+ elem_dtype,
+ block_size,
+ target_dtype,
+ scale_dtype=None,
+ return_scale=False,
+):
+ orig_shape = data_lp.shape
+ last_dim = orig_shape[-1]
+ data_lp = data_lp.reshape(-1, last_dim)
+ result_shape = orig_shape[:-1] + (last_dim * 2,)
+ assert data_lp.is_contiguous(), f"Data must be contiguous, got {data_lp.stride()}"
+
+ assert elem_dtype == "fp4_e2m1", f"Expected 'fp4_e2m1', got {elem_dtype}"
+
+ m, half_n = data_lp.shape
+ n = half_n * 2
+ data_hp = unpack_fp4_from_uint8(data_lp, m, n, dtype=target_dtype)
+
+ data_hp = data_hp.reshape(-1, block_size)
+
+ if scale_dtype is None:
+ scale_dtype = target_dtype
+ s_fp = get_fp_scale(scale_e8m0).reshape(-1, 1).to(scale_dtype)
+ if return_scale:
+ return data_hp.reshape(result_shape), s_fp
+
+ data_hp = data_hp * s_fp
+ data_hp = data_hp.reshape(result_shape)
+
+ return data_hp
+
+
+def run_mxfp4_emulations(
+ x: torch.Tensor,
+ weight: torch.Tensor,
+ weight_scale: torch.Tensor,
+ bias: torch.Tensor | None = None,
+):
+ # TODO: select the rounding mode based on config
+ group_size = 32
+ # quantize input to (FP4 and interleaved block scale)
+ input_scale, x_q = to_mxfp4_rceil(
+ data_hp=x,
+ elem_dtype="fp4_e2m1",
+ block_size=group_size,
+ )
+
+ # dequantize input
+ x_dq = to_dtype(
+ data_lp=x_q,
+ scale_e8m0=input_scale,
+ elem_dtype="fp4_e2m1",
+ block_size=group_size,
+ target_dtype=x.dtype,
+ )
+
+ # dequantize weight
+ w_dq = to_dtype(
+ data_lp=weight,
+ scale_e8m0=weight_scale,
+ elem_dtype="fp4_e2m1",
+ block_size=group_size,
+ target_dtype=x.dtype,
+ )
+
+ # matmul
+ out = torch.matmul(x_dq, w_dq.t())
+ if bias is not None:
+ out += bias
+ return out
+
+
+def dequant_mxfp4_to_fp8(data_lp, scale_e8m0):
+ data_fp8, scale_float = to_dtype(
+ data_lp=data_lp,
+ scale_e8m0=scale_e8m0,
+ elem_dtype="fp4_e2m1",
+ block_size=32,
+ target_dtype=torch.float8_e4m3fn,
+ scale_dtype=torch.bfloat16,
+ return_scale=True,
+ )
+ return data_fp8, scale_float
+
+
+def mxfp4_fp8_weight_to_bf16(weight_fp8, scale_bf16):
+ origin_shape = weight_fp8.shape
+ weight_fp8 = weight_fp8.reshape(-1, 32)
+ assert weight_fp8.shape[0] == scale_bf16.shape[0], f"shape mismatch: {weight_fp8.shape} vs {scale_bf16.shape}"
+ dequant_weight_bf16 = weight_fp8.to(torch.bfloat16) * scale_bf16
+ dequant_weight_bf16 = dequant_weight_bf16.reshape(origin_shape)
+ return dequant_weight_bf16
+
+
+def mxfp4_gemm_with_unpacked_weight(x, weight_fp8, weight_scale_bf16, bias=None):
+ x = qdq_mxfp4(x)
+
+ # dequantize weight
+ w_dq = mxfp4_fp8_weight_to_bf16(weight_fp8, weight_scale_bf16)
+ # matmul
+ out = torch.matmul(x, w_dq.t())
+ if bias is not None:
+ out += bias
+ return out
+
+
+def fp4_121_positive(x: torch.Tensor) -> torch.Tensor:
+ step1 = torch.round(2.0 * x) / 2.0
+ step2 = torch.round(x)
+ step3 = 2.0 * torch.round(x / 2.0)
+
+ mask1 = x < 2.0
+ mask2 = x < 4.0
+
+ return step1 * mask1 + step2 * (~mask1) * mask2 + step3 * (~mask1) * (~mask2)
+
+
+def fp4_121_scaled_even_rounding(x: torch.Tensor) -> torch.Tensor:
+ sign = x.sign()
+ x_abs = x.abs()
+ amax_x = x_abs.max(dim=-1, keepdim=True)[0]
+ scale_tmp = torch.floor(torch.log2(amax_x)) - 2.0
+ scale_clamp = torch.clamp(scale_tmp, min=-127, max=127)
+ scale = torch.pow(2.0, scale_clamp)
+
+ scale = torch.where((0 < scale) * (scale < torch.inf), scale, 1.0)
+
+ x_fp4_abs = fp4_121_positive(x_abs / scale) * scale
+ return sign * x_fp4_abs
+
+
+# https://github.com/Anonymous1252022/fp4-all-the-way/blob/main/experimental/fp4.py
+def qdq_mxfp4(
+ x: torch.Tensor,
+) -> torch.Tensor:
+ # TODO: select the rounding mode based on config
+ block_size = 32
+ shape = x.shape
+ x = x.reshape(-1, block_size)
+
+ x = fp4_121_scaled_even_rounding(x)
+
+ x = x.reshape(shape)
+
+ return x
diff --git a/auto_round_extension/vllm_ext/quant_method_moe.py b/auto_round_extension/vllm_ext/quant_method_moe.py
new file mode 100644
index 000000000..ae7b81775
--- /dev/null
+++ b/auto_round_extension/vllm_ext/quant_method_moe.py
@@ -0,0 +1,84 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+
+import torch
+from vllm.logger import init_logger
+from vllm.model_executor.layers.fused_moe import (
+ FusedMoEConfig,
+ FusedMoEMethodBase,
+)
+from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
+from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig
+
+from auto_round.schemes import QuantizationScheme
+
+logger = init_logger(__name__)
+
+
+QMOE_METHODS_DISPATCH_TABLE = {}
+
+
+def _is_mxfp4_w4a4(scheme: QuantizationScheme):
+ # FIXME: below impl is incomplete
+ return scheme.bits == 4 and scheme.group_size == 32
+
+
+class AutoRoundMoEMethod(FusedMoEMethodBase):
+ def __init__(self, moe: FusedMoEConfig):
+ super().__init__(moe)
+
+ @staticmethod
+ def get_moe_method(
+ quant_config: AutoRoundConfig,
+ layer: torch.nn.Module,
+ prefix: str,
+ ) -> "AutoRoundMoEMethod":
+
+ def get_scheme(quant_config: AutoRoundConfig, prefix: str):
+ # Check extra_config first
+ layer_schemes = quant_config.layer_schemes
+ # FIXME: make more robust
+ for name, scheme in layer_schemes.items():
+ if prefix.startswith(name):
+ return scheme
+ # If not found, use default
+ return quant_config.quant_scheme
+
+ def check_quantized(weight_bits: int) -> bool:
+ return weight_bits < 16
+
+ def get_impl(scheme: QuantizationScheme):
+ if not check_quantized(scheme.bits):
+ from vllm.model_executor.layers.fused_moe.layer import (
+ UnquantizedFusedMoEMethod,
+ )
+
+ return UnquantizedFusedMoEMethod(layer.moe_config)
+
+ elif _is_mxfp4_w4a4(scheme):
+ from auto_round_extension.vllm_ext.moe_impl_mxfp4 import AutoRoundMoEMethodMXFp4Impl
+
+ return AutoRoundMoEMethodMXFp4Impl(quant_config, layer.moe_config)
+
+ raise ValueError(f"Unsupported FusedMoe scheme: {scheme}")
+
+ layer_scheme = get_scheme(quant_config, prefix)
+ impl = get_impl(layer_scheme)
+ logger.debug("Apply %s to %s", impl.__class__.__name__, prefix)
+ return impl
+
+ def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> Optional[FusedMoEQuantConfig]:
+ return self.impl.get_fused_moe_quant_config(layer)
diff --git a/auto_round_extension/vllm_ext/sitecustomize.py b/auto_round_extension/vllm_ext/sitecustomize.py
new file mode 100644
index 000000000..8c22d1ab2
--- /dev/null
+++ b/auto_round_extension/vllm_ext/sitecustomize.py
@@ -0,0 +1,41 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+VLLM_ENABLE_AR_EXT = os.environ.get("VLLM_ENABLE_AR_EXT", "") in [
+ "1",
+ "true",
+ "True",
+]
+
+if VLLM_ENABLE_AR_EXT:
+ print("*****************************************************************************")
+ print(f"* !!! VLLM_ENABLE_AR_EXT is set to {VLLM_ENABLE_AR_EXT}, applying auto_round_vllm_extension *")
+ print("*****************************************************************************")
+
+ import vllm.model_executor.layers.quantization.auto_round as auto_round_module
+
+ from auto_round_extension.vllm_ext.auto_round_ext import AutoRoundExtensionConfig
+
+ auto_round_module.AutoRoundConfig = AutoRoundExtensionConfig
+ from auto_round_extension.vllm_ext.envs_ext import extra_environment_variables
+
+
+else:
+ print("*****************************************************************************")
+ print(
+ f"* Sitecustomize is loaded, but VLLM_ENABLE_AR_EXT is set to {VLLM_ENABLE_AR_EXT}, skipping auto_round_vllm_extension *"
+ )
+ print("*****************************************************************************")
diff --git a/auto_round_extension/vllm_ext/tests/conftest.py b/auto_round_extension/vllm_ext/tests/conftest.py
new file mode 100644
index 000000000..fcdd97a49
--- /dev/null
+++ b/auto_round_extension/vllm_ext/tests/conftest.py
@@ -0,0 +1,1396 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import contextlib
+import pathlib
+from copy import deepcopy
+
+from tblib import pickling_support
+
+# ruff: noqa
+
+# Install support for pickling exceptions so that we can nicely propagate
+# failures from tests running in a subprocess.
+# This should be run before any custom exception subclasses are defined.
+pickling_support.install()
+
+import http.server
+import json
+import math
+import mimetypes
+import os
+import socket
+import tempfile
+import threading
+from collections.abc import Generator
+from contextlib import nullcontext
+from enum import Enum
+from typing import Any, Callable, TypedDict, TypeVar, cast
+
+import numpy as np
+import pytest
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from huggingface_hub import snapshot_download
+from PIL import Image
+from transformers import (
+ AutoConfig,
+ AutoModelForCausalLM,
+ AutoTokenizer,
+ BatchEncoding,
+ BatchFeature,
+)
+from transformers.models.auto.auto_factory import _BaseAutoModelClass
+
+# from tests.models.utils import TokensTextLogprobs, TokensTextLogprobsPromptLogprobs
+from vllm import LLM, SamplingParams, envs
+from vllm.assets.audio import AudioAsset
+from vllm.assets.image import ImageAsset
+from vllm.assets.video import VideoAsset
+from vllm.config.model import ConvertOption, RunnerOption, _get_and_verify_dtype
+from vllm.connections import global_http_connection
+from vllm.distributed import (
+ cleanup_dist_env_and_memory,
+ init_distributed_environment,
+ initialize_model_parallel,
+)
+from vllm.logger import init_logger
+from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
+from vllm.multimodal.utils import fetch_image
+from vllm.outputs import RequestOutput
+from vllm.sampling_params import BeamSearchParams
+from vllm.transformers_utils.utils import maybe_model_redirect
+from vllm.utils.collection_utils import is_list_of
+from vllm.utils.torch_utils import set_default_torch_num_threads
+
+# Representation of generated sequence as a tuple of
+# * Token ID list
+# * String
+# * List of top sample logprobs for each sampled token
+#
+# Assumes prompt logprobs were not requested.
+TokensTextLogprobs = tuple[list[int], str, list[dict[int, float]] | SampleLogprobs | None]
+
+# Allow for tokens to be represented as str's rather than IDs;
+# tuple of
+# * Token string representations list
+# * String
+# * Optional list of top sample logprobs for each sampled token
+#
+# Assumes prompt logprobs were not requested.
+TextTextLogprobs = tuple[list[str], str, list[dict[str, float]] | list[dict[str, Logprob]] | None]
+
+# Representation of generated sequence as a tuple of
+# * Token ID list
+# * String
+# * Optional list of top sample logprobs for each sampled token
+# * Optional list of top prompt logprobs for each prompt token
+#
+# Allows prompt logprobs to be requested.
+TokensTextLogprobsPromptLogprobs = tuple[
+ list[int],
+ str,
+ list[dict[int, float]] | SampleLogprobs | None,
+ list[dict[int, float] | None] | PromptLogprobs | None,
+]
+
+
+logger = init_logger(__name__)
+
+_TEST_DIR = os.path.dirname(__file__)
+_TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
+_LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
+_SYS_MSG = os.path.join(_TEST_DIR, "system_messages", "sonnet3.5_nov2024.txt")
+
+_M = TypeVar("_M")
+
+_PromptMultiModalInput = list[_M] | list[list[_M]]
+
+PromptImageInput = _PromptMultiModalInput[Image.Image]
+PromptAudioInput = _PromptMultiModalInput[tuple[np.ndarray, int]]
+PromptVideoInput = _PromptMultiModalInput[np.ndarray]
+
+
+def _read_prompts(filename: str) -> list[str]:
+ with open(filename) as f:
+ prompts = f.readlines()
+ return prompts
+
+
+class ImageAssetPrompts(TypedDict):
+ stop_sign: str
+ cherry_blossom: str
+
+
+class ImageTestAssets(list[ImageAsset]):
+ def __init__(self) -> None:
+ super().__init__(
+ [
+ ImageAsset("stop_sign"),
+ ImageAsset("cherry_blossom"),
+ ]
+ )
+
+ def prompts(self, prompts: ImageAssetPrompts) -> list[str]:
+ """
+ Convenience method to define the prompt for each test image.
+
+ The order of the returned prompts matches the order of the
+ assets when iterating through this object.
+ """
+ return [prompts["stop_sign"], prompts["cherry_blossom"]]
+
+
+class VideoAssetPrompts(TypedDict):
+ baby_reading: str
+
+
+class VideoTestAssets(list[VideoAsset]):
+ def __init__(self) -> None:
+ super().__init__(
+ [
+ VideoAsset("baby_reading"),
+ ]
+ )
+
+ def prompts(self, prompts: VideoAssetPrompts) -> list[str]:
+ return [prompts["baby_reading"]]
+
+
+class AudioAssetPrompts(TypedDict):
+ mary_had_lamb: str
+ winning_call: str
+
+
+class AudioTestAssets(list[AudioAsset]):
+ def __init__(self) -> None:
+ super().__init__(
+ [
+ AudioAsset("mary_had_lamb"),
+ AudioAsset("winning_call"),
+ ]
+ )
+
+ def prompts(self, prompts: AudioAssetPrompts) -> list[str]:
+ return [prompts["mary_had_lamb"], prompts["winning_call"]]
+
+
+IMAGE_ASSETS = ImageTestAssets()
+"""Singleton instance of {class}`ImageTestAssets`."""
+VIDEO_ASSETS = VideoTestAssets()
+"""Singleton instance of {class}`VideoTestAssets`."""
+AUDIO_ASSETS = AudioTestAssets()
+"""Singleton instance of {class}`AudioTestAssets`."""
+
+
[email protected](scope="function", autouse=True)
+def cleanup_VLLM_USE_V1(monkeypatch):
+ """
+ The V1 oracle sets "VLLM_USE_V1" during loading. This means
+ that each invocation of a test change the env variable.
+
+ If we touch "VLLM_USE_V1" with monkeypatch, then any changes
+ made during the test run by vLLM will be cleaned up.
+
+ This fixture is used by every test.
+ """
+
+ # If VLLM_USE_V1 is not set, set then delete. This will
+ # cause monkeypatch to clean up VLLM_USE_V1 upon exit
+ # if VLLM modifies the value of envs.VLLM_USE_V1.
+ if "VLLM_USE_V1" not in os.environ:
+ monkeypatch.setenv("VLLM_USE_V1", "")
+ monkeypatch.delenv("VLLM_USE_V1")
+
+
[email protected](autouse=True)
+def init_test_http_connection():
+ # pytest_asyncio may use a different event loop per test
+ # so we need to make sure the async client is created anew
+ global_http_connection.reuse_client = False
+
+
+def dist_init():
+ temp_file = tempfile.mkstemp()[1]
+ init_distributed_environment(
+ world_size=1,
+ rank=0,
+ distributed_init_method=f"file://{temp_file}",
+ local_rank=0,
+ backend="nccl",
+ )
+ initialize_model_parallel(1, 1)
+ yield
+ cleanup_dist_env_and_memory()
+
+
+def should_do_global_cleanup_after_test(request) -> bool:
+ """Allow subdirectories to skip global cleanup by overriding this fixture.
+ This can provide a ~10x speedup for non-GPU unit tests since they don't need
+ to initialize torch.
+ """
+
+ return not request.node.get_closest_marker("skip_global_cleanup")
+
+
[email protected](autouse=True)
+def cleanup_fixture(should_do_global_cleanup_after_test: bool):
+ yield
+ if should_do_global_cleanup_after_test:
+ cleanup_dist_env_and_memory()
+
+
[email protected](autouse=True)
+def dynamo_reset():
+ yield
+ torch._dynamo.reset()
+
+
+def example_prompts() -> list[str]:
+ prompts = []
+ for filename in _TEST_PROMPTS:
+ prompts += _read_prompts(filename)
+ return prompts
+
+
+def example_system_message() -> str:
+ with open(_SYS_MSG) as f:
+ return f.read()
+
+
+class DecoderPromptType(Enum):
+ """For encoder/decoder models only."""
+
+ CUSTOM = 1
+ NONE = 2
+ EMPTY_STR = 3
+
+
+def example_long_prompts() -> list[str]:
+ prompts = []
+ for filename in _LONG_PROMPTS:
+ prompts += _read_prompts(filename)
+ return prompts
+
+
[email protected](scope="session")
+def image_assets() -> ImageTestAssets:
+ return IMAGE_ASSETS
+
+
[email protected](scope="session")
+def video_assets() -> VideoTestAssets:
+ return VIDEO_ASSETS
+
+
[email protected](scope="session")
+def audio_assets() -> AudioTestAssets:
+ return AUDIO_ASSETS
+
+
+_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature, dict)
+_R = TypeVar("_R")
+
+
+class HfRunner:
+ def get_default_device(self):
+ from vllm.platforms import current_platform
+
+ return "cpu" if current_platform.is_cpu() else current_platform.device_type
+
+ def wrap_device(self, x: _T, device: str | None = None) -> _T:
+ if x is None or isinstance(x, (bool,)):
+ return x
+
+ if device is None:
+ device = self.device
+
+ if isinstance(x, dict):
+ return {k: self.wrap_device(v, device) for k, v in x.items()}
+
+ if hasattr(x, "device") and x.device.type == device:
+ return x
+
+ return x.to(device)
+
+ def __init__(
+ self,
+ model_name: str,
+ dtype: str = "auto",
+ *,
+ model_kwargs: dict[str, Any] | None = None,
+ trust_remote_code: bool = True,
+ is_sentence_transformer: bool = False,
+ is_cross_encoder: bool = False,
+ skip_tokenizer_init: bool = False,
+ auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,
+ # Set this to avoid hanging issue
+ default_torch_num_threads: int | None = None,
+ ) -> None:
+ init_ctx = (
+ nullcontext()
+ if default_torch_num_threads is None
+ else set_default_torch_num_threads(default_torch_num_threads)
+ )
+
+ with init_ctx:
+ self._init(
+ model_name=model_name,
+ dtype=dtype,
+ model_kwargs=model_kwargs,
+ trust_remote_code=trust_remote_code,
+ is_sentence_transformer=is_sentence_transformer,
+ is_cross_encoder=is_cross_encoder,
+ skip_tokenizer_init=skip_tokenizer_init,
+ auto_cls=auto_cls,
+ )
+
+ def _init(
+ self,
+ model_name: str,
+ dtype: str = "auto",
+ *,
+ model_kwargs: dict[str, Any] | None = None,
+ trust_remote_code: bool = True,
+ is_sentence_transformer: bool = False,
+ is_cross_encoder: bool = False,
+ skip_tokenizer_init: bool = False,
+ auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,
+ ) -> None:
+ model_name = maybe_model_redirect(model_name)
+ self.model_name = model_name
+
+ self.config = AutoConfig.from_pretrained(
+ model_name,
+ trust_remote_code=trust_remote_code,
+ )
+ self.device = self.get_default_device()
+ self.dtype = dtype = _get_and_verify_dtype(
+ self.model_name,
+ self.config,
+ dtype=dtype,
+ is_pooling_model=is_sentence_transformer or is_cross_encoder,
+ )
+
+ model_kwargs = model_kwargs if model_kwargs is not None else {}
+ model_kwargs.setdefault("dtype", dtype)
+
+ if is_sentence_transformer:
+ # Lazy init required for AMD CI
+ from sentence_transformers import SentenceTransformer
+
+ self.model = SentenceTransformer(
+ model_name,
+ device=self.device,
+ model_kwargs=model_kwargs,
+ trust_remote_code=trust_remote_code,
+ )
+ elif is_cross_encoder:
+ # Lazy init required for AMD CI
+ from sentence_transformers import CrossEncoder
+
+ self.model = CrossEncoder(
+ model_name,
+ device=self.device,
+ automodel_args=model_kwargs,
+ trust_remote_code=trust_remote_code,
+ )
+ else:
+ model = auto_cls.from_pretrained(
+ model_name,
+ trust_remote_code=trust_remote_code,
+ **model_kwargs,
+ )
+
+ # in case some unquantized custom models are not in same dtype
+ if getattr(model, "quantization_method", None) is None and any(
+ p.dtype != self.dtype for p in model.parameters()
+ ):
+ model = model.to(dtype=self.dtype)
+
+ if (
+ getattr(model, "quantization_method", None) != "bitsandbytes"
+ and len({p.device for p in model.parameters()}) < 2
+ ):
+ model = model.to(device=self.device)
+
+ self.model = model
+
+ if not skip_tokenizer_init:
+ self.tokenizer = AutoTokenizer.from_pretrained(
+ model_name,
+ dtype=dtype,
+ trust_remote_code=trust_remote_code,
+ )
+
+ # don't put this import at the top level
+ # it will call torch.cuda.device_count()
+ from transformers import AutoProcessor # noqa: F401
+
+ self.processor = AutoProcessor.from_pretrained(
+ model_name,
+ dtype=dtype,
+ trust_remote_code=trust_remote_code,
+ )
+ if skip_tokenizer_init:
+ self.tokenizer = self.processor.tokenizer
+
+ def get_inputs(
+ self,
+ prompts: list[str] | list[list[int]],
+ images: PromptImageInput | None = None,
+ videos: PromptVideoInput | None = None,
+ audios: PromptAudioInput | None = None,
+ ) -> list[BatchFeature | BatchEncoding | dict[str, torch.Tensor]]:
+ if images is not None:
+ assert len(prompts) == len(images)
+
+ if videos is not None:
+ assert len(prompts) == len(videos)
+
+ if audios is not None:
+ assert len(prompts) == len(audios)
+
+ all_inputs: list[BatchFeature | BatchEncoding | dict[str, torch.Tensor]] = []
+ for i, prompt in enumerate(prompts):
+ if isinstance(prompt, str):
+ processor_kwargs: dict[str, Any] = {
+ "text": prompt,
+ "return_tensors": "pt",
+ }
+ if images is not None and (image := images[i]) is not None:
+ processor_kwargs["images"] = image
+ if videos is not None and (video := videos[i]) is not None:
+ processor_kwargs["videos"] = video
+ if audios is not None and (audio_inputs := audios[i]) is not None:
+ # HACK - not all processors take sampling_rate; we should
+ # clean this up in the future.
+ if len(audio_inputs) == 2:
+ audio, sr = audio_inputs
+ processor_kwargs["audio"] = audio
+ processor_kwargs["sampling_rate"] = sr
+ else:
+ processor_kwargs["audio"] = audio_inputs
+
+ inputs = self.processor(**processor_kwargs)
+ if isinstance(inputs, BatchFeature):
+ inputs = inputs.to(dtype=self.dtype)
+ all_inputs.append(inputs)
+ else:
+ # check that prompt is (batched) list of integers (token ids)
+ if not is_list_of(prompt, typ=int, check="all"):
+ raise ValueError("Prompt must be a list of ints corresponding to the prompt token ids.")
+ # check that no multimodal input is provided
+ if images or videos or audios:
+ raise ValueError("When providing prompt token ids multimodal inputs are not supported.")
+ input_dict = {
+ "input_ids": torch.tensor(prompt, dtype=torch.long).unsqueeze(0),
+ }
+ all_inputs.append(input_dict)
+
+ return all_inputs
+
+ def get_prompt_embeddings(self, prompts: list[str]) -> list[torch.Tensor]:
+ all_inputs = self.get_inputs(prompts)
+ embeddings = []
+ for inputs in all_inputs:
+ input_ids = self.wrap_device(inputs)["input_ids"]
+ embedding = self.model.get_input_embeddings()(input_ids).squeeze(0)
+ embeddings.append(embedding)
+ return embeddings
+
+ def classify(self, prompts: list[str]) -> list[str]:
+ # output is final logits
+ all_inputs = self.get_inputs(prompts)
+ outputs = []
+ problem_type = getattr(self.config, "problem_type", "")
+
+ for inputs in all_inputs:
+ output = self.model(**self.wrap_device(inputs))
+ if problem_type == "regression":
+ logits = output.logits[0].tolist()
+ elif problem_type == "multi_label_classification":
+ logits = output.logits.sigmoid()[0].tolist()
+ else:
+ logits = output.logits.softmax(dim=-1)[0].tolist()
+ outputs.append(logits)
+
+ return outputs
+
+ def generate(
+ self,
+ prompts: list[str] | list[list[int]],
+ images: PromptImageInput | None = None,
+ videos: PromptVideoInput | None = None,
+ audios: PromptAudioInput | None = None,
+ **kwargs: Any,
+ ) -> list[tuple[list[list[int]], list[str]]]:
+ all_inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)
+
+ outputs: list[tuple[list[list[int]], list[str]]] = []
+ for inputs in all_inputs:
+ output_ids = self.model.generate(
+ **self.wrap_device(inputs),
+ use_cache=True,
+ **kwargs,
+ )
+ output_str = self.processor.batch_decode(
+ output_ids,
+ skip_special_tokens=True,
+ clean_up_tokenization_spaces=False,
+ )
+ output_ids = output_ids.cpu().tolist()
+ outputs.append((output_ids, output_str))
+ return outputs
+
+ def generate_greedy(
+ self,
+ prompts: list[str] | list[list[int]],
+ max_tokens: int,
+ images: PromptImageInput | None = None,
+ videos: PromptVideoInput | None = None,
+ audios: PromptAudioInput | None = None,
+ **kwargs: Any,
+ ) -> list[tuple[list[int], str]]:
+ outputs = self.generate(
+ prompts,
+ do_sample=False,
+ max_new_tokens=max_tokens,
+ images=images,
+ videos=videos,
+ audios=audios,
+ **kwargs,
+ )
+
+ return [(output_ids[0], output_str[0]) for output_ids, output_str in outputs]
+
+ def generate_beam_search(
+ self,
+ prompts: list[str],
+ beam_width: int,
+ max_tokens: int,
+ images: PromptImageInput | None = None,
+ videos: PromptVideoInput | None = None,
+ audios: PromptAudioInput | None = None,
+ ) -> list[tuple[list[list[int]], list[str]]]:
+ outputs = self.generate(
+ prompts,
+ do_sample=False,
+ max_new_tokens=max_tokens,
+ num_beams=beam_width,
+ num_return_sequences=beam_width,
+ images=images,
+ videos=videos,
+ audios=audios,
+ )
+
+ for i in range(len(outputs)):
+ output_ids, output_str = outputs[i]
+ for j in range(len(output_ids)):
+ output_ids[j] = [x for x in output_ids[j] if x != self.tokenizer.pad_token_id]
+ outputs[i] = (output_ids, output_str)
+ return outputs
+
+ def generate_greedy_logprobs(
+ self,
+ prompts: list[str],
+ max_tokens: int,
+ images: PromptImageInput | None = None,
+ videos: PromptVideoInput | None = None,
+ audios: PromptAudioInput | None = None,
+ **kwargs: Any,
+ ) -> list[list[torch.Tensor]]:
+ all_inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)
+
+ all_logprobs: list[list[torch.Tensor]] = []
+ for inputs in all_inputs:
+ output = self.model.generate(
+ **self.wrap_device(inputs),
+ use_cache=True,
+ do_sample=False,
+ max_new_tokens=max_tokens,
+ output_hidden_states=True,
+ return_dict_in_generate=True,
+ **kwargs,
+ )
+ seq_logprobs = self._hidden_states_to_seq_logprobs(output.hidden_states)
+ all_logprobs.append(seq_logprobs)
+ return all_logprobs
+
+ def _hidden_states_to_seq_logprobs(
+ self,
+ hidden_states: tuple[tuple[torch.Tensor, ...], ...],
+ ) -> list[torch.Tensor]:
+ output_embeddings = self.model.get_output_embeddings()
+
+ seq_logprobs: list[torch.Tensor] = []
+ for _, hidden_state in enumerate(hidden_states):
+ last_hidden_states = hidden_state[-1][0]
+ logits = torch.matmul(
+ last_hidden_states.to(
+ device=output_embeddings.weight.device,
+ dtype=output_embeddings.weight.dtype,
+ ),
+ output_embeddings.weight.t(),
+ )
+ if getattr(output_embeddings, "bias", None) is not None:
+ logits += output_embeddings.bias.unsqueeze(0)
+ logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
+ seq_logprobs.append(logprobs)
+
+ return seq_logprobs
+
+ def _hidden_states_to_logprobs(
+ self,
+ hidden_states: tuple[tuple[torch.Tensor, ...], ...],
+ num_logprobs: int | None,
+ ) -> tuple[list[dict[int, float]], int]:
+ seq_logprobs = self._hidden_states_to_seq_logprobs(hidden_states)
+ output_len = len(hidden_states)
+
+ # convert to dict
+ seq_logprobs_lst: list[dict[int, float]] = []
+ for tok_idx, tok_logprobs in enumerate(seq_logprobs):
+ # drop prompt logprobs
+ if tok_idx == 0:
+ tok_logprobs = tok_logprobs[-1, :].reshape(1, -1)
+ topk = tok_logprobs.topk(num_logprobs)
+
+ tok_logprobs_dct = {}
+ for token_id, logprob in zip(topk.indices[0], topk.values[0]):
+ tok_logprobs_dct[token_id.item()] = logprob.item()
+
+ seq_logprobs_lst.append(tok_logprobs_dct)
+
+ return (
+ seq_logprobs_lst,
+ output_len,
+ )
+
+ def generate_greedy_logprobs_limit(
+ self,
+ prompts: list[str],
+ max_tokens: int,
+ num_logprobs: int | None,
+ images: PromptImageInput | None = None,
+ audios: PromptAudioInput | None = None,
+ videos: PromptVideoInput | None = None,
+ **kwargs: Any,
+ ) -> list[TokensTextLogprobs]:
+ all_inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)
+
+ all_logprobs: list[list[dict[int, float]]] = []
+ all_output_ids: list[list[int]] = []
+ all_output_strs: list[str] = []
+
+ for inputs in all_inputs:
+ output = self.model.generate(
+ **self.wrap_device(inputs),
+ use_cache=True,
+ do_sample=False,
+ max_new_tokens=max_tokens,
+ output_hidden_states=True,
+ return_dict_in_generate=True,
+ **kwargs,
+ )
+
+ (
+ seq_logprobs_lst,
+ output_len,
+ ) = self._hidden_states_to_logprobs(output.hidden_states, num_logprobs)
+
+ all_logprobs.append(seq_logprobs_lst)
+ seq_ids = output.sequences[0]
+ output_len = len(seq_logprobs_lst)
+ output_ids = seq_ids[-output_len:]
+ all_output_ids.append(output_ids.tolist())
+ all_output_strs.append(self.tokenizer.decode(output_ids))
+
+ outputs = zip(all_output_ids, all_output_strs, all_logprobs)
+ return [(output_ids, output_str, output_logprobs) for output_ids, output_str, output_logprobs in outputs]
+
+ def encode(self, prompts: list[str], *args, **kwargs) -> list[list[torch.Tensor]]:
+ return self.model.encode(prompts, *args, **kwargs)
+
+ def predict(self, prompts: list[list[str]], *args, **kwargs) -> torch.Tensor:
+ return self.model.predict(prompts, *args, convert_to_tensor=True, **kwargs)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ del self.model
+ cleanup_dist_env_and_memory()
+
+
[email protected](scope="session")
+def hf_runner():
+ return HfRunner
+
+
+class VllmRunner:
+ """
+ The default value of some arguments have been modified from
+ {class}`~vllm.LLM` as follows:
+
+ - `trust_remote_code`: Set to `True` instead of `False` for convenience.
+ - `seed`: Set to `0` instead of `None` for test reproducibility.
+ - `max_model_len`: Set to `1024` instead of `None` to reduce memory usage.
+ - `block_size`: To reduce memory usage, set default to `64` if on XPU
+ devices, otherwise default to `16`.
+ - `enable_chunked_prefill`: Set to `False` instead of `None` for
+ test reproducibility.
+ - `enforce_eager`: Set to `False` to test CUDA graph.
+ """
+
+ def __init__(
+ self,
+ model_name: str,
+ runner: RunnerOption = "auto",
+ convert: ConvertOption = "auto",
+ tokenizer_name: str | None = None,
+ tokenizer_mode: str = "auto",
+ trust_remote_code: bool = True,
+ seed: int | None = 0,
+ max_model_len: int | None = 1024,
+ dtype: str = "auto",
+ disable_log_stats: bool = True,
+ tensor_parallel_size: int = 1,
+ block_size: int = 16 if not torch.xpu.is_available() else 64,
+ enable_chunked_prefill: bool | None = False,
+ swap_space: int = 4,
+ enforce_eager: bool | None = False,
+ # Set this to avoid hanging issue
+ default_torch_num_threads: int | None = None,
+ **kwargs,
+ ) -> None:
+ init_ctx = (
+ nullcontext()
+ if default_torch_num_threads is None
+ else set_default_torch_num_threads(default_torch_num_threads)
+ )
+
+ if not kwargs.get("compilation_config", None):
+ # Note(@tdoublep): This is set to 4 because some tests (e.g., hybrid
+ # model tests) may set max_num_seqs=4. If min cudagraph_capture_size is
+ # set to larger than max_num_seqs, then it will lead to *no* graphs
+ # being captured which can trigger edge cases that we don't handle yet.
+ kwargs["compilation_config"] = {"cudagraph_capture_sizes": [4]}
+
+ with init_ctx:
+ self.llm = LLM(
+ model=model_name,
+ runner=runner,
+ convert=convert,
+ tokenizer=tokenizer_name,
+ tokenizer_mode=tokenizer_mode,
+ trust_remote_code=trust_remote_code,
+ dtype=dtype,
+ seed=seed,
+ swap_space=swap_space,
+ enforce_eager=enforce_eager,
+ disable_log_stats=disable_log_stats,
+ tensor_parallel_size=tensor_parallel_size,
+ max_model_len=max_model_len,
+ block_size=block_size,
+ enable_chunked_prefill=enable_chunked_prefill,
+ **kwargs,
+ )
+
+ def get_inputs(
+ self,
+ prompts: list[str] | list[torch.Tensor] | list[list[int]],
+ images: PromptImageInput | None = None,
+ videos: PromptVideoInput | None = None,
+ audios: PromptAudioInput | None = None,
+ ) -> list[dict[str, Any]]:
+ if any(x is not None and len(x) != len(prompts) for x in [images, videos, audios]):
+ raise ValueError("All non-None multimodal inputs must have the same length as prompts")
+
+ inputs = list[dict[str, Any]]()
+ for i, prompt in enumerate(prompts):
+ prompt_dict = dict[str, Any]()
+ if isinstance(prompt, str):
+ prompt_dict["prompt"] = prompt
+ elif isinstance(prompt, list):
+ prompt_dict["prompt_token_ids"] = prompt
+ else:
+ prompt_dict["prompt_embeds"] = prompt
+
+ multi_modal_data = dict[str, Any]()
+ if images is not None and (image := images[i]) is not None:
+ multi_modal_data["image"] = image
+ if videos is not None and (video := videos[i]) is not None:
+ multi_modal_data["video"] = video
+ if audios is not None and (audio := audios[i]) is not None:
+ multi_modal_data["audio"] = audio
+
+ if multi_modal_data:
+ prompt_dict["multi_modal_data"] = multi_modal_data
+
+ inputs.append(prompt_dict)
+
+ return inputs
+
+ def generate(
+ self,
+ prompts: list[str] | list[torch.Tensor] | list[list[int]],
+ sampling_params: SamplingParams,
+ images: PromptImageInput | None = None,
+ videos: PromptVideoInput | None = None,
+ audios: PromptAudioInput | None = None,
+ **kwargs: Any,
+ ) -> list[tuple[list[list[int]], list[str]]]:
+ inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)
+
+ req_outputs = self.llm.generate(inputs, sampling_params=sampling_params, **kwargs)
+
+ outputs: list[tuple[list[list[int]], list[str]]] = []
+ for req_output in req_outputs:
+ prompt_str = req_output.prompt
+ prompt_ids = req_output.prompt_token_ids
+ req_sample_output_ids: list[list[int]] = []
+ req_sample_output_strs: list[str] = []
+ for sample in req_output.outputs:
+ output_str = sample.text
+ output_ids = list(sample.token_ids)
+ req_sample_output_ids.append(prompt_ids + output_ids)
+ req_sample_output_strs.append((prompt_str or "") + output_str)
+ outputs.append((req_sample_output_ids, req_sample_output_strs))
+ return outputs
+
+ @staticmethod
+ def _final_steps_generate_w_logprobs(
+ req_outputs: list[RequestOutput],
+ ) -> list[TokensTextLogprobsPromptLogprobs]:
+ outputs: list[TokensTextLogprobsPromptLogprobs] = []
+ for req_output in req_outputs:
+ assert len(req_output.outputs) > 0
+ for sample in req_output.outputs:
+ output_str = sample.text
+ output_ids = list(sample.token_ids)
+ output_logprobs = sample.logprobs
+ outputs.append((output_ids, output_str, output_logprobs, req_output.prompt_logprobs))
+ return outputs
+
+ def generate_w_logprobs(
+ self,
+ prompts: list[str],
+ sampling_params: SamplingParams,
+ images: PromptImageInput | None = None,
+ audios: PromptAudioInput | None = None,
+ videos: PromptVideoInput | None = None,
+ **kwargs: Any,
+ ) -> list[TokensTextLogprobs] | list[TokensTextLogprobsPromptLogprobs]:
+ inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)
+
+ req_outputs = self.llm.generate(inputs, sampling_params=sampling_params, **kwargs)
+
+ toks_str_logsprobs_prompt_logprobs = self._final_steps_generate_w_logprobs(req_outputs)
+ # Omit prompt logprobs if not required by sampling params
+ return (
+ [x[0:-1] for x in toks_str_logsprobs_prompt_logprobs]
+ if sampling_params.prompt_logprobs is None
+ else toks_str_logsprobs_prompt_logprobs
+ )
+
+ def generate_greedy(
+ self,
+ prompts: list[str] | list[torch.Tensor] | list[list[int]],
+ max_tokens: int,
+ images: PromptImageInput | None = None,
+ videos: PromptVideoInput | None = None,
+ audios: PromptAudioInput | None = None,
+ **kwargs: Any,
+ ) -> list[tuple[list[int], str]]:
+ greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
+ outputs = self.generate(
+ prompts,
+ greedy_params,
+ images=images,
+ videos=videos,
+ audios=audios,
+ **kwargs,
+ )
+ return [(output_ids[0], output_str[0]) for output_ids, output_str in outputs]
+
+ def generate_greedy_logprobs(
+ self,
+ prompts: list[str],
+ max_tokens: int,
+ num_logprobs: int | None,
+ num_prompt_logprobs: int | None = None,
+ images: PromptImageInput | None = None,
+ audios: PromptAudioInput | None = None,
+ videos: PromptVideoInput | None = None,
+ stop_token_ids: list[int] | None = None,
+ stop: list[str] | None = None,
+ **kwargs: Any,
+ ) -> list[TokensTextLogprobs] | list[TokensTextLogprobsPromptLogprobs]:
+ greedy_logprobs_params = SamplingParams(
+ temperature=0.0,
+ max_tokens=max_tokens,
+ logprobs=num_logprobs,
+ prompt_logprobs=num_prompt_logprobs,
+ stop_token_ids=stop_token_ids,
+ stop=stop,
+ )
+
+ return self.generate_w_logprobs(
+ prompts,
+ greedy_logprobs_params,
+ images=images,
+ audios=audios,
+ videos=videos,
+ **kwargs,
+ )
+
+ def generate_prompt_perplexity(self, prompts: list[str]) -> list[float]:
+ """
+ Return the perplexity score associated with generating the prompts
+
+ :param prompts: list of prompts to score
+ :return: perplexity score of each prompt
+ """
+ outputs = self.generate_greedy_logprobs(prompts, max_tokens=1, num_logprobs=None, num_prompt_logprobs=0)
+
+ perplexities = []
+ for output in outputs:
+ output = cast(TokensTextLogprobsPromptLogprobs, output)
+ token_data = cast(list[dict[int, Logprob] | None], output[3])
+ assert token_data[0] is None
+ token_log_probs = []
+ for token_data in token_data[1:]:
+ assert token_data is not None
+ assert len(token_data) == 1
+ token_log_prob = list(token_data.values())[0].logprob
+ token_log_probs.append(token_log_prob)
+
+ perplexity = math.exp(-sum(token_log_probs) / len(token_log_probs))
+ perplexities.append(perplexity)
+
+ return perplexities
+
+ def generate_beam_search(
+ self,
+ prompts: list[str],
+ beam_width: int,
+ max_tokens: int,
+ images: PromptImageInput | None = None,
+ videos: PromptVideoInput | None = None,
+ audios: PromptAudioInput | None = None,
+ concurrency_limit: int | None = None,
+ ) -> list[tuple[list[list[int]], list[str]]]:
+ inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)
+
+ outputs = self.llm.beam_search(
+ inputs,
+ BeamSearchParams(beam_width=beam_width, max_tokens=max_tokens),
+ concurrency_limit=concurrency_limit,
+ )
+ returned_outputs = []
+ for output in outputs:
+ token_ids = [x.tokens for x in output.sequences]
+ texts = [x.text for x in output.sequences]
+ returned_outputs.append((token_ids, texts))
+ return returned_outputs
+
+ def classify(self, prompts: list[str]) -> list[list[float]]:
+ req_outputs = self.llm.classify(prompts)
+ return [req_output.outputs.probs for req_output in req_outputs]
+
+ def embed(
+ self,
+ prompts: list[str],
+ images: PromptImageInput | None = None,
+ videos: PromptVideoInput | None = None,
+ audios: PromptAudioInput | None = None,
+ *args,
+ **kwargs,
+ ) -> list[list[float]]:
+ inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)
+
+ req_outputs = self.llm.embed(inputs, *args, **kwargs)
+ return [req_output.outputs.embedding for req_output in req_outputs]
+
+ def token_embed(self, prompts: list[str]) -> list[list[float]]:
+ req_outputs = self.llm.encode(prompts, pooling_task="token_embed")
+ return [req_output.outputs.data for req_output in req_outputs]
+
+ def token_classify(self, prompts: list[str]) -> list[list[float]]:
+ req_outputs = self.llm.encode(prompts, pooling_task="token_classify")
+ return [req_output.outputs.data for req_output in req_outputs]
+
+ def reward(self, prompts: list[str]) -> list[list[float]]:
+ req_outputs = self.llm.reward(prompts)
+ return [req_output.outputs.data for req_output in req_outputs]
+
+ def score(
+ self,
+ text_1: list[str] | str,
+ text_2: list[str] | str,
+ *args,
+ **kwargs,
+ ) -> list[float]:
+ req_outputs = self.llm.score(text_1, text_2, *args, **kwargs)
+ return [req_output.outputs.score for req_output in req_outputs]
+
+ def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
+ return self.llm.apply_model(func)
+
+ def get_llm(self) -> LLM:
+ return self.llm
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ del self.llm
+ cleanup_dist_env_and_memory()
+
+
[email protected](scope="session")
+def vllm_runner():
+ return VllmRunner
+
+
+def temporary_enable_log_propagate():
+ import logging
+
+ logger = logging.getLogger("vllm")
+ logger.propagate = True
+ yield
+ logger.propagate = False
+
+
+def caplog_vllm(temporary_enable_log_propagate, caplog):
+ # To capture vllm log, we should enable propagate=True temporarily
+ # because caplog depends on logs propagated to the root logger.
+ yield caplog
+
+
+def caplog_mp_fork():
+ """
+ This fixture enables capturing logs from a forked MP subprocess.
+ It should be used in conjunction with caplog_vllm.
+
+ By default, subprocess logs do not go through the parent process.
+ We instead create a queue listener in the parent process which
+ forwards logs to the logger's other handlers, and add a QueueHandler
+ to the root logger. Forked subprocesses will inherit the root logger
+ and pass their messages to the queue, which the listener will forward
+ to the root logger, which can be captured by caplog.
+
+ Note that this workaround only works for fork; with spawn, the subprocess
+ reinitializes logging and does not automatically inherit the queue.
+ We'd have to manually pass the queue to the subprocess at the spawn point.
+ See caplog_mp_spawn below.
+ """
+
+ @contextlib.contextmanager
+ def ctx():
+ import logging.handlers
+ import multiprocessing as mp
+
+ logger_queue: mp.Queue[logging.LogRecord] = mp.Queue()
+ logger = logging.getLogger()
+ handlers = logger.handlers
+
+ # The listener works on a background thread, not inherited by the child.
+ queue_listener = logging.handlers.QueueListener(logger_queue, *handlers)
+ queue_listener.start()
+
+ # Add queue handler after creating the listener to avoid cycle
+ logger.addHandler(logging.handlers.QueueHandler(logger_queue))
+ yield
+ queue_listener.stop()
+
+ return ctx
+
+
+class LogHolder:
+ def __init__(self):
+ self.text = None
+
+
+def caplog_mp_spawn(tmp_path, monkeypatch):
+ """
+ This fixture enables capturing logs from a forked MP subprocess.
+ It does not require caplog_vllm (but it only contains logs from the child).
+
+ By default, subprocess logs do not go through the parent process.
+ We instead add a FileHandler to the config so the spawned child process
+ writes its logs to a temp file.
+ In the parent, we read the file and return the contents.
+
+ Note: this method could be extended to fork by either reconfiguring logging
+ in the parent or using a SocketHandler:
+ https://docs.python.org/3/howto/logging-cookbook.html#sending-and-receiving-logging-events-across-a-network # noqa: E501
+ """
+
+ @contextlib.contextmanager
+ def ctx(level: int | str):
+ from vllm.logger import DEFAULT_LOGGING_CONFIG
+
+ config_path = tmp_path / "vllm_logging_config.json"
+ log_path = tmp_path / "vllm.log"
+ log_holder = LogHolder()
+
+ config = deepcopy(DEFAULT_LOGGING_CONFIG)
+ if envs.VLLM_LOGGING_CONFIG_PATH:
+ path = pathlib.Path(envs.VLLM_LOGGING_CONFIG_PATH)
+ assert path.exists()
+ config = json.loads(path.read_text())
+
+ config["loggers"]["vllm"]["handlers"] += ["vllm_file"]
+ config["handlers"]["vllm_file"] = {
+ "class": "logging.FileHandler",
+ "formatter": "vllm",
+ "level": level,
+ "filename": log_path.as_posix(),
+ }
+
+ config_path.write_text(json.dumps(config))
+
+ with monkeypatch.context() as monkeypatch_ctx:
+ monkeypatch_ctx.setenv("VLLM_LOGGING_CONFIG_PATH", config_path.as_posix())
+ monkeypatch_ctx.setenv("VLLM_CONFIGURE_LOGGING", "1")
+ yield log_holder
+
+ log_holder.text = log_path.read_text()
+
+ return ctx
+
+
[email protected](scope="session")
+def num_gpus_available():
+ """Get number of GPUs without initializing the CUDA context
+ in current process."""
+
+ from vllm.platforms import current_platform
+
+ return current_platform.device_count()
+
+
+temp_dir = tempfile.gettempdir()
+_dummy_opt_path = os.path.join(temp_dir, "dummy_opt")
+_dummy_llava_path = os.path.join(temp_dir, "dummy_llava")
+_dummy_gemma2_embedding_path = os.path.join(temp_dir, "dummy_gemma2_embedding")
+
+
+def dummy_opt_path():
+ json_path = os.path.join(_dummy_opt_path, "config.json")
+ if not os.path.exists(_dummy_opt_path):
+ snapshot_download(
+ repo_id="facebook/opt-125m",
+ local_dir=_dummy_opt_path,
+ ignore_patterns=["*.bin", "*.bin.index.json", "*.pt", "*.h5", "*.msgpack"],
+ )
+ assert os.path.exists(json_path)
+ with open(json_path) as f:
+ config = json.load(f)
+ config["architectures"] = ["MyOPTForCausalLM"]
+ with open(json_path, "w") as f:
+ json.dump(config, f)
+ return _dummy_opt_path
+
+
+def dummy_llava_path():
+ json_path = os.path.join(_dummy_llava_path, "config.json")
+ if not os.path.exists(_dummy_llava_path):
+ snapshot_download(
+ repo_id="llava-hf/llava-1.5-7b-hf",
+ local_dir=_dummy_llava_path,
+ ignore_patterns=[
+ "*.bin",
+ "*.bin.index.json",
+ "*.pt",
+ "*.h5",
+ "*.msgpack",
+ "*.safetensors",
+ ],
+ )
+ assert os.path.exists(json_path)
+ with open(json_path) as f:
+ config = json.load(f)
+ config["architectures"] = ["MyLlava"]
+ with open(json_path, "w") as f:
+ json.dump(config, f)
+ return _dummy_llava_path
+
+
+def dummy_gemma2_embedding_path():
+ json_path = os.path.join(_dummy_gemma2_embedding_path, "config.json")
+ if not os.path.exists(_dummy_gemma2_embedding_path):
+ snapshot_download(
+ repo_id="BAAI/bge-multilingual-gemma2",
+ local_dir=_dummy_gemma2_embedding_path,
+ ignore_patterns=[
+ "*.bin",
+ "*.bin.index.json",
+ "*.pt",
+ "*.h5",
+ "*.msgpack",
+ "*.safetensors",
+ ],
+ )
+ assert os.path.exists(json_path)
+ with open(json_path) as f:
+ config = json.load(f)
+ config["architectures"] = ["MyGemma2Embedding"]
+ with open(json_path, "w") as f:
+ json.dump(config, f)
+ return _dummy_gemma2_embedding_path
+
+
+# Add the flag `--optional` to allow run tests
+# that are marked with @pytest.mark.optional
+def pytest_addoption(parser):
+ parser.addoption("--optional", action="store_true", default=False, help="run optional test")
+
+
+def pytest_collection_modifyitems(config, items):
+ if config.getoption("--optional"):
+ # --optional given in cli: do not skip optional tests
+ return
+ skip_optional = pytest.mark.skip(reason="need --optional option to run")
+ for item in items:
+ if "optional" in item.keywords:
+ item.add_marker(skip_optional)
+
+
[email protected](scope="session")
+def cli_config_file():
+ """Return the path to the CLI config file."""
+ return os.path.join(_TEST_DIR, "config", "test_config.yaml")
+
+
[email protected](scope="session")
+def cli_config_file_with_model():
+ """Return the path to the CLI config file with model."""
+ return os.path.join(_TEST_DIR, "config", "test_config_with_model.yaml")
+
+
+class AssetHandler(http.server.BaseHTTPRequestHandler):
+ # _IMAGE_CACHE : Dict[str, bytes] = {}
+
+ def log_message(self, *args, **kwargs):
+ pass
+
+ def do_GET(self):
+ # Accepts paths like: /1280px-Venn_diagram_rgb.jpg
+ filename = self.path.lstrip("/")
+ if not filename or "." not in filename:
+ self.send_error(404, "Missing filename (expected /<name>.<ext>)")
+ return
+
+ base, ext = filename.rsplit(".", 1)
+ ext = ext.lower()
+
+ if ext not in ["jpg", "png"]:
+ self.send_error(404, f"Unsupported extension: .{ext}")
+ return
+
+ try:
+ data = ImageAsset(base).read_bytes(ext=ext)
+ except Exception as e:
+ self.send_error(500, f"Failed to load asset: {ext} {base} {e} ")
+ return
+
+ ctype, _ = mimetypes.guess_type(filename)
+ if ctype is None:
+ ctype = {"jpg": "image/jpg", "png": "image/png"}[ext]
+ self.send_response(200)
+ self.send_header("Content-Type", ctype)
+ self.send_header("Content-Length", str(len(data)))
+ self.end_headers()
+ self.wfile.write(data)
+
+
+def _find_free_port() -> int:
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ return s.getsockname()[1]
+
+
+class LocalAssetServer:
+ address: str
+ port: int
+ server: http.server.ThreadingHTTPServer | None
+ thread: threading.Thread | None
+
+ def __init__(self, address: str = "127.0.0.1") -> None:
+ self.address = address
+ self.port = -1
+ self.server = None
+ self.thread = None
+
+ def __enter__(self):
+ self.port = _find_free_port()
+ self.server = http.server.ThreadingHTTPServer((self.address, self.port), AssetHandler)
+ self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
+ self.thread.start()
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ if self.server:
+ self.server.shutdown()
+ del self.server
+
+ if self.thread:
+ self.thread.join()
+ del self.thread
+
+ if exc_type is None:
+ return None
+
+ return False
+
+ @property
+ def base_url(self) -> str:
+ assert self.port is not None
+ return f"http://{self.address}:{self.port}"
+
+ def url_for(self, name: str) -> str:
+ """e.g., name='RGBA_comp.png' -> 'http://127.0.0.1:PORT/RGBA_comp.png'"""
+ return f"{self.base_url}/{name}"
+
+ def get_image_asset(self, name: str) -> Image.Image:
+ return fetch_image(self.url_for(name))
+
+
[email protected](scope="session")
+def local_asset_server() -> Generator[LocalAssetServer, None, None]:
+ """
+ Starts a thread based HTTP server bound to 127.0.0.1 on a random free port.
+ The server currently servers images at:
+ http://127.0.0.1:<port>/<name>.<ext>
+ """
+ with LocalAssetServer() as srv:
+ yield srv
+
+
+def image_url(request, local_asset_server) -> str:
+ # request.param is one of the IMAGE_ASSETS filenames
+ name = request.param
+ return local_asset_server.url_for(name)
+
+
+def image_urls(request, local_asset_server) -> list[str]:
+ """Indirect fixture: takes a list of names, returns list of full URLs."""
+ names: list[str] = request.param
+ return [local_asset_server.url_for(name) for name in names]
diff --git a/auto_round_extension/vllm_ext/tests/test_mxfp4_moe.py b/auto_round_extension/vllm_ext/tests/test_mxfp4_moe.py
new file mode 100644
index 000000000..a2de32832
--- /dev/null
+++ b/auto_round_extension/vllm_ext/tests/test_mxfp4_moe.py
@@ -0,0 +1,34 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+from vllm.platforms import current_platform
+
+MODELS = [
+ # "/data5/yliu7/HF_HOME/unsloth-gpt-oss-20b-BF16-ar-MXFP4/"
+ # "/data5/yliu7/HF_HOME/Qwen2.5-0.5B-Instruct-test-FP8_STATIC-fp8kv/"
+ "/data6/yiliu4/Qwen3-15B-A2B-Base-MXFP4"
+]
+
+
+ not current_platform.is_cuda(),
+ reason="only supports CUDA backend.",
+)
[email protected]("model", MODELS)
+def test_auto_round(vllm_runner, model):
+ with vllm_runner(model, enforce_eager=True) as llm:
+ output = llm.generate_greedy(["The capital of France is"], max_tokens=8)
+ assert output
+ print(f"output is: {output[0][1]}")
diff --git a/auto_round_extension/vllm_ext/utils.py b/auto_round_extension/vllm_ext/utils.py
new file mode 100644
index 000000000..f98eb2325
--- /dev/null
+++ b/auto_round_extension/vllm_ext/utils.py
@@ -0,0 +1,81 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+E8M0_EXPONENT_BIAS = 127
+E8M0_EXPONENT_NAN_VAL = 255
+
+
+def get_fp_scale(scale_e8m0):
+ scale_e8m0 = scale_e8m0.view(torch.uint8)
+ s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS
+ # TODO(later): it would be nice if there was a way to do the 2^x operation
+ # in PyTorch without creating a tensor of twos
+ two = torch.full(s_offset.size(), 2.0, device=scale_e8m0.device)
+ # pow(two, s_offset) can be out of range of floating point formats.
+ # TODO(later): handle this for float16 if we decide to support float16
+ # scales.
+ s_fp = torch.pow(two, s_offset)
+
+ # If a block exponent was 255, set values of that block to NaN
+ s_fp = torch.where(scale_e8m0 != E8M0_EXPONENT_NAN_VAL, s_fp, float("nan"))
+
+ return s_fp
+
+
+def _to_mx_rceil(
+ data_hp: torch.Tensor,
+ max_abs: torch.Tensor,
+ max_pos: float,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ """
+ A prototype implementation of MXFP scale factor derivation method described in
+ https://docs.nvidia.com/cuda/cublas/#d-block-quantization
+
+ For Nvidia GPU with Blackwell+ architecture, the scale factor derivation method
+ could be accelerated by the `cvt.rp.satfinite.ue8m0x2.f32` instruction.
+
+ Args:
+ data_hp: High precision data.
+ max_abs: Maximum absolute value for data_hp along specified dimension/block_size.
+ max_pos: The maximum value of the low precision data type.
+
+ Returns:
+ exponent: The biased exponent with dtype E8M0 in uint8 container.
+ data_lp: The targeted low precision data, in high precision container
+ (requires cast to low precision data type).
+ """
+ descale = max_abs / max_pos
+ # TODO: nan/inf needs to be set for any value
+ # of nan/inf in input not just amax.
+ exponent = torch.where(
+ torch.isnan(descale),
+ 0xFF, # Handle biased exponent for nan
+ # NOTE: descale < (torch.finfo(torch.float32).smallest_normal / 2) is handled through clamping
+ (
+ torch.clamp(
+ torch.ceil(torch.log2(descale)),
+ min=-E8M0_EXPONENT_BIAS,
+ max=E8M0_EXPONENT_BIAS,
+ )
+ + E8M0_EXPONENT_BIAS
+ ).to(torch.uint8),
+ )
+
+ descale_fp = torch.where(exponent == 0, 1.0, torch.exp2(E8M0_EXPONENT_BIAS - exponent.to(torch.float32)))
+
+ # scale and saturated cast the data elements to max of target dtype
+ data_lp = torch.clamp(data_hp * descale_fp.unsqueeze(1), min=-1 * max_pos, max=max_pos)
+ return exponent, data_lp