PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[1/N] Initial vllm-ext evaluation support (MXFP4 MOE) (#935)
Yi Liue8bc3536d22e
diff --git a/auto_round_extension/vllm_ext/__init__.py b/auto_round_extension/vllm_ext/__init__.pynew file mode 100644index 000000000..3e145334f--- /dev/null+++ b/auto_round_extension/vllm_ext/__init__.py@@ -0,0 +1,26 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# ==---------------------------------------------------------------------------==+# Apply the extension+# ==---------------------------------------------------------------------------==+++def apply():+ import vllm.model_executor.layers.quantization.auto_round as auto_round_module++ from auto_round_extension.vllm_ext.auto_round_ext import AutoRoundExtensionConfig++ auto_round_module.AutoRoundConfig = AutoRoundExtensionConfig+ from auto_round_extension.vllm_ext.envs_ext import extra_environment_variablesdiff --git a/auto_round_extension/vllm_ext/apply_ext.sh b/auto_round_extension/vllm_ext/apply_ext.shnew file mode 100644index 000000000..b327ec22c--- /dev/null+++ b/auto_round_extension/vllm_ext/apply_ext.sh@@ -0,0 +1,46 @@+#!/bin/bash++# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# Define the relative path for the `auto-round` installation+AUTO_ROUND_PATH="auto_round/../auto_round_extension/vllm_ext/sitecustomize.py"++# Try to find the pip installation location+PIP_LOCATION=$(pip show auto-round 2>/dev/null | grep "Location:" | awk '{print $2}')++if [ -n "$PIP_LOCATION" ]; then+ SITE_CUSTOMIZE_PATH="$PIP_LOCATION/$AUTO_ROUND_PATH"+ echo "Checking for sitecustomize.py at: $SITE_CUSTOMIZE_PATH"++ if [ -f "$SITE_CUSTOMIZE_PATH" ]; then+ echo "Found sitecustomize.py at: $SITE_CUSTOMIZE_PATH"+ export PYTHONPATH=$(dirname "$SITE_CUSTOMIZE_PATH"):$PYTHONPATH+ echo "PYTHONPATH set to: $PYTHONPATH"+ return 0 2>/dev/null || true+ fi+fi++# Fallback: check current directory+LOCAL_SITE_CUSTOMIZE="./sitecustomize.py"+if [ -f "$LOCAL_SITE_CUSTOMIZE" ]; then+ echo "Found sitecustomize.py at current directory."+ export PYTHONPATH=$(pwd):$PYTHONPATH+ echo "PYTHONPATH set to: $PYTHONPATH"+ return 0 2>/dev/null || true+fi++echo "Warning: sitecustomize.py not found in pip installation or current directory."+# Do not exit the shell+return 1 2>/dev/null || true\ No newline at end of filediff --git a/auto_round_extension/vllm_ext/auto_round_ext.py b/auto_round_extension/vllm_ext/auto_round_ext.pynew file mode 100644index 000000000..cce2c912d--- /dev/null+++ b/auto_round_extension/vllm_ext/auto_round_ext.py@@ -0,0 +1,63 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from typing import Any++import torch+from vllm.logger import init_logger+from vllm.model_executor.layers.fused_moe import FusedMoE+from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod+from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig++from auto_round.schemes import QuantizationScheme+from auto_round_extension.vllm_ext.quant_method_moe import AutoRoundMoEMethod++logger = init_logger(__name__)+++class AutoRoundExtensionConfig(AutoRoundConfig):+ SUPPORTED_DTYPES = AutoRoundConfig.SUPPORTED_DTYPES.union({"mx_fp"})+ SUPPORTED_FORMATS = AutoRoundConfig.SUPPORTED_FORMATS.union({"auto_round:llm_compressor"})++ def get_quant_method(self, layer: torch.nn.Module, prefix: str):+ # FIXME: (yi) make it compatible with `AutoRoundConfig`+ if isinstance(layer, FusedMoE):+ quant_method = AutoRoundMoEMethod.get_moe_method(self, layer, prefix)+ return quant_method+ elif isinstance(layer, LinearBase):+ return UnquantizedLinearMethod()+ else:+ return None++ @staticmethod+ def _parse_quant_scheme(config: dict):+ quant_scheme_attrs = QuantizationScheme.get_attributes()+ filter_config = {key: value for key, value in config.items() if key in quant_scheme_attrs}+ quant_scheme = QuantizationScheme.from_dict(filter_config)+ return quant_scheme++ @classmethod+ def from_config(cls, config: dict[str, Any]) -> AutoRoundConfig:+ ar_config = super().from_config(config)+ # TODO: (yi) refine below implementation+ quant_scheme = AutoRoundExtensionConfig._parse_quant_scheme(config)+ layer_schemes = {}+ layer_schemes = {} # ensure dict+ extra_config = getattr(ar_config, "extra_config", None)+ if extra_config is not None:+ for layer_name, layer_config in extra_config.items():+ layer_schemes[layer_name] = AutoRoundExtensionConfig._parse_quant_scheme(layer_config)+ ar_config.quant_scheme = quant_scheme+ ar_config.layer_schemes = layer_schemes+ return ar_configdiff --git a/auto_round_extension/vllm_ext/envs_ext.py b/auto_round_extension/vllm_ext/envs_ext.pynew file mode 100644index 000000000..325b9de7c--- /dev/null+++ b/auto_round_extension/vllm_ext/envs_ext.py@@ -0,0 +1,39 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import os+from typing import Any, Callable++from vllm.logger import init_logger++logger = init_logger(__name__)++# Define extra environment variables+extra_environment_variables: dict[str, Callable[[], Any]] = {+ "VLLM_MXFP4_PRE_UNPACK_WEIGHTS": lambda: os.getenv("VLLM_MXFP4_PRE_UNPACK_WEIGHTS", "1") in ("1", "true", "True"),+ "VLLM_ENABLE_STATIC_MOE": lambda: os.getenv("VLLM_ENABLE_STATIC_MOE", "1") in ("1", "true", "True"),+ "VLLM_AR_MXFP4_MODULAR_MOE": lambda: os.getenv("VLLM_AR_MXFP4_MODULAR_MOE", "0") in ("1", "true", "True"),+}+# Add the extra environment variables to vllm.envs+import vllm.envs as envs+from vllm.envs import environment_variables++# Merge the environment variables+all_environment_variables = {**environment_variables, **extra_environment_variables}+++for name, value_fn in extra_environment_variables.items():+ setattr(envs, name, value_fn())++logger.warning_once(f"Added extra environment variables: {list(extra_environment_variables.keys())}")diff --git a/auto_round_extension/vllm_ext/fp4_utils.py b/auto_round_extension/vllm_ext/fp4_utils.pynew file mode 100644index 000000000..adb67d190--- /dev/null+++ b/auto_round_extension/vllm_ext/fp4_utils.py@@ -0,0 +1,113 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from typing import Optional++import torch++# Module-level device tensor cache to fix cuda graph issue+_DEVICE_E2M1_TENSORS = {}++# Constants for FP4 values (E2M1 format)+_E2M1_VALUES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]+++def get_e2m1_tensor(device):+ """Get device-specific E2M1 lookup tensor, creating it if needed."""+ device_str = str(device)+ if device_str not in _DEVICE_E2M1_TENSORS:+ _DEVICE_E2M1_TENSORS[device_str] = torch.tensor(_E2M1_VALUES, dtype=torch.float32, device=device)+ return _DEVICE_E2M1_TENSORS[device_str]+++def pack_fp4_to_uint8(x: torch.Tensor) -> torch.Tensor:+ m, n = x.shape+ device = x.device++ # Create lookup table for FP4 values to indices+ # Map the absolute values to 0-7 indices+ kE2M1 = get_e2m1_tensor(x.device)++ # Find closest valid FP4 value index for each element+ abs_x = torch.abs(x)+ abs_diff_x = torch.abs(abs_x.unsqueeze(-1) - kE2M1) # [m, n, 8]+ abs_indices = torch.argmin(abs_diff_x, dim=-1) # [m, n]++ # Apply sign bit (bit 3) to get final 4-bit representation+ indices = abs_indices + (torch.signbit(x).to(torch.long) << 3)++ # Reshape to prepare for packing pairs of values+ indices = indices.reshape(-1)++ # Handle odd length by padding if necessary+ assert indices.numel() % 2 != 0, f"Expected even number of elements, got {indices.numel()}"++ # Reshape to pair consecutive elements+ indices = indices.reshape(-1, 2)++ # Pack pairs of 4-bit values into 8-bit values+ packed = (indices[:, 0] | (indices[:, 1] << 4)).to(torch.uint8)++ return packed.reshape(m, n // 2)+++def unpack_fp4_from_uint8(+ a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16+) -> torch.Tensor:+ """+ Unpacks uint8 values into fp4. Each uint8 consists of two fp4 values+ (i.e. first four bits correspond to one fp4 value, last four correspond to a+ consecutive fp4 value). The bits represent an index, which are mapped to an fp4+ value.++ :param a: tensor to unpack+ :param m: original dim 0 size of the unpacked tensor+ :param n: original dim 1 size of the unpacked tensor+ :param dtype: dense dtype to cast the unpacked tensor to+ """+ assert a.dtype == torch.uint8, f"expected uint8, got {a.dtype}"++ # Vectorized nibble processing+ a_flat = a.flatten()+ high = (a_flat & 0xF0) >> 4 # Upper nibbles+ low = a_flat & 0x0F # Lower nibbles++ # Combine nibbles for batch processing+ combined = torch.stack((low, high), dim=1).flatten()++ # Vectorized sign and magnitude extraction+ signs = (combined & 0x08).to(torch.bool) # Sign bits+ abs_vals = (combined & 0x07).to(torch.long) # Magnitude indices++ # Device-aware lookup and sign application+ kE2M1 = get_e2m1_tensor(a.device)++ values = kE2M1[abs_vals] * torch.where(signs, -1.0, 1.0)++ # Reshape to final form+ return values.reshape(m, n).to(dtype=dtype)+++def cast_to_fp4(x):+ sign = torch.sign(x)+ x = torch.abs(x)+ x[(x >= 0.0) & (x <= 0.25)] = 0.0+ x[(x > 0.25) & (x < 0.75)] = 0.5+ x[(x >= 0.75) & (x <= 1.25)] = 1.0+ x[(x > 1.25) & (x < 1.75)] = 1.5+ x[(x >= 1.75) & (x <= 2.5)] = 2.0+ x[(x > 2.5) & (x < 3.5)] = 3.0+ x[(x >= 3.5) & (x <= 5.0)] = 4.0+ x[x > 5.0] = 6.0+ return x * signdiff --git a/auto_round_extension/vllm_ext/moe_impl_mxfp4.py b/auto_round_extension/vllm_ext/moe_impl_mxfp4.pynew file mode 100644index 000000000..77ec6570d--- /dev/null+++ b/auto_round_extension/vllm_ext/moe_impl_mxfp4.py@@ -0,0 +1,539 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from typing import Callable, Optional, Union++import torch+import torch.nn.functional as F+import vllm.envs as envs+from vllm.distributed import get_tensor_model_parallel_rank+from vllm.logger import init_logger+from vllm.model_executor.layers.fused_moe import (+ FusedMoE,+ FusedMoEConfig,+ FusedMoeWeightScaleSupported,+)+from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig+from vllm.model_executor.utils import set_weight_attrs++import auto_round_extension.vllm_ext.mxfp4_qdq_utils as mxfp4_utils+from auto_round_extension.vllm_ext.mxfp4_qdq_utils import (+ dequant_mxfp4_to_fp8,+ mxfp4_gemm_with_unpacked_weight,+ run_mxfp4_emulations,+)+from auto_round_extension.vllm_ext.quant_method_moe import AutoRoundMoEMethod++logger = init_logger(__name__)+++def apply_act(local_w1_out: torch.Tensor, local_w3_out: torch.Tensor, activation: str) -> torch.Tensor:+ if activation == "silu":+ act_fn = F.silu+ w13_out = act_fn(local_w1_out) * local_w3_out+ elif activation == "swigluoai":+ limit = 7.0+ alpha = 1.702+ local_w1_out = local_w1_out.clamp(min=None, max=limit)+ local_w3_out = local_w3_out.clamp(min=-limit, max=limit)+ glu = (local_w1_out) * F.sigmoid(local_w1_out * alpha)+ w13_out = (local_w3_out + 1) * glu+ else:+ raise NotImplementedError(f"Activation {activation} is not implemented.")+ return w13_out+++class AutoRoundMoEMethodMXFp4Impl(AutoRoundMoEMethod):+ def __init__(+ self,+ quant_config: "AutoRoundConfig", # type: ignore # noqa E501+ moe: FusedMoEConfig,+ ):+ super().__init__(moe)+ self.use_marlin = False+ self.group_size = 32+ self.quant_config = quant_config+ self.has_bias = self.moe.has_bias+ self.mask_weights_buffer = None+ self.experts_mask_buffer = None+ self.num_all_tokens_threshold = 16 * 1024+ # self.num_all_tokens_threshold = 0++ def create_weights(+ self,+ layer: torch.nn.Module,+ num_experts: int,+ hidden_size: int,+ intermediate_size_per_partition: int,+ params_dtype: torch.dtype,+ **extra_weight_attrs,+ ):+ layer.num_experts = num_experts+ layer.params_dtype = params_dtype++ w13_weight = torch.nn.Parameter(+ torch.empty(+ num_experts,+ 2 * intermediate_size_per_partition,+ # 2 fp4 items are packed in the input dimension+ hidden_size // 2,+ requires_grad=False,+ dtype=torch.uint8,+ ),+ requires_grad=False,+ )+ layer.register_parameter("w13_weight_packed", w13_weight)+ set_weight_attrs(w13_weight, extra_weight_attrs)++ w2_weight = torch.nn.Parameter(+ torch.empty(+ num_experts,+ hidden_size,+ # 2 fp4 items are packed in the input dimension+ intermediate_size_per_partition // 2,+ dtype=torch.uint8,+ ),+ requires_grad=False,+ )+ layer.register_parameter("w2_weight_packed", w2_weight)+ set_weight_attrs(w2_weight, extra_weight_attrs)++ # Weight Scales+ w13_weight_scale = torch.nn.Parameter(+ torch.empty(+ num_experts,+ 2 * intermediate_size_per_partition,+ # 2 fp4 items are packed in the input dimension+ hidden_size // self.group_size,+ dtype=torch.uint8,+ ),+ requires_grad=False,+ )+ layer.register_parameter("w13_weight_scale", w13_weight_scale)+ extra_weight_attrs.update({"quant_method": FusedMoeWeightScaleSupported.GROUP.value})+ set_weight_attrs(w13_weight_scale, extra_weight_attrs)++ w2_weight_scale = torch.nn.Parameter(+ torch.empty(+ num_experts,+ hidden_size,+ # 2 fp4 items are packed in the input dimension+ intermediate_size_per_partition // self.group_size,+ dtype=torch.uint8,+ ),+ requires_grad=False,+ )+ layer.register_parameter("w2_weight_scale", w2_weight_scale)+ extra_weight_attrs.update({"quant_method": FusedMoeWeightScaleSupported.GROUP.value})+ set_weight_attrs(w2_weight_scale, extra_weight_attrs)++ E = num_experts+ H = hidden_size+ IN = intermediate_size_per_partition+ if self.has_bias:+ # TODO: yiliu30 use the dtype in CK+ bias_dtype = torch.bfloat16+ w13_bias = torch.nn.Parameter(torch.zeros(E, 2 * IN, dtype=bias_dtype), requires_grad=False)+ layer.register_parameter("w13_bias", w13_bias)+ set_weight_attrs(w13_bias, extra_weight_attrs)++ w2_bias = torch.nn.Parameter(+ torch.zeros(E, H, dtype=bias_dtype),+ requires_grad=False,+ )+ layer.register_parameter("w2_bias", w2_bias)+ set_weight_attrs(w2_bias, extra_weight_attrs)++ def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> Optional[FusedMoEQuantConfig]:+ if envs.VLLM_AR_MXFP4_MODULAR_MOE:+ from vllm.model_executor.layers.fused_moe.config import (+ ocp_mx_moe_quant_config,+ )++ self.input_dtype = "mxfp4"+ self.weight_dtype = "mxfp4"+ return ocp_mx_moe_quant_config(+ quant_dtype=self.input_dtype,+ weight_dtype=self.weight_dtype,+ w1_scale=layer.w13_weight_scale,+ w2_scale=layer.w2_weight_scale,+ a1_scale=None,+ a2_scale=None,+ w1_bias=layer.w13_bias if self.has_bias else None,+ w2_bias=layer.w2_bias if self.has_bias else None,+ block_shape=None,+ )+ return None++ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:+ if envs.VLLM_ENABLE_STATIC_MOE:+ if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:+ weight_name_lst = ["w13_weight", "w2_weight"]++ for weight_name_prefix in weight_name_lst:+ weight_name = f"{weight_name_prefix}_packed"+ weight = getattr(layer, weight_name)+ weight_scale_name = f"{weight_name_prefix}_scale"+ weight_scale = getattr(layer, weight_scale_name)+ new_weight_name = f"{weight_name_prefix}_unpacked"+ new_scale_name = weight_scale_name+ num_experts, _, _ = weight.shape+ unpacked_weight_lst = []+ scale_list = []+ for expert_index in range(num_experts):+ weight_fp8, scale_bf16 = dequant_mxfp4_to_fp8(+ data_lp=weight[expert_index],+ scale_e8m0=weight_scale[expert_index],+ )++ unpacked_weight_lst.append(weight_fp8)+ scale_list.append(scale_bf16)+ unpacked_weight_fp8 = torch.stack(unpacked_weight_lst, dim=0)+ scale_bf16 = torch.stack(scale_list, dim=0)+ assert unpacked_weight_fp8.shape[0] == num_experts, (+ f"Expected {num_experts} unpacked weights, got " f"{unpacked_weight_fp8.shape[0]}"+ )+ delattr(layer, weight_name)+ delattr(layer, weight_scale_name)+ layer.register_parameter(+ new_weight_name,+ torch.nn.Parameter(+ unpacked_weight_fp8,+ requires_grad=False,+ ),+ )+ layer.register_parameter(+ new_scale_name,+ torch.nn.Parameter(+ scale_bf16,+ requires_grad=False,+ ),+ )++ elif envs.VLLM_AR_MXFP4_MODULAR_MOE:++ def revert_interleaved_bias(bias):+ """+ Convert from blocked bias format to interleaved format.++ Args:+ bias: Tensor of shape [E, intermediate_size*2] where the first half contains+ w1 biases and second half contains w3 biases for each expert++ Returns:+ Tensor with interleaved w1 and w3 biases+ """+ # loaded bias[0]: [e0_w1_bias_0, e0_w1_bias_1, ..., e0_w3_bias_0, e0_w3_bias_1...]+ # Expected bias[0]: [e0_w1_bias_0, e0_w3_bias_0, e0_w1_bias_1, e0_w3_bias_1...]+ # The expected bias order is used in triton kernel https://github.com/vllm-project/vllm/pull/22508+ revert_bias = torch.zeros_like(bias, device=bias.device)+ E, two_IN = bias.shape++ # Verify the shape is as expected+ if two_IN % 2 != 0:+ raise ValueError(f"Expected even number of bias elements, got {two_IN}")++ revert_bias[..., ::2] = bias[..., : two_IN // 2]+ revert_bias[..., 1::2] = bias[..., two_IN // 2 :]++ return revert_bias++ # breakpoint()+ w13_bias_swapped = revert_interleaved_bias(layer.w13_bias)+ layer.w13_bias.data.copy_(w13_bias_swapped)++ if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:++ w1 = layer.w13_weight_packed+ w1_scale = layer.w13_weight_scale+ w1 = mxfp4_utils.to_dtype(+ data_lp=w1,+ scale_e8m0=w1_scale,+ elem_dtype="fp4_e2m1",+ block_size=32,+ target_dtype=torch.bfloat16,+ )++ def revert_interleaved_w1(w1):+ new_w1 = torch.zeros_like(w1)+ E, N, H = w1.shape+ new_w1[:, ::2, :] = w1[:, : N // 2, :]+ new_w1[:, 1::2, :] = w1[:, N // 2 :, :]+ return new_w1++ w1 = revert_interleaved_w1(w1)++ w1_scale = None+ w2 = layer.w2_weight_packed+ w2_scale = layer.w2_weight_scale+ w2 = mxfp4_utils.to_dtype(+ data_lp=w2,+ scale_e8m0=w2_scale,+ elem_dtype="fp4_e2m1",+ block_size=32,+ target_dtype=torch.bfloat16,+ )+ w2_scale = None+ del layer.w13_weight_packed+ del layer.w13_weight_scale+ del layer.w2_weight_packed+ del layer.w2_weight_scale+ layer.w13_weight_scale = None+ layer.w2_weight_scale = None+ layer.register_parameter(+ "w13_weight_unpacked",+ torch.nn.Parameter(+ w1,+ requires_grad=False,+ ),+ )+ layer.register_parameter(+ "w2_weight_unpacked",+ torch.nn.Parameter(+ w2,+ requires_grad=False,+ ),+ )++ else:+ raise NotImplementedError("process_weights_after_loading is not implemented for now.")++ @torch.inference_mode()+ def apply(+ self,+ layer: torch.nn.Module,+ x: torch.Tensor,+ router_logits: torch.Tensor,+ top_k: int,+ renormalize: bool,+ use_grouped_topk: bool = False,+ topk_group: Optional[int] = None,+ num_expert_group: Optional[int] = None,+ global_num_experts: int = -1,+ expert_map: Optional[torch.Tensor] = None,+ custom_routing_function: Optional[Callable] = None,+ scoring_func: str = "softmax",+ routed_scaling_factor: float = 1.0,+ e_score_correction_bias: Optional[torch.Tensor] = None,+ apply_router_weight_on_input: bool = False,+ activation: str = "silu",+ enable_eplb: bool = False,+ expert_load_view: Optional[torch.Tensor] = None,+ logical_to_physical_map: Optional[torch.Tensor] = None,+ logical_replica_count: Optional[torch.Tensor] = None,+ ) -> Union[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:+ topk_weights, topk_ids, _ = FusedMoE.select_experts(+ hidden_states=x,+ router_logits=router_logits,+ use_grouped_topk=use_grouped_topk,+ top_k=top_k,+ renormalize=renormalize,+ topk_group=topk_group,+ num_expert_group=num_expert_group,+ custom_routing_function=custom_routing_function,+ scoring_func=scoring_func,+ e_score_correction_bias=e_score_correction_bias,+ )+ assert self.fused_experts is None++ # There are three implementations:++ if envs.VLLM_AR_MXFP4_MODULAR_MOE:+ from vllm.model_executor.layers.fused_moe import fused_experts++ if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:+ w1 = layer.w13_weight_unpacked+ w2 = layer.w2_weight_unpacked+ else:+ w1 = layer.w13_weight_packed+ w2 = layer.w2_weight_packed+ out = fused_experts(+ x,+ w1,+ w2,+ topk_weights=topk_weights,+ topk_ids=topk_ids,+ inplace=True,+ activation=activation,+ global_num_experts=global_num_experts,+ apply_router_weight_on_input=apply_router_weight_on_input,+ expert_map=expert_map,+ quant_config=self.moe_quant_config,+ )+ return out++ num_all_tokens, hidden_dim = x.shape+ num_experts = layer.local_num_experts+ total_num_experts = router_logits.size(-1)+ if (+ self.mask_weights_buffer is None+ or self.mask_weights_buffer.dtype != x.dtype+ or self.mask_weights_buffer.device != x.device+ or self.mask_weights_buffer.shape[0] < num_all_tokens+ or self.mask_weights_buffer.shape[1] < total_num_experts+ ):+ if num_all_tokens > self.num_all_tokens_threshold:+ mask_weights = torch.zeros((num_all_tokens, total_num_experts), dtype=x.dtype, device=x.device)+ if self.mask_weights_buffer is None and self.num_all_tokens_threshold != 0:+ self.mask_weights_buffer = torch.zeros(+ (self.num_all_tokens_threshold, total_num_experts),+ dtype=x.dtype,+ device=x.device,+ )+ self.experts_mask_buffer = torch.zeros(+ (self.num_all_tokens_threshold, total_num_experts),+ dtype=x.dtype,+ device=x.device,+ )+ else:+ self.mask_weights_buffer = torch.zeros(+ (num_all_tokens, total_num_experts), dtype=x.dtype, device=x.device+ )+ self.experts_mask_buffer = torch.zeros(+ (num_all_tokens, total_num_experts), dtype=x.dtype, device=x.device+ )+ mask_weights = self.mask_weights_buffer+ experts_mask = self.experts_mask_buffer+ else:+ self.mask_weights_buffer.zero_()+ mask_weights = self.mask_weights_buffer+ self.experts_mask_buffer.zero_()+ experts_mask = self.experts_mask_buffer++ topk_ids = topk_ids.to(torch.int64)+ topk_weights = topk_weights.to(x.dtype)+ experts_mask.scatter_(-1, topk_ids, topk_weights)+ mask_weights.scatter_(-1, topk_ids, 1)+ mask_weights = mask_weights[:num_all_tokens, :total_num_experts]+ mask_weights = mask_weights.transpose(0, 1)+ experts_mask = experts_mask[:num_all_tokens, :total_num_experts]+ experts_mask = experts_mask.transpose(0, 1)+ # Note: ep_size equal tp_size+ ep_rank = get_tensor_model_parallel_rank() if expert_map is not None else 0+ ep_shift = ep_rank * num_experts++ if envs.VLLM_ENABLE_STATIC_MOE and not envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:+ num_experts, intermediate_size_per_partition_x2, _ = layer.w13_weight_packed.shape+ intermediate_size_per_partition = intermediate_size_per_partition_x2 // 2+ for expert_index in range(num_experts):+ mask_weight = mask_weights[expert_index + ep_shift].unsqueeze(1)+ current_state_static = x * mask_weight++ local_w13_packed = layer.w13_weight_packed[expert_index]+ local_w13_scale = layer.w13_weight_scale[expert_index]+ local_w2_packed = layer.w2_weight_packed[expert_index]+ local_w2_scale = layer.w2_weight_scale[expert_index]++ local_w1_packed = local_w13_packed[:intermediate_size_per_partition, ...]+ local_w1_scale = local_w13_scale[:intermediate_size_per_partition, ...]++ local_w3_packed = local_w13_packed[intermediate_size_per_partition:, ...]+ local_w3_scale = local_w13_scale[intermediate_size_per_partition:, ...]++ local_w1_bias = None+ local_w2_bias = None+ local_w3_bias = None+ if self.has_bias:+ local_w13_bias = layer.w13_bias[expert_index]+ local_w1_bias = local_w13_bias[:intermediate_size_per_partition]+ local_w3_bias = local_w13_bias[intermediate_size_per_partition:]+ local_w2_bias = layer.w2_bias[expert_index]++ local_w1_out = run_mxfp4_emulations(+ x=current_state_static,+ weight=local_w1_packed,+ weight_scale=local_w1_scale,+ bias=local_w1_bias,+ )+ local_w3_out = run_mxfp4_emulations(+ x=current_state_static,+ weight=local_w3_packed,+ weight_scale=local_w3_scale,+ bias=local_w3_bias,+ )++ w13_out = apply_act(local_w1_out, local_w3_out, activation)++ local_w2_out = run_mxfp4_emulations(+ x=w13_out,+ weight=local_w2_packed,+ weight_scale=local_w2_scale,+ bias=local_w2_bias,+ )+ padded_weight = experts_mask[expert_index + ep_shift].unsqueeze(1)+ local_w2_out = local_w2_out * padded_weight+ if expert_index == 0:+ final_hidden_states = local_w2_out+ else:+ final_hidden_states += local_w2_out+ return final_hidden_states+ if envs.VLLM_ENABLE_STATIC_MOE and envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:+ num_experts, intermediate_size_per_partition_x2, _ = layer.w13_weight_unpacked.shape+ intermediate_size_per_partition = intermediate_size_per_partition_x2 // 2++ for expert_index in range(num_experts):+ mask_weight = mask_weights[expert_index + ep_shift].unsqueeze(1)+ current_state_static = x * mask_weight++ local_unpacked_w13 = layer.w13_weight_unpacked[expert_index]+ local_w13_scale = layer.w13_weight_scale[expert_index]++ local_unpacked_w2 = layer.w2_weight_unpacked[expert_index]+ local_w2_scale = layer.w2_weight_scale[expert_index]++ local_unpacked_w1 = local_unpacked_w13[:intermediate_size_per_partition, ...]+ half_scale = local_w13_scale.shape[0] // 2+ local_w1_scale = local_w13_scale[:half_scale, ...]+ local_unpacked_w3 = local_unpacked_w13[intermediate_size_per_partition:, ...]+ local_w3_scale = local_w13_scale[half_scale:, ...]++ local_w1_bias = None+ local_w2_bias = None+ local_w3_bias = None+ if self.has_bias:+ local_w13_bias = layer.w13_bias[expert_index]+ local_w1_bias = local_w13_bias[:intermediate_size_per_partition]+ local_w3_bias = local_w13_bias[intermediate_size_per_partition:]+ local_w2_bias = layer.w2_bias[expert_index]++ local_w1_out = mxfp4_gemm_with_unpacked_weight(+ x=current_state_static,+ weight_fp8=local_unpacked_w1,+ weight_scale_bf16=local_w1_scale,+ bias=local_w1_bias,+ )+ local_w3_out = mxfp4_gemm_with_unpacked_weight(+ x=current_state_static,+ weight_fp8=local_unpacked_w3,+ weight_scale_bf16=local_w3_scale,+ bias=local_w3_bias,+ )++ w13_out = apply_act(local_w1_out, local_w3_out, activation)++ local_w2_out = mxfp4_gemm_with_unpacked_weight(+ x=w13_out,+ weight_fp8=local_unpacked_w2,+ weight_scale_bf16=local_w2_scale,+ bias=local_w2_bias,+ )++ padded_weight = experts_mask[expert_index + ep_shift].unsqueeze(1)+ local_w2_out = local_w2_out * padded_weight+ if expert_index == 0:+ final_hidden_states = local_w2_out+ else:+ final_hidden_states += local_w2_out+ return final_hidden_states+ raise NotImplementedError("Not implemented for now.")diff --git a/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py b/auto_round_extension/vllm_ext/mxfp4_qdq_utils.pynew file mode 100644index 000000000..197d757fa--- /dev/null+++ b/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py@@ -0,0 +1,220 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from typing import Union++import torch++from auto_round_extension.vllm_ext.fp4_utils import cast_to_fp4, pack_fp4_to_uint8, unpack_fp4_from_uint8+from auto_round_extension.vllm_ext.utils import _to_mx_rceil, get_fp_scale++F4_E2M1_MAX = 6.0+F32_EXP_BIAS = 127++F32_MIN_NORMAL = 2 ** (-F32_EXP_BIAS + 1)+++def to_mxfp4_rceil(+ data_hp: torch.Tensor,+ elem_dtype: Union[torch.dtype, str],+ block_size: int,+):++ assert data_hp.dtype in (+ torch.bfloat16,+ torch.float,+ ), f"{data_hp.dtype} is not supported yet"+ # TODO(future PR): consider supporting padding+ assert data_hp.numel() % block_size == 0, f"data size must be multiple of block_size={block_size}"+ assert data_hp.is_contiguous(), f"data must be contiguous, got {data_hp.stride()}"++ # calculate the scale in e8m0 format++ orig_shape = data_hp.shape+ data_hp = data_hp.reshape(-1, block_size)++ # find max value of the data+ # Note: this only implements the `minimally supported` version of+ # https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf+ # section 6.3.+ max_abs = torch.amax(torch.abs(data_hp), 1)++ # Add an epsilon to prevent the log2 function call for returning -inf+ # where the values are zero.+ eps = F32_MIN_NORMAL * (max_abs == 0).type(max_abs.dtype)++ # Set X to be the largest power-of-two less than or equal to+ # max_abs(v), divided by the largest power of two representable+ # in the element data type, and get the mbits at the same time++ max_pos = F4_E2M1_MAX+ scale_e8m0_biased, data_lp = _to_mx_rceil(data_hp, max_abs, max_pos)++ data_lp = data_lp.reshape(orig_shape)+ orig_shape = [*orig_shape[:-1], orig_shape[-1] // 2]+ data_lp = cast_to_fp4(data_lp)+ data_lp = pack_fp4_to_uint8(data_lp)++ scale_e8m0_biased = scale_e8m0_biased.view(torch.uint8)+ return scale_e8m0_biased, data_lp+++def to_dtype(+ data_lp,+ scale_e8m0,+ elem_dtype,+ block_size,+ target_dtype,+ scale_dtype=None,+ return_scale=False,+):+ orig_shape = data_lp.shape+ last_dim = orig_shape[-1]+ data_lp = data_lp.reshape(-1, last_dim)+ result_shape = orig_shape[:-1] + (last_dim * 2,)+ assert data_lp.is_contiguous(), f"Data must be contiguous, got {data_lp.stride()}"++ assert elem_dtype == "fp4_e2m1", f"Expected 'fp4_e2m1', got {elem_dtype}"++ m, half_n = data_lp.shape+ n = half_n * 2+ data_hp = unpack_fp4_from_uint8(data_lp, m, n, dtype=target_dtype)++ data_hp = data_hp.reshape(-1, block_size)++ if scale_dtype is None:+ scale_dtype = target_dtype+ s_fp = get_fp_scale(scale_e8m0).reshape(-1, 1).to(scale_dtype)+ if return_scale:+ return data_hp.reshape(result_shape), s_fp++ data_hp = data_hp * s_fp+ data_hp = data_hp.reshape(result_shape)++ return data_hp+++def run_mxfp4_emulations(+ x: torch.Tensor,+ weight: torch.Tensor,+ weight_scale: torch.Tensor,+ bias: torch.Tensor | None = None,+):+ # TODO: select the rounding mode based on config+ group_size = 32+ # quantize input to (FP4 and interleaved block scale)+ input_scale, x_q = to_mxfp4_rceil(+ data_hp=x,+ elem_dtype="fp4_e2m1",+ block_size=group_size,+ )++ # dequantize input+ x_dq = to_dtype(+ data_lp=x_q,+ scale_e8m0=input_scale,+ elem_dtype="fp4_e2m1",+ block_size=group_size,+ target_dtype=x.dtype,+ )++ # dequantize weight+ w_dq = to_dtype(+ data_lp=weight,+ scale_e8m0=weight_scale,+ elem_dtype="fp4_e2m1",+ block_size=group_size,+ target_dtype=x.dtype,+ )++ # matmul+ out = torch.matmul(x_dq, w_dq.t())+ if bias is not None:+ out += bias+ return out+++def dequant_mxfp4_to_fp8(data_lp, scale_e8m0):+ data_fp8, scale_float = to_dtype(+ data_lp=data_lp,+ scale_e8m0=scale_e8m0,+ elem_dtype="fp4_e2m1",+ block_size=32,+ target_dtype=torch.float8_e4m3fn,+ scale_dtype=torch.bfloat16,+ return_scale=True,+ )+ return data_fp8, scale_float+++def mxfp4_fp8_weight_to_bf16(weight_fp8, scale_bf16):+ origin_shape = weight_fp8.shape+ weight_fp8 = weight_fp8.reshape(-1, 32)+ assert weight_fp8.shape[0] == scale_bf16.shape[0], f"shape mismatch: {weight_fp8.shape} vs {scale_bf16.shape}"+ dequant_weight_bf16 = weight_fp8.to(torch.bfloat16) * scale_bf16+ dequant_weight_bf16 = dequant_weight_bf16.reshape(origin_shape)+ return dequant_weight_bf16+++def mxfp4_gemm_with_unpacked_weight(x, weight_fp8, weight_scale_bf16, bias=None):+ x = qdq_mxfp4(x)++ # dequantize weight+ w_dq = mxfp4_fp8_weight_to_bf16(weight_fp8, weight_scale_bf16)+ # matmul+ out = torch.matmul(x, w_dq.t())+ if bias is not None:+ out += bias+ return out+++def fp4_121_positive(x: torch.Tensor) -> torch.Tensor:+ step1 = torch.round(2.0 * x) / 2.0+ step2 = torch.round(x)+ step3 = 2.0 * torch.round(x / 2.0)++ mask1 = x < 2.0+ mask2 = x < 4.0++ return step1 * mask1 + step2 * (~mask1) * mask2 + step3 * (~mask1) * (~mask2)+++def fp4_121_scaled_even_rounding(x: torch.Tensor) -> torch.Tensor:+ sign = x.sign()+ x_abs = x.abs()+ amax_x = x_abs.max(dim=-1, keepdim=True)[0]+ scale_tmp = torch.floor(torch.log2(amax_x)) - 2.0+ scale_clamp = torch.clamp(scale_tmp, min=-127, max=127)+ scale = torch.pow(2.0, scale_clamp)++ scale = torch.where((0 < scale) * (scale < torch.inf), scale, 1.0)++ x_fp4_abs = fp4_121_positive(x_abs / scale) * scale+ return sign * x_fp4_abs+++# https://github.com/Anonymous1252022/fp4-all-the-way/blob/main/experimental/fp4.py+def qdq_mxfp4(+ x: torch.Tensor,+) -> torch.Tensor:+ # TODO: select the rounding mode based on config+ block_size = 32+ shape = x.shape+ x = x.reshape(-1, block_size)++ x = fp4_121_scaled_even_rounding(x)++ x = x.reshape(shape)++ return xdiff --git a/auto_round_extension/vllm_ext/quant_method_moe.py b/auto_round_extension/vllm_ext/quant_method_moe.pynew file mode 100644index 000000000..ae7b81775--- /dev/null+++ b/auto_round_extension/vllm_ext/quant_method_moe.py@@ -0,0 +1,84 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from typing import Optional++import torch+from vllm.logger import init_logger+from vllm.model_executor.layers.fused_moe import (+ FusedMoEConfig,+ FusedMoEMethodBase,+)+from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig+from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig++from auto_round.schemes import QuantizationScheme++logger = init_logger(__name__)+++QMOE_METHODS_DISPATCH_TABLE = {}+++def _is_mxfp4_w4a4(scheme: QuantizationScheme):+ # FIXME: below impl is incomplete+ return scheme.bits == 4 and scheme.group_size == 32+++class AutoRoundMoEMethod(FusedMoEMethodBase):+ def __init__(self, moe: FusedMoEConfig):+ super().__init__(moe)++ @staticmethod+ def get_moe_method(+ quant_config: AutoRoundConfig,+ layer: torch.nn.Module,+ prefix: str,+ ) -> "AutoRoundMoEMethod":++ def get_scheme(quant_config: AutoRoundConfig, prefix: str):+ # Check extra_config first+ layer_schemes = quant_config.layer_schemes+ # FIXME: make more robust+ for name, scheme in layer_schemes.items():+ if prefix.startswith(name):+ return scheme+ # If not found, use default+ return quant_config.quant_scheme++ def check_quantized(weight_bits: int) -> bool:+ return weight_bits < 16++ def get_impl(scheme: QuantizationScheme):+ if not check_quantized(scheme.bits):+ from vllm.model_executor.layers.fused_moe.layer import (+ UnquantizedFusedMoEMethod,+ )++ return UnquantizedFusedMoEMethod(layer.moe_config)++ elif _is_mxfp4_w4a4(scheme):+ from auto_round_extension.vllm_ext.moe_impl_mxfp4 import AutoRoundMoEMethodMXFp4Impl++ return AutoRoundMoEMethodMXFp4Impl(quant_config, layer.moe_config)++ raise ValueError(f"Unsupported FusedMoe scheme: {scheme}")++ layer_scheme = get_scheme(quant_config, prefix)+ impl = get_impl(layer_scheme)+ logger.debug("Apply %s to %s", impl.__class__.__name__, prefix)+ return impl++ def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> Optional[FusedMoEQuantConfig]:+ return self.impl.get_fused_moe_quant_config(layer)diff --git a/auto_round_extension/vllm_ext/sitecustomize.py b/auto_round_extension/vllm_ext/sitecustomize.pynew file mode 100644index 000000000..8c22d1ab2--- /dev/null+++ b/auto_round_extension/vllm_ext/sitecustomize.py@@ -0,0 +1,41 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import os++VLLM_ENABLE_AR_EXT = os.environ.get("VLLM_ENABLE_AR_EXT", "") in [+ "1",+ "true",+ "True",+]++if VLLM_ENABLE_AR_EXT:+ print("*****************************************************************************")+ print(f"* !!! VLLM_ENABLE_AR_EXT is set to {VLLM_ENABLE_AR_EXT}, applying auto_round_vllm_extension *")+ print("*****************************************************************************")++ import vllm.model_executor.layers.quantization.auto_round as auto_round_module++ from auto_round_extension.vllm_ext.auto_round_ext import AutoRoundExtensionConfig++ auto_round_module.AutoRoundConfig = AutoRoundExtensionConfig+ from auto_round_extension.vllm_ext.envs_ext import extra_environment_variables+++else:+ print("*****************************************************************************")+ print(+ f"* Sitecustomize is loaded, but VLLM_ENABLE_AR_EXT is set to {VLLM_ENABLE_AR_EXT}, skipping auto_round_vllm_extension *"+ )+ print("*****************************************************************************")diff --git a/auto_round_extension/vllm_ext/tests/conftest.py b/auto_round_extension/vllm_ext/tests/conftest.pynew file mode 100644index 000000000..fcdd97a49--- /dev/null+++ b/auto_round_extension/vllm_ext/tests/conftest.py@@ -0,0 +1,1396 @@+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project+import contextlib+import pathlib+from copy import deepcopy++from tblib import pickling_support++# ruff: noqa++# Install support for pickling exceptions so that we can nicely propagate+# failures from tests running in a subprocess.+# This should be run before any custom exception subclasses are defined.+pickling_support.install()++import http.server+import json+import math+import mimetypes+import os+import socket+import tempfile+import threading+from collections.abc import Generator+from contextlib import nullcontext+from enum import Enum+from typing import Any, Callable, TypedDict, TypeVar, cast++import numpy as np+import pytest+import torch+import torch.nn as nn+import torch.nn.functional as F+from huggingface_hub import snapshot_download+from PIL import Image+from transformers import (+ AutoConfig,+ AutoModelForCausalLM,+ AutoTokenizer,+ BatchEncoding,+ BatchFeature,+)+from transformers.models.auto.auto_factory import _BaseAutoModelClass++# from tests.models.utils import TokensTextLogprobs, TokensTextLogprobsPromptLogprobs+from vllm import LLM, SamplingParams, envs+from vllm.assets.audio import AudioAsset+from vllm.assets.image import ImageAsset+from vllm.assets.video import VideoAsset+from vllm.config.model import ConvertOption, RunnerOption, _get_and_verify_dtype+from vllm.connections import global_http_connection+from vllm.distributed import (+ cleanup_dist_env_and_memory,+ init_distributed_environment,+ initialize_model_parallel,+)+from vllm.logger import init_logger+from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs+from vllm.multimodal.utils import fetch_image+from vllm.outputs import RequestOutput+from vllm.sampling_params import BeamSearchParams+from vllm.transformers_utils.utils import maybe_model_redirect+from vllm.utils.collection_utils import is_list_of+from vllm.utils.torch_utils import set_default_torch_num_threads++# Representation of generated sequence as a tuple of+# * Token ID list+# * String+# * List of top sample logprobs for each sampled token+#+# Assumes prompt logprobs were not requested.+TokensTextLogprobs = tuple[list[int], str, list[dict[int, float]] | SampleLogprobs | None]++# Allow for tokens to be represented as str's rather than IDs;+# tuple of+# * Token string representations list+# * String+# * Optional list of top sample logprobs for each sampled token+#+# Assumes prompt logprobs were not requested.+TextTextLogprobs = tuple[list[str], str, list[dict[str, float]] | list[dict[str, Logprob]] | None]++# Representation of generated sequence as a tuple of+# * Token ID list+# * String+# * Optional list of top sample logprobs for each sampled token+# * Optional list of top prompt logprobs for each prompt token+#+# Allows prompt logprobs to be requested.+TokensTextLogprobsPromptLogprobs = tuple[+ list[int],+ str,+ list[dict[int, float]] | SampleLogprobs | None,+ list[dict[int, float] | None] | PromptLogprobs | None,+]+++logger = init_logger(__name__)++_TEST_DIR = os.path.dirname(__file__)+_TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]+_LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]+_SYS_MSG = os.path.join(_TEST_DIR, "system_messages", "sonnet3.5_nov2024.txt")++_M = TypeVar("_M")++_PromptMultiModalInput = list[_M] | list[list[_M]]++PromptImageInput = _PromptMultiModalInput[Image.Image]+PromptAudioInput = _PromptMultiModalInput[tuple[np.ndarray, int]]+PromptVideoInput = _PromptMultiModalInput[np.ndarray]+++def _read_prompts(filename: str) -> list[str]:+ with open(filename) as f:+ prompts = f.readlines()+ return prompts+++class ImageAssetPrompts(TypedDict):+ stop_sign: str+ cherry_blossom: str+++class ImageTestAssets(list[ImageAsset]):+ def __init__(self) -> None:+ super().__init__(+ [+ ImageAsset("stop_sign"),+ ImageAsset("cherry_blossom"),+ ]+ )++ def prompts(self, prompts: ImageAssetPrompts) -> list[str]:+ """+ Convenience method to define the prompt for each test image.++ The order of the returned prompts matches the order of the+ assets when iterating through this object.+ """+ return [prompts["stop_sign"], prompts["cherry_blossom"]]+++class VideoAssetPrompts(TypedDict):+ baby_reading: str+++class VideoTestAssets(list[VideoAsset]):+ def __init__(self) -> None:+ super().__init__(+ [+ VideoAsset("baby_reading"),+ ]+ )++ def prompts(self, prompts: VideoAssetPrompts) -> list[str]:+ return [prompts["baby_reading"]]+++class AudioAssetPrompts(TypedDict):+ mary_had_lamb: str+ winning_call: str+++class AudioTestAssets(list[AudioAsset]):+ def __init__(self) -> None:+ super().__init__(+ [+ AudioAsset("mary_had_lamb"),+ AudioAsset("winning_call"),+ ]+ )++ def prompts(self, prompts: AudioAssetPrompts) -> list[str]:+ return [prompts["mary_had_lamb"], prompts["winning_call"]]+++IMAGE_ASSETS = ImageTestAssets()+"""Singleton instance of {class}`ImageTestAssets`."""+VIDEO_ASSETS = VideoTestAssets()+"""Singleton instance of {class}`VideoTestAssets`."""+AUDIO_ASSETS = AudioTestAssets()+"""Singleton instance of {class}`AudioTestAssets`."""++[email protected](scope="function", autouse=True)+def cleanup_VLLM_USE_V1(monkeypatch):+ """+ The V1 oracle sets "VLLM_USE_V1" during loading. This means+ that each invocation of a test change the env variable.++ If we touch "VLLM_USE_V1" with monkeypatch, then any changes+ made during the test run by vLLM will be cleaned up.++ This fixture is used by every test.+ """++ # If VLLM_USE_V1 is not set, set then delete. This will+ # cause monkeypatch to clean up VLLM_USE_V1 upon exit+ # if VLLM modifies the value of envs.VLLM_USE_V1.+ if "VLLM_USE_V1" not in os.environ:+ monkeypatch.setenv("VLLM_USE_V1", "")+ monkeypatch.delenv("VLLM_USE_V1")++[email protected](autouse=True)+def init_test_http_connection():+ # pytest_asyncio may use a different event loop per test+ # so we need to make sure the async client is created anew+ global_http_connection.reuse_client = False+++def dist_init():+ temp_file = tempfile.mkstemp()[1]+ init_distributed_environment(+ world_size=1,+ rank=0,+ distributed_init_method=f"file://{temp_file}",+ local_rank=0,+ backend="nccl",+ )+ initialize_model_parallel(1, 1)+ yield+ cleanup_dist_env_and_memory()+++def should_do_global_cleanup_after_test(request) -> bool:+ """Allow subdirectories to skip global cleanup by overriding this fixture.+ This can provide a ~10x speedup for non-GPU unit tests since they don't need+ to initialize torch.+ """++ return not request.node.get_closest_marker("skip_global_cleanup")++[email protected](autouse=True)+def cleanup_fixture(should_do_global_cleanup_after_test: bool):+ yield+ if should_do_global_cleanup_after_test:+ cleanup_dist_env_and_memory()++[email protected](autouse=True)+def dynamo_reset():+ yield+ torch._dynamo.reset()+++def example_prompts() -> list[str]:+ prompts = []+ for filename in _TEST_PROMPTS:+ prompts += _read_prompts(filename)+ return prompts+++def example_system_message() -> str:+ with open(_SYS_MSG) as f:+ return f.read()+++class DecoderPromptType(Enum):+ """For encoder/decoder models only."""++ CUSTOM = 1+ NONE = 2+ EMPTY_STR = 3+++def example_long_prompts() -> list[str]:+ prompts = []+ for filename in _LONG_PROMPTS:+ prompts += _read_prompts(filename)+ return prompts++[email protected](scope="session")+def image_assets() -> ImageTestAssets:+ return IMAGE_ASSETS++[email protected](scope="session")+def video_assets() -> VideoTestAssets:+ return VIDEO_ASSETS++[email protected](scope="session")+def audio_assets() -> AudioTestAssets:+ return AUDIO_ASSETS+++_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature, dict)+_R = TypeVar("_R")+++class HfRunner:+ def get_default_device(self):+ from vllm.platforms import current_platform++ return "cpu" if current_platform.is_cpu() else current_platform.device_type++ def wrap_device(self, x: _T, device: str | None = None) -> _T:+ if x is None or isinstance(x, (bool,)):+ return x++ if device is None:+ device = self.device++ if isinstance(x, dict):+ return {k: self.wrap_device(v, device) for k, v in x.items()}++ if hasattr(x, "device") and x.device.type == device:+ return x++ return x.to(device)++ def __init__(+ self,+ model_name: str,+ dtype: str = "auto",+ *,+ model_kwargs: dict[str, Any] | None = None,+ trust_remote_code: bool = True,+ is_sentence_transformer: bool = False,+ is_cross_encoder: bool = False,+ skip_tokenizer_init: bool = False,+ auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,+ # Set this to avoid hanging issue+ default_torch_num_threads: int | None = None,+ ) -> None:+ init_ctx = (+ nullcontext()+ if default_torch_num_threads is None+ else set_default_torch_num_threads(default_torch_num_threads)+ )++ with init_ctx:+ self._init(+ model_name=model_name,+ dtype=dtype,+ model_kwargs=model_kwargs,+ trust_remote_code=trust_remote_code,+ is_sentence_transformer=is_sentence_transformer,+ is_cross_encoder=is_cross_encoder,+ skip_tokenizer_init=skip_tokenizer_init,+ auto_cls=auto_cls,+ )++ def _init(+ self,+ model_name: str,+ dtype: str = "auto",+ *,+ model_kwargs: dict[str, Any] | None = None,+ trust_remote_code: bool = True,+ is_sentence_transformer: bool = False,+ is_cross_encoder: bool = False,+ skip_tokenizer_init: bool = False,+ auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,+ ) -> None:+ model_name = maybe_model_redirect(model_name)+ self.model_name = model_name++ self.config = AutoConfig.from_pretrained(+ model_name,+ trust_remote_code=trust_remote_code,+ )+ self.device = self.get_default_device()+ self.dtype = dtype = _get_and_verify_dtype(+ self.model_name,+ self.config,+ dtype=dtype,+ is_pooling_model=is_sentence_transformer or is_cross_encoder,+ )++ model_kwargs = model_kwargs if model_kwargs is not None else {}+ model_kwargs.setdefault("dtype", dtype)++ if is_sentence_transformer:+ # Lazy init required for AMD CI+ from sentence_transformers import SentenceTransformer++ self.model = SentenceTransformer(+ model_name,+ device=self.device,+ model_kwargs=model_kwargs,+ trust_remote_code=trust_remote_code,+ )+ elif is_cross_encoder:+ # Lazy init required for AMD CI+ from sentence_transformers import CrossEncoder++ self.model = CrossEncoder(+ model_name,+ device=self.device,+ automodel_args=model_kwargs,+ trust_remote_code=trust_remote_code,+ )+ else:+ model = auto_cls.from_pretrained(+ model_name,+ trust_remote_code=trust_remote_code,+ **model_kwargs,+ )++ # in case some unquantized custom models are not in same dtype+ if getattr(model, "quantization_method", None) is None and any(+ p.dtype != self.dtype for p in model.parameters()+ ):+ model = model.to(dtype=self.dtype)++ if (+ getattr(model, "quantization_method", None) != "bitsandbytes"+ and len({p.device for p in model.parameters()}) < 2+ ):+ model = model.to(device=self.device)++ self.model = model++ if not skip_tokenizer_init:+ self.tokenizer = AutoTokenizer.from_pretrained(+ model_name,+ dtype=dtype,+ trust_remote_code=trust_remote_code,+ )++ # don't put this import at the top level+ # it will call torch.cuda.device_count()+ from transformers import AutoProcessor # noqa: F401++ self.processor = AutoProcessor.from_pretrained(+ model_name,+ dtype=dtype,+ trust_remote_code=trust_remote_code,+ )+ if skip_tokenizer_init:+ self.tokenizer = self.processor.tokenizer++ def get_inputs(+ self,+ prompts: list[str] | list[list[int]],+ images: PromptImageInput | None = None,+ videos: PromptVideoInput | None = None,+ audios: PromptAudioInput | None = None,+ ) -> list[BatchFeature | BatchEncoding | dict[str, torch.Tensor]]:+ if images is not None:+ assert len(prompts) == len(images)++ if videos is not None:+ assert len(prompts) == len(videos)++ if audios is not None:+ assert len(prompts) == len(audios)++ all_inputs: list[BatchFeature | BatchEncoding | dict[str, torch.Tensor]] = []+ for i, prompt in enumerate(prompts):+ if isinstance(prompt, str):+ processor_kwargs: dict[str, Any] = {+ "text": prompt,+ "return_tensors": "pt",+ }+ if images is not None and (image := images[i]) is not None:+ processor_kwargs["images"] = image+ if videos is not None and (video := videos[i]) is not None:+ processor_kwargs["videos"] = video+ if audios is not None and (audio_inputs := audios[i]) is not None:+ # HACK - not all processors take sampling_rate; we should+ # clean this up in the future.+ if len(audio_inputs) == 2:+ audio, sr = audio_inputs+ processor_kwargs["audio"] = audio+ processor_kwargs["sampling_rate"] = sr+ else:+ processor_kwargs["audio"] = audio_inputs++ inputs = self.processor(**processor_kwargs)+ if isinstance(inputs, BatchFeature):+ inputs = inputs.to(dtype=self.dtype)+ all_inputs.append(inputs)+ else:+ # check that prompt is (batched) list of integers (token ids)+ if not is_list_of(prompt, typ=int, check="all"):+ raise ValueError("Prompt must be a list of ints corresponding to the prompt token ids.")+ # check that no multimodal input is provided+ if images or videos or audios:+ raise ValueError("When providing prompt token ids multimodal inputs are not supported.")+ input_dict = {+ "input_ids": torch.tensor(prompt, dtype=torch.long).unsqueeze(0),+ }+ all_inputs.append(input_dict)++ return all_inputs++ def get_prompt_embeddings(self, prompts: list[str]) -> list[torch.Tensor]:+ all_inputs = self.get_inputs(prompts)+ embeddings = []+ for inputs in all_inputs:+ input_ids = self.wrap_device(inputs)["input_ids"]+ embedding = self.model.get_input_embeddings()(input_ids).squeeze(0)+ embeddings.append(embedding)+ return embeddings++ def classify(self, prompts: list[str]) -> list[str]:+ # output is final logits+ all_inputs = self.get_inputs(prompts)+ outputs = []+ problem_type = getattr(self.config, "problem_type", "")++ for inputs in all_inputs:+ output = self.model(**self.wrap_device(inputs))+ if problem_type == "regression":+ logits = output.logits[0].tolist()+ elif problem_type == "multi_label_classification":+ logits = output.logits.sigmoid()[0].tolist()+ else:+ logits = output.logits.softmax(dim=-1)[0].tolist()+ outputs.append(logits)++ return outputs++ def generate(+ self,+ prompts: list[str] | list[list[int]],+ images: PromptImageInput | None = None,+ videos: PromptVideoInput | None = None,+ audios: PromptAudioInput | None = None,+ **kwargs: Any,+ ) -> list[tuple[list[list[int]], list[str]]]:+ all_inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)++ outputs: list[tuple[list[list[int]], list[str]]] = []+ for inputs in all_inputs:+ output_ids = self.model.generate(+ **self.wrap_device(inputs),+ use_cache=True,+ **kwargs,+ )+ output_str = self.processor.batch_decode(+ output_ids,+ skip_special_tokens=True,+ clean_up_tokenization_spaces=False,+ )+ output_ids = output_ids.cpu().tolist()+ outputs.append((output_ids, output_str))+ return outputs++ def generate_greedy(+ self,+ prompts: list[str] | list[list[int]],+ max_tokens: int,+ images: PromptImageInput | None = None,+ videos: PromptVideoInput | None = None,+ audios: PromptAudioInput | None = None,+ **kwargs: Any,+ ) -> list[tuple[list[int], str]]:+ outputs = self.generate(+ prompts,+ do_sample=False,+ max_new_tokens=max_tokens,+ images=images,+ videos=videos,+ audios=audios,+ **kwargs,+ )++ return [(output_ids[0], output_str[0]) for output_ids, output_str in outputs]++ def generate_beam_search(+ self,+ prompts: list[str],+ beam_width: int,+ max_tokens: int,+ images: PromptImageInput | None = None,+ videos: PromptVideoInput | None = None,+ audios: PromptAudioInput | None = None,+ ) -> list[tuple[list[list[int]], list[str]]]:+ outputs = self.generate(+ prompts,+ do_sample=False,+ max_new_tokens=max_tokens,+ num_beams=beam_width,+ num_return_sequences=beam_width,+ images=images,+ videos=videos,+ audios=audios,+ )++ for i in range(len(outputs)):+ output_ids, output_str = outputs[i]+ for j in range(len(output_ids)):+ output_ids[j] = [x for x in output_ids[j] if x != self.tokenizer.pad_token_id]+ outputs[i] = (output_ids, output_str)+ return outputs++ def generate_greedy_logprobs(+ self,+ prompts: list[str],+ max_tokens: int,+ images: PromptImageInput | None = None,+ videos: PromptVideoInput | None = None,+ audios: PromptAudioInput | None = None,+ **kwargs: Any,+ ) -> list[list[torch.Tensor]]:+ all_inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)++ all_logprobs: list[list[torch.Tensor]] = []+ for inputs in all_inputs:+ output = self.model.generate(+ **self.wrap_device(inputs),+ use_cache=True,+ do_sample=False,+ max_new_tokens=max_tokens,+ output_hidden_states=True,+ return_dict_in_generate=True,+ **kwargs,+ )+ seq_logprobs = self._hidden_states_to_seq_logprobs(output.hidden_states)+ all_logprobs.append(seq_logprobs)+ return all_logprobs++ def _hidden_states_to_seq_logprobs(+ self,+ hidden_states: tuple[tuple[torch.Tensor, ...], ...],+ ) -> list[torch.Tensor]:+ output_embeddings = self.model.get_output_embeddings()++ seq_logprobs: list[torch.Tensor] = []+ for _, hidden_state in enumerate(hidden_states):+ last_hidden_states = hidden_state[-1][0]+ logits = torch.matmul(+ last_hidden_states.to(+ device=output_embeddings.weight.device,+ dtype=output_embeddings.weight.dtype,+ ),+ output_embeddings.weight.t(),+ )+ if getattr(output_embeddings, "bias", None) is not None:+ logits += output_embeddings.bias.unsqueeze(0)+ logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)+ seq_logprobs.append(logprobs)++ return seq_logprobs++ def _hidden_states_to_logprobs(+ self,+ hidden_states: tuple[tuple[torch.Tensor, ...], ...],+ num_logprobs: int | None,+ ) -> tuple[list[dict[int, float]], int]:+ seq_logprobs = self._hidden_states_to_seq_logprobs(hidden_states)+ output_len = len(hidden_states)++ # convert to dict+ seq_logprobs_lst: list[dict[int, float]] = []+ for tok_idx, tok_logprobs in enumerate(seq_logprobs):+ # drop prompt logprobs+ if tok_idx == 0:+ tok_logprobs = tok_logprobs[-1, :].reshape(1, -1)+ topk = tok_logprobs.topk(num_logprobs)++ tok_logprobs_dct = {}+ for token_id, logprob in zip(topk.indices[0], topk.values[0]):+ tok_logprobs_dct[token_id.item()] = logprob.item()++ seq_logprobs_lst.append(tok_logprobs_dct)++ return (+ seq_logprobs_lst,+ output_len,+ )++ def generate_greedy_logprobs_limit(+ self,+ prompts: list[str],+ max_tokens: int,+ num_logprobs: int | None,+ images: PromptImageInput | None = None,+ audios: PromptAudioInput | None = None,+ videos: PromptVideoInput | None = None,+ **kwargs: Any,+ ) -> list[TokensTextLogprobs]:+ all_inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)++ all_logprobs: list[list[dict[int, float]]] = []+ all_output_ids: list[list[int]] = []+ all_output_strs: list[str] = []++ for inputs in all_inputs:+ output = self.model.generate(+ **self.wrap_device(inputs),+ use_cache=True,+ do_sample=False,+ max_new_tokens=max_tokens,+ output_hidden_states=True,+ return_dict_in_generate=True,+ **kwargs,+ )++ (+ seq_logprobs_lst,+ output_len,+ ) = self._hidden_states_to_logprobs(output.hidden_states, num_logprobs)++ all_logprobs.append(seq_logprobs_lst)+ seq_ids = output.sequences[0]+ output_len = len(seq_logprobs_lst)+ output_ids = seq_ids[-output_len:]+ all_output_ids.append(output_ids.tolist())+ all_output_strs.append(self.tokenizer.decode(output_ids))++ outputs = zip(all_output_ids, all_output_strs, all_logprobs)+ return [(output_ids, output_str, output_logprobs) for output_ids, output_str, output_logprobs in outputs]++ def encode(self, prompts: list[str], *args, **kwargs) -> list[list[torch.Tensor]]:+ return self.model.encode(prompts, *args, **kwargs)++ def predict(self, prompts: list[list[str]], *args, **kwargs) -> torch.Tensor:+ return self.model.predict(prompts, *args, convert_to_tensor=True, **kwargs)++ def __enter__(self):+ return self++ def __exit__(self, exc_type, exc_value, traceback):+ del self.model+ cleanup_dist_env_and_memory()++[email protected](scope="session")+def hf_runner():+ return HfRunner+++class VllmRunner:+ """+ The default value of some arguments have been modified from+ {class}`~vllm.LLM` as follows:++ - `trust_remote_code`: Set to `True` instead of `False` for convenience.+ - `seed`: Set to `0` instead of `None` for test reproducibility.+ - `max_model_len`: Set to `1024` instead of `None` to reduce memory usage.+ - `block_size`: To reduce memory usage, set default to `64` if on XPU+ devices, otherwise default to `16`.+ - `enable_chunked_prefill`: Set to `False` instead of `None` for+ test reproducibility.+ - `enforce_eager`: Set to `False` to test CUDA graph.+ """++ def __init__(+ self,+ model_name: str,+ runner: RunnerOption = "auto",+ convert: ConvertOption = "auto",+ tokenizer_name: str | None = None,+ tokenizer_mode: str = "auto",+ trust_remote_code: bool = True,+ seed: int | None = 0,+ max_model_len: int | None = 1024,+ dtype: str = "auto",+ disable_log_stats: bool = True,+ tensor_parallel_size: int = 1,+ block_size: int = 16 if not torch.xpu.is_available() else 64,+ enable_chunked_prefill: bool | None = False,+ swap_space: int = 4,+ enforce_eager: bool | None = False,+ # Set this to avoid hanging issue+ default_torch_num_threads: int | None = None,+ **kwargs,+ ) -> None:+ init_ctx = (+ nullcontext()+ if default_torch_num_threads is None+ else set_default_torch_num_threads(default_torch_num_threads)+ )++ if not kwargs.get("compilation_config", None):+ # Note(@tdoublep): This is set to 4 because some tests (e.g., hybrid+ # model tests) may set max_num_seqs=4. If min cudagraph_capture_size is+ # set to larger than max_num_seqs, then it will lead to *no* graphs+ # being captured which can trigger edge cases that we don't handle yet.+ kwargs["compilation_config"] = {"cudagraph_capture_sizes": [4]}++ with init_ctx:+ self.llm = LLM(+ model=model_name,+ runner=runner,+ convert=convert,+ tokenizer=tokenizer_name,+ tokenizer_mode=tokenizer_mode,+ trust_remote_code=trust_remote_code,+ dtype=dtype,+ seed=seed,+ swap_space=swap_space,+ enforce_eager=enforce_eager,+ disable_log_stats=disable_log_stats,+ tensor_parallel_size=tensor_parallel_size,+ max_model_len=max_model_len,+ block_size=block_size,+ enable_chunked_prefill=enable_chunked_prefill,+ **kwargs,+ )++ def get_inputs(+ self,+ prompts: list[str] | list[torch.Tensor] | list[list[int]],+ images: PromptImageInput | None = None,+ videos: PromptVideoInput | None = None,+ audios: PromptAudioInput | None = None,+ ) -> list[dict[str, Any]]:+ if any(x is not None and len(x) != len(prompts) for x in [images, videos, audios]):+ raise ValueError("All non-None multimodal inputs must have the same length as prompts")++ inputs = list[dict[str, Any]]()+ for i, prompt in enumerate(prompts):+ prompt_dict = dict[str, Any]()+ if isinstance(prompt, str):+ prompt_dict["prompt"] = prompt+ elif isinstance(prompt, list):+ prompt_dict["prompt_token_ids"] = prompt+ else:+ prompt_dict["prompt_embeds"] = prompt++ multi_modal_data = dict[str, Any]()+ if images is not None and (image := images[i]) is not None:+ multi_modal_data["image"] = image+ if videos is not None and (video := videos[i]) is not None:+ multi_modal_data["video"] = video+ if audios is not None and (audio := audios[i]) is not None:+ multi_modal_data["audio"] = audio++ if multi_modal_data:+ prompt_dict["multi_modal_data"] = multi_modal_data++ inputs.append(prompt_dict)++ return inputs++ def generate(+ self,+ prompts: list[str] | list[torch.Tensor] | list[list[int]],+ sampling_params: SamplingParams,+ images: PromptImageInput | None = None,+ videos: PromptVideoInput | None = None,+ audios: PromptAudioInput | None = None,+ **kwargs: Any,+ ) -> list[tuple[list[list[int]], list[str]]]:+ inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)++ req_outputs = self.llm.generate(inputs, sampling_params=sampling_params, **kwargs)++ outputs: list[tuple[list[list[int]], list[str]]] = []+ for req_output in req_outputs:+ prompt_str = req_output.prompt+ prompt_ids = req_output.prompt_token_ids+ req_sample_output_ids: list[list[int]] = []+ req_sample_output_strs: list[str] = []+ for sample in req_output.outputs:+ output_str = sample.text+ output_ids = list(sample.token_ids)+ req_sample_output_ids.append(prompt_ids + output_ids)+ req_sample_output_strs.append((prompt_str or "") + output_str)+ outputs.append((req_sample_output_ids, req_sample_output_strs))+ return outputs++ @staticmethod+ def _final_steps_generate_w_logprobs(+ req_outputs: list[RequestOutput],+ ) -> list[TokensTextLogprobsPromptLogprobs]:+ outputs: list[TokensTextLogprobsPromptLogprobs] = []+ for req_output in req_outputs:+ assert len(req_output.outputs) > 0+ for sample in req_output.outputs:+ output_str = sample.text+ output_ids = list(sample.token_ids)+ output_logprobs = sample.logprobs+ outputs.append((output_ids, output_str, output_logprobs, req_output.prompt_logprobs))+ return outputs++ def generate_w_logprobs(+ self,+ prompts: list[str],+ sampling_params: SamplingParams,+ images: PromptImageInput | None = None,+ audios: PromptAudioInput | None = None,+ videos: PromptVideoInput | None = None,+ **kwargs: Any,+ ) -> list[TokensTextLogprobs] | list[TokensTextLogprobsPromptLogprobs]:+ inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)++ req_outputs = self.llm.generate(inputs, sampling_params=sampling_params, **kwargs)++ toks_str_logsprobs_prompt_logprobs = self._final_steps_generate_w_logprobs(req_outputs)+ # Omit prompt logprobs if not required by sampling params+ return (+ [x[0:-1] for x in toks_str_logsprobs_prompt_logprobs]+ if sampling_params.prompt_logprobs is None+ else toks_str_logsprobs_prompt_logprobs+ )++ def generate_greedy(+ self,+ prompts: list[str] | list[torch.Tensor] | list[list[int]],+ max_tokens: int,+ images: PromptImageInput | None = None,+ videos: PromptVideoInput | None = None,+ audios: PromptAudioInput | None = None,+ **kwargs: Any,+ ) -> list[tuple[list[int], str]]:+ greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)+ outputs = self.generate(+ prompts,+ greedy_params,+ images=images,+ videos=videos,+ audios=audios,+ **kwargs,+ )+ return [(output_ids[0], output_str[0]) for output_ids, output_str in outputs]++ def generate_greedy_logprobs(+ self,+ prompts: list[str],+ max_tokens: int,+ num_logprobs: int | None,+ num_prompt_logprobs: int | None = None,+ images: PromptImageInput | None = None,+ audios: PromptAudioInput | None = None,+ videos: PromptVideoInput | None = None,+ stop_token_ids: list[int] | None = None,+ stop: list[str] | None = None,+ **kwargs: Any,+ ) -> list[TokensTextLogprobs] | list[TokensTextLogprobsPromptLogprobs]:+ greedy_logprobs_params = SamplingParams(+ temperature=0.0,+ max_tokens=max_tokens,+ logprobs=num_logprobs,+ prompt_logprobs=num_prompt_logprobs,+ stop_token_ids=stop_token_ids,+ stop=stop,+ )++ return self.generate_w_logprobs(+ prompts,+ greedy_logprobs_params,+ images=images,+ audios=audios,+ videos=videos,+ **kwargs,+ )++ def generate_prompt_perplexity(self, prompts: list[str]) -> list[float]:+ """+ Return the perplexity score associated with generating the prompts++ :param prompts: list of prompts to score+ :return: perplexity score of each prompt+ """+ outputs = self.generate_greedy_logprobs(prompts, max_tokens=1, num_logprobs=None, num_prompt_logprobs=0)++ perplexities = []+ for output in outputs:+ output = cast(TokensTextLogprobsPromptLogprobs, output)+ token_data = cast(list[dict[int, Logprob] | None], output[3])+ assert token_data[0] is None+ token_log_probs = []+ for token_data in token_data[1:]:+ assert token_data is not None+ assert len(token_data) == 1+ token_log_prob = list(token_data.values())[0].logprob+ token_log_probs.append(token_log_prob)++ perplexity = math.exp(-sum(token_log_probs) / len(token_log_probs))+ perplexities.append(perplexity)++ return perplexities++ def generate_beam_search(+ self,+ prompts: list[str],+ beam_width: int,+ max_tokens: int,+ images: PromptImageInput | None = None,+ videos: PromptVideoInput | None = None,+ audios: PromptAudioInput | None = None,+ concurrency_limit: int | None = None,+ ) -> list[tuple[list[list[int]], list[str]]]:+ inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)++ outputs = self.llm.beam_search(+ inputs,+ BeamSearchParams(beam_width=beam_width, max_tokens=max_tokens),+ concurrency_limit=concurrency_limit,+ )+ returned_outputs = []+ for output in outputs:+ token_ids = [x.tokens for x in output.sequences]+ texts = [x.text for x in output.sequences]+ returned_outputs.append((token_ids, texts))+ return returned_outputs++ def classify(self, prompts: list[str]) -> list[list[float]]:+ req_outputs = self.llm.classify(prompts)+ return [req_output.outputs.probs for req_output in req_outputs]++ def embed(+ self,+ prompts: list[str],+ images: PromptImageInput | None = None,+ videos: PromptVideoInput | None = None,+ audios: PromptAudioInput | None = None,+ *args,+ **kwargs,+ ) -> list[list[float]]:+ inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)++ req_outputs = self.llm.embed(inputs, *args, **kwargs)+ return [req_output.outputs.embedding for req_output in req_outputs]++ def token_embed(self, prompts: list[str]) -> list[list[float]]:+ req_outputs = self.llm.encode(prompts, pooling_task="token_embed")+ return [req_output.outputs.data for req_output in req_outputs]++ def token_classify(self, prompts: list[str]) -> list[list[float]]:+ req_outputs = self.llm.encode(prompts, pooling_task="token_classify")+ return [req_output.outputs.data for req_output in req_outputs]++ def reward(self, prompts: list[str]) -> list[list[float]]:+ req_outputs = self.llm.reward(prompts)+ return [req_output.outputs.data for req_output in req_outputs]++ def score(+ self,+ text_1: list[str] | str,+ text_2: list[str] | str,+ *args,+ **kwargs,+ ) -> list[float]:+ req_outputs = self.llm.score(text_1, text_2, *args, **kwargs)+ return [req_output.outputs.score for req_output in req_outputs]++ def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:+ return self.llm.apply_model(func)++ def get_llm(self) -> LLM:+ return self.llm++ def __enter__(self):+ return self++ def __exit__(self, exc_type, exc_value, traceback):+ del self.llm+ cleanup_dist_env_and_memory()++[email protected](scope="session")+def vllm_runner():+ return VllmRunner+++def temporary_enable_log_propagate():+ import logging++ logger = logging.getLogger("vllm")+ logger.propagate = True+ yield+ logger.propagate = False+++def caplog_vllm(temporary_enable_log_propagate, caplog):+ # To capture vllm log, we should enable propagate=True temporarily+ # because caplog depends on logs propagated to the root logger.+ yield caplog+++def caplog_mp_fork():+ """+ This fixture enables capturing logs from a forked MP subprocess.+ It should be used in conjunction with caplog_vllm.++ By default, subprocess logs do not go through the parent process.+ We instead create a queue listener in the parent process which+ forwards logs to the logger's other handlers, and add a QueueHandler+ to the root logger. Forked subprocesses will inherit the root logger+ and pass their messages to the queue, which the listener will forward+ to the root logger, which can be captured by caplog.++ Note that this workaround only works for fork; with spawn, the subprocess+ reinitializes logging and does not automatically inherit the queue.+ We'd have to manually pass the queue to the subprocess at the spawn point.+ See caplog_mp_spawn below.+ """++ @contextlib.contextmanager+ def ctx():+ import logging.handlers+ import multiprocessing as mp++ logger_queue: mp.Queue[logging.LogRecord] = mp.Queue()+ logger = logging.getLogger()+ handlers = logger.handlers++ # The listener works on a background thread, not inherited by the child.+ queue_listener = logging.handlers.QueueListener(logger_queue, *handlers)+ queue_listener.start()++ # Add queue handler after creating the listener to avoid cycle+ logger.addHandler(logging.handlers.QueueHandler(logger_queue))+ yield+ queue_listener.stop()++ return ctx+++class LogHolder:+ def __init__(self):+ self.text = None+++def caplog_mp_spawn(tmp_path, monkeypatch):+ """+ This fixture enables capturing logs from a forked MP subprocess.+ It does not require caplog_vllm (but it only contains logs from the child).++ By default, subprocess logs do not go through the parent process.+ We instead add a FileHandler to the config so the spawned child process+ writes its logs to a temp file.+ In the parent, we read the file and return the contents.++ Note: this method could be extended to fork by either reconfiguring logging+ in the parent or using a SocketHandler:+ https://docs.python.org/3/howto/logging-cookbook.html#sending-and-receiving-logging-events-across-a-network # noqa: E501+ """++ @contextlib.contextmanager+ def ctx(level: int | str):+ from vllm.logger import DEFAULT_LOGGING_CONFIG++ config_path = tmp_path / "vllm_logging_config.json"+ log_path = tmp_path / "vllm.log"+ log_holder = LogHolder()++ config = deepcopy(DEFAULT_LOGGING_CONFIG)+ if envs.VLLM_LOGGING_CONFIG_PATH:+ path = pathlib.Path(envs.VLLM_LOGGING_CONFIG_PATH)+ assert path.exists()+ config = json.loads(path.read_text())++ config["loggers"]["vllm"]["handlers"] += ["vllm_file"]+ config["handlers"]["vllm_file"] = {+ "class": "logging.FileHandler",+ "formatter": "vllm",+ "level": level,+ "filename": log_path.as_posix(),+ }++ config_path.write_text(json.dumps(config))++ with monkeypatch.context() as monkeypatch_ctx:+ monkeypatch_ctx.setenv("VLLM_LOGGING_CONFIG_PATH", config_path.as_posix())+ monkeypatch_ctx.setenv("VLLM_CONFIGURE_LOGGING", "1")+ yield log_holder++ log_holder.text = log_path.read_text()++ return ctx++[email protected](scope="session")+def num_gpus_available():+ """Get number of GPUs without initializing the CUDA context+ in current process."""++ from vllm.platforms import current_platform++ return current_platform.device_count()+++temp_dir = tempfile.gettempdir()+_dummy_opt_path = os.path.join(temp_dir, "dummy_opt")+_dummy_llava_path = os.path.join(temp_dir, "dummy_llava")+_dummy_gemma2_embedding_path = os.path.join(temp_dir, "dummy_gemma2_embedding")+++def dummy_opt_path():+ json_path = os.path.join(_dummy_opt_path, "config.json")+ if not os.path.exists(_dummy_opt_path):+ snapshot_download(+ repo_id="facebook/opt-125m",+ local_dir=_dummy_opt_path,+ ignore_patterns=["*.bin", "*.bin.index.json", "*.pt", "*.h5", "*.msgpack"],+ )+ assert os.path.exists(json_path)+ with open(json_path) as f:+ config = json.load(f)+ config["architectures"] = ["MyOPTForCausalLM"]+ with open(json_path, "w") as f:+ json.dump(config, f)+ return _dummy_opt_path+++def dummy_llava_path():+ json_path = os.path.join(_dummy_llava_path, "config.json")+ if not os.path.exists(_dummy_llava_path):+ snapshot_download(+ repo_id="llava-hf/llava-1.5-7b-hf",+ local_dir=_dummy_llava_path,+ ignore_patterns=[+ "*.bin",+ "*.bin.index.json",+ "*.pt",+ "*.h5",+ "*.msgpack",+ "*.safetensors",+ ],+ )+ assert os.path.exists(json_path)+ with open(json_path) as f:+ config = json.load(f)+ config["architectures"] = ["MyLlava"]+ with open(json_path, "w") as f:+ json.dump(config, f)+ return _dummy_llava_path+++def dummy_gemma2_embedding_path():+ json_path = os.path.join(_dummy_gemma2_embedding_path, "config.json")+ if not os.path.exists(_dummy_gemma2_embedding_path):+ snapshot_download(+ repo_id="BAAI/bge-multilingual-gemma2",+ local_dir=_dummy_gemma2_embedding_path,+ ignore_patterns=[+ "*.bin",+ "*.bin.index.json",+ "*.pt",+ "*.h5",+ "*.msgpack",+ "*.safetensors",+ ],+ )+ assert os.path.exists(json_path)+ with open(json_path) as f:+ config = json.load(f)+ config["architectures"] = ["MyGemma2Embedding"]+ with open(json_path, "w") as f:+ json.dump(config, f)+ return _dummy_gemma2_embedding_path+++# Add the flag `--optional` to allow run tests+# that are marked with @pytest.mark.optional+def pytest_addoption(parser):+ parser.addoption("--optional", action="store_true", default=False, help="run optional test")+++def pytest_collection_modifyitems(config, items):+ if config.getoption("--optional"):+ # --optional given in cli: do not skip optional tests+ return+ skip_optional = pytest.mark.skip(reason="need --optional option to run")+ for item in items:+ if "optional" in item.keywords:+ item.add_marker(skip_optional)++[email protected](scope="session")+def cli_config_file():+ """Return the path to the CLI config file."""+ return os.path.join(_TEST_DIR, "config", "test_config.yaml")++[email protected](scope="session")+def cli_config_file_with_model():+ """Return the path to the CLI config file with model."""+ return os.path.join(_TEST_DIR, "config", "test_config_with_model.yaml")+++class AssetHandler(http.server.BaseHTTPRequestHandler):+ # _IMAGE_CACHE : Dict[str, bytes] = {}++ def log_message(self, *args, **kwargs):+ pass++ def do_GET(self):+ # Accepts paths like: /1280px-Venn_diagram_rgb.jpg+ filename = self.path.lstrip("/")+ if not filename or "." not in filename:+ self.send_error(404, "Missing filename (expected /<name>.<ext>)")+ return++ base, ext = filename.rsplit(".", 1)+ ext = ext.lower()++ if ext not in ["jpg", "png"]:+ self.send_error(404, f"Unsupported extension: .{ext}")+ return++ try:+ data = ImageAsset(base).read_bytes(ext=ext)+ except Exception as e:+ self.send_error(500, f"Failed to load asset: {ext} {base} {e} ")+ return++ ctype, _ = mimetypes.guess_type(filename)+ if ctype is None:+ ctype = {"jpg": "image/jpg", "png": "image/png"}[ext]+ self.send_response(200)+ self.send_header("Content-Type", ctype)+ self.send_header("Content-Length", str(len(data)))+ self.end_headers()+ self.wfile.write(data)+++def _find_free_port() -> int:+ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ return s.getsockname()[1]+++class LocalAssetServer:+ address: str+ port: int+ server: http.server.ThreadingHTTPServer | None+ thread: threading.Thread | None++ def __init__(self, address: str = "127.0.0.1") -> None:+ self.address = address+ self.port = -1+ self.server = None+ self.thread = None++ def __enter__(self):+ self.port = _find_free_port()+ self.server = http.server.ThreadingHTTPServer((self.address, self.port), AssetHandler)+ self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)+ self.thread.start()+ return self++ def __exit__(self, exc_type, exc_value, traceback):+ if self.server:+ self.server.shutdown()+ del self.server++ if self.thread:+ self.thread.join()+ del self.thread++ if exc_type is None:+ return None++ return False++ @property+ def base_url(self) -> str:+ assert self.port is not None+ return f"http://{self.address}:{self.port}"++ def url_for(self, name: str) -> str:+ """e.g., name='RGBA_comp.png' -> 'http://127.0.0.1:PORT/RGBA_comp.png'"""+ return f"{self.base_url}/{name}"++ def get_image_asset(self, name: str) -> Image.Image:+ return fetch_image(self.url_for(name))++[email protected](scope="session")+def local_asset_server() -> Generator[LocalAssetServer, None, None]:+ """+ Starts a thread based HTTP server bound to 127.0.0.1 on a random free port.+ The server currently servers images at:+ http://127.0.0.1:<port>/<name>.<ext>+ """+ with LocalAssetServer() as srv:+ yield srv+++def image_url(request, local_asset_server) -> str:+ # request.param is one of the IMAGE_ASSETS filenames+ name = request.param+ return local_asset_server.url_for(name)+++def image_urls(request, local_asset_server) -> list[str]:+ """Indirect fixture: takes a list of names, returns list of full URLs."""+ names: list[str] = request.param+ return [local_asset_server.url_for(name) for name in names]diff --git a/auto_round_extension/vllm_ext/tests/test_mxfp4_moe.py b/auto_round_extension/vllm_ext/tests/test_mxfp4_moe.pynew file mode 100644index 000000000..a2de32832--- /dev/null+++ b/auto_round_extension/vllm_ext/tests/test_mxfp4_moe.py@@ -0,0 +1,34 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import pytest+from vllm.platforms import current_platform++MODELS = [+ # "/data5/yliu7/HF_HOME/unsloth-gpt-oss-20b-BF16-ar-MXFP4/"+ # "/data5/yliu7/HF_HOME/Qwen2.5-0.5B-Instruct-test-FP8_STATIC-fp8kv/"+ "/data6/yiliu4/Qwen3-15B-A2B-Base-MXFP4"+]+++ not current_platform.is_cuda(),+ reason="only supports CUDA backend.",+)[email protected]("model", MODELS)+def test_auto_round(vllm_runner, model):+ with vllm_runner(model, enforce_eager=True) as llm:+ output = llm.generate_greedy(["The capital of France is"], max_tokens=8)+ assert output+ print(f"output is: {output[0][1]}")diff --git a/auto_round_extension/vllm_ext/utils.py b/auto_round_extension/vllm_ext/utils.pynew file mode 100644index 000000000..f98eb2325--- /dev/null+++ b/auto_round_extension/vllm_ext/utils.py@@ -0,0 +1,81 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import torch++E8M0_EXPONENT_BIAS = 127+E8M0_EXPONENT_NAN_VAL = 255+++def get_fp_scale(scale_e8m0):+ scale_e8m0 = scale_e8m0.view(torch.uint8)+ s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS+ # TODO(later): it would be nice if there was a way to do the 2^x operation+ # in PyTorch without creating a tensor of twos+ two = torch.full(s_offset.size(), 2.0, device=scale_e8m0.device)+ # pow(two, s_offset) can be out of range of floating point formats.+ # TODO(later): handle this for float16 if we decide to support float16+ # scales.+ s_fp = torch.pow(two, s_offset)++ # If a block exponent was 255, set values of that block to NaN+ s_fp = torch.where(scale_e8m0 != E8M0_EXPONENT_NAN_VAL, s_fp, float("nan"))++ return s_fp+++def _to_mx_rceil(+ data_hp: torch.Tensor,+ max_abs: torch.Tensor,+ max_pos: float,+) -> tuple[torch.Tensor, torch.Tensor]:+ """+ A prototype implementation of MXFP scale factor derivation method described in+ https://docs.nvidia.com/cuda/cublas/#d-block-quantization++ For Nvidia GPU with Blackwell+ architecture, the scale factor derivation method+ could be accelerated by the `cvt.rp.satfinite.ue8m0x2.f32` instruction.++ Args:+ data_hp: High precision data.+ max_abs: Maximum absolute value for data_hp along specified dimension/block_size.+ max_pos: The maximum value of the low precision data type.++ Returns:+ exponent: The biased exponent with dtype E8M0 in uint8 container.+ data_lp: The targeted low precision data, in high precision container+ (requires cast to low precision data type).+ """+ descale = max_abs / max_pos+ # TODO: nan/inf needs to be set for any value+ # of nan/inf in input not just amax.+ exponent = torch.where(+ torch.isnan(descale),+ 0xFF, # Handle biased exponent for nan+ # NOTE: descale < (torch.finfo(torch.float32).smallest_normal / 2) is handled through clamping+ (+ torch.clamp(+ torch.ceil(torch.log2(descale)),+ min=-E8M0_EXPONENT_BIAS,+ max=E8M0_EXPONENT_BIAS,+ )+ + E8M0_EXPONENT_BIAS+ ).to(torch.uint8),+ )++ descale_fp = torch.where(exponent == 0, 1.0, torch.exp2(E8M0_EXPONENT_BIAS - exponent.to(torch.float32)))++ # scale and saturated cast the data elements to max of target dtype+ data_lp = torch.clamp(data_hp * descale_fp.unsqueeze(1), min=-1 * max_pos, max=max_pos)+ return exponent, data_lp