PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Remove deprecated files (#2373)
chen, suyuefd077878e6ff
diff --git a/auto_round/experimental/qmodules/fp4_utils.py b/auto_round/experimental/qmodules/fp4_utils.pyindex aa9cae421..f2dafa1c8 100644--- a/auto_round/experimental/qmodules/fp4_utils.py+++ b/auto_round/experimental/qmodules/fp4_utils.py@@ -45,6 +45,19 @@ def get_e2m1_tensor(device):return _DEVICE_E2M1_TENSORS[device_str]+def pack_fp4_to_uint8(x: torch.Tensor) -> torch.Tensor:+ """Pack pairs of E2M1 values into uint8, with the first value in the low nibble."""+ rows, columns = x.shape+ assert columns % 2 == 0, f"Expected an even number of columns, got {columns}"++ values = get_e2m1_tensor(x.device)+ magnitude = torch.argmin(torch.abs(torch.abs(x).unsqueeze(-1) - values), dim=-1)+ indices = magnitude + (torch.signbit(x).to(torch.long) << 3)+ pairs = indices.reshape(-1, 2)+ packed = (pairs[:, 0] | (pairs[:, 1] << 4)).to(torch.uint8)+ return packed.reshape(rows, columns // 2)++def unpack_fp4_from_uint8(a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16) -> torch.Tensor:diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.pyindex 5a96f9efa..0c321c34e 100644--- a/auto_round/utils/model_free_utils.py+++ b/auto_round/utils/model_free_utils.py@@ -1654,6 +1654,57 @@ def _modelopt_nvfp4_bases_from_weight_map(weight_map: dict[str, str]) -> set[str}+def get_fp_scale(scale_e8m0: torch.Tensor) -> torch.Tensor:+ """Convert biased E8M0 exponents to floating-point scales."""+ exponent = scale_e8m0.view(torch.uint8).to(torch.int16) - 127+ return torch.pow(2.0, exponent.to(torch.float32))+++def dequant_mxfp4(+ data_lp: torch.Tensor,+ scale_e8m0: torch.Tensor,+ elem_dtype: str,+ block_size: int,+ target_dtype: torch.dtype,+ scale_dtype: torch.dtype | None = None,+ return_scale: bool = False,+) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:+ """Dequantize packed MXFP4 data to ``target_dtype``."""+ from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8++ original_shape = data_lp.shape+ last_dim = original_shape[-1]+ data_lp = data_lp.reshape(-1, last_dim)+ result_shape = original_shape[:-1] + (last_dim * 2,)+ assert data_lp.is_contiguous(), f"Data must be contiguous, got {data_lp.stride()}"+ assert elem_dtype == "fp4_e2m1", f"Expected 'fp4_e2m1', got {elem_dtype}"++ rows, half_columns = data_lp.shape+ data_hp = unpack_fp4_from_uint8(data_lp, rows, half_columns * 2, dtype=target_dtype)+ data_hp = data_hp.reshape(-1, block_size)++ if scale_dtype is None:+ scale_dtype = target_dtype+ scale = get_fp_scale(scale_e8m0).reshape(-1, 1).to(scale_dtype)+ if return_scale:+ return data_hp.reshape(result_shape), scale++ return (data_hp * scale).reshape(result_shape)+++def dequant_mx_fp8(+ weight_fp8: torch.Tensor,+ scale_e8m0: torch.Tensor,+ block_size: int,+ target_dtype: torch.dtype,+) -> torch.Tensor:+ """Dequantize MXFP8 data using its per-block E8M0 scales."""+ original_shape = weight_fp8.shape+ weight = weight_fp8.to(torch.bfloat16).reshape(-1, block_size)+ scale = get_fp_scale(scale_e8m0).reshape(-1, 1)+ return (weight * scale).reshape(original_shape).to(target_dtype)++def _hydrate_and_clean_modelopt_nvfp4_aux(raw_tensors: dict[str, torch.Tensor],shard_path: str,@@ -1773,21 +1824,17 @@ def _dequant_mxfp_tensors() -> dict[str, torch.Tensor]:"""Dequantize llm-compressor MXFP8 / MXFP4 weight tensors to bfloat16.- Detection is purely by *name* and *dtype*, reusing the dequant kernels in- :mod:`auto_round_extension.vllm_ext`:+ Detection is purely by *name* and *dtype*:* ``<layer>.weight`` (``float8_e4m3fn``) + ``<layer>.weight_scale`` → MXFP8,- dequantized via :func:`~auto_round_extension.vllm_ext.mxfp8_qdq_utils.dequant_mx_fp8`.+ dequantized via :func:`dequant_mx_fp8`.* ``<layer>.weight_packed`` (``uint8``) + ``<layer>.weight_scale`` → MXFP4,- dequantized via :func:`~auto_round_extension.vllm_ext.mxfp4_qdq_utils.to_dtype`.+ dequantized via :func:`dequant_mxfp4`.The dequantized weight is written back under ``<layer>.weight`` and thescale (and any ``weight_packed``) tensor is removed, so the downstream RTNpath can requantize the layer to the requested target scheme."""- from auto_round_extension.vllm_ext.mxfp4_qdq_utils import to_dtype- from auto_round_extension.vllm_ext.mxfp8_qdq_utils import dequant_mx_fp8-# Tuple layout: (layer_name, weight_key, scale_key, bits)entries = _collect_mxfp_source_entries(raw_tensors)@@ -1831,14 +1878,14 @@ def _dequant_mxfp_tensors(shard_prefix=shard_prefix,op_name="MXFP dequant",tensor_label=layer_name,- on_device=lambda weight=weight, scale=scale: to_dtype(+ on_device=lambda weight=weight, scale=scale: dequant_mxfp4(data_lp=weight.view(torch.uint8).contiguous().to(dequant_device, non_blocking=True),scale_e8m0=scale.to(dequant_device, non_blocking=True),elem_dtype="fp4_e2m1",block_size=32,target_dtype=torch.bfloat16,).to("cpu"),- on_cpu=lambda weight=weight, scale=scale: to_dtype(+ on_cpu=lambda weight=weight, scale=scale: dequant_mxfp4(data_lp=weight.view(torch.uint8).contiguous(),scale_e8m0=scale,elem_dtype="fp4_e2m1",diff --git a/auto_round/utils/weight_handler.py b/auto_round/utils/weight_handler.pyindex 5e2248218..abfca6ba4 100755--- a/auto_round/utils/weight_handler.py+++ b/auto_round/utils/weight_handler.py@@ -701,8 +701,7 @@ class MXFP4Handler(WeightTypeHandler):layer = layer.to(device)- # MXFP4 dequantization using to_dtype from mxfp4_qdq_utils- from auto_round_extension.vllm_ext.mxfp4_qdq_utils import to_dtype+ from auto_round.utils.model_free_utils import dequant_mxfp4# Get packed weight and scale from layer# MXFP4 weights are stored as packed uint8 in weight_packed@@ -716,7 +715,7 @@ class MXFP4Handler(WeightTypeHandler):raise ValueError("MXFP4 layer must have weight_packed and weight_scale attributes")# Dequantize using to_dtype function- dq_weight = to_dtype(+ dq_weight = dequant_mxfp4(data_lp=weight_packed,scale_e8m0=weight_scale,elem_dtype="fp4_e2m1",@@ -790,8 +789,7 @@ class MXFP8Handler(WeightTypeHandler):layer = layer.to(device)- # MXFP8 dequantization using dequant_mx_fp8 from mxfp8_qdq_utils- from auto_round_extension.vllm_ext.mxfp8_qdq_utils import dequant_mx_fp8+ from auto_round.utils.model_free_utils import dequant_mx_fp8# Get weight and scale from layerweight = layer.weightdiff --git a/auto_round_extension/ark/auto_round_kernel/mxfp4_hadamard.py b/auto_round_extension/ark/auto_round_kernel/mxfp4_hadamard.pyindex e6ab5d2d9..6cc57f4d5 100644--- a/auto_round_extension/ark/auto_round_kernel/mxfp4_hadamard.py+++ b/auto_round_extension/ark/auto_round_kernel/mxfp4_hadamard.py@@ -29,12 +29,12 @@ Frozen MVP contract (Phase 0):is ``2 ** (e8m0 - 127)`` (standard E8M0, always a power of two);* ``q = y * 2 ** -(e8m0 - 127)`` is encoded as ``signbit(q) << 3 | magnitude``with FP4 (E2M1) magnitude levels ``0, 0.5, 1, 1.5, 2, 3, 4, 6`` and the- nearest-even thresholds of ``vllm_ext/fp4_utils.py::cast_to_fp4``;+ nearest-even thresholds of the frozen ``_E2M1_THRESHOLDS`` table below;* **zero is canonicalised**: whenever the magnitude index is 0 the sign bit isdropped, so the code is ``0x0`` and never ``0x8`` (negative zero). See"Canonical zero" below for why this rule is required rather than optional;* two codes share one byte, the even element occupying the low nibble- (identical to ``vllm_ext/fp4_utils.py::pack_fp4_to_uint8``);+ (identical to ``auto_round.experimental.qmodules.fp4_utils::pack_fp4_to_uint8``);* an all-zero group produces ``e8m0 = 0`` and all-zero codes;* NaN/Inf are outside the supported input domain. The reference always rejectsthem; the XPU entry point only does so under ``check_finite=True``, because@@ -92,7 +92,7 @@ the mathematical value rather than of the rounding residue. Both this referenceand the kernel therefore drop the sign bit whenever the magnitude index is 0.This is a deliberate, documented deviation from-``vllm_ext/fp4_utils.py::pack_fp4_to_uint8``, which applies ``signbit``+``auto_round.experimental.qmodules.fp4_utils::pack_fp4_to_uint8``, which applies ``signbit``unconditionally: that helper encodes already-clean dequantized values, where anegative zero can only appear if the caller supplied one."""@@ -122,7 +122,7 @@ SUPPORTED_HADAMARD_DIMS = tuple(GROUP_SIZE * (1 << i) for i in range(MAX_LANES_PE2M1_VALUES = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)# (threshold, is_closed_interval, magnitude_index), first match wins, mirroring-# the early-out ladder of ``cast_to_fp4`` including its alternating ``<=`` / ``<``+# the early-out ladder of ``_E2M1_THRESHOLDS'`` including its alternating ``<=`` / ``<``# boundary operators. ``_encode_fp4`` walks the table in reverse with# ``torch.where``, so each earlier entry overrides every later one._E2M1_THRESHOLDS = (diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard.hppindex 00b34c6f8..cabe45157 100644--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard.hpp+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard.hpp@@ -117,7 +117,7 @@ class XpuMxfp4Hadamard {// FP4 (E2M1) magnitude levels: 0, 0.5, 1, 1.5, 2, 3, 4, 6.// Thresholds and boundary comparison operators are taken verbatim from the- // PyTorch reference (auto_round_extension/vllm_ext/fp4_utils.py::cast_to_fp4)+ // PyTorch reference (auto_round_extension/ark/auto_round_kernel/mxfp4_hadamard.py::_E2M1_THRESHOLDS)// so that the kernel is bit-exact with it.//// Written branchless, as a descending count rather than the reference'sdiff --git a/auto_round_extension/ark/test/test_mxfp4_hadamard.py b/auto_round_extension/ark/test/test_mxfp4_hadamard.pyindex 218fadcb7..9c19f35ee 100644--- a/auto_round_extension/ark/test/test_mxfp4_hadamard.py+++ b/auto_round_extension/ark/test/test_mxfp4_hadamard.py@@ -105,8 +105,8 @@ class TestReferenceContract:supported D by :class:`TestHadamardDimReferenceParity`."""- def test_packing_matches_vllm_ext_fp4_utils(self):- fp4_utils = pytest.importorskip("auto_round_extension.vllm_ext.fp4_utils")+ def test_packing_matches_fp4_utils(self):+ fp4_utils = pytest.importorskip("auto_round.experimental.qmodules.fp4_utils")torch.manual_seed(0)codes = torch.randint(0, 16, (4, 64), dtype=torch.uint8)# Build the FP4 values the codes represent, then pack them with thediff --git a/auto_round_extension/vllm_ext/README.md b/auto_round_extension/vllm_ext/README.mddeleted file mode 100644index 1f0cb832f..000000000--- a/auto_round_extension/vllm_ext/README.md+++ /dev/null@@ -1,12 +0,0 @@-- Build and Install vLLM--```-git clone --branch fused-moe-ar https://github.com/yiliu30/vllm-fork.git-VLLM_USE_PRECOMPILED=1 pip install --editable . -vvv-```---- Enable vLLM-Ext at Runtime-```bash-VLLM_ENABLE_AR_EXT=1 vllm serve ...-```\ No newline at end of filediff --git a/auto_round_extension/vllm_ext/__init__.py b/auto_round_extension/vllm_ext/__init__.pydeleted file mode 100644index d05b3c731..000000000--- a/auto_round_extension/vllm_ext/__init__.py+++ /dev/null@@ -1,27 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--# ==---------------------------------------------------------------------------==-# Apply the extension-# ==---------------------------------------------------------------------------==---def apply():- import auto_round_extension.vllm_ext.auto_round_ext- import auto_round_extension.vllm_ext.envs_ext- import auto_round_extension.vllm_ext.vllm_oot_patches-- print("*****************************************************************************")- print("* !!! VLLM_ENABLE_AR_EXT is set to 1, applying auto_round_vllm_extension *")- print("*****************************************************************************")diff --git a/auto_round_extension/vllm_ext/auto_round_ext.py b/auto_round_extension/vllm_ext/auto_round_ext.pydeleted file mode 100644index 408d0d6a6..000000000--- a/auto_round_extension/vllm_ext/auto_round_ext.py+++ /dev/null@@ -1,133 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--from copy import deepcopy-from typing import Any--import torch-from vllm.logger import init_logger-from vllm.model_executor.layers.fused_moe import FusedMoE-from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod-from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig as _BaseAutoRoundConfig--from auto_round.schemes import QuantizationScheme-from auto_round_extension.vllm_ext.quant_method_linear import AutoRoundQuantLinearMethod-from auto_round_extension.vllm_ext.quant_method_moe import AutoRoundMoEMethod--logger = init_logger(__name__)---class AutoRoundExtensionConfig(_BaseAutoRoundConfig):- SUPPORTED_DTYPES = _BaseAutoRoundConfig.SUPPORTED_DTYPES.union({"mx_fp"})- SUPPORTED_FORMATS = _BaseAutoRoundConfig.SUPPORTED_FORMATS.union({"auto_round:llm_compressor"})-- def __init__(- self,- weight_bits: int,- group_size: int,- sym: bool = True,- packing_format: str = "auto_round:auto_gptq",- block_name_to_quantize: str | list[str] | None = None,- extra_config: dict[str, Any] | None = None,- data_type: str = "int",- backend: str = "auto",- act_bits: int = None,- act_data_type: int = None,- act_dynamic: bool = None,- act_group_size: int = None,- act_sym: bool = None,- ) -> None:- super().__init__(- weight_bits=weight_bits,- group_size=group_size,- sym=sym,- packing_format=packing_format,- block_name_to_quantize=block_name_to_quantize,- extra_config=extra_config,- data_type=data_type,- backend=backend,- )- self.act_bits = act_bits- self.act_data_type = act_data_type- self.act_dynamic = act_dynamic- self.act_group_size = act_group_size- self.act_sym = act_sym-- def get_quant_method(self, layer: torch.nn.Module, prefix: str):- # FIXME: (yi) make it compatible with `AutoRoundConfig`- from vllm.attention.layer import Attention, MLAAttention-- if isinstance(layer, (Attention, MLAAttention)):-- from auto_round_extension.vllm_ext.kv_cache import AutoRoundKVCacheMethod, AutoRoundKVCacheMethodForMLA-- if isinstance(layer, MLAAttention):- return AutoRoundKVCacheMethodForMLA(self)-- return AutoRoundKVCacheMethod(self)- if isinstance(layer, FusedMoE):- quant_method = AutoRoundMoEMethod.get_moe_method(self, layer, prefix)- return quant_method- elif isinstance(layer, LinearBase):- return AutoRoundQuantLinearMethod.get_method(self, layer, prefix)- else:- return None-- @staticmethod- def _parse_quant_scheme(config: dict):- quant_scheme_attrs = QuantizationScheme.get_attributes()- filter_config = {key: value for key, value in config.items() if key in quant_scheme_attrs}- quant_scheme = QuantizationScheme.from_dict(filter_config)- return quant_scheme-- @classmethod- def from_config(cls, config: dict[str, Any]) -> _BaseAutoRoundConfig:- ar_config = cls(- weight_bits=cls.get_from_keys(config, ["bits"]),- group_size=cls.get_from_keys(config, ["group_size"]),- sym=cls.get_from_keys(config, ["sym"]),- packing_format=cls.get_from_keys_or(config, ["packing_format"], "auto_round:auto_gptq"),- block_name_to_quantize=cls.get_from_keys_or(- config, ["block_name_to_quantize", "to_quant_block_names"], None- ),- extra_config=cls.get_from_keys_or(config, ["extra_config"], None),- data_type=cls.get_from_keys_or(config, ["data_type"], "int"),- backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),- act_bits=cls.get_from_keys(config, ["act_bits"]),- act_data_type=cls.get_from_keys(config, ["act_data_type"]),- act_dynamic=cls.get_from_keys(config, ["act_dynamic"]),- act_group_size=cls.get_from_keys(config, ["act_group_size"]),- act_sym=cls.get_from_keys(config, ["act_sym"]),- )- # TODO: (yi) refine below implementation- quant_scheme = AutoRoundExtensionConfig._parse_quant_scheme(config)- layer_schemes = {}- layer_schemes = {} # ensure dict- extra_config = getattr(ar_config, "extra_config", None)- if extra_config is not None:- for layer_name, layer_config in extra_config.items():- layer_scheme = deepcopy(quant_scheme)- for key, val in layer_config.items():- if key in layer_scheme.keys() and val != layer_scheme[key]:- layer_scheme[key] = val- layer_schemes[layer_name] = layer_scheme- ar_config.quant_scheme = quant_scheme- ar_config.layer_schemes = layer_schemes- return ar_config---# Patch vLLM’s AutoRoundConfig at import time-import vllm.model_executor.layers.quantization.auto_round as _auto_round_module--_auto_round_module.AutoRoundConfig = AutoRoundExtensionConfigdiff --git a/auto_round_extension/vllm_ext/envs_ext.py b/auto_round_extension/vllm_ext/envs_ext.pydeleted file mode 100644index eb251ccef..000000000--- a/auto_round_extension/vllm_ext/envs_ext.py+++ /dev/null@@ -1,42 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--import os-from typing import Any, Callable--from vllm.logger import init_logger--logger = init_logger(__name__)--# Define extra environment variables-extra_environment_variables: dict[str, Callable[[], Any]] = {- "VLLM_MXFP4_PRE_UNPACK_WEIGHTS": lambda: os.getenv("VLLM_MXFP4_PRE_UNPACK_WEIGHTS", "0") in ("1", "true", "True"),- "VLLM_MXFP4_PRE_UNPACK_TO_FP8": lambda: os.getenv("VLLM_MXFP4_PRE_UNPACK_TO_FP8", "1") in ("1", "true", "True"),- "VLLM_ENABLE_STATIC_MOE": lambda: os.getenv("VLLM_ENABLE_STATIC_MOE", "0") in ("1", "true", "True"),- "VLLM_AR_MXFP4_MODULAR_MOE": lambda: os.getenv("VLLM_AR_MXFP4_MODULAR_MOE", "1") in ("1", "true", "True"),- "VLLM_AR_POST_PROCESS_GPTOSS": lambda: os.getenv("VLLM_AR_POST_PROCESS_GPTOSS", "0") in ("1", "true", "True"),- "VLLM_AR_FLASHINFER_WORKSPACE_BUFFER_SIZE": lambda: int(os.getenv("VLLM_AR_FLASHINFER_WORKSPACE_BUFFER_SIZE", "0")),-}-# Add the extra environment variables to vllm.envs-import vllm.envs as envs-from vllm.envs import environment_variables--# Merge the environment variables-all_environment_variables = {**environment_variables, **extra_environment_variables}---for name, value_fn in extra_environment_variables.items():- setattr(envs, name, value_fn())--logger.warning_once(f"Added extra environment variables: {list(extra_environment_variables.keys())}")diff --git a/auto_round_extension/vllm_ext/fp4_utils.py b/auto_round_extension/vllm_ext/fp4_utils.pydeleted file mode 100644index fb7b6e935..000000000--- a/auto_round_extension/vllm_ext/fp4_utils.py+++ /dev/null@@ -1,113 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--from typing import Optional--import torch--# Module-level device tensor cache to fix cuda graph issue-_DEVICE_E2M1_TENSORS = {}--# Constants for FP4 values (E2M1 format)-_E2M1_VALUES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]---def get_e2m1_tensor(device):- """Get device-specific E2M1 lookup tensor, creating it if needed."""- device_str = str(device)- if device_str not in _DEVICE_E2M1_TENSORS:- _DEVICE_E2M1_TENSORS[device_str] = torch.tensor(_E2M1_VALUES, dtype=torch.float32, device=device)- return _DEVICE_E2M1_TENSORS[device_str]---def pack_fp4_to_uint8(x: torch.Tensor) -> torch.Tensor:- m, n = x.shape- device = x.device-- # Create lookup table for FP4 values to indices- # Map the absolute values to 0-7 indices- kE2M1 = get_e2m1_tensor(x.device)-- # Find closest valid FP4 value index for each element- abs_x = torch.abs(x)- abs_diff_x = torch.abs(abs_x.unsqueeze(-1) - kE2M1) # [m, n, 8]- abs_indices = torch.argmin(abs_diff_x, dim=-1) # [m, n]-- # Apply sign bit (bit 3) to get final 4-bit representation- indices = abs_indices + (torch.signbit(x).to(torch.long) << 3)-- # Reshape to prepare for packing pairs of values- indices = indices.reshape(-1)-- # Handle odd length by padding if necessary- # assert indices.numel() % 2 != 0, f"Expected even number of elements, got {indices.numel()}"-- # Reshape to pair consecutive elements- indices = indices.reshape(-1, 2)-- # Pack pairs of 4-bit values into 8-bit values- packed = (indices[:, 0] | (indices[:, 1] << 4)).to(torch.uint8)-- return packed.reshape(m, n // 2)---def unpack_fp4_from_uint8(- a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16-) -> torch.Tensor:- """- Unpacks uint8 values into fp4. Each uint8 consists of two fp4 values- (i.e. first four bits correspond to one fp4 value, last four correspond to a- consecutive fp4 value). The bits represent an index, which are mapped to an fp4- value.-- :param a: tensor to unpack- :param m: original dim 0 size of the unpacked tensor- :param n: original dim 1 size of the unpacked tensor- :param dtype: dense dtype to cast the unpacked tensor to- """- assert a.dtype == torch.uint8, f"expected uint8, got {a.dtype}"-- # Vectorized nibble processing- a_flat = a.flatten()- high = (a_flat & 0xF0) >> 4 # Upper nibbles- low = a_flat & 0x0F # Lower nibbles-- # Combine nibbles for batch processing- combined = torch.stack((low, high), dim=1).flatten()-- # Vectorized sign and magnitude extraction- signs = (combined & 0x08).to(torch.bool) # Sign bits- abs_vals = (combined & 0x07).to(torch.long) # Magnitude indices-- # Device-aware lookup and sign application- kE2M1 = get_e2m1_tensor(a.device)-- values = kE2M1[abs_vals] * torch.where(signs, -1.0, 1.0)-- # Reshape to final form- return values.reshape(m, n).to(dtype=dtype)---def cast_to_fp4(x):- sign = torch.sign(x)- x = torch.abs(x)- x[(x >= 0.0) & (x <= 0.25)] = 0.0- x[(x > 0.25) & (x < 0.75)] = 0.5- x[(x >= 0.75) & (x <= 1.25)] = 1.0- x[(x > 1.25) & (x < 1.75)] = 1.5- x[(x >= 1.75) & (x <= 2.5)] = 2.0- x[(x > 2.5) & (x < 3.5)] = 3.0- x[(x >= 3.5) & (x <= 5.0)] = 4.0- x[x > 5.0] = 6.0- return x * signdiff --git a/auto_round_extension/vllm_ext/kv_cache.py b/auto_round_extension/vllm_ext/kv_cache.pydeleted file mode 100644index 9b8572b06..000000000--- a/auto_round_extension/vllm_ext/kv_cache.py+++ /dev/null@@ -1,51 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.---from typing import TYPE_CHECKING, Any, Literal, Optional, cast--import torch-from vllm.logger import init_logger-from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod--logger = init_logger(__name__)---class AutoRoundKVCacheMethod(BaseKVCacheMethod):- """- Supports loading kv-cache scaling factors from compressed-tensors- checkpoints.- """-- def __init__(self, quant_config):- self.validate_kv_cache_scheme(quant_config)- super().__init__(quant_config)-- @staticmethod- def validate_kv_cache_scheme(quant_config):- # FIXME: parse from quant_config- return True---class AutoRoundKVCacheMethodForMLA(BaseKVCacheMethod):- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:- super().process_weights_after_loading(layer)- k_scale = layer._k_scale- v_scale = layer._v_scale- aligned_kv_scale = max(k_scale, v_scale)- layer._k_scale.data.fill_(aligned_kv_scale)- layer._v_scale.data.fill_(aligned_kv_scale)- layer._k_scale_float = aligned_kv_scale.item()- layer._v_scale_float = aligned_kv_scale.item()- logger.warning_once("Aligned MLA k/v scales for MLA")diff --git a/auto_round_extension/vllm_ext/linear_impl_mxfp4.py b/auto_round_extension/vllm_ext/linear_impl_mxfp4.pydeleted file mode 100644index e544fdbbf..000000000--- a/auto_round_extension/vllm_ext/linear_impl_mxfp4.py+++ /dev/null@@ -1,124 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--# SPDX-License-Identifier: Apache-2.0-from typing import Callable, Optional--import torch-import vllm.envs as envs-from torch.nn.parameter import Parameter-from vllm.logger import init_logger-from vllm.model_executor.parameter import GroupQuantScaleParameter, ModelWeightParameter, PerTensorScaleParameter-from vllm.platforms import current_platform--from auto_round_extension.vllm_ext.mxfp4_qdq_utils import (- dequant_mxfp4_to_fp8,- mxfp4_gemm_with_unpacked_weight,- run_mxfp4_emulations,-)--logger = init_logger(__name__)--__all__ = ["AutoRoundMXFP4LinearImpl"]--from auto_round_extension.vllm_ext.quant_impl import AutoRoundQuantImpl---class AutoRoundMXFP4LinearImpl(AutoRoundQuantImpl):- def __init__(self, quant_scheme):- self.quant_scheme = quant_scheme- self.group_size = 32-- @classmethod- def get_min_capability(cls) -> int:- if envs.VLLM_USE_MXFP4_CT_EMULATIONS:- return 80- return 100-- def create_weights(- self,- layer: torch.nn.Module,- output_partition_sizes: list[int],- input_size_per_partition: int,- params_dtype: torch.dtype,- weight_loader: Callable,- **kwargs,- ):- output_size_per_partition = sum(output_partition_sizes)- layer.logical_widths = output_partition_sizes- layer.input_size_per_partition = input_size_per_partition- layer.output_size_per_partition = output_size_per_partition-- # Weight- weight = ModelWeightParameter(- data=torch.empty(sum(output_partition_sizes), input_size_per_partition // 2, dtype=torch.uint8),- input_dim=1,- output_dim=0,- weight_loader=weight_loader,- )- layer.register_parameter("weight_packed", weight)-- # Per Group Weight Scale- weight_scale = GroupQuantScaleParameter(- data=torch.empty(- sum(output_partition_sizes),- input_size_per_partition // self.group_size,- # dtype=torch.uint8,- dtype=torch.uint8,- ),- input_dim=1,- output_dim=0,- weight_loader=weight_loader,- )-- layer.register_parameter("weight_scale", weight_scale)-- def process_weights_after_loading(self, layer) -> None:- # FIXME: may dequant to bf16- if envs.VLLM_MXFP4_PRE_UNPACK_TO_FP8:- weight_fp8, scale_bf16 = dequant_mxfp4_to_fp8(- data_lp=layer.weight_packed,- scale_e8m0=layer.weight_scale,- )- del layer.weight_packed- del layer.weight_scale- layer.weight_packed = None- layer.weight_scale = None- layer.register_parameter(- "weight_unpacked_fp8",- torch.nn.Parameter(- weight_fp8,- requires_grad=False,- ),- )- layer.register_parameter(- "weight_scale_bf16",- torch.nn.Parameter(- scale_bf16,- requires_grad=False,- ),- )- else:- raise NotImplementedError("Only VLLM_MXFP4_PRE_UNPACK_TO_FP8 is supported now.")-- def apply_weights(- self, layer: torch.nn.Module, x: torch.Tensor, bias: Optional[torch.Tensor] = None- ) -> torch.Tensor:- out = mxfp4_gemm_with_unpacked_weight(- x=x,- weight_fp8=layer.weight_unpacked_fp8,- weight_scale_bf16=layer.weight_scale_bf16,- bias=bias,- )- return outdiff --git a/auto_round_extension/vllm_ext/linear_impl_mxfp8.py b/auto_round_extension/vllm_ext/linear_impl_mxfp8.pydeleted file mode 100644index cf1382054..000000000--- a/auto_round_extension/vllm_ext/linear_impl_mxfp8.py+++ /dev/null@@ -1,112 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--from typing import Callable, Optional--import torch-import vllm.envs as envs-from vllm.model_executor.parameter import (- GroupQuantScaleParameter,- ModelWeightParameter,- PerTensorScaleParameter,-)--from auto_round_extension.vllm_ext.mxfp8_qdq_utils import dequant_mx_fp8, quant_mx_fp8-from auto_round_extension.vllm_ext.quant_impl import AutoRoundQuantImpl---class AutoRoundMXFP8LinearImpl(AutoRoundQuantImpl):- def __init__(self, quant_scheme):- self.quant_scheme = quant_scheme- self.strategy = "TENSOR_GROUP"- self.out_dtype = torch.get_default_dtype()- self.group_size = 32-- @classmethod- def get_min_capability(cls) -> int:- return 80-- def process_weights_after_loading(self, layer) -> None:- return-- def create_weights(- self,- layer: torch.nn.Module,- output_partition_sizes: list[int],- input_size_per_partition: int,- params_dtype: torch.dtype,- weight_loader: Callable,- **kwargs,- ):- # maybe_create_device_identity()-- output_size_per_partition = sum(output_partition_sizes)- layer.logical_widths = output_partition_sizes-- # WEIGHT- weight = ModelWeightParameter(- data=torch.empty(- output_size_per_partition,- input_size_per_partition,- dtype=torch.float8_e4m3fn,- ),- input_dim=1,- output_dim=0,- weight_loader=weight_loader,- )- layer.register_parameter("weight", weight)-- # WEIGHT SCALE- # Per Group Weight Scale- weight_scale = GroupQuantScaleParameter(- data=torch.empty(- sum(output_partition_sizes),- input_size_per_partition // self.group_size,- dtype=torch.uint8, # E8M0 for MXFP8 scale- ),- input_dim=1,- output_dim=0,- weight_loader=weight_loader,- )- layer.register_parameter("weight_scale", weight_scale)-- def apply_weights(- self,- layer: torch.nn.Module,- x: torch.Tensor,- bias: Optional[torch.Tensor] = None,- ) -> torch.Tensor:- # dequant weight- weight = layer.weight- weight_scale = layer.weight_scale- dequnat_weight = dequant_mx_fp8(- weight_fp8=weight.data,- scale_e8m0=weight_scale.data,- block_size=self.group_size,- target_dtype=x.dtype,- )- dequnat_weight = dequnat_weight.to(x.dtype)- # if not envs.VLLM_AR_MXFP8_DISABLE_INPUT_QDQ:- # q-dq input- x_scale, x_quant = quant_mx_fp8(x)- dequant_x = dequant_mx_fp8(- weight_fp8=x_quant,- scale_e8m0=x_scale,- block_size=self.group_size,- target_dtype=x.dtype,- )- x = dequant_x.to(x.dtype)-- out = x @ dequnat_weight.t()- return out.to(x.dtype) + (bias if bias is not None else 0)diff --git a/auto_round_extension/vllm_ext/moe_impl_mxfp4.py b/auto_round_extension/vllm_ext/moe_impl_mxfp4.pydeleted file mode 100644index 75e298acd..000000000--- a/auto_round_extension/vllm_ext/moe_impl_mxfp4.py+++ /dev/null@@ -1,592 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--from typing import Callable, Optional, Union--import torch-import torch.nn.functional as F-import vllm.envs as envs-from vllm.distributed import get_tensor_model_parallel_rank-from vllm.logger import init_logger-from vllm.model_executor.layers.fused_moe import (- FusedMoE,- FusedMoEConfig,- FusedMoeWeightScaleSupported,-)-from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig-from vllm.model_executor.utils import set_weight_attrs--import auto_round_extension.vllm_ext.mxfp4_qdq_utils as mxfp4_utils-from auto_round_extension.vllm_ext.mxfp4_qdq_utils import (- dequant_mxfp4_to_fp8,- mxfp4_gemm_with_unpacked_weight,- run_mxfp4_emulations,-)-from auto_round_extension.vllm_ext.quant_method_moe import AutoRoundMoEMethod--logger = init_logger(__name__)---def apply_act(local_w1_out: torch.Tensor, local_w3_out: torch.Tensor, activation: str) -> torch.Tensor:- if activation == "silu":- act_fn = F.silu- w13_out = act_fn(local_w1_out) * local_w3_out- elif activation == "swigluoai":- limit = 7.0- alpha = 1.702- local_w1_out = local_w1_out.clamp(min=None, max=limit)- local_w3_out = local_w3_out.clamp(min=-limit, max=limit)- glu = (local_w1_out) * F.sigmoid(local_w1_out * alpha)- w13_out = (local_w3_out + 1) * glu- else:- raise NotImplementedError(f"Activation {activation} is not implemented.")- return w13_out---class AutoRoundMoEMethodMXFp4Impl(AutoRoundMoEMethod):- def __init__(- self,- quant_config: "AutoRoundConfig", # type: ignore # noqa E501- moe: FusedMoEConfig,- ):- super().__init__(moe)- self.use_marlin = False- self.group_size = 32- self.quant_config = quant_config- self.has_bias = self.moe.has_bias- self.mask_weights_buffer = None- self.experts_mask_buffer = None- self.num_all_tokens_threshold = 16 * 1024- # self.num_all_tokens_threshold = 0-- def create_weights(- self,- layer: torch.nn.Module,- num_experts: int,- hidden_size: int,- intermediate_size_per_partition: int,- params_dtype: torch.dtype,- **extra_weight_attrs,- ):- layer.num_experts = num_experts- layer.params_dtype = params_dtype-- w13_weight = torch.nn.Parameter(- torch.empty(- num_experts,- 2 * intermediate_size_per_partition,- # 2 fp4 items are packed in the input dimension- hidden_size // 2,- requires_grad=False,- dtype=torch.uint8,- ),- requires_grad=False,- )- layer.register_parameter("w13_weight_packed", w13_weight)- set_weight_attrs(w13_weight, extra_weight_attrs)-- w2_weight = torch.nn.Parameter(- torch.empty(- num_experts,- hidden_size,- # 2 fp4 items are packed in the input dimension- intermediate_size_per_partition // 2,- dtype=torch.uint8,- ),- requires_grad=False,- )- layer.register_parameter("w2_weight_packed", w2_weight)- set_weight_attrs(w2_weight, extra_weight_attrs)-- # Weight Scales- w13_weight_scale = torch.nn.Parameter(- torch.empty(- num_experts,- 2 * intermediate_size_per_partition,- # 2 fp4 items are packed in the input dimension- hidden_size // self.group_size,- dtype=torch.uint8,- ),- requires_grad=False,- )- layer.register_parameter("w13_weight_scale", w13_weight_scale)- extra_weight_attrs.update({"quant_method": FusedMoeWeightScaleSupported.GROUP.value})- set_weight_attrs(w13_weight_scale, extra_weight_attrs)-- w2_weight_scale = torch.nn.Parameter(- torch.empty(- num_experts,- hidden_size,- # 2 fp4 items are packed in the input dimension- intermediate_size_per_partition // self.group_size,- dtype=torch.uint8,- ),- requires_grad=False,- )- layer.register_parameter("w2_weight_scale", w2_weight_scale)- extra_weight_attrs.update({"quant_method": FusedMoeWeightScaleSupported.GROUP.value})- set_weight_attrs(w2_weight_scale, extra_weight_attrs)-- E = num_experts- H = hidden_size- IN = intermediate_size_per_partition- if self.has_bias:- # TODO: yiliu30 use the dtype in CK- bias_dtype = torch.bfloat16- w13_bias = torch.nn.Parameter(torch.zeros(E, 2 * IN, dtype=bias_dtype), requires_grad=False)- layer.register_parameter("w13_bias", w13_bias)- set_weight_attrs(w13_bias, extra_weight_attrs)-- w2_bias = torch.nn.Parameter(- torch.zeros(E, H, dtype=bias_dtype),- requires_grad=False,- )- layer.register_parameter("w2_bias", w2_bias)- set_weight_attrs(w2_bias, extra_weight_attrs)-- def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> Optional[FusedMoEQuantConfig]:- if envs.VLLM_AR_MXFP4_MODULAR_MOE:- from vllm.model_executor.layers.fused_moe.config import (- ocp_mx_moe_quant_config,- )-- if envs.VLLM_MXFP4_PRE_UNPACK_TO_FP8:- self.input_dtype = "mxfp8_e4m3"- self.weight_dtype = "mxfp8_e4m3"- ocp_config = ocp_mx_moe_quant_config(- quant_dtype=self.input_dtype,- weight_dtype=self.weight_dtype,- w1_scale=layer.w13_weight_scale,- w2_scale=layer.w2_weight_scale,- a1_scale=None,- a2_scale=None,- w1_bias=layer.w13_bias if self.has_bias else None,- w2_bias=layer.w2_bias if self.has_bias else None,- block_shape=None,- )- from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (- OCP_MX_DTYPES,- OCP_MX_Scheme,- )-- ocp_config.ocp_mx_scheme = OCP_MX_Scheme.from_quant_dtype(- input_dtype="mxfp8_e4m3",- weight_dtype="mxfp8_e4m3",- original_dtype="mxfp4",- )- logger.warning_once(f"Set OCP_MX_Scheme to {ocp_config.ocp_mx_scheme}")- return ocp_config-- self.input_dtype = "mxfp4"- self.weight_dtype = "mxfp4"- return ocp_mx_moe_quant_config(- quant_dtype=self.input_dtype,- weight_dtype=self.weight_dtype,- w1_scale=layer.w13_weight_scale,- w2_scale=layer.w2_weight_scale,- a1_scale=None,- a2_scale=None,- w1_bias=layer.w13_bias if self.has_bias else None,- w2_bias=layer.w2_bias if self.has_bias else None,- block_shape=None,- )- return None-- def _dequant_fp4_to_fp8(self, layer):- weight_name_lst = ["w13_weight", "w2_weight"]-- for weight_name_prefix in weight_name_lst:- weight_name = f"{weight_name_prefix}_packed"- weight = getattr(layer, weight_name)- weight_scale_name = f"{weight_name_prefix}_scale"- weight_scale = getattr(layer, weight_scale_name)- new_weight_name = f"{weight_name_prefix}_unpacked"- new_scale_name = weight_scale_name- num_experts, _, _ = weight.shape- unpacked_weight_lst = []- scale_list = []- for expert_index in range(num_experts):- weight_fp8, scale_bf16 = dequant_mxfp4_to_fp8(- data_lp=weight[expert_index],- scale_e8m0=weight_scale[expert_index],- )-- unpacked_weight_lst.append(weight_fp8)- scale_list.append(scale_bf16)- unpacked_weight_fp8 = torch.stack(unpacked_weight_lst, dim=0)- scale_bf16 = torch.stack(scale_list, dim=0)- assert unpacked_weight_fp8.shape[0] == num_experts, (- f"Expected {num_experts} unpacked weights, got " f"{unpacked_weight_fp8.shape[0]}"- )- delattr(layer, weight_name)- delattr(layer, weight_scale_name)- layer.register_parameter(- new_weight_name,- torch.nn.Parameter(- unpacked_weight_fp8,- requires_grad=False,- ),- )- layer.register_parameter(- new_scale_name,- torch.nn.Parameter(- scale_bf16,- requires_grad=False,- ),- )-- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:- logger.debug(f"Processing weights after loading for layer: {layer._prefix}")- if envs.VLLM_ENABLE_STATIC_MOE:- if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:- self._dequant_fp4_to_fp8(layer)- return- elif envs.VLLM_AR_MXFP4_MODULAR_MOE:- if envs.VLLM_MXFP4_PRE_UNPACK_TO_FP8:- self._dequant_fp4_to_fp8(layer)- return-- def revert_interleaved_bias(bias):- """- Convert from blocked bias format to interleaved format.-- Args:- bias: Tensor of shape [E, intermediate_size*2] where the first half contains- w1 biases and second half contains w3 biases for each expert-- Returns:- Tensor with interleaved w1 and w3 biases- """- # loaded bias[0]: [e0_w1_bias_0, e0_w1_bias_1, ..., e0_w3_bias_0, e0_w3_bias_1...]- # Expected bias[0]: [e0_w1_bias_0, e0_w3_bias_0, e0_w1_bias_1, e0_w3_bias_1...]- # The expected bias order is used in triton kernel https://github.com/vllm-project/vllm/pull/22508- revert_bias = torch.zeros_like(bias, device=bias.device)- E, two_IN = bias.shape-- # Verify the shape is as expected- if two_IN % 2 != 0:- raise ValueError(f"Expected even number of bias elements, got {two_IN}")-- revert_bias[..., ::2] = bias[..., : two_IN // 2]- revert_bias[..., 1::2] = bias[..., two_IN // 2 :]-- return revert_bias-- if self.has_bias:- if envs.VLLM_AR_POST_PROCESS_GPTOSS:- w13_bias_swapped = revert_interleaved_bias(layer.w13_bias)- layer.w13_bias.data.copy_(w13_bias_swapped)-- if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:-- w1 = layer.w13_weight_packed- w1_scale = layer.w13_weight_scale- w1 = mxfp4_utils.to_dtype(- data_lp=w1,- scale_e8m0=w1_scale,- elem_dtype="fp4_e2m1",- block_size=32,- target_dtype=torch.bfloat16,- )-- def revert_interleaved_w1(w1):- new_w1 = torch.zeros_like(w1)- E, N, H = w1.shape- new_w1[:, ::2, :] = w1[:, : N // 2, :]- new_w1[:, 1::2, :] = w1[:, N // 2 :, :]- return new_w1-- if envs.VLLM_AR_POST_PROCESS_GPTOSS:- w1 = revert_interleaved_w1(w1)-- w1_scale = None- w2 = layer.w2_weight_packed- w2_scale = layer.w2_weight_scale- w2 = mxfp4_utils.to_dtype(- data_lp=w2,- scale_e8m0=w2_scale,- elem_dtype="fp4_e2m1",- block_size=32,- target_dtype=torch.bfloat16,- )- w2_scale = None- del layer.w13_weight_packed- del layer.w13_weight_scale- del layer.w2_weight_packed- del layer.w2_weight_scale- layer.w13_weight_scale = None- layer.w2_weight_scale = None- layer.register_parameter(- "w13_weight_unpacked",- torch.nn.Parameter(- w1,- requires_grad=False,- ),- )- layer.register_parameter(- "w2_weight_unpacked",- torch.nn.Parameter(- w2,- requires_grad=False,- ),- )-- else:- raise NotImplementedError("process_weights_after_loading is not implemented for now.")-- @torch.inference_mode()- def apply(- self,- layer: torch.nn.Module,- x: torch.Tensor,- router_logits: torch.Tensor,- top_k: int,- renormalize: bool,- use_grouped_topk: bool = False,- topk_group: Optional[int] = None,- num_expert_group: Optional[int] = None,- global_num_experts: int = -1,- expert_map: Optional[torch.Tensor] = None,- custom_routing_function: Optional[Callable] = None,- scoring_func: str = "softmax",- routed_scaling_factor: float = 1.0,- e_score_correction_bias: Optional[torch.Tensor] = None,- apply_router_weight_on_input: bool = False,- activation: str = "silu",- enable_eplb: bool = False,- expert_load_view: Optional[torch.Tensor] = None,- logical_to_physical_map: Optional[torch.Tensor] = None,- logical_replica_count: Optional[torch.Tensor] = None,- ) -> Union[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:- topk_weights, topk_ids, _ = FusedMoE.select_experts(- hidden_states=x,- router_logits=router_logits,- use_grouped_topk=use_grouped_topk,- top_k=top_k,- renormalize=renormalize,- topk_group=topk_group,- num_expert_group=num_expert_group,- custom_routing_function=custom_routing_function,- scoring_func=scoring_func,- e_score_correction_bias=e_score_correction_bias,- )- assert self.fused_experts is None-- # There are three implementations:-- if envs.VLLM_AR_MXFP4_MODULAR_MOE:- from vllm.model_executor.layers.fused_moe import fused_experts-- if envs.VLLM_MXFP4_PRE_UNPACK_TO_FP8:- w1 = layer.w13_weight_unpacked- w2 = layer.w2_weight_unpacked- out = fused_experts(- x,- w1,- w2,- topk_weights=topk_weights,- topk_ids=topk_ids,- inplace=True,- activation=activation,- global_num_experts=global_num_experts,- apply_router_weight_on_input=apply_router_weight_on_input,- expert_map=expert_map,- quant_config=self.moe_quant_config,- )- return out- if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:- w1 = layer.w13_weight_unpacked- w2 = layer.w2_weight_unpacked- else:- w1 = layer.w13_weight_packed- w2 = layer.w2_weight_packed- out = fused_experts(- x,- w1,- w2,- topk_weights=topk_weights,- topk_ids=topk_ids,- inplace=True,- activation=activation,- global_num_experts=global_num_experts,- apply_router_weight_on_input=apply_router_weight_on_input,- expert_map=expert_map,- quant_config=self.moe_quant_config,- )- return out-- num_all_tokens, hidden_dim = x.shape- num_experts = layer.local_num_experts- total_num_experts = router_logits.size(-1)- if (- self.mask_weights_buffer is None- or self.mask_weights_buffer.dtype != x.dtype- or self.mask_weights_buffer.device != x.device- or self.mask_weights_buffer.shape[0] < num_all_tokens- or self.mask_weights_buffer.shape[1] < total_num_experts- ):- if num_all_tokens > self.num_all_tokens_threshold:- mask_weights = torch.zeros((num_all_tokens, total_num_experts), dtype=x.dtype, device=x.device)- if self.mask_weights_buffer is None and self.num_all_tokens_threshold != 0:- self.mask_weights_buffer = torch.zeros(- (self.num_all_tokens_threshold, total_num_experts),- dtype=x.dtype,- device=x.device,- )- self.experts_mask_buffer = torch.zeros(- (self.num_all_tokens_threshold, total_num_experts),- dtype=x.dtype,- device=x.device,- )- else:- self.mask_weights_buffer = torch.zeros(- (num_all_tokens, total_num_experts), dtype=x.dtype, device=x.device- )- self.experts_mask_buffer = torch.zeros(- (num_all_tokens, total_num_experts), dtype=x.dtype, device=x.device- )- mask_weights = self.mask_weights_buffer- experts_mask = self.experts_mask_buffer- else:- self.mask_weights_buffer.zero_()- mask_weights = self.mask_weights_buffer- self.experts_mask_buffer.zero_()- experts_mask = self.experts_mask_buffer-- topk_ids = topk_ids.to(torch.int64)- topk_weights = topk_weights.to(x.dtype)- experts_mask.scatter_(-1, topk_ids, topk_weights)- mask_weights.scatter_(-1, topk_ids, 1)- mask_weights = mask_weights[:num_all_tokens, :total_num_experts]- mask_weights = mask_weights.transpose(0, 1)- experts_mask = experts_mask[:num_all_tokens, :total_num_experts]- experts_mask = experts_mask.transpose(0, 1)- # Note: ep_size equal tp_size- ep_rank = get_tensor_model_parallel_rank() if expert_map is not None else 0- ep_shift = ep_rank * num_experts-- if envs.VLLM_ENABLE_STATIC_MOE and not envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:- num_experts, intermediate_size_per_partition_x2, _ = layer.w13_weight_packed.shape- intermediate_size_per_partition = intermediate_size_per_partition_x2 // 2- for expert_index in range(num_experts):- mask_weight = mask_weights[expert_index + ep_shift].unsqueeze(1)- current_state_static = x * mask_weight-- local_w13_packed = layer.w13_weight_packed[expert_index]- local_w13_scale = layer.w13_weight_scale[expert_index]- local_w2_packed = layer.w2_weight_packed[expert_index]- local_w2_scale = layer.w2_weight_scale[expert_index]-- local_w1_packed = local_w13_packed[:intermediate_size_per_partition, ...]- local_w1_scale = local_w13_scale[:intermediate_size_per_partition, ...]-- local_w3_packed = local_w13_packed[intermediate_size_per_partition:, ...]- local_w3_scale = local_w13_scale[intermediate_size_per_partition:, ...]-- local_w1_bias = None- local_w2_bias = None- local_w3_bias = None- if self.has_bias:- local_w13_bias = layer.w13_bias[expert_index]- local_w1_bias = local_w13_bias[:intermediate_size_per_partition]- local_w3_bias = local_w13_bias[intermediate_size_per_partition:]- local_w2_bias = layer.w2_bias[expert_index]-- local_w1_out = run_mxfp4_emulations(- x=current_state_static,- weight=local_w1_packed,- weight_scale=local_w1_scale,- bias=local_w1_bias,- )- local_w3_out = run_mxfp4_emulations(- x=current_state_static,- weight=local_w3_packed,- weight_scale=local_w3_scale,- bias=local_w3_bias,- )-- w13_out = apply_act(local_w1_out, local_w3_out, activation)-- local_w2_out = run_mxfp4_emulations(- x=w13_out,- weight=local_w2_packed,- weight_scale=local_w2_scale,- bias=local_w2_bias,- )- padded_weight = experts_mask[expert_index + ep_shift].unsqueeze(1)- local_w2_out = local_w2_out * padded_weight- if expert_index == 0:- final_hidden_states = local_w2_out- else:- final_hidden_states += local_w2_out- return final_hidden_states- if envs.VLLM_ENABLE_STATIC_MOE and envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:- num_experts, intermediate_size_per_partition_x2, _ = layer.w13_weight_unpacked.shape- intermediate_size_per_partition = intermediate_size_per_partition_x2 // 2-- for expert_index in range(num_experts):- mask_weight = mask_weights[expert_index + ep_shift].unsqueeze(1)- current_state_static = x * mask_weight-- local_unpacked_w13 = layer.w13_weight_unpacked[expert_index]- local_w13_scale = layer.w13_weight_scale[expert_index]-- local_unpacked_w2 = layer.w2_weight_unpacked[expert_index]- local_w2_scale = layer.w2_weight_scale[expert_index]-- local_unpacked_w1 = local_unpacked_w13[:intermediate_size_per_partition, ...]- half_scale = local_w13_scale.shape[0] // 2- local_w1_scale = local_w13_scale[:half_scale, ...]- local_unpacked_w3 = local_unpacked_w13[intermediate_size_per_partition:, ...]- local_w3_scale = local_w13_scale[half_scale:, ...]-- local_w1_bias = None- local_w2_bias = None- local_w3_bias = None- if self.has_bias:- local_w13_bias = layer.w13_bias[expert_index]- local_w1_bias = local_w13_bias[:intermediate_size_per_partition]- local_w3_bias = local_w13_bias[intermediate_size_per_partition:]- local_w2_bias = layer.w2_bias[expert_index]-- local_w1_out = mxfp4_gemm_with_unpacked_weight(- x=current_state_static,- weight_fp8=local_unpacked_w1,- weight_scale_bf16=local_w1_scale,- bias=local_w1_bias,- )- local_w3_out = mxfp4_gemm_with_unpacked_weight(- x=current_state_static,- weight_fp8=local_unpacked_w3,- weight_scale_bf16=local_w3_scale,- bias=local_w3_bias,- )-- w13_out = apply_act(local_w1_out, local_w3_out, activation)-- local_w2_out = mxfp4_gemm_with_unpacked_weight(- x=w13_out,- weight_fp8=local_unpacked_w2,- weight_scale_bf16=local_w2_scale,- bias=local_w2_bias,- )-- padded_weight = experts_mask[expert_index + ep_shift].unsqueeze(1)- local_w2_out = local_w2_out * padded_weight- if expert_index == 0:- final_hidden_states = local_w2_out- else:- final_hidden_states += local_w2_out- return final_hidden_states- raise NotImplementedError("Not implemented for now.")diff --git a/auto_round_extension/vllm_ext/moe_impl_mxfp8.py b/auto_round_extension/vllm_ext/moe_impl_mxfp8.pydeleted file mode 100644index ecd4183ff..000000000--- a/auto_round_extension/vllm_ext/moe_impl_mxfp8.py+++ /dev/null@@ -1,208 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--from typing import Callable, Optional, Union--import torch-import torch.nn.functional as F-from vllm.logger import init_logger-from vllm.model_executor.layers.fused_moe import (- FusedMoE,- FusedMoEConfig,- FusedMoeWeightScaleSupported,-)-from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig-from vllm.model_executor.utils import set_weight_attrs--from auto_round_extension.vllm_ext.quant_method_moe import AutoRoundMoEMethod--logger = init_logger(__name__)---def apply_act(local_w1_out: torch.Tensor, local_w3_out: torch.Tensor, activation: str) -> torch.Tensor:- if activation == "silu":- act_fn = F.silu- w13_out = act_fn(local_w1_out) * local_w3_out- elif activation == "swigluoai":- limit = 7.0- alpha = 1.702- local_w1_out = local_w1_out.clamp(min=None, max=limit)- local_w3_out = local_w3_out.clamp(min=-limit, max=limit)- glu = (local_w1_out) * F.sigmoid(local_w1_out * alpha)- w13_out = (local_w3_out + 1) * glu- else:- raise NotImplementedError(f"Activation {activation} is not implemented.")- return w13_out---class AutoRoundMoEMethodMXFp8Impl(AutoRoundMoEMethod):- def __init__(- self,- quant_config: "AutoRoundConfig", # type: ignore # noqa E501- moe: FusedMoEConfig,- ):- super().__init__(moe)- self.use_marlin = False- self.group_size = 32- self.quant_config = quant_config- self.has_bias = self.moe.has_bias-- def create_weights(- self,- layer: torch.nn.Module,- num_experts: int,- hidden_size: int,- intermediate_size_per_partition: int,- params_dtype: torch.dtype,- **extra_weight_attrs,- ):- layer.intermediate_size_per_partition = intermediate_size_per_partition- layer.hidden_size = hidden_size- layer.num_experts = num_experts- layer.orig_dtype = params_dtype- layer.weight_block_size = None-- params_dtype = torch.float8_e4m3fn-- # WEIGHTS- w13_weight = torch.nn.Parameter(- torch.empty(- num_experts,- 2 * intermediate_size_per_partition,- hidden_size,- dtype=params_dtype,- ),- requires_grad=False,- )- layer.register_parameter("w13_weight", w13_weight)- set_weight_attrs(w13_weight, extra_weight_attrs)-- w2_weight = torch.nn.Parameter(- torch.empty(- num_experts,- hidden_size,- intermediate_size_per_partition,- dtype=params_dtype,- ),- requires_grad=False,- )- layer.register_parameter("w2_weight", w2_weight)- set_weight_attrs(w2_weight, extra_weight_attrs)-- # WEIGHT_SCALES- w13_weight_scale = torch.nn.Parameter(- data=torch.empty(- num_experts,- 2 * intermediate_size_per_partition,- hidden_size // self.group_size,- dtype=torch.uint8, # E8M0 for MXFP8 scale- ),- requires_grad=False,- )- layer.register_parameter("w13_weight_scale", w13_weight_scale)- # w2- w2_weight_scale = torch.nn.Parameter(- data=torch.empty(- num_experts,- hidden_size,- intermediate_size_per_partition // self.group_size,- dtype=torch.uint8, # E8M0 for MXFP8 scale- ),- requires_grad=False,- )- layer.register_parameter("w2_weight_scale", w2_weight_scale)- # Add PER-TENSORGROUP quantization for FusedMoE.weight_loader.- extra_weight_attrs.update({"quant_method": FusedMoeWeightScaleSupported.GROUP.value})- set_weight_attrs(w13_weight_scale, extra_weight_attrs)- set_weight_attrs(w2_weight_scale, extra_weight_attrs)-- def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> Optional[FusedMoEQuantConfig]:- from vllm.model_executor.layers.fused_moe.config import (- ocp_mx_moe_quant_config,- )-- self.input_dtype = "mxfp8_e4m3"- self.weight_dtype = "mxfp8_e4m3"- return ocp_mx_moe_quant_config(- quant_dtype=self.input_dtype,- weight_dtype=self.weight_dtype,- w1_scale=layer.w13_weight_scale,- w2_scale=layer.w2_weight_scale,- a1_scale=None,- a2_scale=None,- w1_bias=layer.w13_bias if self.has_bias else None,- w2_bias=layer.w2_bias if self.has_bias else None,- block_shape=None,- )- return None-- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:- return-- @torch.inference_mode()- def apply(- self,- layer: torch.nn.Module,- x: torch.Tensor,- router_logits: torch.Tensor,- top_k: int,- renormalize: bool,- use_grouped_topk: bool = False,- topk_group: Optional[int] = None,- num_expert_group: Optional[int] = None,- global_num_experts: int = -1,- expert_map: Optional[torch.Tensor] = None,- custom_routing_function: Optional[Callable] = None,- scoring_func: str = "softmax",- routed_scaling_factor: float = 1.0,- e_score_correction_bias: Optional[torch.Tensor] = None,- apply_router_weight_on_input: bool = False,- activation: str = "silu",- enable_eplb: bool = False,- expert_load_view: Optional[torch.Tensor] = None,- logical_to_physical_map: Optional[torch.Tensor] = None,- logical_replica_count: Optional[torch.Tensor] = None,- ) -> Union[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:- topk_weights, topk_ids, _ = FusedMoE.select_experts(- hidden_states=x,- router_logits=router_logits,- use_grouped_topk=use_grouped_topk,- top_k=top_k,- renormalize=renormalize,- topk_group=topk_group,- num_expert_group=num_expert_group,- custom_routing_function=custom_routing_function,- scoring_func=scoring_func,- e_score_correction_bias=e_score_correction_bias,- )- assert self.fused_experts is None-- from vllm.model_executor.layers.fused_moe import fused_experts-- w1 = layer.w13_weight- w2 = layer.w2_weight- out = fused_experts(- x,- w1,- w2,- topk_weights=topk_weights,- topk_ids=topk_ids,- inplace=True,- activation=activation,- global_num_experts=global_num_experts,- apply_router_weight_on_input=apply_router_weight_on_input,- expert_map=expert_map,- quant_config=self.moe_quant_config,- )- return outdiff --git a/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py b/auto_round_extension/vllm_ext/mxfp4_qdq_utils.pydeleted file mode 100644index 2a068924f..000000000--- a/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py+++ /dev/null@@ -1,221 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--from typing import Union--import torch--from auto_round_extension.vllm_ext.fp4_utils import cast_to_fp4, pack_fp4_to_uint8, unpack_fp4_from_uint8-from auto_round_extension.vllm_ext.utils import _to_mx_rceil, get_fp_scale--F4_E2M1_MAX = 6.0-F32_EXP_BIAS = 127--F32_MIN_NORMAL = 2 ** (-F32_EXP_BIAS + 1)---def to_mxfp4_rceil(- data_hp: torch.Tensor,- elem_dtype: Union[torch.dtype, str],- block_size: int,-):-- assert data_hp.dtype in (- torch.bfloat16,- torch.float,- ), f"{data_hp.dtype} is not supported yet"- # TODO(future PR): consider supporting padding- assert data_hp.numel() % block_size == 0, f"data size must be multiple of block_size={block_size}"- data_hp = data_hp.contiguous()- # calculate the scale in e8m0 format-- orig_shape = data_hp.shape- data_hp = data_hp.reshape(-1, block_size)-- # find max value of the data- # Note: this only implements the `minimally supported` version of- # https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf- # section 6.3.- max_abs = torch.amax(torch.abs(data_hp), 1)-- # Add an epsilon to prevent the log2 function call for returning -inf- # where the values are zero.- eps = F32_MIN_NORMAL * (max_abs == 0).type(max_abs.dtype)-- # Set X to be the largest power-of-two less than or equal to- # max_abs(v), divided by the largest power of two representable- # in the element data type, and get the mbits at the same time-- max_pos = F4_E2M1_MAX- scale_e8m0_biased, data_lp = _to_mx_rceil(data_hp, max_abs, max_pos)-- data_lp = data_lp.reshape(orig_shape)- orig_shape = [*orig_shape[:-1], orig_shape[-1] // 2]- data_lp = cast_to_fp4(data_lp)- data_lp = pack_fp4_to_uint8(data_lp)-- scale_e8m0_biased = scale_e8m0_biased.view(torch.uint8)- return scale_e8m0_biased, data_lp---def to_dtype(- data_lp,- scale_e8m0,- elem_dtype,- block_size,- target_dtype,- scale_dtype=None,- return_scale=False,-):- orig_shape = data_lp.shape- last_dim = orig_shape[-1]- data_lp = data_lp.reshape(-1, last_dim)- result_shape = orig_shape[:-1] + (last_dim * 2,)- assert data_lp.is_contiguous(), f"Data must be contiguous, got {data_lp.stride()}"-- assert elem_dtype == "fp4_e2m1", f"Expected 'fp4_e2m1', got {elem_dtype}"-- m, half_n = data_lp.shape- n = half_n * 2- data_hp = unpack_fp4_from_uint8(data_lp, m, n, dtype=target_dtype)-- data_hp = data_hp.reshape(-1, block_size)-- if scale_dtype is None:- scale_dtype = target_dtype- s_fp = get_fp_scale(scale_e8m0).reshape(-1, 1).to(scale_dtype)- if return_scale:- return data_hp.reshape(result_shape), s_fp-- data_hp = data_hp * s_fp- data_hp = data_hp.reshape(result_shape)-- return data_hp---def run_mxfp4_emulations(- x: torch.Tensor,- weight: torch.Tensor,- weight_scale: torch.Tensor,- bias: torch.Tensor | None = None,-):- # TODO: select the rounding mode based on config- group_size = 32- # quantize input to (FP4 and interleaved block scale)- input_scale, x_q = to_mxfp4_rceil(- data_hp=x,- elem_dtype="fp4_e2m1",- block_size=group_size,- )-- # dequantize input- x_dq = to_dtype(- data_lp=x_q,- scale_e8m0=input_scale,- elem_dtype="fp4_e2m1",- block_size=group_size,- target_dtype=x.dtype,- )-- # dequantize weight- w_dq = to_dtype(- data_lp=weight,- scale_e8m0=weight_scale,- elem_dtype="fp4_e2m1",- block_size=group_size,- target_dtype=x.dtype,- )-- # matmul- out = torch.matmul(x_dq, w_dq.t())- if bias is not None:- out += bias- return out---def dequant_mxfp4_to_fp8(data_lp, scale_e8m0):- data_fp8, scale_float = to_dtype(- data_lp=data_lp,- scale_e8m0=scale_e8m0,- elem_dtype="fp4_e2m1",- block_size=32,- target_dtype=torch.float8_e4m3fn,- scale_dtype=torch.bfloat16,- return_scale=True,- )- return data_fp8, scale_float---def mxfp4_fp8_weight_to_bf16(weight_fp8, scale_bf16):-- origin_shape = weight_fp8.shape- weight_fp8 = weight_fp8.reshape(-1, 32)- scale_bf16 = scale_bf16.reshape(-1, 1)- assert weight_fp8.shape[0] == scale_bf16.shape[0], f"shape mismatch: {weight_fp8.shape} vs {scale_bf16.shape}"- dequant_weight_bf16 = weight_fp8.to(torch.bfloat16) * scale_bf16- dequant_weight_bf16 = dequant_weight_bf16.reshape(origin_shape)- return dequant_weight_bf16---def mxfp4_gemm_with_unpacked_weight(x, weight_fp8, weight_scale_bf16, bias=None):- x = qdq_mxfp4(x)-- # dequantize weight- w_dq = mxfp4_fp8_weight_to_bf16(weight_fp8, weight_scale_bf16)- # matmul- out = torch.matmul(x, w_dq.t())- if bias is not None:- out += bias- return out---def fp4_121_positive(x: torch.Tensor) -> torch.Tensor:- step1 = torch.round(2.0 * x) / 2.0- step2 = torch.round(x)- step3 = 2.0 * torch.round(x / 2.0)-- mask1 = x < 2.0- mask2 = x < 4.0-- return step1 * mask1 + step2 * (~mask1) * mask2 + step3 * (~mask1) * (~mask2)---def fp4_121_scaled_even_rounding(x: torch.Tensor) -> torch.Tensor:- sign = x.sign()- x_abs = x.abs()- amax_x = x_abs.max(dim=-1, keepdim=True)[0]- scale_tmp = torch.floor(torch.log2(amax_x)) - 2.0- scale_clamp = torch.clamp(scale_tmp, min=-127, max=127)- scale = torch.pow(2.0, scale_clamp)-- scale = torch.where((0 < scale) * (scale < torch.inf), scale, 1.0)-- x_fp4_abs = fp4_121_positive(x_abs / scale) * scale- return sign * x_fp4_abs---# https://github.com/Anonymous1252022/fp4-all-the-way/blob/main/experimental/fp4.py-def qdq_mxfp4(- x: torch.Tensor,-) -> torch.Tensor:- # TODO: select the rounding mode based on config- block_size = 32- shape = x.shape- x = x.reshape(-1, block_size)-- x = fp4_121_scaled_even_rounding(x)-- x = x.reshape(shape)-- return xdiff --git a/auto_round_extension/vllm_ext/mxfp8_qdq_utils.py b/auto_round_extension/vllm_ext/mxfp8_qdq_utils.pydeleted file mode 100644index 482d77f85..000000000--- a/auto_round_extension/vllm_ext/mxfp8_qdq_utils.py+++ /dev/null@@ -1,63 +0,0 @@-# SPDX-License-Identifier: Apache-2.0-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project-import torch--__all__ = ["get_fp_scale", "dequant_mx_fp8", "quant_mx_fp8"]---# def get_fp_scale(scale_e8m0):-# # https://github.com/pytorch/ao/blob/994a4ba6c869854fcaa6ca7e118fcbd75e6c28cc/torchao/prototype/mx_formats/mx_tensor.py#L337-# assert scale_e8m0.dtype == torch.uint8, f"Expected uint8, got {scale_e8m0.dtype}"-# E8M0_EXPONENT_BIAS = 127-# scale_e8m0 = scale_e8m0.view(torch.uint8)-# s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS-# # TODO(later): it would be nice if there was a way to do the 2^x operation-# # in PyTorch without creating a tensor of twos-# two = torch.full(s_offset.size(), 2.0, device=scale_e8m0.device)-# # pow(two, s_offset) can be out of range of floating point formats.-# # TODO(later): handle this for float16 if we decide to support float16-# # scales.-# s_fp = torch.pow(two, s_offset)--# return s_fp---def get_fp_scale(scale_e8m0):- # https://github.com/pytorch/ao/blob/994a4ba6c869854fcaa6ca7e118fcbd75e6c28cc/torchao/prototype/mx_formats/mx_tensor.py#L337- E8M0_EXPONENT_BIAS = 127-- scale_e8m0 = scale_e8m0.view(torch.uint8)- s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS- # TODO(later): it would be nice if there was a way to do the 2^x operation- # in PyTorch without creating a tensor of twos- # two = torch.full(s_offset.size(), 2.0, device=scale_e8m0.device)- # pow(two, s_offset) can be out of range of floating point formats.- # TODO(later): handle this for float16 if we decide to support float16- # scales.- # s_fp = torch.pow(two, s_offset)- # !!!!NOTE Critical: fixed the OoM issue when using HPU graph- s_fp = torch.pow(2.0, s_offset.to(torch.float))-- return s_fp---def dequant_mx_fp8(weight_fp8, scale_e8m0, block_size, target_dtype):- scale_float = get_fp_scale(scale_e8m0)- weight_bf16 = weight_fp8.to(torch.bfloat16)- weight_original_shape = weight_bf16.shape- weight_bf16 = weight_bf16.reshape(-1, block_size)- scale_float = scale_float.reshape(-1, 1)- dequant_weight = weight_bf16 * scale_float- dequant_weight = dequant_weight.reshape(weight_original_shape)- return dequant_weight.to(target_dtype)---def quant_mx_fp8(tensor):- from auto_round_extension.vllm_ext.utils import to_mx_fp8e4m3-- scale_e8m0_biased, data_lp = to_mx_fp8e4m3(- data_hp=tensor,- elem_dtype=torch.float8_e4m3fn,- block_size=32,- )- return scale_e8m0_biased, data_lpdiff --git a/auto_round_extension/vllm_ext/quant_impl.py b/auto_round_extension/vllm_ext/quant_impl.pydeleted file mode 100644index b88843462..000000000--- a/auto_round_extension/vllm_ext/quant_impl.py+++ /dev/null@@ -1,45 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--from abc import ABC, abstractmethod-from typing import Optional--import torch---class AutoRoundQuantImpl(ABC):- @classmethod- @abstractmethod- def get_min_capability(cls) -> int:- """- Get minimum device capability.- """- raise NotImplementedError-- @abstractmethod- def create_weights(self, *args, **kwargs):- raise NotImplementedError-- @abstractmethod- def apply_weights(- self,- layer: torch.nn.Module,- x: torch.Tensor,- bias: Optional[torch.Tensor],- ):- raise NotImplementedError-- @abstractmethod- def process_weights_after_loading(self, layer: torch.nn.Module):- raise NotImplementedErrordiff --git a/auto_round_extension/vllm_ext/quant_method_linear.py b/auto_round_extension/vllm_ext/quant_method_linear.pydeleted file mode 100644index eecc97902..000000000--- a/auto_round_extension/vllm_ext/quant_method_linear.py+++ /dev/null@@ -1,112 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--from typing import Optional--import torch-from vllm.logger import init_logger-from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase, UnquantizedLinearMethod-from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig--from auto_round.schemes import QuantizationScheme-from auto_round_extension.vllm_ext.utils import _is_mxfp4_w4a4, _is_mxfp8_w8a8, get_scheme, need_quantize--logger = init_logger(__name__)---QLINEAR_METHODS_DISPATCH_TABLE = {}---class AutoRoundQuantLinearMethod(LinearMethodBase):-- def __init__(self, impl, config=None, scheme=None):- self.config = config- self.impl = impl- self.scheme = scheme-- @staticmethod- def get_method(- quant_config: AutoRoundConfig,- layer: torch.nn.Module,- prefix: str,- ) -> "AutoRoundQuantLinearMethod":-- def get_impl(scheme: QuantizationScheme):- if _is_mxfp8_w8a8(scheme):- from auto_round_extension.vllm_ext.linear_impl_mxfp8 import AutoRoundMXFP8LinearImpl-- return AutoRoundMXFP8LinearImpl(quant_config)-- elif _is_mxfp4_w4a4(scheme):- from auto_round_extension.vllm_ext.linear_impl_mxfp4 import AutoRoundMXFP4LinearImpl-- return AutoRoundMXFP4LinearImpl(quant_config)-- raise ValueError(f"Unsupported Linear scheme: {scheme}, layer: {prefix}")-- packed_modules_mapping = quant_config.packed_modules_mapping- for packed_name, child_names in packed_modules_mapping.items():- if prefix.endswith(packed_name):- prefix = prefix.replace(packed_name, child_names[0])- break-- # TODO: use a more robust way to map layer names- if prefix.endswith("gate_up_proj"):- # update gate_up_proj to gate_proj, assume both gate and up share the same quantization scheme- prefix = prefix.replace("gate_up_proj", "up_proj")- if prefix.endswith("qkv_proj"):- # update qkv_proj to q_proj, assume all qkv share the same quantization scheme- prefix = prefix.replace("qkv_proj", "q_proj")- layer_scheme = get_scheme(quant_config, prefix)- if not need_quantize(layer_scheme.bits):- return UnquantizedLinearMethod()- impl = get_impl(layer_scheme)- logger.info("Apply %s to %s", impl.__class__.__name__, prefix)- return AutoRoundQuantLinearMethod(impl=impl)-- @classmethod- def get_min_capability(cls) -> int:- return cls.impl.get_min_capability()-- def create_weights(- self,- layer: torch.nn.Module,- input_size_per_partition: int,- output_partition_sizes: list[int],- input_size: int,- output_size: int,- params_dtype: torch.dtype,- **extra_weight_attrs,- ):- weight_loader = extra_weight_attrs.get("weight_loader")- return self.impl.create_weights(- layer=layer,- input_size=input_size,- input_size_per_partition=input_size_per_partition,- output_partition_sizes=output_partition_sizes,- output_size=output_size,- params_dtype=params_dtype,- weight_loader=weight_loader,- )-- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:- return self.impl.process_weights_after_loading(layer)-- def apply(- self,- layer: torch.nn.Module,- x: torch.Tensor,- bias: Optional[torch.Tensor] = None,- ):- return self.impl.apply_weights(layer, x, bias=bias)diff --git a/auto_round_extension/vllm_ext/quant_method_moe.py b/auto_round_extension/vllm_ext/quant_method_moe.pydeleted file mode 100644index ef5c75291..000000000--- a/auto_round_extension/vllm_ext/quant_method_moe.py+++ /dev/null@@ -1,73 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--from typing import Optional--import torch-from vllm.logger import init_logger-from vllm.model_executor.layers.fused_moe import (- FusedMoEConfig,- FusedMoEMethodBase,-)-from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig-from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig--from auto_round.schemes import QuantizationScheme-from auto_round_extension.vllm_ext.utils import _is_mxfp4_w4a4, _is_mxfp8_w8a8, get_scheme, need_quantize--logger = init_logger(__name__)---QMOE_METHODS_DISPATCH_TABLE = {}---class AutoRoundMoEMethod(FusedMoEMethodBase):- def __init__(self, moe: FusedMoEConfig):- super().__init__(moe)-- @staticmethod- def get_moe_method(- quant_config: AutoRoundConfig,- layer: torch.nn.Module,- prefix: str,- ) -> "AutoRoundMoEMethod":-- def get_impl(scheme: QuantizationScheme):- if not need_quantize(scheme.bits):- from vllm.model_executor.layers.fused_moe.layer import (- UnquantizedFusedMoEMethod,- )-- return UnquantizedFusedMoEMethod(layer.moe_config)-- elif _is_mxfp4_w4a4(scheme):- from auto_round_extension.vllm_ext.moe_impl_mxfp4 import AutoRoundMoEMethodMXFp4Impl-- return AutoRoundMoEMethodMXFp4Impl(quant_config, layer.moe_config)-- elif _is_mxfp8_w8a8(scheme):- from auto_round_extension.vllm_ext.moe_impl_mxfp8 import AutoRoundMoEMethodMXFp8Impl-- return AutoRoundMoEMethodMXFp8Impl(quant_config, layer.moe_config)-- raise ValueError(f"Unsupported FusedMoe scheme: {scheme}")-- layer_scheme = get_scheme(quant_config, prefix)- impl = get_impl(layer_scheme)- layer._prefix = prefix- logger.debug("Apply %s to %s", impl.__class__.__name__, prefix)- return impl-- def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> Optional[FusedMoEQuantConfig]:- return self.impl.get_fused_moe_quant_config(layer)diff --git a/auto_round_extension/vllm_ext/sitecustomize.py b/auto_round_extension/vllm_ext/sitecustomize.pydeleted file mode 100644index 8c22d1ab2..000000000--- a/auto_round_extension/vllm_ext/sitecustomize.py+++ /dev/null@@ -1,41 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--import os--VLLM_ENABLE_AR_EXT = os.environ.get("VLLM_ENABLE_AR_EXT", "") in [- "1",- "true",- "True",-]--if VLLM_ENABLE_AR_EXT:- print("*****************************************************************************")- print(f"* !!! VLLM_ENABLE_AR_EXT is set to {VLLM_ENABLE_AR_EXT}, applying auto_round_vllm_extension *")- print("*****************************************************************************")-- import vllm.model_executor.layers.quantization.auto_round as auto_round_module-- from auto_round_extension.vllm_ext.auto_round_ext import AutoRoundExtensionConfig-- auto_round_module.AutoRoundConfig = AutoRoundExtensionConfig- from auto_round_extension.vllm_ext.envs_ext import extra_environment_variables---else:- print("*****************************************************************************")- print(- f"* Sitecustomize is loaded, but VLLM_ENABLE_AR_EXT is set to {VLLM_ENABLE_AR_EXT}, skipping auto_round_vllm_extension *"- )- print("*****************************************************************************")diff --git a/auto_round_extension/vllm_ext/tests/conftest.py b/auto_round_extension/vllm_ext/tests/conftest.pydeleted file mode 100644index fcdd97a49..000000000--- a/auto_round_extension/vllm_ext/tests/conftest.py+++ /dev/null@@ -1,1396 +0,0 @@-# SPDX-License-Identifier: Apache-2.0-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project-import contextlib-import pathlib-from copy import deepcopy--from tblib import pickling_support--# ruff: noqa--# Install support for pickling exceptions so that we can nicely propagate-# failures from tests running in a subprocess.-# This should be run before any custom exception subclasses are defined.-pickling_support.install()--import http.server-import json-import math-import mimetypes-import os-import socket-import tempfile-import threading-from collections.abc import Generator-from contextlib import nullcontext-from enum import Enum-from typing import Any, Callable, TypedDict, TypeVar, cast--import numpy as np-import pytest-import torch-import torch.nn as nn-import torch.nn.functional as F-from huggingface_hub import snapshot_download-from PIL import Image-from transformers import (- AutoConfig,- AutoModelForCausalLM,- AutoTokenizer,- BatchEncoding,- BatchFeature,-)-from transformers.models.auto.auto_factory import _BaseAutoModelClass--# from tests.models.utils import TokensTextLogprobs, TokensTextLogprobsPromptLogprobs-from vllm import LLM, SamplingParams, envs-from vllm.assets.audio import AudioAsset-from vllm.assets.image import ImageAsset-from vllm.assets.video import VideoAsset-from vllm.config.model import ConvertOption, RunnerOption, _get_and_verify_dtype-from vllm.connections import global_http_connection-from vllm.distributed import (- cleanup_dist_env_and_memory,- init_distributed_environment,- initialize_model_parallel,-)-from vllm.logger import init_logger-from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs-from vllm.multimodal.utils import fetch_image-from vllm.outputs import RequestOutput-from vllm.sampling_params import BeamSearchParams-from vllm.transformers_utils.utils import maybe_model_redirect-from vllm.utils.collection_utils import is_list_of-from vllm.utils.torch_utils import set_default_torch_num_threads--# Representation of generated sequence as a tuple of-# * Token ID list-# * String-# * List of top sample logprobs for each sampled token-#-# Assumes prompt logprobs were not requested.-TokensTextLogprobs = tuple[list[int], str, list[dict[int, float]] | SampleLogprobs | None]--# Allow for tokens to be represented as str's rather than IDs;-# tuple of-# * Token string representations list-# * String-# * Optional list of top sample logprobs for each sampled token-#-# Assumes prompt logprobs were not requested.-TextTextLogprobs = tuple[list[str], str, list[dict[str, float]] | list[dict[str, Logprob]] | None]--# Representation of generated sequence as a tuple of-# * Token ID list-# * String-# * Optional list of top sample logprobs for each sampled token-# * Optional list of top prompt logprobs for each prompt token-#-# Allows prompt logprobs to be requested.-TokensTextLogprobsPromptLogprobs = tuple[- list[int],- str,- list[dict[int, float]] | SampleLogprobs | None,- list[dict[int, float] | None] | PromptLogprobs | None,-]---logger = init_logger(__name__)--_TEST_DIR = os.path.dirname(__file__)-_TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]-_LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]-_SYS_MSG = os.path.join(_TEST_DIR, "system_messages", "sonnet3.5_nov2024.txt")--_M = TypeVar("_M")--_PromptMultiModalInput = list[_M] | list[list[_M]]--PromptImageInput = _PromptMultiModalInput[Image.Image]-PromptAudioInput = _PromptMultiModalInput[tuple[np.ndarray, int]]-PromptVideoInput = _PromptMultiModalInput[np.ndarray]---def _read_prompts(filename: str) -> list[str]:- with open(filename) as f:- prompts = f.readlines()- return prompts---class ImageAssetPrompts(TypedDict):- stop_sign: str- cherry_blossom: str---class ImageTestAssets(list[ImageAsset]):- def __init__(self) -> None:- super().__init__(- [- ImageAsset("stop_sign"),- ImageAsset("cherry_blossom"),- ]- )-- def prompts(self, prompts: ImageAssetPrompts) -> list[str]:- """- Convenience method to define the prompt for each test image.-- The order of the returned prompts matches the order of the- assets when iterating through this object.- """- return [prompts["stop_sign"], prompts["cherry_blossom"]]---class VideoAssetPrompts(TypedDict):- baby_reading: str---class VideoTestAssets(list[VideoAsset]):- def __init__(self) -> None:- super().__init__(- [- VideoAsset("baby_reading"),- ]- )-- def prompts(self, prompts: VideoAssetPrompts) -> list[str]:- return [prompts["baby_reading"]]---class AudioAssetPrompts(TypedDict):- mary_had_lamb: str- winning_call: str---class AudioTestAssets(list[AudioAsset]):- def __init__(self) -> None:- super().__init__(- [- AudioAsset("mary_had_lamb"),- AudioAsset("winning_call"),- ]- )-- def prompts(self, prompts: AudioAssetPrompts) -> list[str]:- return [prompts["mary_had_lamb"], prompts["winning_call"]]---IMAGE_ASSETS = ImageTestAssets()-"""Singleton instance of {class}`ImageTestAssets`."""-VIDEO_ASSETS = VideoTestAssets()-"""Singleton instance of {class}`VideoTestAssets`."""-AUDIO_ASSETS = AudioTestAssets()-"""Singleton instance of {class}`AudioTestAssets`."""--[email protected](scope="function", autouse=True)-def cleanup_VLLM_USE_V1(monkeypatch):- """- The V1 oracle sets "VLLM_USE_V1" during loading. This means- that each invocation of a test change the env variable.-- If we touch "VLLM_USE_V1" with monkeypatch, then any changes- made during the test run by vLLM will be cleaned up.-- This fixture is used by every test.- """-- # If VLLM_USE_V1 is not set, set then delete. This will- # cause monkeypatch to clean up VLLM_USE_V1 upon exit- # if VLLM modifies the value of envs.VLLM_USE_V1.- if "VLLM_USE_V1" not in os.environ:- monkeypatch.setenv("VLLM_USE_V1", "")- monkeypatch.delenv("VLLM_USE_V1")--[email protected](autouse=True)-def init_test_http_connection():- # pytest_asyncio may use a different event loop per test- # so we need to make sure the async client is created anew- global_http_connection.reuse_client = False---def dist_init():- temp_file = tempfile.mkstemp()[1]- init_distributed_environment(- world_size=1,- rank=0,- distributed_init_method=f"file://{temp_file}",- local_rank=0,- backend="nccl",- )- initialize_model_parallel(1, 1)- yield- cleanup_dist_env_and_memory()---def should_do_global_cleanup_after_test(request) -> bool:- """Allow subdirectories to skip global cleanup by overriding this fixture.- This can provide a ~10x speedup for non-GPU unit tests since they don't need- to initialize torch.- """-- return not request.node.get_closest_marker("skip_global_cleanup")--[email protected](autouse=True)-def cleanup_fixture(should_do_global_cleanup_after_test: bool):- yield- if should_do_global_cleanup_after_test:- cleanup_dist_env_and_memory()--[email protected](autouse=True)-def dynamo_reset():- yield- torch._dynamo.reset()---def example_prompts() -> list[str]:- prompts = []- for filename in _TEST_PROMPTS:- prompts += _read_prompts(filename)- return prompts---def example_system_message() -> str:- with open(_SYS_MSG) as f:- return f.read()---class DecoderPromptType(Enum):- """For encoder/decoder models only."""-- CUSTOM = 1- NONE = 2- EMPTY_STR = 3---def example_long_prompts() -> list[str]:- prompts = []- for filename in _LONG_PROMPTS:- prompts += _read_prompts(filename)- return prompts--[email protected](scope="session")-def image_assets() -> ImageTestAssets:- return IMAGE_ASSETS--[email protected](scope="session")-def video_assets() -> VideoTestAssets:- return VIDEO_ASSETS--[email protected](scope="session")-def audio_assets() -> AudioTestAssets:- return AUDIO_ASSETS---_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature, dict)-_R = TypeVar("_R")---class HfRunner:- def get_default_device(self):- from vllm.platforms import current_platform-- return "cpu" if current_platform.is_cpu() else current_platform.device_type-- def wrap_device(self, x: _T, device: str | None = None) -> _T:- if x is None or isinstance(x, (bool,)):- return x-- if device is None:- device = self.device-- if isinstance(x, dict):- return {k: self.wrap_device(v, device) for k, v in x.items()}-- if hasattr(x, "device") and x.device.type == device:- return x-- return x.to(device)-- def __init__(- self,- model_name: str,- dtype: str = "auto",- *,- model_kwargs: dict[str, Any] | None = None,- trust_remote_code: bool = True,- is_sentence_transformer: bool = False,- is_cross_encoder: bool = False,- skip_tokenizer_init: bool = False,- auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,- # Set this to avoid hanging issue- default_torch_num_threads: int | None = None,- ) -> None:- init_ctx = (- nullcontext()- if default_torch_num_threads is None- else set_default_torch_num_threads(default_torch_num_threads)- )-- with init_ctx:- self._init(- model_name=model_name,- dtype=dtype,- model_kwargs=model_kwargs,- trust_remote_code=trust_remote_code,- is_sentence_transformer=is_sentence_transformer,- is_cross_encoder=is_cross_encoder,- skip_tokenizer_init=skip_tokenizer_init,- auto_cls=auto_cls,- )-- def _init(- self,- model_name: str,- dtype: str = "auto",- *,- model_kwargs: dict[str, Any] | None = None,- trust_remote_code: bool = True,- is_sentence_transformer: bool = False,- is_cross_encoder: bool = False,- skip_tokenizer_init: bool = False,- auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,- ) -> None:- model_name = maybe_model_redirect(model_name)- self.model_name = model_name-- self.config = AutoConfig.from_pretrained(- model_name,- trust_remote_code=trust_remote_code,- )- self.device = self.get_default_device()- self.dtype = dtype = _get_and_verify_dtype(- self.model_name,- self.config,- dtype=dtype,- is_pooling_model=is_sentence_transformer or is_cross_encoder,- )-- model_kwargs = model_kwargs if model_kwargs is not None else {}- model_kwargs.setdefault("dtype", dtype)-- if is_sentence_transformer:- # Lazy init required for AMD CI- from sentence_transformers import SentenceTransformer-- self.model = SentenceTransformer(- model_name,- device=self.device,- model_kwargs=model_kwargs,- trust_remote_code=trust_remote_code,- )- elif is_cross_encoder:- # Lazy init required for AMD CI- from sentence_transformers import CrossEncoder-- self.model = CrossEncoder(- model_name,- device=self.device,- automodel_args=model_kwargs,- trust_remote_code=trust_remote_code,- )- else:- model = auto_cls.from_pretrained(- model_name,- trust_remote_code=trust_remote_code,- **model_kwargs,- )-- # in case some unquantized custom models are not in same dtype- if getattr(model, "quantization_method", None) is None and any(- p.dtype != self.dtype for p in model.parameters()- ):- model = model.to(dtype=self.dtype)-- if (- getattr(model, "quantization_method", None) != "bitsandbytes"- and len({p.device for p in model.parameters()}) < 2- ):- model = model.to(device=self.device)-- self.model = model-- if not skip_tokenizer_init:- self.tokenizer = AutoTokenizer.from_pretrained(- model_name,- dtype=dtype,- trust_remote_code=trust_remote_code,- )-- # don't put this import at the top level- # it will call torch.cuda.device_count()- from transformers import AutoProcessor # noqa: F401-- self.processor = AutoProcessor.from_pretrained(- model_name,- dtype=dtype,- trust_remote_code=trust_remote_code,- )- if skip_tokenizer_init:- self.tokenizer = self.processor.tokenizer-- def get_inputs(- self,- prompts: list[str] | list[list[int]],- images: PromptImageInput | None = None,- videos: PromptVideoInput | None = None,- audios: PromptAudioInput | None = None,- ) -> list[BatchFeature | BatchEncoding | dict[str, torch.Tensor]]:- if images is not None:- assert len(prompts) == len(images)-- if videos is not None:- assert len(prompts) == len(videos)-- if audios is not None:- assert len(prompts) == len(audios)-- all_inputs: list[BatchFeature | BatchEncoding | dict[str, torch.Tensor]] = []- for i, prompt in enumerate(prompts):- if isinstance(prompt, str):- processor_kwargs: dict[str, Any] = {- "text": prompt,- "return_tensors": "pt",- }- if images is not None and (image := images[i]) is not None:- processor_kwargs["images"] = image- if videos is not None and (video := videos[i]) is not None:- processor_kwargs["videos"] = video- if audios is not None and (audio_inputs := audios[i]) is not None:- # HACK - not all processors take sampling_rate; we should- # clean this up in the future.- if len(audio_inputs) == 2:- audio, sr = audio_inputs- processor_kwargs["audio"] = audio- processor_kwargs["sampling_rate"] = sr- else:- processor_kwargs["audio"] = audio_inputs-- inputs = self.processor(**processor_kwargs)- if isinstance(inputs, BatchFeature):- inputs = inputs.to(dtype=self.dtype)- all_inputs.append(inputs)- else:- # check that prompt is (batched) list of integers (token ids)- if not is_list_of(prompt, typ=int, check="all"):- raise ValueError("Prompt must be a list of ints corresponding to the prompt token ids.")- # check that no multimodal input is provided- if images or videos or audios:- raise ValueError("When providing prompt token ids multimodal inputs are not supported.")- input_dict = {- "input_ids": torch.tensor(prompt, dtype=torch.long).unsqueeze(0),- }- all_inputs.append(input_dict)-- return all_inputs-- def get_prompt_embeddings(self, prompts: list[str]) -> list[torch.Tensor]:- all_inputs = self.get_inputs(prompts)- embeddings = []- for inputs in all_inputs:- input_ids = self.wrap_device(inputs)["input_ids"]- embedding = self.model.get_input_embeddings()(input_ids).squeeze(0)- embeddings.append(embedding)- return embeddings-- def classify(self, prompts: list[str]) -> list[str]:- # output is final logits- all_inputs = self.get_inputs(prompts)- outputs = []- problem_type = getattr(self.config, "problem_type", "")-- for inputs in all_inputs:- output = self.model(**self.wrap_device(inputs))- if problem_type == "regression":- logits = output.logits[0].tolist()- elif problem_type == "multi_label_classification":- logits = output.logits.sigmoid()[0].tolist()- else:- logits = output.logits.softmax(dim=-1)[0].tolist()- outputs.append(logits)-- return outputs-- def generate(- self,- prompts: list[str] | list[list[int]],- images: PromptImageInput | None = None,- videos: PromptVideoInput | None = None,- audios: PromptAudioInput | None = None,- **kwargs: Any,- ) -> list[tuple[list[list[int]], list[str]]]:- all_inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)-- outputs: list[tuple[list[list[int]], list[str]]] = []- for inputs in all_inputs:- output_ids = self.model.generate(- **self.wrap_device(inputs),- use_cache=True,- **kwargs,- )- output_str = self.processor.batch_decode(- output_ids,- skip_special_tokens=True,- clean_up_tokenization_spaces=False,- )- output_ids = output_ids.cpu().tolist()- outputs.append((output_ids, output_str))- return outputs-- def generate_greedy(- self,- prompts: list[str] | list[list[int]],- max_tokens: int,- images: PromptImageInput | None = None,- videos: PromptVideoInput | None = None,- audios: PromptAudioInput | None = None,- **kwargs: Any,- ) -> list[tuple[list[int], str]]:- outputs = self.generate(- prompts,- do_sample=False,- max_new_tokens=max_tokens,- images=images,- videos=videos,- audios=audios,- **kwargs,- )-- return [(output_ids[0], output_str[0]) for output_ids, output_str in outputs]-- def generate_beam_search(- self,- prompts: list[str],- beam_width: int,- max_tokens: int,- images: PromptImageInput | None = None,- videos: PromptVideoInput | None = None,- audios: PromptAudioInput | None = None,- ) -> list[tuple[list[list[int]], list[str]]]:- outputs = self.generate(- prompts,- do_sample=False,- max_new_tokens=max_tokens,- num_beams=beam_width,- num_return_sequences=beam_width,- images=images,- videos=videos,- audios=audios,- )-- for i in range(len(outputs)):- output_ids, output_str = outputs[i]- for j in range(len(output_ids)):- output_ids[j] = [x for x in output_ids[j] if x != self.tokenizer.pad_token_id]- outputs[i] = (output_ids, output_str)- return outputs-- def generate_greedy_logprobs(- self,- prompts: list[str],- max_tokens: int,- images: PromptImageInput | None = None,- videos: PromptVideoInput | None = None,- audios: PromptAudioInput | None = None,- **kwargs: Any,- ) -> list[list[torch.Tensor]]:- all_inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)-- all_logprobs: list[list[torch.Tensor]] = []- for inputs in all_inputs:- output = self.model.generate(- **self.wrap_device(inputs),- use_cache=True,- do_sample=False,- max_new_tokens=max_tokens,- output_hidden_states=True,- return_dict_in_generate=True,- **kwargs,- )- seq_logprobs = self._hidden_states_to_seq_logprobs(output.hidden_states)- all_logprobs.append(seq_logprobs)- return all_logprobs-- def _hidden_states_to_seq_logprobs(- self,- hidden_states: tuple[tuple[torch.Tensor, ...], ...],- ) -> list[torch.Tensor]:- output_embeddings = self.model.get_output_embeddings()-- seq_logprobs: list[torch.Tensor] = []- for _, hidden_state in enumerate(hidden_states):- last_hidden_states = hidden_state[-1][0]- logits = torch.matmul(- last_hidden_states.to(- device=output_embeddings.weight.device,- dtype=output_embeddings.weight.dtype,- ),- output_embeddings.weight.t(),- )- if getattr(output_embeddings, "bias", None) is not None:- logits += output_embeddings.bias.unsqueeze(0)- logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)- seq_logprobs.append(logprobs)-- return seq_logprobs-- def _hidden_states_to_logprobs(- self,- hidden_states: tuple[tuple[torch.Tensor, ...], ...],- num_logprobs: int | None,- ) -> tuple[list[dict[int, float]], int]:- seq_logprobs = self._hidden_states_to_seq_logprobs(hidden_states)- output_len = len(hidden_states)-- # convert to dict- seq_logprobs_lst: list[dict[int, float]] = []- for tok_idx, tok_logprobs in enumerate(seq_logprobs):- # drop prompt logprobs- if tok_idx == 0:- tok_logprobs = tok_logprobs[-1, :].reshape(1, -1)- topk = tok_logprobs.topk(num_logprobs)-- tok_logprobs_dct = {}- for token_id, logprob in zip(topk.indices[0], topk.values[0]):- tok_logprobs_dct[token_id.item()] = logprob.item()-- seq_logprobs_lst.append(tok_logprobs_dct)-- return (- seq_logprobs_lst,- output_len,- )-- def generate_greedy_logprobs_limit(- self,- prompts: list[str],- max_tokens: int,- num_logprobs: int | None,- images: PromptImageInput | None = None,- audios: PromptAudioInput | None = None,- videos: PromptVideoInput | None = None,- **kwargs: Any,- ) -> list[TokensTextLogprobs]:- all_inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)-- all_logprobs: list[list[dict[int, float]]] = []- all_output_ids: list[list[int]] = []- all_output_strs: list[str] = []-- for inputs in all_inputs:- output = self.model.generate(- **self.wrap_device(inputs),- use_cache=True,- do_sample=False,- max_new_tokens=max_tokens,- output_hidden_states=True,- return_dict_in_generate=True,- **kwargs,- )-- (- seq_logprobs_lst,- output_len,- ) = self._hidden_states_to_logprobs(output.hidden_states, num_logprobs)-- all_logprobs.append(seq_logprobs_lst)- seq_ids = output.sequences[0]- output_len = len(seq_logprobs_lst)- output_ids = seq_ids[-output_len:]- all_output_ids.append(output_ids.tolist())- all_output_strs.append(self.tokenizer.decode(output_ids))-- outputs = zip(all_output_ids, all_output_strs, all_logprobs)- return [(output_ids, output_str, output_logprobs) for output_ids, output_str, output_logprobs in outputs]-- def encode(self, prompts: list[str], *args, **kwargs) -> list[list[torch.Tensor]]:- return self.model.encode(prompts, *args, **kwargs)-- def predict(self, prompts: list[list[str]], *args, **kwargs) -> torch.Tensor:- return self.model.predict(prompts, *args, convert_to_tensor=True, **kwargs)-- def __enter__(self):- return self-- def __exit__(self, exc_type, exc_value, traceback):- del self.model- cleanup_dist_env_and_memory()--[email protected](scope="session")-def hf_runner():- return HfRunner---class VllmRunner:- """- The default value of some arguments have been modified from- {class}`~vllm.LLM` as follows:-- - `trust_remote_code`: Set to `True` instead of `False` for convenience.- - `seed`: Set to `0` instead of `None` for test reproducibility.- - `max_model_len`: Set to `1024` instead of `None` to reduce memory usage.- - `block_size`: To reduce memory usage, set default to `64` if on XPU- devices, otherwise default to `16`.- - `enable_chunked_prefill`: Set to `False` instead of `None` for- test reproducibility.- - `enforce_eager`: Set to `False` to test CUDA graph.- """-- def __init__(- self,- model_name: str,- runner: RunnerOption = "auto",- convert: ConvertOption = "auto",- tokenizer_name: str | None = None,- tokenizer_mode: str = "auto",- trust_remote_code: bool = True,- seed: int | None = 0,- max_model_len: int | None = 1024,- dtype: str = "auto",- disable_log_stats: bool = True,- tensor_parallel_size: int = 1,- block_size: int = 16 if not torch.xpu.is_available() else 64,- enable_chunked_prefill: bool | None = False,- swap_space: int = 4,- enforce_eager: bool | None = False,- # Set this to avoid hanging issue- default_torch_num_threads: int | None = None,- **kwargs,- ) -> None:- init_ctx = (- nullcontext()- if default_torch_num_threads is None- else set_default_torch_num_threads(default_torch_num_threads)- )-- if not kwargs.get("compilation_config", None):- # Note(@tdoublep): This is set to 4 because some tests (e.g., hybrid- # model tests) may set max_num_seqs=4. If min cudagraph_capture_size is- # set to larger than max_num_seqs, then it will lead to *no* graphs- # being captured which can trigger edge cases that we don't handle yet.- kwargs["compilation_config"] = {"cudagraph_capture_sizes": [4]}-- with init_ctx:- self.llm = LLM(- model=model_name,- runner=runner,- convert=convert,- tokenizer=tokenizer_name,- tokenizer_mode=tokenizer_mode,- trust_remote_code=trust_remote_code,- dtype=dtype,- seed=seed,- swap_space=swap_space,- enforce_eager=enforce_eager,- disable_log_stats=disable_log_stats,- tensor_parallel_size=tensor_parallel_size,- max_model_len=max_model_len,- block_size=block_size,- enable_chunked_prefill=enable_chunked_prefill,- **kwargs,- )-- def get_inputs(- self,- prompts: list[str] | list[torch.Tensor] | list[list[int]],- images: PromptImageInput | None = None,- videos: PromptVideoInput | None = None,- audios: PromptAudioInput | None = None,- ) -> list[dict[str, Any]]:- if any(x is not None and len(x) != len(prompts) for x in [images, videos, audios]):- raise ValueError("All non-None multimodal inputs must have the same length as prompts")-- inputs = list[dict[str, Any]]()- for i, prompt in enumerate(prompts):- prompt_dict = dict[str, Any]()- if isinstance(prompt, str):- prompt_dict["prompt"] = prompt- elif isinstance(prompt, list):- prompt_dict["prompt_token_ids"] = prompt- else:- prompt_dict["prompt_embeds"] = prompt-- multi_modal_data = dict[str, Any]()- if images is not None and (image := images[i]) is not None:- multi_modal_data["image"] = image- if videos is not None and (video := videos[i]) is not None:- multi_modal_data["video"] = video- if audios is not None and (audio := audios[i]) is not None:- multi_modal_data["audio"] = audio-- if multi_modal_data:- prompt_dict["multi_modal_data"] = multi_modal_data-- inputs.append(prompt_dict)-- return inputs-- def generate(- self,- prompts: list[str] | list[torch.Tensor] | list[list[int]],- sampling_params: SamplingParams,- images: PromptImageInput | None = None,- videos: PromptVideoInput | None = None,- audios: PromptAudioInput | None = None,- **kwargs: Any,- ) -> list[tuple[list[list[int]], list[str]]]:- inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)-- req_outputs = self.llm.generate(inputs, sampling_params=sampling_params, **kwargs)-- outputs: list[tuple[list[list[int]], list[str]]] = []- for req_output in req_outputs:- prompt_str = req_output.prompt- prompt_ids = req_output.prompt_token_ids- req_sample_output_ids: list[list[int]] = []- req_sample_output_strs: list[str] = []- for sample in req_output.outputs:- output_str = sample.text- output_ids = list(sample.token_ids)- req_sample_output_ids.append(prompt_ids + output_ids)- req_sample_output_strs.append((prompt_str or "") + output_str)- outputs.append((req_sample_output_ids, req_sample_output_strs))- return outputs-- @staticmethod- def _final_steps_generate_w_logprobs(- req_outputs: list[RequestOutput],- ) -> list[TokensTextLogprobsPromptLogprobs]:- outputs: list[TokensTextLogprobsPromptLogprobs] = []- for req_output in req_outputs:- assert len(req_output.outputs) > 0- for sample in req_output.outputs:- output_str = sample.text- output_ids = list(sample.token_ids)- output_logprobs = sample.logprobs- outputs.append((output_ids, output_str, output_logprobs, req_output.prompt_logprobs))- return outputs-- def generate_w_logprobs(- self,- prompts: list[str],- sampling_params: SamplingParams,- images: PromptImageInput | None = None,- audios: PromptAudioInput | None = None,- videos: PromptVideoInput | None = None,- **kwargs: Any,- ) -> list[TokensTextLogprobs] | list[TokensTextLogprobsPromptLogprobs]:- inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)-- req_outputs = self.llm.generate(inputs, sampling_params=sampling_params, **kwargs)-- toks_str_logsprobs_prompt_logprobs = self._final_steps_generate_w_logprobs(req_outputs)- # Omit prompt logprobs if not required by sampling params- return (- [x[0:-1] for x in toks_str_logsprobs_prompt_logprobs]- if sampling_params.prompt_logprobs is None- else toks_str_logsprobs_prompt_logprobs- )-- def generate_greedy(- self,- prompts: list[str] | list[torch.Tensor] | list[list[int]],- max_tokens: int,- images: PromptImageInput | None = None,- videos: PromptVideoInput | None = None,- audios: PromptAudioInput | None = None,- **kwargs: Any,- ) -> list[tuple[list[int], str]]:- greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)- outputs = self.generate(- prompts,- greedy_params,- images=images,- videos=videos,- audios=audios,- **kwargs,- )- return [(output_ids[0], output_str[0]) for output_ids, output_str in outputs]-- def generate_greedy_logprobs(- self,- prompts: list[str],- max_tokens: int,- num_logprobs: int | None,- num_prompt_logprobs: int | None = None,- images: PromptImageInput | None = None,- audios: PromptAudioInput | None = None,- videos: PromptVideoInput | None = None,- stop_token_ids: list[int] | None = None,- stop: list[str] | None = None,- **kwargs: Any,- ) -> list[TokensTextLogprobs] | list[TokensTextLogprobsPromptLogprobs]:- greedy_logprobs_params = SamplingParams(- temperature=0.0,- max_tokens=max_tokens,- logprobs=num_logprobs,- prompt_logprobs=num_prompt_logprobs,- stop_token_ids=stop_token_ids,- stop=stop,- )-- return self.generate_w_logprobs(- prompts,- greedy_logprobs_params,- images=images,- audios=audios,- videos=videos,- **kwargs,- )-- def generate_prompt_perplexity(self, prompts: list[str]) -> list[float]:- """- Return the perplexity score associated with generating the prompts-- :param prompts: list of prompts to score- :return: perplexity score of each prompt- """- outputs = self.generate_greedy_logprobs(prompts, max_tokens=1, num_logprobs=None, num_prompt_logprobs=0)-- perplexities = []- for output in outputs:- output = cast(TokensTextLogprobsPromptLogprobs, output)- token_data = cast(list[dict[int, Logprob] | None], output[3])- assert token_data[0] is None- token_log_probs = []- for token_data in token_data[1:]:- assert token_data is not None- assert len(token_data) == 1- token_log_prob = list(token_data.values())[0].logprob- token_log_probs.append(token_log_prob)-- perplexity = math.exp(-sum(token_log_probs) / len(token_log_probs))- perplexities.append(perplexity)-- return perplexities-- def generate_beam_search(- self,- prompts: list[str],- beam_width: int,- max_tokens: int,- images: PromptImageInput | None = None,- videos: PromptVideoInput | None = None,- audios: PromptAudioInput | None = None,- concurrency_limit: int | None = None,- ) -> list[tuple[list[list[int]], list[str]]]:- inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)-- outputs = self.llm.beam_search(- inputs,- BeamSearchParams(beam_width=beam_width, max_tokens=max_tokens),- concurrency_limit=concurrency_limit,- )- returned_outputs = []- for output in outputs:- token_ids = [x.tokens for x in output.sequences]- texts = [x.text for x in output.sequences]- returned_outputs.append((token_ids, texts))- return returned_outputs-- def classify(self, prompts: list[str]) -> list[list[float]]:- req_outputs = self.llm.classify(prompts)- return [req_output.outputs.probs for req_output in req_outputs]-- def embed(- self,- prompts: list[str],- images: PromptImageInput | None = None,- videos: PromptVideoInput | None = None,- audios: PromptAudioInput | None = None,- *args,- **kwargs,- ) -> list[list[float]]:- inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)-- req_outputs = self.llm.embed(inputs, *args, **kwargs)- return [req_output.outputs.embedding for req_output in req_outputs]-- def token_embed(self, prompts: list[str]) -> list[list[float]]:- req_outputs = self.llm.encode(prompts, pooling_task="token_embed")- return [req_output.outputs.data for req_output in req_outputs]-- def token_classify(self, prompts: list[str]) -> list[list[float]]:- req_outputs = self.llm.encode(prompts, pooling_task="token_classify")- return [req_output.outputs.data for req_output in req_outputs]-- def reward(self, prompts: list[str]) -> list[list[float]]:- req_outputs = self.llm.reward(prompts)- return [req_output.outputs.data for req_output in req_outputs]-- def score(- self,- text_1: list[str] | str,- text_2: list[str] | str,- *args,- **kwargs,- ) -> list[float]:- req_outputs = self.llm.score(text_1, text_2, *args, **kwargs)- return [req_output.outputs.score for req_output in req_outputs]-- def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:- return self.llm.apply_model(func)-- def get_llm(self) -> LLM:- return self.llm-- def __enter__(self):- return self-- def __exit__(self, exc_type, exc_value, traceback):- del self.llm- cleanup_dist_env_and_memory()--[email protected](scope="session")-def vllm_runner():- return VllmRunner---def temporary_enable_log_propagate():- import logging-- logger = logging.getLogger("vllm")- logger.propagate = True- yield- logger.propagate = False---def caplog_vllm(temporary_enable_log_propagate, caplog):- # To capture vllm log, we should enable propagate=True temporarily- # because caplog depends on logs propagated to the root logger.- yield caplog---def caplog_mp_fork():- """- This fixture enables capturing logs from a forked MP subprocess.- It should be used in conjunction with caplog_vllm.-- By default, subprocess logs do not go through the parent process.- We instead create a queue listener in the parent process which- forwards logs to the logger's other handlers, and add a QueueHandler- to the root logger. Forked subprocesses will inherit the root logger- and pass their messages to the queue, which the listener will forward- to the root logger, which can be captured by caplog.-- Note that this workaround only works for fork; with spawn, the subprocess- reinitializes logging and does not automatically inherit the queue.- We'd have to manually pass the queue to the subprocess at the spawn point.- See caplog_mp_spawn below.- """-- @contextlib.contextmanager- def ctx():- import logging.handlers- import multiprocessing as mp-- logger_queue: mp.Queue[logging.LogRecord] = mp.Queue()- logger = logging.getLogger()- handlers = logger.handlers-- # The listener works on a background thread, not inherited by the child.- queue_listener = logging.handlers.QueueListener(logger_queue, *handlers)- queue_listener.start()-- # Add queue handler after creating the listener to avoid cycle- logger.addHandler(logging.handlers.QueueHandler(logger_queue))- yield- queue_listener.stop()-- return ctx---class LogHolder:- def __init__(self):- self.text = None---def caplog_mp_spawn(tmp_path, monkeypatch):- """- This fixture enables capturing logs from a forked MP subprocess.- It does not require caplog_vllm (but it only contains logs from the child).-- By default, subprocess logs do not go through the parent process.- We instead add a FileHandler to the config so the spawned child process- writes its logs to a temp file.- In the parent, we read the file and return the contents.-- Note: this method could be extended to fork by either reconfiguring logging- in the parent or using a SocketHandler:- https://docs.python.org/3/howto/logging-cookbook.html#sending-and-receiving-logging-events-across-a-network # noqa: E501- """-- @contextlib.contextmanager- def ctx(level: int | str):- from vllm.logger import DEFAULT_LOGGING_CONFIG-- config_path = tmp_path / "vllm_logging_config.json"- log_path = tmp_path / "vllm.log"- log_holder = LogHolder()-- config = deepcopy(DEFAULT_LOGGING_CONFIG)- if envs.VLLM_LOGGING_CONFIG_PATH:- path = pathlib.Path(envs.VLLM_LOGGING_CONFIG_PATH)- assert path.exists()- config = json.loads(path.read_text())-- config["loggers"]["vllm"]["handlers"] += ["vllm_file"]- config["handlers"]["vllm_file"] = {- "class": "logging.FileHandler",- "formatter": "vllm",- "level": level,- "filename": log_path.as_posix(),- }-- config_path.write_text(json.dumps(config))-- with monkeypatch.context() as monkeypatch_ctx:- monkeypatch_ctx.setenv("VLLM_LOGGING_CONFIG_PATH", config_path.as_posix())- monkeypatch_ctx.setenv("VLLM_CONFIGURE_LOGGING", "1")- yield log_holder-- log_holder.text = log_path.read_text()-- return ctx--[email protected](scope="session")-def num_gpus_available():- """Get number of GPUs without initializing the CUDA context- in current process."""-- from vllm.platforms import current_platform-- return current_platform.device_count()---temp_dir = tempfile.gettempdir()-_dummy_opt_path = os.path.join(temp_dir, "dummy_opt")-_dummy_llava_path = os.path.join(temp_dir, "dummy_llava")-_dummy_gemma2_embedding_path = os.path.join(temp_dir, "dummy_gemma2_embedding")---def dummy_opt_path():- json_path = os.path.join(_dummy_opt_path, "config.json")- if not os.path.exists(_dummy_opt_path):- snapshot_download(- repo_id="facebook/opt-125m",- local_dir=_dummy_opt_path,- ignore_patterns=["*.bin", "*.bin.index.json", "*.pt", "*.h5", "*.msgpack"],- )- assert os.path.exists(json_path)- with open(json_path) as f:- config = json.load(f)- config["architectures"] = ["MyOPTForCausalLM"]- with open(json_path, "w") as f:- json.dump(config, f)- return _dummy_opt_path---def dummy_llava_path():- json_path = os.path.join(_dummy_llava_path, "config.json")- if not os.path.exists(_dummy_llava_path):- snapshot_download(- repo_id="llava-hf/llava-1.5-7b-hf",- local_dir=_dummy_llava_path,- ignore_patterns=[- "*.bin",- "*.bin.index.json",- "*.pt",- "*.h5",- "*.msgpack",- "*.safetensors",- ],- )- assert os.path.exists(json_path)- with open(json_path) as f:- config = json.load(f)- config["architectures"] = ["MyLlava"]- with open(json_path, "w") as f:- json.dump(config, f)- return _dummy_llava_path---def dummy_gemma2_embedding_path():- json_path = os.path.join(_dummy_gemma2_embedding_path, "config.json")- if not os.path.exists(_dummy_gemma2_embedding_path):- snapshot_download(- repo_id="BAAI/bge-multilingual-gemma2",- local_dir=_dummy_gemma2_embedding_path,- ignore_patterns=[- "*.bin",- "*.bin.index.json",- "*.pt",- "*.h5",- "*.msgpack",- "*.safetensors",- ],- )- assert os.path.exists(json_path)- with open(json_path) as f:- config = json.load(f)- config["architectures"] = ["MyGemma2Embedding"]- with open(json_path, "w") as f:- json.dump(config, f)- return _dummy_gemma2_embedding_path---# Add the flag `--optional` to allow run tests-# that are marked with @pytest.mark.optional-def pytest_addoption(parser):- parser.addoption("--optional", action="store_true", default=False, help="run optional test")---def pytest_collection_modifyitems(config, items):- if config.getoption("--optional"):- # --optional given in cli: do not skip optional tests- return- skip_optional = pytest.mark.skip(reason="need --optional option to run")- for item in items:- if "optional" in item.keywords:- item.add_marker(skip_optional)--[email protected](scope="session")-def cli_config_file():- """Return the path to the CLI config file."""- return os.path.join(_TEST_DIR, "config", "test_config.yaml")--[email protected](scope="session")-def cli_config_file_with_model():- """Return the path to the CLI config file with model."""- return os.path.join(_TEST_DIR, "config", "test_config_with_model.yaml")---class AssetHandler(http.server.BaseHTTPRequestHandler):- # _IMAGE_CACHE : Dict[str, bytes] = {}-- def log_message(self, *args, **kwargs):- pass-- def do_GET(self):- # Accepts paths like: /1280px-Venn_diagram_rgb.jpg- filename = self.path.lstrip("/")- if not filename or "." not in filename:- self.send_error(404, "Missing filename (expected /<name>.<ext>)")- return-- base, ext = filename.rsplit(".", 1)- ext = ext.lower()-- if ext not in ["jpg", "png"]:- self.send_error(404, f"Unsupported extension: .{ext}")- return-- try:- data = ImageAsset(base).read_bytes(ext=ext)- except Exception as e:- self.send_error(500, f"Failed to load asset: {ext} {base} {e} ")- return-- ctype, _ = mimetypes.guess_type(filename)- if ctype is None:- ctype = {"jpg": "image/jpg", "png": "image/png"}[ext]- self.send_response(200)- self.send_header("Content-Type", ctype)- self.send_header("Content-Length", str(len(data)))- self.end_headers()- self.wfile.write(data)---def _find_free_port() -> int:- with socket.socket() as s:- s.bind(("127.0.0.1", 0))- return s.getsockname()[1]---class LocalAssetServer:- address: str- port: int- server: http.server.ThreadingHTTPServer | None- thread: threading.Thread | None-- def __init__(self, address: str = "127.0.0.1") -> None:- self.address = address- self.port = -1- self.server = None- self.thread = None-- def __enter__(self):- self.port = _find_free_port()- self.server = http.server.ThreadingHTTPServer((self.address, self.port), AssetHandler)- self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)- self.thread.start()- return self-- def __exit__(self, exc_type, exc_value, traceback):- if self.server:- self.server.shutdown()- del self.server-- if self.thread:- self.thread.join()- del self.thread-- if exc_type is None:- return None-- return False-- @property- def base_url(self) -> str:- assert self.port is not None- return f"http://{self.address}:{self.port}"-- def url_for(self, name: str) -> str:- """e.g., name='RGBA_comp.png' -> 'http://127.0.0.1:PORT/RGBA_comp.png'"""- return f"{self.base_url}/{name}"-- def get_image_asset(self, name: str) -> Image.Image:- return fetch_image(self.url_for(name))--[email protected](scope="session")-def local_asset_server() -> Generator[LocalAssetServer, None, None]:- """- Starts a thread based HTTP server bound to 127.0.0.1 on a random free port.- The server currently servers images at:- http://127.0.0.1:<port>/<name>.<ext>- """- with LocalAssetServer() as srv:- yield srv---def image_url(request, local_asset_server) -> str:- # request.param is one of the IMAGE_ASSETS filenames- name = request.param- return local_asset_server.url_for(name)---def image_urls(request, local_asset_server) -> list[str]:- """Indirect fixture: takes a list of names, returns list of full URLs."""- names: list[str] = request.param- return [local_asset_server.url_for(name) for name in names]diff --git a/auto_round_extension/vllm_ext/tests/test_fp8kv.py b/auto_round_extension/vllm_ext/tests/test_fp8kv.pydeleted file mode 100644index 709308f9a..000000000--- a/auto_round_extension/vllm_ext/tests/test_fp8kv.py+++ /dev/null@@ -1,61 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-import pytest-import torch-from vllm.platforms import current_platform---def cuda_capability_at_least(major, minor):- device_capability = torch.cuda.get_device_capability()- return device_capability[0] >= major or (device_capability[0] == major and device_capability[1] >= minor)---MODELS = ["/home/yiliu7/workspace/auto-round/examples/Qwen2.5-0.5B-Instruct-ar-MXFP4-fp8"]--[email protected](autouse=True)-def set_vllm_ar_env(monkeypatch):- monkeypatch.setenv("VLLM_AR_MXFP4_MODULAR_MOE", "1")- monkeypatch.setenv("VLLM_MXFP4_PRE_UNPACK_TO_FP8", "1")- monkeypatch.setenv("VLLM_MXFP4_PRE_UNPACK_WEIGHTS", "0")- monkeypatch.setenv("VLLM_ENABLE_STATIC_MOE", "0")- monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "0")- monkeypatch.setenv("VLLM_ENABLE_AR_EXT", "1")- monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")- monkeypatch.setenv("VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION", "1")- monkeypatch.setenv("VLLM_ATTENTION_BACKEND", "FLASHINFER")--- not current_platform.is_cuda(),- reason="only supports CUDA backend.",-)- not cuda_capability_at_least(10, 0), reason="FP8 KV cache only supported on CUDA with compute capability >= 10.0"-)[email protected]("model", MODELS)-def test_auto_fp8_kv(vllm_runner, model):- with vllm_runner(- model,- # enforce_eager=True,- kv_cache_dtype="fp8",- gpu_memory_utilization=0.1,- ) as llm:- output = llm.generate_greedy(["The capital of France is"], max_tokens=8)- assert (- llm.llm.llm_engine.engine_core.engine_core.model_executor.driver_worker.worker.model_runner.kv_cache_dtype- == torch.uint8- ), f"Expected kv_cache_dtype to be torch.uint8, but got {llm.llm.llm_engine.engine_core.engine_core.model_executor.driver_worker.worker.model_runner.kv_cache_dtype}"- assert output- print(f"output is: {output[0][1]}")diff --git a/auto_round_extension/vllm_ext/tests/test_models.py b/auto_round_extension/vllm_ext/tests/test_models.pydeleted file mode 100644index faa783cde..000000000--- a/auto_round_extension/vllm_ext/tests/test_models.py+++ /dev/null@@ -1,48 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--import pytest-from vllm.platforms import current_platform--MODELS = [- # "/data5/yliu7/HF_HOME/unsloth-gpt-oss-20b-BF16-ar-MXFP4/"- # "/data5/yliu7/HF_HOME/Qwen2.5-0.5B-Instruct-test-FP8_STATIC-fp8kv/"- # "/data6/yiliu4/Qwen3-15B-A2B-Base-MXFP4",- # "/data6/yiliu4/Llama-3.2-1B-Instruct-MXFP4-fp8attention",- # "/data6/yiliu4/Llama-3.2-1B-Instruct-MXFP8"- "/storage/yiliu7/ar_vllm_ext/quantized_model_qwen_mxfp4",- "/storage/yiliu7/ar_vllm_ext/quantized_model_qwen_mxfp8",-]--[email protected](autouse=True)-def set_vllm_ar_env(monkeypatch):- monkeypatch.setenv("VLLM_AR_MXFP4_MODULAR_MOE", "1")- monkeypatch.setenv("VLLM_MXFP4_PRE_UNPACK_TO_FP8", "1")- monkeypatch.setenv("VLLM_MXFP4_PRE_UNPACK_WEIGHTS", "0")- monkeypatch.setenv("VLLM_ENABLE_STATIC_MOE", "0")- monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "0")- monkeypatch.setenv("VLLM_ENABLE_AR_EXT", "1")--- not current_platform.is_cuda(),- reason="only supports CUDA backend.",-)[email protected]("model", MODELS)-def test_auto_round(vllm_runner, model):- with vllm_runner(model, enforce_eager=True) as llm:- output = llm.generate_greedy(["The capital of France is"], max_tokens=8)- assert output- print(f"output is: {output[0][1]}")diff --git a/auto_round_extension/vllm_ext/utils.py b/auto_round_extension/vllm_ext/utils.pydeleted file mode 100644index db5a0dff1..000000000--- a/auto_round_extension/vllm_ext/utils.py+++ /dev/null@@ -1,167 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--from typing import Union--import torch--from auto_round.schemes import QuantizationScheme--E8M0_EXPONENT_BIAS = 127-E8M0_EXPONENT_NAN_VAL = 255---def get_scheme(quant_config, prefix: str):- # Check extra_config first- layer_schemes = quant_config.layer_schemes- # FIXME: make more robust- for name, scheme in layer_schemes.items():- if prefix.startswith(name):- return scheme- # If not found, use default- return quant_config.quant_scheme---def need_quantize(weight_bits: int) -> bool:- return weight_bits < 16---def _is_mxfp4_w4a4(scheme: QuantizationScheme):- # FIXME: below impl is incomplete- return scheme.bits == 4 and scheme.group_size == 32---def _is_mxfp8_w8a8(scheme: QuantizationScheme):- # FIXME: below impl is incomplete- return scheme.bits == 8 and scheme.group_size == 32---def get_fp_scale(scale_e8m0):- scale_e8m0 = scale_e8m0.view(torch.uint8)- s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS- # TODO(later): it would be nice if there was a way to do the 2^x operation- # in PyTorch without creating a tensor of twos- two = torch.full(s_offset.size(), 2.0, device=scale_e8m0.device)- # pow(two, s_offset) can be out of range of floating point formats.- # TODO(later): handle this for float16 if we decide to support float16- # scales.- s_fp = torch.pow(two, s_offset)-- # If a block exponent was 255, set values of that block to NaN- s_fp = torch.where(scale_e8m0 != E8M0_EXPONENT_NAN_VAL, s_fp, float("nan"))-- return s_fp---def _to_mx_rceil(- data_hp: torch.Tensor,- max_abs: torch.Tensor,- max_pos: float,-) -> tuple[torch.Tensor, torch.Tensor]:- """- A prototype implementation of MXFP scale factor derivation method described in- https://docs.nvidia.com/cuda/cublas/#d-block-quantization-- For Nvidia GPU with Blackwell+ architecture, the scale factor derivation method- could be accelerated by the `cvt.rp.satfinite.ue8m0x2.f32` instruction.-- Args:- data_hp: High precision data.- max_abs: Maximum absolute value for data_hp along specified dimension/block_size.- max_pos: The maximum value of the low precision data type.-- Returns:- exponent: The biased exponent with dtype E8M0 in uint8 container.- data_lp: The targeted low precision data, in high precision container- (requires cast to low precision data type).- """- descale = max_abs / max_pos- # TODO: nan/inf needs to be set for any value- # of nan/inf in input not just amax.- exponent = torch.where(- torch.isnan(descale),- 0xFF, # Handle biased exponent for nan- # NOTE: descale < (torch.finfo(torch.float32).smallest_normal / 2) is handled through clamping- (- torch.clamp(- torch.ceil(torch.log2(descale)),- min=-E8M0_EXPONENT_BIAS,- max=E8M0_EXPONENT_BIAS,- )- + E8M0_EXPONENT_BIAS- ).to(torch.uint8),- )-- descale_fp = torch.where(exponent == 0, 1.0, torch.exp2(E8M0_EXPONENT_BIAS - exponent.to(torch.float32)))-- # scale and saturated cast the data elements to max of target dtype- data_lp = torch.clamp(data_hp * descale_fp.unsqueeze(1), min=-1 * max_pos, max=max_pos)- return exponent, data_lp---def to_mx_fp8e4m3(- data_hp: torch.Tensor,- elem_dtype: Union[torch.dtype, str],- block_size: int,-):- """- Takes a high precision tensor and converts to MX scale and raw data, in- naive layout (scale and raw data are separate tensors).- """-- assert data_hp.dtype in (- torch.bfloat16,- torch.float,- ), f"{data_hp.dtype} is not supported yet"- # TODO(future PR): consider supporting padding- data_hp = data_hp.contiguous()-- # calculate the scale in e8m0 format- orig_shape = data_hp.shape- data_hp = data_hp.reshape(-1, block_size)-- # find max value of the data- # Note: this only implements the `minimally supported` version of- # https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf- # section 6.3.- max_abs = torch.amax(torch.abs(data_hp), 1)-- # Set X to be the largest power-of-two less than or equal to- # max_abs(v), divided by the largest power of two representable- # in the element data type, and get the mbits at the same time- assert elem_dtype == torch.float8_e4m3fn, f"only float8_e4m3fn is supported now, got {elem_dtype}"-- max_pos = torch.finfo(torch.float8_e4m3fn).max # 448.0- # TODO: add support for other rounding modes- scale_e8m0_biased, data_lp = _to_mx_rceil(data_hp, max_abs, max_pos)-- data_lp = data_lp.to(elem_dtype)- # need to reshape at the end to help inductor fuse things- data_lp = data_lp.reshape(orig_shape)-- # scale_e8m0_biased = scale_e8m0_biased.view(torch.float8_e8m0fnu)- return scale_e8m0_biased, data_lp---def down_size(size):- assert size[-1] % 2 == 0, f"{size} last dim not divisible by two"- return (*size[:-1], size[-1] // 2)---def pack_uint4(uint8_data: torch.Tensor) -> torch.Tensor:- # converting to uint8 for operations- shape = uint8_data.shape- assert shape[-1] % 2 == 0- uint8_data = uint8_data.contiguous().view(-1)- return (uint8_data[::2] << 4 | uint8_data[1::2]).view(down_size(shape))diff --git a/auto_round_extension/vllm_ext/vllm_oot_patches.py b/auto_round_extension/vllm_ext/vllm_oot_patches.pydeleted file mode 100644index cf229e596..000000000--- a/auto_round_extension/vllm_ext/vllm_oot_patches.py+++ /dev/null@@ -1,131 +0,0 @@-# Copyright (c) 2025 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--import vllm-import vllm.envs as envs-from vllm.logger import init_logger--logger = init_logger(__name__)---def update_flashinfer_workspace_buffer_size(new_size: int):- import vllm.v1.attention.backends.flashinfer as flashinfer_backend-- flashinfer_backend.WORKSPACE_BUFFER_SIZE = new_size- logger.info(f"Updated FlashInfer WORKSPACE_BUFFER_SIZE to {new_size}")---if envs.VLLM_AR_FLASHINFER_WORKSPACE_BUFFER_SIZE > 0:- update_flashinfer_workspace_buffer_size(envs.VLLM_AR_FLASHINFER_WORKSPACE_BUFFER_SIZE)---def oot_maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None:- """Remap the name of FP8 k/v_scale parameters.-- This function handles the remapping of FP8 k/v_scale parameter names.- It detects if the given name ends with a suffix and attempts to remap- it to the expected name format in the model. If the remapped name is not- found in the params_dict, a warning is printed and None is returned.-- Args:- name (str): The original loaded checkpoint parameter name.- params_dict (dict): Dictionary containing the model's named parameters.-- Returns:- str: The remapped parameter name if successful, or the original name- if no remapping is needed.- None: If the remapped name is not found in params_dict.- """-- if name.endswith(".kv_scale"):- logger.warning_once(- "DEPRECATED. Found kv_scale in the checkpoint. "- "This format is deprecated in favor of separate k_scale and "- "v_scale tensors and will be removed in a future release. "- "Functionally, we will remap kv_scale to k_scale and duplicate "- "k_scale to v_scale"- )- # NOTE: we remap the deprecated kv_scale to k_scale- remapped_name = name.replace(".kv_scale", ".attn.k_scale")- if remapped_name not in params_dict:- logger.warning_once(- "Found kv_scale in the checkpoint (e.g. %s), but not found the expected name in the model (e.g. %s). kv_scale is not loaded.", # noqa: E501- name,- remapped_name,- )- return None- return remapped_name-- if any("mla_attn" in key for key in params_dict):- attn_str = "mla_attn.mla_attn"- logger.debug_once(- f"Found mla_attn with k_scale and v_scale in " f"the checkpoint, using {attn_str} as attn_str"- )- else:- attn_str = "attn"- # Define scale name mapping patterns in order of precedence- scale_mapping_patterns = [- # AR format:- # .self_attn.{q,k,v}_scale ->- # .attn.{attn_str}.{q,k,v}_scale- (- r"\.self_attn\.([qkv])_scale$",- rf".self_attn.{attn_str}.\1_scale",- ),- (- r"\.self_attn\.([kv])_proj\.([kv])_scale$",- rf".self_attn.{attn_str}.\2_scale",- ),- # ModelOpt format: .self_attn.{k,v}_proj.{k,v}_scale ->- # .self_attn.attn.{k,v}_scale- (- r"\.self_attn\.([kv])_proj\.([kv])_scale$",- rf".self_attn.{attn_str}.\2_scale",- ),- # QKV proj format: .self_attn.qkv_proj.{k,v}_scale ->- # .self_attn.attn.{k,v}_scale- (r"\.self_attn\.qkv_proj\.([kv])_scale$", r".self_attn.attn.\1_scale"),- # Qwen3 MoE format: .self_attn.qkqkv_proj.{k,v}_scale ->- # .self_attn.attn.{k,v}_scale- (r"\.self_attn\.qkqkv_proj\.([kv])_scale$", r".self_attn.attn.\1_scale"),- # Default format: .{k,v}_scale -> .attn.{k,v}_scale- (r"\.([kv])_scale$", r".attn.\1_scale"),- ]-- # Check if name ends with k_scale or v_scale- if name.endswith((".k_scale", ".v_scale", ".q_scale")):- import regex as re-- for pattern, replacement in scale_mapping_patterns:- if re.search(pattern, name):- remapped_name = re.sub(pattern, replacement, name)- if remapped_name not in params_dict:- scale_type = name.split(".")[-1]- logger.warning_once(- "Found %s in the checkpoint (e.g. %s), but not found the expected name in the model (e.g. %s). %s is not loaded.", # noqa: E501- scale_type,- name,- remapped_name,- scale_type,- )- return None- return remapped_name-- # If there were no matches, return the untouched param name- return name---import vllm.model_executor.model_loader.weight_utils as vllm_weight_utils--vllm_weight_utils.maybe_remap_kv_scale_name = oot_maybe_remap_kv_scale_namediff --git a/test/unit/test_cpu/utils/test_model_free_utils.py b/test/unit/test_cpu/utils/test_model_free_utils.pyindex 6d4cf5577..f4e36613a 100644--- a/test/unit/test_cpu/utils/test_model_free_utils.py+++ b/test/unit/test_cpu/utils/test_model_free_utils.py@@ -50,6 +50,9 @@ from auto_round.utils.model_free_utils import (_quantize_single_tensor,_quantize_weight_mxfp,_validate_auto_scheme_options,+ dequant_mx_fp8,+ dequant_mxfp4,+ get_fp_scale,)from auto_round.utils.model_free_utils import (handle_model_type_low_precision_source_tensors as _handle_model_type_low_precision_source_tensors,@@ -906,6 +909,24 @@ class TestHandleMXFPSourceTensors:assert "layer.weight_scale" not in out+class TestMXFPDequantization:+ def test_get_fp_scale(self):+ scale = torch.tensor([126, 127, 128], dtype=torch.uint8)+ assert torch.equal(get_fp_scale(scale), torch.tensor([0.5, 1.0, 2.0]))++ def test_dequant_mxfp4(self):+ packed = torch.tensor([[0x10, 0x32]], dtype=torch.uint8)+ scale = torch.tensor([[128]], dtype=torch.uint8)+ result = dequant_mxfp4(packed, scale, "fp4_e2m1", block_size=4, target_dtype=torch.float32)+ assert torch.equal(result, torch.tensor([[0.0, 1.0, 2.0, 3.0]]))++ def test_dequant_mx_fp8(self):+ weight = torch.tensor([[1.0, 2.0, 3.0, 4.0]], dtype=torch.float8_e4m3fn)+ scale = torch.tensor([[126, 128]], dtype=torch.uint8)+ result = dequant_mx_fp8(weight, scale, block_size=2, target_dtype=torch.float32)+ assert torch.equal(result, torch.tensor([[0.5, 1.0, 6.0, 8.0]]))++# ===========================================================================# AutoScheme helpers# ===========================================================================