Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Remove deprecated files (#2373)

chen, suyuefd077878e6ff
diff --git a/auto_round/experimental/qmodules/fp4_utils.py b/auto_round/experimental/qmodules/fp4_utils.py
index aa9cae421..f2dafa1c8 100644
--- a/auto_round/experimental/qmodules/fp4_utils.py
+++ b/auto_round/experimental/qmodules/fp4_utils.py
@@ -45,6 +45,19 @@ def get_e2m1_tensor(device):
return _DEVICE_E2M1_TENSORS[device_str]
+def pack_fp4_to_uint8(x: torch.Tensor) -> torch.Tensor:
+ """Pack pairs of E2M1 values into uint8, with the first value in the low nibble."""
+ rows, columns = x.shape
+ assert columns % 2 == 0, f"Expected an even number of columns, got {columns}"
+
+ values = get_e2m1_tensor(x.device)
+ magnitude = torch.argmin(torch.abs(torch.abs(x).unsqueeze(-1) - values), dim=-1)
+ indices = magnitude + (torch.signbit(x).to(torch.long) << 3)
+ pairs = indices.reshape(-1, 2)
+ packed = (pairs[:, 0] | (pairs[:, 1] << 4)).to(torch.uint8)
+ return packed.reshape(rows, columns // 2)
+
+
def unpack_fp4_from_uint8(
a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16
) -> torch.Tensor:
diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.py
index 5a96f9efa..0c321c34e 100644
--- a/auto_round/utils/model_free_utils.py
+++ b/auto_round/utils/model_free_utils.py
@@ -1654,6 +1654,57 @@ def _modelopt_nvfp4_bases_from_weight_map(weight_map: dict[str, str]) -> set[str
}
+def get_fp_scale(scale_e8m0: torch.Tensor) -> torch.Tensor:
+ """Convert biased E8M0 exponents to floating-point scales."""
+ exponent = scale_e8m0.view(torch.uint8).to(torch.int16) - 127
+ return torch.pow(2.0, exponent.to(torch.float32))
+
+
+def dequant_mxfp4(
+ data_lp: torch.Tensor,
+ scale_e8m0: torch.Tensor,
+ elem_dtype: str,
+ block_size: int,
+ target_dtype: torch.dtype,
+ scale_dtype: torch.dtype | None = None,
+ return_scale: bool = False,
+) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
+ """Dequantize packed MXFP4 data to ``target_dtype``."""
+ from auto_round.experimental.qmodules.fp4_utils import unpack_fp4_from_uint8
+
+ original_shape = data_lp.shape
+ last_dim = original_shape[-1]
+ data_lp = data_lp.reshape(-1, last_dim)
+ result_shape = original_shape[:-1] + (last_dim * 2,)
+ assert data_lp.is_contiguous(), f"Data must be contiguous, got {data_lp.stride()}"
+ assert elem_dtype == "fp4_e2m1", f"Expected 'fp4_e2m1', got {elem_dtype}"
+
+ rows, half_columns = data_lp.shape
+ data_hp = unpack_fp4_from_uint8(data_lp, rows, half_columns * 2, dtype=target_dtype)
+ data_hp = data_hp.reshape(-1, block_size)
+
+ if scale_dtype is None:
+ scale_dtype = target_dtype
+ scale = get_fp_scale(scale_e8m0).reshape(-1, 1).to(scale_dtype)
+ if return_scale:
+ return data_hp.reshape(result_shape), scale
+
+ return (data_hp * scale).reshape(result_shape)
+
+
+def dequant_mx_fp8(
+ weight_fp8: torch.Tensor,
+ scale_e8m0: torch.Tensor,
+ block_size: int,
+ target_dtype: torch.dtype,
+) -> torch.Tensor:
+ """Dequantize MXFP8 data using its per-block E8M0 scales."""
+ original_shape = weight_fp8.shape
+ weight = weight_fp8.to(torch.bfloat16).reshape(-1, block_size)
+ scale = get_fp_scale(scale_e8m0).reshape(-1, 1)
+ return (weight * scale).reshape(original_shape).to(target_dtype)
+
+
def _hydrate_and_clean_modelopt_nvfp4_aux(
raw_tensors: dict[str, torch.Tensor],
shard_path: str,
@@ -1773,21 +1824,17 @@ def _dequant_mxfp_tensors(
) -> dict[str, torch.Tensor]:
"""Dequantize llm-compressor MXFP8 / MXFP4 weight tensors to bfloat16.
- Detection is purely by *name* and *dtype*, reusing the dequant kernels in
- :mod:`auto_round_extension.vllm_ext`:
+ Detection is purely by *name* and *dtype*:
* ``<layer>.weight`` (``float8_e4m3fn``) + ``<layer>.weight_scale`` → MXFP8,
- dequantized via :func:`~auto_round_extension.vllm_ext.mxfp8_qdq_utils.dequant_mx_fp8`.
+ dequantized via :func:`dequant_mx_fp8`.
* ``<layer>.weight_packed`` (``uint8``) + ``<layer>.weight_scale`` → MXFP4,
- dequantized via :func:`~auto_round_extension.vllm_ext.mxfp4_qdq_utils.to_dtype`.
+ dequantized via :func:`dequant_mxfp4`.
The dequantized weight is written back under ``<layer>.weight`` and the
scale (and any ``weight_packed``) tensor is removed, so the downstream RTN
path can requantize the layer to the requested target scheme.
"""
- from auto_round_extension.vllm_ext.mxfp4_qdq_utils import to_dtype
- from auto_round_extension.vllm_ext.mxfp8_qdq_utils import dequant_mx_fp8
-
# Tuple layout: (layer_name, weight_key, scale_key, bits)
entries = _collect_mxfp_source_entries(raw_tensors)
@@ -1831,14 +1878,14 @@ def _dequant_mxfp_tensors(
shard_prefix=shard_prefix,
op_name="MXFP dequant",
tensor_label=layer_name,
- on_device=lambda weight=weight, scale=scale: to_dtype(
+ on_device=lambda weight=weight, scale=scale: dequant_mxfp4(
data_lp=weight.view(torch.uint8).contiguous().to(dequant_device, non_blocking=True),
scale_e8m0=scale.to(dequant_device, non_blocking=True),
elem_dtype="fp4_e2m1",
block_size=32,
target_dtype=torch.bfloat16,
).to("cpu"),
- on_cpu=lambda weight=weight, scale=scale: to_dtype(
+ on_cpu=lambda weight=weight, scale=scale: dequant_mxfp4(
data_lp=weight.view(torch.uint8).contiguous(),
scale_e8m0=scale,
elem_dtype="fp4_e2m1",
diff --git a/auto_round/utils/weight_handler.py b/auto_round/utils/weight_handler.py
index 5e2248218..abfca6ba4 100755
--- a/auto_round/utils/weight_handler.py
+++ b/auto_round/utils/weight_handler.py
@@ -701,8 +701,7 @@ class MXFP4Handler(WeightTypeHandler):
layer = layer.to(device)
- # MXFP4 dequantization using to_dtype from mxfp4_qdq_utils
- from auto_round_extension.vllm_ext.mxfp4_qdq_utils import to_dtype
+ from auto_round.utils.model_free_utils import dequant_mxfp4
# Get packed weight and scale from layer
# MXFP4 weights are stored as packed uint8 in weight_packed
@@ -716,7 +715,7 @@ class MXFP4Handler(WeightTypeHandler):
raise ValueError("MXFP4 layer must have weight_packed and weight_scale attributes")
# Dequantize using to_dtype function
- dq_weight = to_dtype(
+ dq_weight = dequant_mxfp4(
data_lp=weight_packed,
scale_e8m0=weight_scale,
elem_dtype="fp4_e2m1",
@@ -790,8 +789,7 @@ class MXFP8Handler(WeightTypeHandler):
layer = layer.to(device)
- # MXFP8 dequantization using dequant_mx_fp8 from mxfp8_qdq_utils
- from auto_round_extension.vllm_ext.mxfp8_qdq_utils import dequant_mx_fp8
+ from auto_round.utils.model_free_utils import dequant_mx_fp8
# Get weight and scale from layer
weight = layer.weight
diff --git a/auto_round_extension/ark/auto_round_kernel/mxfp4_hadamard.py b/auto_round_extension/ark/auto_round_kernel/mxfp4_hadamard.py
index e6ab5d2d9..6cc57f4d5 100644
--- a/auto_round_extension/ark/auto_round_kernel/mxfp4_hadamard.py
+++ b/auto_round_extension/ark/auto_round_kernel/mxfp4_hadamard.py
@@ -29,12 +29,12 @@ Frozen MVP contract (Phase 0):
is ``2 ** (e8m0 - 127)`` (standard E8M0, always a power of two);
* ``q = y * 2 ** -(e8m0 - 127)`` is encoded as ``signbit(q) << 3 | magnitude``
with FP4 (E2M1) magnitude levels ``0, 0.5, 1, 1.5, 2, 3, 4, 6`` and the
- nearest-even thresholds of ``vllm_ext/fp4_utils.py::cast_to_fp4``;
+ nearest-even thresholds of the frozen ``_E2M1_THRESHOLDS`` table below;
* **zero is canonicalised**: whenever the magnitude index is 0 the sign bit is
dropped, so the code is ``0x0`` and never ``0x8`` (negative zero). See
"Canonical zero" below for why this rule is required rather than optional;
* two codes share one byte, the even element occupying the low nibble
- (identical to ``vllm_ext/fp4_utils.py::pack_fp4_to_uint8``);
+ (identical to ``auto_round.experimental.qmodules.fp4_utils::pack_fp4_to_uint8``);
* an all-zero group produces ``e8m0 = 0`` and all-zero codes;
* NaN/Inf are outside the supported input domain. The reference always rejects
them; the XPU entry point only does so under ``check_finite=True``, because
@@ -92,7 +92,7 @@ the mathematical value rather than of the rounding residue. Both this reference
and the kernel therefore drop the sign bit whenever the magnitude index is 0.
This is a deliberate, documented deviation from
-``vllm_ext/fp4_utils.py::pack_fp4_to_uint8``, which applies ``signbit``
+``auto_round.experimental.qmodules.fp4_utils::pack_fp4_to_uint8``, which applies ``signbit``
unconditionally: that helper encodes already-clean dequantized values, where a
negative zero can only appear if the caller supplied one.
"""
@@ -122,7 +122,7 @@ SUPPORTED_HADAMARD_DIMS = tuple(GROUP_SIZE * (1 << i) for i in range(MAX_LANES_P
E2M1_VALUES = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
# (threshold, is_closed_interval, magnitude_index), first match wins, mirroring
-# the early-out ladder of ``cast_to_fp4`` including its alternating ``<=`` / ``<``
+# the early-out ladder of ``_E2M1_THRESHOLDS'`` including its alternating ``<=`` / ``<``
# boundary operators. ``_encode_fp4`` walks the table in reverse with
# ``torch.where``, so each earlier entry overrides every later one.
_E2M1_THRESHOLDS = (
diff --git a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard.hpp b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard.hpp
index 00b34c6f8..cabe45157 100644
--- a/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard.hpp
+++ b/auto_round_extension/ark/auto_round_kernel/wrapper/include/xpu_mxfp4_hadamard.hpp
@@ -117,7 +117,7 @@ class XpuMxfp4Hadamard {
// FP4 (E2M1) magnitude levels: 0, 0.5, 1, 1.5, 2, 3, 4, 6.
// Thresholds and boundary comparison operators are taken verbatim from the
- // PyTorch reference (auto_round_extension/vllm_ext/fp4_utils.py::cast_to_fp4)
+ // PyTorch reference (auto_round_extension/ark/auto_round_kernel/mxfp4_hadamard.py::_E2M1_THRESHOLDS)
// so that the kernel is bit-exact with it.
//
// Written branchless, as a descending count rather than the reference's
diff --git a/auto_round_extension/ark/test/test_mxfp4_hadamard.py b/auto_round_extension/ark/test/test_mxfp4_hadamard.py
index 218fadcb7..9c19f35ee 100644
--- a/auto_round_extension/ark/test/test_mxfp4_hadamard.py
+++ b/auto_round_extension/ark/test/test_mxfp4_hadamard.py
@@ -105,8 +105,8 @@ class TestReferenceContract:
supported D by :class:`TestHadamardDimReferenceParity`.
"""
- def test_packing_matches_vllm_ext_fp4_utils(self):
- fp4_utils = pytest.importorskip("auto_round_extension.vllm_ext.fp4_utils")
+ def test_packing_matches_fp4_utils(self):
+ fp4_utils = pytest.importorskip("auto_round.experimental.qmodules.fp4_utils")
torch.manual_seed(0)
codes = torch.randint(0, 16, (4, 64), dtype=torch.uint8)
# Build the FP4 values the codes represent, then pack them with the
diff --git a/auto_round_extension/vllm_ext/README.md b/auto_round_extension/vllm_ext/README.md
deleted file mode 100644
index 1f0cb832f..000000000
--- a/auto_round_extension/vllm_ext/README.md
+++ /dev/null
@@ -1,12 +0,0 @@
-- Build and Install vLLM
-
-```
-git clone --branch fused-moe-ar https://github.com/yiliu30/vllm-fork.git
-VLLM_USE_PRECOMPILED=1 pip install --editable . -vvv
-```
-
-
-- Enable vLLM-Ext at Runtime
-```bash
-VLLM_ENABLE_AR_EXT=1 vllm serve ...
-```
\ No newline at end of file
diff --git a/auto_round_extension/vllm_ext/__init__.py b/auto_round_extension/vllm_ext/__init__.py
deleted file mode 100644
index d05b3c731..000000000
--- a/auto_round_extension/vllm_ext/__init__.py
+++ /dev/null
@@ -1,27 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# ==---------------------------------------------------------------------------==
-# Apply the extension
-# ==---------------------------------------------------------------------------==
-
-
-def apply():
- import auto_round_extension.vllm_ext.auto_round_ext
- import auto_round_extension.vllm_ext.envs_ext
- import auto_round_extension.vllm_ext.vllm_oot_patches
-
- print("*****************************************************************************")
- print("* !!! VLLM_ENABLE_AR_EXT is set to 1, applying auto_round_vllm_extension *")
- print("*****************************************************************************")
diff --git a/auto_round_extension/vllm_ext/auto_round_ext.py b/auto_round_extension/vllm_ext/auto_round_ext.py
deleted file mode 100644
index 408d0d6a6..000000000
--- a/auto_round_extension/vllm_ext/auto_round_ext.py
+++ /dev/null
@@ -1,133 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from copy import deepcopy
-from typing import Any
-
-import torch
-from vllm.logger import init_logger
-from vllm.model_executor.layers.fused_moe import FusedMoE
-from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
-from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig as _BaseAutoRoundConfig
-
-from auto_round.schemes import QuantizationScheme
-from auto_round_extension.vllm_ext.quant_method_linear import AutoRoundQuantLinearMethod
-from auto_round_extension.vllm_ext.quant_method_moe import AutoRoundMoEMethod
-
-logger = init_logger(__name__)
-
-
-class AutoRoundExtensionConfig(_BaseAutoRoundConfig):
- SUPPORTED_DTYPES = _BaseAutoRoundConfig.SUPPORTED_DTYPES.union({"mx_fp"})
- SUPPORTED_FORMATS = _BaseAutoRoundConfig.SUPPORTED_FORMATS.union({"auto_round:llm_compressor"})
-
- def __init__(
- self,
- weight_bits: int,
- group_size: int,
- sym: bool = True,
- packing_format: str = "auto_round:auto_gptq",
- block_name_to_quantize: str | list[str] | None = None,
- extra_config: dict[str, Any] | None = None,
- data_type: str = "int",
- backend: str = "auto",
- act_bits: int = None,
- act_data_type: int = None,
- act_dynamic: bool = None,
- act_group_size: int = None,
- act_sym: bool = None,
- ) -> None:
- super().__init__(
- weight_bits=weight_bits,
- group_size=group_size,
- sym=sym,
- packing_format=packing_format,
- block_name_to_quantize=block_name_to_quantize,
- extra_config=extra_config,
- data_type=data_type,
- backend=backend,
- )
- self.act_bits = act_bits
- self.act_data_type = act_data_type
- self.act_dynamic = act_dynamic
- self.act_group_size = act_group_size
- self.act_sym = act_sym
-
- def get_quant_method(self, layer: torch.nn.Module, prefix: str):
- # FIXME: (yi) make it compatible with `AutoRoundConfig`
- from vllm.attention.layer import Attention, MLAAttention
-
- if isinstance(layer, (Attention, MLAAttention)):
-
- from auto_round_extension.vllm_ext.kv_cache import AutoRoundKVCacheMethod, AutoRoundKVCacheMethodForMLA
-
- if isinstance(layer, MLAAttention):
- return AutoRoundKVCacheMethodForMLA(self)
-
- return AutoRoundKVCacheMethod(self)
- if isinstance(layer, FusedMoE):
- quant_method = AutoRoundMoEMethod.get_moe_method(self, layer, prefix)
- return quant_method
- elif isinstance(layer, LinearBase):
- return AutoRoundQuantLinearMethod.get_method(self, layer, prefix)
- else:
- return None
-
- @staticmethod
- def _parse_quant_scheme(config: dict):
- quant_scheme_attrs = QuantizationScheme.get_attributes()
- filter_config = {key: value for key, value in config.items() if key in quant_scheme_attrs}
- quant_scheme = QuantizationScheme.from_dict(filter_config)
- return quant_scheme
-
- @classmethod
- def from_config(cls, config: dict[str, Any]) -> _BaseAutoRoundConfig:
- ar_config = cls(
- weight_bits=cls.get_from_keys(config, ["bits"]),
- group_size=cls.get_from_keys(config, ["group_size"]),
- sym=cls.get_from_keys(config, ["sym"]),
- packing_format=cls.get_from_keys_or(config, ["packing_format"], "auto_round:auto_gptq"),
- block_name_to_quantize=cls.get_from_keys_or(
- config, ["block_name_to_quantize", "to_quant_block_names"], None
- ),
- extra_config=cls.get_from_keys_or(config, ["extra_config"], None),
- data_type=cls.get_from_keys_or(config, ["data_type"], "int"),
- backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),
- act_bits=cls.get_from_keys(config, ["act_bits"]),
- act_data_type=cls.get_from_keys(config, ["act_data_type"]),
- act_dynamic=cls.get_from_keys(config, ["act_dynamic"]),
- act_group_size=cls.get_from_keys(config, ["act_group_size"]),
- act_sym=cls.get_from_keys(config, ["act_sym"]),
- )
- # TODO: (yi) refine below implementation
- quant_scheme = AutoRoundExtensionConfig._parse_quant_scheme(config)
- layer_schemes = {}
- layer_schemes = {} # ensure dict
- extra_config = getattr(ar_config, "extra_config", None)
- if extra_config is not None:
- for layer_name, layer_config in extra_config.items():
- layer_scheme = deepcopy(quant_scheme)
- for key, val in layer_config.items():
- if key in layer_scheme.keys() and val != layer_scheme[key]:
- layer_scheme[key] = val
- layer_schemes[layer_name] = layer_scheme
- ar_config.quant_scheme = quant_scheme
- ar_config.layer_schemes = layer_schemes
- return ar_config
-
-
-# Patch vLLM’s AutoRoundConfig at import time
-import vllm.model_executor.layers.quantization.auto_round as _auto_round_module
-
-_auto_round_module.AutoRoundConfig = AutoRoundExtensionConfig
diff --git a/auto_round_extension/vllm_ext/envs_ext.py b/auto_round_extension/vllm_ext/envs_ext.py
deleted file mode 100644
index eb251ccef..000000000
--- a/auto_round_extension/vllm_ext/envs_ext.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-from typing import Any, Callable
-
-from vllm.logger import init_logger
-
-logger = init_logger(__name__)
-
-# Define extra environment variables
-extra_environment_variables: dict[str, Callable[[], Any]] = {
- "VLLM_MXFP4_PRE_UNPACK_WEIGHTS": lambda: os.getenv("VLLM_MXFP4_PRE_UNPACK_WEIGHTS", "0") in ("1", "true", "True"),
- "VLLM_MXFP4_PRE_UNPACK_TO_FP8": lambda: os.getenv("VLLM_MXFP4_PRE_UNPACK_TO_FP8", "1") in ("1", "true", "True"),
- "VLLM_ENABLE_STATIC_MOE": lambda: os.getenv("VLLM_ENABLE_STATIC_MOE", "0") in ("1", "true", "True"),
- "VLLM_AR_MXFP4_MODULAR_MOE": lambda: os.getenv("VLLM_AR_MXFP4_MODULAR_MOE", "1") in ("1", "true", "True"),
- "VLLM_AR_POST_PROCESS_GPTOSS": lambda: os.getenv("VLLM_AR_POST_PROCESS_GPTOSS", "0") in ("1", "true", "True"),
- "VLLM_AR_FLASHINFER_WORKSPACE_BUFFER_SIZE": lambda: int(os.getenv("VLLM_AR_FLASHINFER_WORKSPACE_BUFFER_SIZE", "0")),
-}
-# Add the extra environment variables to vllm.envs
-import vllm.envs as envs
-from vllm.envs import environment_variables
-
-# Merge the environment variables
-all_environment_variables = {**environment_variables, **extra_environment_variables}
-
-
-for name, value_fn in extra_environment_variables.items():
- setattr(envs, name, value_fn())
-
-logger.warning_once(f"Added extra environment variables: {list(extra_environment_variables.keys())}")
diff --git a/auto_round_extension/vllm_ext/fp4_utils.py b/auto_round_extension/vllm_ext/fp4_utils.py
deleted file mode 100644
index fb7b6e935..000000000
--- a/auto_round_extension/vllm_ext/fp4_utils.py
+++ /dev/null
@@ -1,113 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Optional
-
-import torch
-
-# Module-level device tensor cache to fix cuda graph issue
-_DEVICE_E2M1_TENSORS = {}
-
-# Constants for FP4 values (E2M1 format)
-_E2M1_VALUES = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
-
-
-def get_e2m1_tensor(device):
- """Get device-specific E2M1 lookup tensor, creating it if needed."""
- device_str = str(device)
- if device_str not in _DEVICE_E2M1_TENSORS:
- _DEVICE_E2M1_TENSORS[device_str] = torch.tensor(_E2M1_VALUES, dtype=torch.float32, device=device)
- return _DEVICE_E2M1_TENSORS[device_str]
-
-
-def pack_fp4_to_uint8(x: torch.Tensor) -> torch.Tensor:
- m, n = x.shape
- device = x.device
-
- # Create lookup table for FP4 values to indices
- # Map the absolute values to 0-7 indices
- kE2M1 = get_e2m1_tensor(x.device)
-
- # Find closest valid FP4 value index for each element
- abs_x = torch.abs(x)
- abs_diff_x = torch.abs(abs_x.unsqueeze(-1) - kE2M1) # [m, n, 8]
- abs_indices = torch.argmin(abs_diff_x, dim=-1) # [m, n]
-
- # Apply sign bit (bit 3) to get final 4-bit representation
- indices = abs_indices + (torch.signbit(x).to(torch.long) << 3)
-
- # Reshape to prepare for packing pairs of values
- indices = indices.reshape(-1)
-
- # Handle odd length by padding if necessary
- # assert indices.numel() % 2 != 0, f"Expected even number of elements, got {indices.numel()}"
-
- # Reshape to pair consecutive elements
- indices = indices.reshape(-1, 2)
-
- # Pack pairs of 4-bit values into 8-bit values
- packed = (indices[:, 0] | (indices[:, 1] << 4)).to(torch.uint8)
-
- return packed.reshape(m, n // 2)
-
-
-def unpack_fp4_from_uint8(
- a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16
-) -> torch.Tensor:
- """
- Unpacks uint8 values into fp4. Each uint8 consists of two fp4 values
- (i.e. first four bits correspond to one fp4 value, last four correspond to a
- consecutive fp4 value). The bits represent an index, which are mapped to an fp4
- value.
-
- :param a: tensor to unpack
- :param m: original dim 0 size of the unpacked tensor
- :param n: original dim 1 size of the unpacked tensor
- :param dtype: dense dtype to cast the unpacked tensor to
- """
- assert a.dtype == torch.uint8, f"expected uint8, got {a.dtype}"
-
- # Vectorized nibble processing
- a_flat = a.flatten()
- high = (a_flat & 0xF0) >> 4 # Upper nibbles
- low = a_flat & 0x0F # Lower nibbles
-
- # Combine nibbles for batch processing
- combined = torch.stack((low, high), dim=1).flatten()
-
- # Vectorized sign and magnitude extraction
- signs = (combined & 0x08).to(torch.bool) # Sign bits
- abs_vals = (combined & 0x07).to(torch.long) # Magnitude indices
-
- # Device-aware lookup and sign application
- kE2M1 = get_e2m1_tensor(a.device)
-
- values = kE2M1[abs_vals] * torch.where(signs, -1.0, 1.0)
-
- # Reshape to final form
- return values.reshape(m, n).to(dtype=dtype)
-
-
-def cast_to_fp4(x):
- sign = torch.sign(x)
- x = torch.abs(x)
- x[(x >= 0.0) & (x <= 0.25)] = 0.0
- x[(x > 0.25) & (x < 0.75)] = 0.5
- x[(x >= 0.75) & (x <= 1.25)] = 1.0
- x[(x > 1.25) & (x < 1.75)] = 1.5
- x[(x >= 1.75) & (x <= 2.5)] = 2.0
- x[(x > 2.5) & (x < 3.5)] = 3.0
- x[(x >= 3.5) & (x <= 5.0)] = 4.0
- x[x > 5.0] = 6.0
- return x * sign
diff --git a/auto_round_extension/vllm_ext/kv_cache.py b/auto_round_extension/vllm_ext/kv_cache.py
deleted file mode 100644
index 9b8572b06..000000000
--- a/auto_round_extension/vllm_ext/kv_cache.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-from typing import TYPE_CHECKING, Any, Literal, Optional, cast
-
-import torch
-from vllm.logger import init_logger
-from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
-
-logger = init_logger(__name__)
-
-
-class AutoRoundKVCacheMethod(BaseKVCacheMethod):
- """
- Supports loading kv-cache scaling factors from compressed-tensors
- checkpoints.
- """
-
- def __init__(self, quant_config):
- self.validate_kv_cache_scheme(quant_config)
- super().__init__(quant_config)
-
- @staticmethod
- def validate_kv_cache_scheme(quant_config):
- # FIXME: parse from quant_config
- return True
-
-
-class AutoRoundKVCacheMethodForMLA(BaseKVCacheMethod):
- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
- super().process_weights_after_loading(layer)
- k_scale = layer._k_scale
- v_scale = layer._v_scale
- aligned_kv_scale = max(k_scale, v_scale)
- layer._k_scale.data.fill_(aligned_kv_scale)
- layer._v_scale.data.fill_(aligned_kv_scale)
- layer._k_scale_float = aligned_kv_scale.item()
- layer._v_scale_float = aligned_kv_scale.item()
- logger.warning_once("Aligned MLA k/v scales for MLA")
diff --git a/auto_round_extension/vllm_ext/linear_impl_mxfp4.py b/auto_round_extension/vllm_ext/linear_impl_mxfp4.py
deleted file mode 100644
index e544fdbbf..000000000
--- a/auto_round_extension/vllm_ext/linear_impl_mxfp4.py
+++ /dev/null
@@ -1,124 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# SPDX-License-Identifier: Apache-2.0
-from typing import Callable, Optional
-
-import torch
-import vllm.envs as envs
-from torch.nn.parameter import Parameter
-from vllm.logger import init_logger
-from vllm.model_executor.parameter import GroupQuantScaleParameter, ModelWeightParameter, PerTensorScaleParameter
-from vllm.platforms import current_platform
-
-from auto_round_extension.vllm_ext.mxfp4_qdq_utils import (
- dequant_mxfp4_to_fp8,
- mxfp4_gemm_with_unpacked_weight,
- run_mxfp4_emulations,
-)
-
-logger = init_logger(__name__)
-
-__all__ = ["AutoRoundMXFP4LinearImpl"]
-
-from auto_round_extension.vllm_ext.quant_impl import AutoRoundQuantImpl
-
-
-class AutoRoundMXFP4LinearImpl(AutoRoundQuantImpl):
- def __init__(self, quant_scheme):
- self.quant_scheme = quant_scheme
- self.group_size = 32
-
- @classmethod
- def get_min_capability(cls) -> int:
- if envs.VLLM_USE_MXFP4_CT_EMULATIONS:
- return 80
- return 100
-
- def create_weights(
- self,
- layer: torch.nn.Module,
- output_partition_sizes: list[int],
- input_size_per_partition: int,
- params_dtype: torch.dtype,
- weight_loader: Callable,
- **kwargs,
- ):
- output_size_per_partition = sum(output_partition_sizes)
- layer.logical_widths = output_partition_sizes
- layer.input_size_per_partition = input_size_per_partition
- layer.output_size_per_partition = output_size_per_partition
-
- # Weight
- weight = ModelWeightParameter(
- data=torch.empty(sum(output_partition_sizes), input_size_per_partition // 2, dtype=torch.uint8),
- input_dim=1,
- output_dim=0,
- weight_loader=weight_loader,
- )
- layer.register_parameter("weight_packed", weight)
-
- # Per Group Weight Scale
- weight_scale = GroupQuantScaleParameter(
- data=torch.empty(
- sum(output_partition_sizes),
- input_size_per_partition // self.group_size,
- # dtype=torch.uint8,
- dtype=torch.uint8,
- ),
- input_dim=1,
- output_dim=0,
- weight_loader=weight_loader,
- )
-
- layer.register_parameter("weight_scale", weight_scale)
-
- def process_weights_after_loading(self, layer) -> None:
- # FIXME: may dequant to bf16
- if envs.VLLM_MXFP4_PRE_UNPACK_TO_FP8:
- weight_fp8, scale_bf16 = dequant_mxfp4_to_fp8(
- data_lp=layer.weight_packed,
- scale_e8m0=layer.weight_scale,
- )
- del layer.weight_packed
- del layer.weight_scale
- layer.weight_packed = None
- layer.weight_scale = None
- layer.register_parameter(
- "weight_unpacked_fp8",
- torch.nn.Parameter(
- weight_fp8,
- requires_grad=False,
- ),
- )
- layer.register_parameter(
- "weight_scale_bf16",
- torch.nn.Parameter(
- scale_bf16,
- requires_grad=False,
- ),
- )
- else:
- raise NotImplementedError("Only VLLM_MXFP4_PRE_UNPACK_TO_FP8 is supported now.")
-
- def apply_weights(
- self, layer: torch.nn.Module, x: torch.Tensor, bias: Optional[torch.Tensor] = None
- ) -> torch.Tensor:
- out = mxfp4_gemm_with_unpacked_weight(
- x=x,
- weight_fp8=layer.weight_unpacked_fp8,
- weight_scale_bf16=layer.weight_scale_bf16,
- bias=bias,
- )
- return out
diff --git a/auto_round_extension/vllm_ext/linear_impl_mxfp8.py b/auto_round_extension/vllm_ext/linear_impl_mxfp8.py
deleted file mode 100644
index cf1382054..000000000
--- a/auto_round_extension/vllm_ext/linear_impl_mxfp8.py
+++ /dev/null
@@ -1,112 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Callable, Optional
-
-import torch
-import vllm.envs as envs
-from vllm.model_executor.parameter import (
- GroupQuantScaleParameter,
- ModelWeightParameter,
- PerTensorScaleParameter,
-)
-
-from auto_round_extension.vllm_ext.mxfp8_qdq_utils import dequant_mx_fp8, quant_mx_fp8
-from auto_round_extension.vllm_ext.quant_impl import AutoRoundQuantImpl
-
-
-class AutoRoundMXFP8LinearImpl(AutoRoundQuantImpl):
- def __init__(self, quant_scheme):
- self.quant_scheme = quant_scheme
- self.strategy = "TENSOR_GROUP"
- self.out_dtype = torch.get_default_dtype()
- self.group_size = 32
-
- @classmethod
- def get_min_capability(cls) -> int:
- return 80
-
- def process_weights_after_loading(self, layer) -> None:
- return
-
- def create_weights(
- self,
- layer: torch.nn.Module,
- output_partition_sizes: list[int],
- input_size_per_partition: int,
- params_dtype: torch.dtype,
- weight_loader: Callable,
- **kwargs,
- ):
- # maybe_create_device_identity()
-
- output_size_per_partition = sum(output_partition_sizes)
- layer.logical_widths = output_partition_sizes
-
- # WEIGHT
- weight = ModelWeightParameter(
- data=torch.empty(
- output_size_per_partition,
- input_size_per_partition,
- dtype=torch.float8_e4m3fn,
- ),
- input_dim=1,
- output_dim=0,
- weight_loader=weight_loader,
- )
- layer.register_parameter("weight", weight)
-
- # WEIGHT SCALE
- # Per Group Weight Scale
- weight_scale = GroupQuantScaleParameter(
- data=torch.empty(
- sum(output_partition_sizes),
- input_size_per_partition // self.group_size,
- dtype=torch.uint8, # E8M0 for MXFP8 scale
- ),
- input_dim=1,
- output_dim=0,
- weight_loader=weight_loader,
- )
- layer.register_parameter("weight_scale", weight_scale)
-
- def apply_weights(
- self,
- layer: torch.nn.Module,
- x: torch.Tensor,
- bias: Optional[torch.Tensor] = None,
- ) -> torch.Tensor:
- # dequant weight
- weight = layer.weight
- weight_scale = layer.weight_scale
- dequnat_weight = dequant_mx_fp8(
- weight_fp8=weight.data,
- scale_e8m0=weight_scale.data,
- block_size=self.group_size,
- target_dtype=x.dtype,
- )
- dequnat_weight = dequnat_weight.to(x.dtype)
- # if not envs.VLLM_AR_MXFP8_DISABLE_INPUT_QDQ:
- # q-dq input
- x_scale, x_quant = quant_mx_fp8(x)
- dequant_x = dequant_mx_fp8(
- weight_fp8=x_quant,
- scale_e8m0=x_scale,
- block_size=self.group_size,
- target_dtype=x.dtype,
- )
- x = dequant_x.to(x.dtype)
-
- out = x @ dequnat_weight.t()
- return out.to(x.dtype) + (bias if bias is not None else 0)
diff --git a/auto_round_extension/vllm_ext/moe_impl_mxfp4.py b/auto_round_extension/vllm_ext/moe_impl_mxfp4.py
deleted file mode 100644
index 75e298acd..000000000
--- a/auto_round_extension/vllm_ext/moe_impl_mxfp4.py
+++ /dev/null
@@ -1,592 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Callable, Optional, Union
-
-import torch
-import torch.nn.functional as F
-import vllm.envs as envs
-from vllm.distributed import get_tensor_model_parallel_rank
-from vllm.logger import init_logger
-from vllm.model_executor.layers.fused_moe import (
- FusedMoE,
- FusedMoEConfig,
- FusedMoeWeightScaleSupported,
-)
-from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
-from vllm.model_executor.utils import set_weight_attrs
-
-import auto_round_extension.vllm_ext.mxfp4_qdq_utils as mxfp4_utils
-from auto_round_extension.vllm_ext.mxfp4_qdq_utils import (
- dequant_mxfp4_to_fp8,
- mxfp4_gemm_with_unpacked_weight,
- run_mxfp4_emulations,
-)
-from auto_round_extension.vllm_ext.quant_method_moe import AutoRoundMoEMethod
-
-logger = init_logger(__name__)
-
-
-def apply_act(local_w1_out: torch.Tensor, local_w3_out: torch.Tensor, activation: str) -> torch.Tensor:
- if activation == "silu":
- act_fn = F.silu
- w13_out = act_fn(local_w1_out) * local_w3_out
- elif activation == "swigluoai":
- limit = 7.0
- alpha = 1.702
- local_w1_out = local_w1_out.clamp(min=None, max=limit)
- local_w3_out = local_w3_out.clamp(min=-limit, max=limit)
- glu = (local_w1_out) * F.sigmoid(local_w1_out * alpha)
- w13_out = (local_w3_out + 1) * glu
- else:
- raise NotImplementedError(f"Activation {activation} is not implemented.")
- return w13_out
-
-
-class AutoRoundMoEMethodMXFp4Impl(AutoRoundMoEMethod):
- def __init__(
- self,
- quant_config: "AutoRoundConfig", # type: ignore # noqa E501
- moe: FusedMoEConfig,
- ):
- super().__init__(moe)
- self.use_marlin = False
- self.group_size = 32
- self.quant_config = quant_config
- self.has_bias = self.moe.has_bias
- self.mask_weights_buffer = None
- self.experts_mask_buffer = None
- self.num_all_tokens_threshold = 16 * 1024
- # self.num_all_tokens_threshold = 0
-
- def create_weights(
- self,
- layer: torch.nn.Module,
- num_experts: int,
- hidden_size: int,
- intermediate_size_per_partition: int,
- params_dtype: torch.dtype,
- **extra_weight_attrs,
- ):
- layer.num_experts = num_experts
- layer.params_dtype = params_dtype
-
- w13_weight = torch.nn.Parameter(
- torch.empty(
- num_experts,
- 2 * intermediate_size_per_partition,
- # 2 fp4 items are packed in the input dimension
- hidden_size // 2,
- requires_grad=False,
- dtype=torch.uint8,
- ),
- requires_grad=False,
- )
- layer.register_parameter("w13_weight_packed", w13_weight)
- set_weight_attrs(w13_weight, extra_weight_attrs)
-
- w2_weight = torch.nn.Parameter(
- torch.empty(
- num_experts,
- hidden_size,
- # 2 fp4 items are packed in the input dimension
- intermediate_size_per_partition // 2,
- dtype=torch.uint8,
- ),
- requires_grad=False,
- )
- layer.register_parameter("w2_weight_packed", w2_weight)
- set_weight_attrs(w2_weight, extra_weight_attrs)
-
- # Weight Scales
- w13_weight_scale = torch.nn.Parameter(
- torch.empty(
- num_experts,
- 2 * intermediate_size_per_partition,
- # 2 fp4 items are packed in the input dimension
- hidden_size // self.group_size,
- dtype=torch.uint8,
- ),
- requires_grad=False,
- )
- layer.register_parameter("w13_weight_scale", w13_weight_scale)
- extra_weight_attrs.update({"quant_method": FusedMoeWeightScaleSupported.GROUP.value})
- set_weight_attrs(w13_weight_scale, extra_weight_attrs)
-
- w2_weight_scale = torch.nn.Parameter(
- torch.empty(
- num_experts,
- hidden_size,
- # 2 fp4 items are packed in the input dimension
- intermediate_size_per_partition // self.group_size,
- dtype=torch.uint8,
- ),
- requires_grad=False,
- )
- layer.register_parameter("w2_weight_scale", w2_weight_scale)
- extra_weight_attrs.update({"quant_method": FusedMoeWeightScaleSupported.GROUP.value})
- set_weight_attrs(w2_weight_scale, extra_weight_attrs)
-
- E = num_experts
- H = hidden_size
- IN = intermediate_size_per_partition
- if self.has_bias:
- # TODO: yiliu30 use the dtype in CK
- bias_dtype = torch.bfloat16
- w13_bias = torch.nn.Parameter(torch.zeros(E, 2 * IN, dtype=bias_dtype), requires_grad=False)
- layer.register_parameter("w13_bias", w13_bias)
- set_weight_attrs(w13_bias, extra_weight_attrs)
-
- w2_bias = torch.nn.Parameter(
- torch.zeros(E, H, dtype=bias_dtype),
- requires_grad=False,
- )
- layer.register_parameter("w2_bias", w2_bias)
- set_weight_attrs(w2_bias, extra_weight_attrs)
-
- def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> Optional[FusedMoEQuantConfig]:
- if envs.VLLM_AR_MXFP4_MODULAR_MOE:
- from vllm.model_executor.layers.fused_moe.config import (
- ocp_mx_moe_quant_config,
- )
-
- if envs.VLLM_MXFP4_PRE_UNPACK_TO_FP8:
- self.input_dtype = "mxfp8_e4m3"
- self.weight_dtype = "mxfp8_e4m3"
- ocp_config = ocp_mx_moe_quant_config(
- quant_dtype=self.input_dtype,
- weight_dtype=self.weight_dtype,
- w1_scale=layer.w13_weight_scale,
- w2_scale=layer.w2_weight_scale,
- a1_scale=None,
- a2_scale=None,
- w1_bias=layer.w13_bias if self.has_bias else None,
- w2_bias=layer.w2_bias if self.has_bias else None,
- block_shape=None,
- )
- from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
- OCP_MX_DTYPES,
- OCP_MX_Scheme,
- )
-
- ocp_config.ocp_mx_scheme = OCP_MX_Scheme.from_quant_dtype(
- input_dtype="mxfp8_e4m3",
- weight_dtype="mxfp8_e4m3",
- original_dtype="mxfp4",
- )
- logger.warning_once(f"Set OCP_MX_Scheme to {ocp_config.ocp_mx_scheme}")
- return ocp_config
-
- self.input_dtype = "mxfp4"
- self.weight_dtype = "mxfp4"
- return ocp_mx_moe_quant_config(
- quant_dtype=self.input_dtype,
- weight_dtype=self.weight_dtype,
- w1_scale=layer.w13_weight_scale,
- w2_scale=layer.w2_weight_scale,
- a1_scale=None,
- a2_scale=None,
- w1_bias=layer.w13_bias if self.has_bias else None,
- w2_bias=layer.w2_bias if self.has_bias else None,
- block_shape=None,
- )
- return None
-
- def _dequant_fp4_to_fp8(self, layer):
- weight_name_lst = ["w13_weight", "w2_weight"]
-
- for weight_name_prefix in weight_name_lst:
- weight_name = f"{weight_name_prefix}_packed"
- weight = getattr(layer, weight_name)
- weight_scale_name = f"{weight_name_prefix}_scale"
- weight_scale = getattr(layer, weight_scale_name)
- new_weight_name = f"{weight_name_prefix}_unpacked"
- new_scale_name = weight_scale_name
- num_experts, _, _ = weight.shape
- unpacked_weight_lst = []
- scale_list = []
- for expert_index in range(num_experts):
- weight_fp8, scale_bf16 = dequant_mxfp4_to_fp8(
- data_lp=weight[expert_index],
- scale_e8m0=weight_scale[expert_index],
- )
-
- unpacked_weight_lst.append(weight_fp8)
- scale_list.append(scale_bf16)
- unpacked_weight_fp8 = torch.stack(unpacked_weight_lst, dim=0)
- scale_bf16 = torch.stack(scale_list, dim=0)
- assert unpacked_weight_fp8.shape[0] == num_experts, (
- f"Expected {num_experts} unpacked weights, got " f"{unpacked_weight_fp8.shape[0]}"
- )
- delattr(layer, weight_name)
- delattr(layer, weight_scale_name)
- layer.register_parameter(
- new_weight_name,
- torch.nn.Parameter(
- unpacked_weight_fp8,
- requires_grad=False,
- ),
- )
- layer.register_parameter(
- new_scale_name,
- torch.nn.Parameter(
- scale_bf16,
- requires_grad=False,
- ),
- )
-
- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
- logger.debug(f"Processing weights after loading for layer: {layer._prefix}")
- if envs.VLLM_ENABLE_STATIC_MOE:
- if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:
- self._dequant_fp4_to_fp8(layer)
- return
- elif envs.VLLM_AR_MXFP4_MODULAR_MOE:
- if envs.VLLM_MXFP4_PRE_UNPACK_TO_FP8:
- self._dequant_fp4_to_fp8(layer)
- return
-
- def revert_interleaved_bias(bias):
- """
- Convert from blocked bias format to interleaved format.
-
- Args:
- bias: Tensor of shape [E, intermediate_size*2] where the first half contains
- w1 biases and second half contains w3 biases for each expert
-
- Returns:
- Tensor with interleaved w1 and w3 biases
- """
- # loaded bias[0]: [e0_w1_bias_0, e0_w1_bias_1, ..., e0_w3_bias_0, e0_w3_bias_1...]
- # Expected bias[0]: [e0_w1_bias_0, e0_w3_bias_0, e0_w1_bias_1, e0_w3_bias_1...]
- # The expected bias order is used in triton kernel https://github.com/vllm-project/vllm/pull/22508
- revert_bias = torch.zeros_like(bias, device=bias.device)
- E, two_IN = bias.shape
-
- # Verify the shape is as expected
- if two_IN % 2 != 0:
- raise ValueError(f"Expected even number of bias elements, got {two_IN}")
-
- revert_bias[..., ::2] = bias[..., : two_IN // 2]
- revert_bias[..., 1::2] = bias[..., two_IN // 2 :]
-
- return revert_bias
-
- if self.has_bias:
- if envs.VLLM_AR_POST_PROCESS_GPTOSS:
- w13_bias_swapped = revert_interleaved_bias(layer.w13_bias)
- layer.w13_bias.data.copy_(w13_bias_swapped)
-
- if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:
-
- w1 = layer.w13_weight_packed
- w1_scale = layer.w13_weight_scale
- w1 = mxfp4_utils.to_dtype(
- data_lp=w1,
- scale_e8m0=w1_scale,
- elem_dtype="fp4_e2m1",
- block_size=32,
- target_dtype=torch.bfloat16,
- )
-
- def revert_interleaved_w1(w1):
- new_w1 = torch.zeros_like(w1)
- E, N, H = w1.shape
- new_w1[:, ::2, :] = w1[:, : N // 2, :]
- new_w1[:, 1::2, :] = w1[:, N // 2 :, :]
- return new_w1
-
- if envs.VLLM_AR_POST_PROCESS_GPTOSS:
- w1 = revert_interleaved_w1(w1)
-
- w1_scale = None
- w2 = layer.w2_weight_packed
- w2_scale = layer.w2_weight_scale
- w2 = mxfp4_utils.to_dtype(
- data_lp=w2,
- scale_e8m0=w2_scale,
- elem_dtype="fp4_e2m1",
- block_size=32,
- target_dtype=torch.bfloat16,
- )
- w2_scale = None
- del layer.w13_weight_packed
- del layer.w13_weight_scale
- del layer.w2_weight_packed
- del layer.w2_weight_scale
- layer.w13_weight_scale = None
- layer.w2_weight_scale = None
- layer.register_parameter(
- "w13_weight_unpacked",
- torch.nn.Parameter(
- w1,
- requires_grad=False,
- ),
- )
- layer.register_parameter(
- "w2_weight_unpacked",
- torch.nn.Parameter(
- w2,
- requires_grad=False,
- ),
- )
-
- else:
- raise NotImplementedError("process_weights_after_loading is not implemented for now.")
-
- @torch.inference_mode()
- def apply(
- self,
- layer: torch.nn.Module,
- x: torch.Tensor,
- router_logits: torch.Tensor,
- top_k: int,
- renormalize: bool,
- use_grouped_topk: bool = False,
- topk_group: Optional[int] = None,
- num_expert_group: Optional[int] = None,
- global_num_experts: int = -1,
- expert_map: Optional[torch.Tensor] = None,
- custom_routing_function: Optional[Callable] = None,
- scoring_func: str = "softmax",
- routed_scaling_factor: float = 1.0,
- e_score_correction_bias: Optional[torch.Tensor] = None,
- apply_router_weight_on_input: bool = False,
- activation: str = "silu",
- enable_eplb: bool = False,
- expert_load_view: Optional[torch.Tensor] = None,
- logical_to_physical_map: Optional[torch.Tensor] = None,
- logical_replica_count: Optional[torch.Tensor] = None,
- ) -> Union[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
- topk_weights, topk_ids, _ = FusedMoE.select_experts(
- hidden_states=x,
- router_logits=router_logits,
- use_grouped_topk=use_grouped_topk,
- top_k=top_k,
- renormalize=renormalize,
- topk_group=topk_group,
- num_expert_group=num_expert_group,
- custom_routing_function=custom_routing_function,
- scoring_func=scoring_func,
- e_score_correction_bias=e_score_correction_bias,
- )
- assert self.fused_experts is None
-
- # There are three implementations:
-
- if envs.VLLM_AR_MXFP4_MODULAR_MOE:
- from vllm.model_executor.layers.fused_moe import fused_experts
-
- if envs.VLLM_MXFP4_PRE_UNPACK_TO_FP8:
- w1 = layer.w13_weight_unpacked
- w2 = layer.w2_weight_unpacked
- out = fused_experts(
- x,
- w1,
- w2,
- topk_weights=topk_weights,
- topk_ids=topk_ids,
- inplace=True,
- activation=activation,
- global_num_experts=global_num_experts,
- apply_router_weight_on_input=apply_router_weight_on_input,
- expert_map=expert_map,
- quant_config=self.moe_quant_config,
- )
- return out
- if envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:
- w1 = layer.w13_weight_unpacked
- w2 = layer.w2_weight_unpacked
- else:
- w1 = layer.w13_weight_packed
- w2 = layer.w2_weight_packed
- out = fused_experts(
- x,
- w1,
- w2,
- topk_weights=topk_weights,
- topk_ids=topk_ids,
- inplace=True,
- activation=activation,
- global_num_experts=global_num_experts,
- apply_router_weight_on_input=apply_router_weight_on_input,
- expert_map=expert_map,
- quant_config=self.moe_quant_config,
- )
- return out
-
- num_all_tokens, hidden_dim = x.shape
- num_experts = layer.local_num_experts
- total_num_experts = router_logits.size(-1)
- if (
- self.mask_weights_buffer is None
- or self.mask_weights_buffer.dtype != x.dtype
- or self.mask_weights_buffer.device != x.device
- or self.mask_weights_buffer.shape[0] < num_all_tokens
- or self.mask_weights_buffer.shape[1] < total_num_experts
- ):
- if num_all_tokens > self.num_all_tokens_threshold:
- mask_weights = torch.zeros((num_all_tokens, total_num_experts), dtype=x.dtype, device=x.device)
- if self.mask_weights_buffer is None and self.num_all_tokens_threshold != 0:
- self.mask_weights_buffer = torch.zeros(
- (self.num_all_tokens_threshold, total_num_experts),
- dtype=x.dtype,
- device=x.device,
- )
- self.experts_mask_buffer = torch.zeros(
- (self.num_all_tokens_threshold, total_num_experts),
- dtype=x.dtype,
- device=x.device,
- )
- else:
- self.mask_weights_buffer = torch.zeros(
- (num_all_tokens, total_num_experts), dtype=x.dtype, device=x.device
- )
- self.experts_mask_buffer = torch.zeros(
- (num_all_tokens, total_num_experts), dtype=x.dtype, device=x.device
- )
- mask_weights = self.mask_weights_buffer
- experts_mask = self.experts_mask_buffer
- else:
- self.mask_weights_buffer.zero_()
- mask_weights = self.mask_weights_buffer
- self.experts_mask_buffer.zero_()
- experts_mask = self.experts_mask_buffer
-
- topk_ids = topk_ids.to(torch.int64)
- topk_weights = topk_weights.to(x.dtype)
- experts_mask.scatter_(-1, topk_ids, topk_weights)
- mask_weights.scatter_(-1, topk_ids, 1)
- mask_weights = mask_weights[:num_all_tokens, :total_num_experts]
- mask_weights = mask_weights.transpose(0, 1)
- experts_mask = experts_mask[:num_all_tokens, :total_num_experts]
- experts_mask = experts_mask.transpose(0, 1)
- # Note: ep_size equal tp_size
- ep_rank = get_tensor_model_parallel_rank() if expert_map is not None else 0
- ep_shift = ep_rank * num_experts
-
- if envs.VLLM_ENABLE_STATIC_MOE and not envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:
- num_experts, intermediate_size_per_partition_x2, _ = layer.w13_weight_packed.shape
- intermediate_size_per_partition = intermediate_size_per_partition_x2 // 2
- for expert_index in range(num_experts):
- mask_weight = mask_weights[expert_index + ep_shift].unsqueeze(1)
- current_state_static = x * mask_weight
-
- local_w13_packed = layer.w13_weight_packed[expert_index]
- local_w13_scale = layer.w13_weight_scale[expert_index]
- local_w2_packed = layer.w2_weight_packed[expert_index]
- local_w2_scale = layer.w2_weight_scale[expert_index]
-
- local_w1_packed = local_w13_packed[:intermediate_size_per_partition, ...]
- local_w1_scale = local_w13_scale[:intermediate_size_per_partition, ...]
-
- local_w3_packed = local_w13_packed[intermediate_size_per_partition:, ...]
- local_w3_scale = local_w13_scale[intermediate_size_per_partition:, ...]
-
- local_w1_bias = None
- local_w2_bias = None
- local_w3_bias = None
- if self.has_bias:
- local_w13_bias = layer.w13_bias[expert_index]
- local_w1_bias = local_w13_bias[:intermediate_size_per_partition]
- local_w3_bias = local_w13_bias[intermediate_size_per_partition:]
- local_w2_bias = layer.w2_bias[expert_index]
-
- local_w1_out = run_mxfp4_emulations(
- x=current_state_static,
- weight=local_w1_packed,
- weight_scale=local_w1_scale,
- bias=local_w1_bias,
- )
- local_w3_out = run_mxfp4_emulations(
- x=current_state_static,
- weight=local_w3_packed,
- weight_scale=local_w3_scale,
- bias=local_w3_bias,
- )
-
- w13_out = apply_act(local_w1_out, local_w3_out, activation)
-
- local_w2_out = run_mxfp4_emulations(
- x=w13_out,
- weight=local_w2_packed,
- weight_scale=local_w2_scale,
- bias=local_w2_bias,
- )
- padded_weight = experts_mask[expert_index + ep_shift].unsqueeze(1)
- local_w2_out = local_w2_out * padded_weight
- if expert_index == 0:
- final_hidden_states = local_w2_out
- else:
- final_hidden_states += local_w2_out
- return final_hidden_states
- if envs.VLLM_ENABLE_STATIC_MOE and envs.VLLM_MXFP4_PRE_UNPACK_WEIGHTS:
- num_experts, intermediate_size_per_partition_x2, _ = layer.w13_weight_unpacked.shape
- intermediate_size_per_partition = intermediate_size_per_partition_x2 // 2
-
- for expert_index in range(num_experts):
- mask_weight = mask_weights[expert_index + ep_shift].unsqueeze(1)
- current_state_static = x * mask_weight
-
- local_unpacked_w13 = layer.w13_weight_unpacked[expert_index]
- local_w13_scale = layer.w13_weight_scale[expert_index]
-
- local_unpacked_w2 = layer.w2_weight_unpacked[expert_index]
- local_w2_scale = layer.w2_weight_scale[expert_index]
-
- local_unpacked_w1 = local_unpacked_w13[:intermediate_size_per_partition, ...]
- half_scale = local_w13_scale.shape[0] // 2
- local_w1_scale = local_w13_scale[:half_scale, ...]
- local_unpacked_w3 = local_unpacked_w13[intermediate_size_per_partition:, ...]
- local_w3_scale = local_w13_scale[half_scale:, ...]
-
- local_w1_bias = None
- local_w2_bias = None
- local_w3_bias = None
- if self.has_bias:
- local_w13_bias = layer.w13_bias[expert_index]
- local_w1_bias = local_w13_bias[:intermediate_size_per_partition]
- local_w3_bias = local_w13_bias[intermediate_size_per_partition:]
- local_w2_bias = layer.w2_bias[expert_index]
-
- local_w1_out = mxfp4_gemm_with_unpacked_weight(
- x=current_state_static,
- weight_fp8=local_unpacked_w1,
- weight_scale_bf16=local_w1_scale,
- bias=local_w1_bias,
- )
- local_w3_out = mxfp4_gemm_with_unpacked_weight(
- x=current_state_static,
- weight_fp8=local_unpacked_w3,
- weight_scale_bf16=local_w3_scale,
- bias=local_w3_bias,
- )
-
- w13_out = apply_act(local_w1_out, local_w3_out, activation)
-
- local_w2_out = mxfp4_gemm_with_unpacked_weight(
- x=w13_out,
- weight_fp8=local_unpacked_w2,
- weight_scale_bf16=local_w2_scale,
- bias=local_w2_bias,
- )
-
- padded_weight = experts_mask[expert_index + ep_shift].unsqueeze(1)
- local_w2_out = local_w2_out * padded_weight
- if expert_index == 0:
- final_hidden_states = local_w2_out
- else:
- final_hidden_states += local_w2_out
- return final_hidden_states
- raise NotImplementedError("Not implemented for now.")
diff --git a/auto_round_extension/vllm_ext/moe_impl_mxfp8.py b/auto_round_extension/vllm_ext/moe_impl_mxfp8.py
deleted file mode 100644
index ecd4183ff..000000000
--- a/auto_round_extension/vllm_ext/moe_impl_mxfp8.py
+++ /dev/null
@@ -1,208 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Callable, Optional, Union
-
-import torch
-import torch.nn.functional as F
-from vllm.logger import init_logger
-from vllm.model_executor.layers.fused_moe import (
- FusedMoE,
- FusedMoEConfig,
- FusedMoeWeightScaleSupported,
-)
-from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
-from vllm.model_executor.utils import set_weight_attrs
-
-from auto_round_extension.vllm_ext.quant_method_moe import AutoRoundMoEMethod
-
-logger = init_logger(__name__)
-
-
-def apply_act(local_w1_out: torch.Tensor, local_w3_out: torch.Tensor, activation: str) -> torch.Tensor:
- if activation == "silu":
- act_fn = F.silu
- w13_out = act_fn(local_w1_out) * local_w3_out
- elif activation == "swigluoai":
- limit = 7.0
- alpha = 1.702
- local_w1_out = local_w1_out.clamp(min=None, max=limit)
- local_w3_out = local_w3_out.clamp(min=-limit, max=limit)
- glu = (local_w1_out) * F.sigmoid(local_w1_out * alpha)
- w13_out = (local_w3_out + 1) * glu
- else:
- raise NotImplementedError(f"Activation {activation} is not implemented.")
- return w13_out
-
-
-class AutoRoundMoEMethodMXFp8Impl(AutoRoundMoEMethod):
- def __init__(
- self,
- quant_config: "AutoRoundConfig", # type: ignore # noqa E501
- moe: FusedMoEConfig,
- ):
- super().__init__(moe)
- self.use_marlin = False
- self.group_size = 32
- self.quant_config = quant_config
- self.has_bias = self.moe.has_bias
-
- def create_weights(
- self,
- layer: torch.nn.Module,
- num_experts: int,
- hidden_size: int,
- intermediate_size_per_partition: int,
- params_dtype: torch.dtype,
- **extra_weight_attrs,
- ):
- layer.intermediate_size_per_partition = intermediate_size_per_partition
- layer.hidden_size = hidden_size
- layer.num_experts = num_experts
- layer.orig_dtype = params_dtype
- layer.weight_block_size = None
-
- params_dtype = torch.float8_e4m3fn
-
- # WEIGHTS
- w13_weight = torch.nn.Parameter(
- torch.empty(
- num_experts,
- 2 * intermediate_size_per_partition,
- hidden_size,
- dtype=params_dtype,
- ),
- requires_grad=False,
- )
- layer.register_parameter("w13_weight", w13_weight)
- set_weight_attrs(w13_weight, extra_weight_attrs)
-
- w2_weight = torch.nn.Parameter(
- torch.empty(
- num_experts,
- hidden_size,
- intermediate_size_per_partition,
- dtype=params_dtype,
- ),
- requires_grad=False,
- )
- layer.register_parameter("w2_weight", w2_weight)
- set_weight_attrs(w2_weight, extra_weight_attrs)
-
- # WEIGHT_SCALES
- w13_weight_scale = torch.nn.Parameter(
- data=torch.empty(
- num_experts,
- 2 * intermediate_size_per_partition,
- hidden_size // self.group_size,
- dtype=torch.uint8, # E8M0 for MXFP8 scale
- ),
- requires_grad=False,
- )
- layer.register_parameter("w13_weight_scale", w13_weight_scale)
- # w2
- w2_weight_scale = torch.nn.Parameter(
- data=torch.empty(
- num_experts,
- hidden_size,
- intermediate_size_per_partition // self.group_size,
- dtype=torch.uint8, # E8M0 for MXFP8 scale
- ),
- requires_grad=False,
- )
- layer.register_parameter("w2_weight_scale", w2_weight_scale)
- # Add PER-TENSORGROUP quantization for FusedMoE.weight_loader.
- extra_weight_attrs.update({"quant_method": FusedMoeWeightScaleSupported.GROUP.value})
- set_weight_attrs(w13_weight_scale, extra_weight_attrs)
- set_weight_attrs(w2_weight_scale, extra_weight_attrs)
-
- def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> Optional[FusedMoEQuantConfig]:
- from vllm.model_executor.layers.fused_moe.config import (
- ocp_mx_moe_quant_config,
- )
-
- self.input_dtype = "mxfp8_e4m3"
- self.weight_dtype = "mxfp8_e4m3"
- return ocp_mx_moe_quant_config(
- quant_dtype=self.input_dtype,
- weight_dtype=self.weight_dtype,
- w1_scale=layer.w13_weight_scale,
- w2_scale=layer.w2_weight_scale,
- a1_scale=None,
- a2_scale=None,
- w1_bias=layer.w13_bias if self.has_bias else None,
- w2_bias=layer.w2_bias if self.has_bias else None,
- block_shape=None,
- )
- return None
-
- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
- return
-
- @torch.inference_mode()
- def apply(
- self,
- layer: torch.nn.Module,
- x: torch.Tensor,
- router_logits: torch.Tensor,
- top_k: int,
- renormalize: bool,
- use_grouped_topk: bool = False,
- topk_group: Optional[int] = None,
- num_expert_group: Optional[int] = None,
- global_num_experts: int = -1,
- expert_map: Optional[torch.Tensor] = None,
- custom_routing_function: Optional[Callable] = None,
- scoring_func: str = "softmax",
- routed_scaling_factor: float = 1.0,
- e_score_correction_bias: Optional[torch.Tensor] = None,
- apply_router_weight_on_input: bool = False,
- activation: str = "silu",
- enable_eplb: bool = False,
- expert_load_view: Optional[torch.Tensor] = None,
- logical_to_physical_map: Optional[torch.Tensor] = None,
- logical_replica_count: Optional[torch.Tensor] = None,
- ) -> Union[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
- topk_weights, topk_ids, _ = FusedMoE.select_experts(
- hidden_states=x,
- router_logits=router_logits,
- use_grouped_topk=use_grouped_topk,
- top_k=top_k,
- renormalize=renormalize,
- topk_group=topk_group,
- num_expert_group=num_expert_group,
- custom_routing_function=custom_routing_function,
- scoring_func=scoring_func,
- e_score_correction_bias=e_score_correction_bias,
- )
- assert self.fused_experts is None
-
- from vllm.model_executor.layers.fused_moe import fused_experts
-
- w1 = layer.w13_weight
- w2 = layer.w2_weight
- out = fused_experts(
- x,
- w1,
- w2,
- topk_weights=topk_weights,
- topk_ids=topk_ids,
- inplace=True,
- activation=activation,
- global_num_experts=global_num_experts,
- apply_router_weight_on_input=apply_router_weight_on_input,
- expert_map=expert_map,
- quant_config=self.moe_quant_config,
- )
- return out
diff --git a/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py b/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py
deleted file mode 100644
index 2a068924f..000000000
--- a/auto_round_extension/vllm_ext/mxfp4_qdq_utils.py
+++ /dev/null
@@ -1,221 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Union
-
-import torch
-
-from auto_round_extension.vllm_ext.fp4_utils import cast_to_fp4, pack_fp4_to_uint8, unpack_fp4_from_uint8
-from auto_round_extension.vllm_ext.utils import _to_mx_rceil, get_fp_scale
-
-F4_E2M1_MAX = 6.0
-F32_EXP_BIAS = 127
-
-F32_MIN_NORMAL = 2 ** (-F32_EXP_BIAS + 1)
-
-
-def to_mxfp4_rceil(
- data_hp: torch.Tensor,
- elem_dtype: Union[torch.dtype, str],
- block_size: int,
-):
-
- assert data_hp.dtype in (
- torch.bfloat16,
- torch.float,
- ), f"{data_hp.dtype} is not supported yet"
- # TODO(future PR): consider supporting padding
- assert data_hp.numel() % block_size == 0, f"data size must be multiple of block_size={block_size}"
- data_hp = data_hp.contiguous()
- # calculate the scale in e8m0 format
-
- orig_shape = data_hp.shape
- data_hp = data_hp.reshape(-1, block_size)
-
- # find max value of the data
- # Note: this only implements the `minimally supported` version of
- # https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf
- # section 6.3.
- max_abs = torch.amax(torch.abs(data_hp), 1)
-
- # Add an epsilon to prevent the log2 function call for returning -inf
- # where the values are zero.
- eps = F32_MIN_NORMAL * (max_abs == 0).type(max_abs.dtype)
-
- # Set X to be the largest power-of-two less than or equal to
- # max_abs(v), divided by the largest power of two representable
- # in the element data type, and get the mbits at the same time
-
- max_pos = F4_E2M1_MAX
- scale_e8m0_biased, data_lp = _to_mx_rceil(data_hp, max_abs, max_pos)
-
- data_lp = data_lp.reshape(orig_shape)
- orig_shape = [*orig_shape[:-1], orig_shape[-1] // 2]
- data_lp = cast_to_fp4(data_lp)
- data_lp = pack_fp4_to_uint8(data_lp)
-
- scale_e8m0_biased = scale_e8m0_biased.view(torch.uint8)
- return scale_e8m0_biased, data_lp
-
-
-def to_dtype(
- data_lp,
- scale_e8m0,
- elem_dtype,
- block_size,
- target_dtype,
- scale_dtype=None,
- return_scale=False,
-):
- orig_shape = data_lp.shape
- last_dim = orig_shape[-1]
- data_lp = data_lp.reshape(-1, last_dim)
- result_shape = orig_shape[:-1] + (last_dim * 2,)
- assert data_lp.is_contiguous(), f"Data must be contiguous, got {data_lp.stride()}"
-
- assert elem_dtype == "fp4_e2m1", f"Expected 'fp4_e2m1', got {elem_dtype}"
-
- m, half_n = data_lp.shape
- n = half_n * 2
- data_hp = unpack_fp4_from_uint8(data_lp, m, n, dtype=target_dtype)
-
- data_hp = data_hp.reshape(-1, block_size)
-
- if scale_dtype is None:
- scale_dtype = target_dtype
- s_fp = get_fp_scale(scale_e8m0).reshape(-1, 1).to(scale_dtype)
- if return_scale:
- return data_hp.reshape(result_shape), s_fp
-
- data_hp = data_hp * s_fp
- data_hp = data_hp.reshape(result_shape)
-
- return data_hp
-
-
-def run_mxfp4_emulations(
- x: torch.Tensor,
- weight: torch.Tensor,
- weight_scale: torch.Tensor,
- bias: torch.Tensor | None = None,
-):
- # TODO: select the rounding mode based on config
- group_size = 32
- # quantize input to (FP4 and interleaved block scale)
- input_scale, x_q = to_mxfp4_rceil(
- data_hp=x,
- elem_dtype="fp4_e2m1",
- block_size=group_size,
- )
-
- # dequantize input
- x_dq = to_dtype(
- data_lp=x_q,
- scale_e8m0=input_scale,
- elem_dtype="fp4_e2m1",
- block_size=group_size,
- target_dtype=x.dtype,
- )
-
- # dequantize weight
- w_dq = to_dtype(
- data_lp=weight,
- scale_e8m0=weight_scale,
- elem_dtype="fp4_e2m1",
- block_size=group_size,
- target_dtype=x.dtype,
- )
-
- # matmul
- out = torch.matmul(x_dq, w_dq.t())
- if bias is not None:
- out += bias
- return out
-
-
-def dequant_mxfp4_to_fp8(data_lp, scale_e8m0):
- data_fp8, scale_float = to_dtype(
- data_lp=data_lp,
- scale_e8m0=scale_e8m0,
- elem_dtype="fp4_e2m1",
- block_size=32,
- target_dtype=torch.float8_e4m3fn,
- scale_dtype=torch.bfloat16,
- return_scale=True,
- )
- return data_fp8, scale_float
-
-
-def mxfp4_fp8_weight_to_bf16(weight_fp8, scale_bf16):
-
- origin_shape = weight_fp8.shape
- weight_fp8 = weight_fp8.reshape(-1, 32)
- scale_bf16 = scale_bf16.reshape(-1, 1)
- assert weight_fp8.shape[0] == scale_bf16.shape[0], f"shape mismatch: {weight_fp8.shape} vs {scale_bf16.shape}"
- dequant_weight_bf16 = weight_fp8.to(torch.bfloat16) * scale_bf16
- dequant_weight_bf16 = dequant_weight_bf16.reshape(origin_shape)
- return dequant_weight_bf16
-
-
-def mxfp4_gemm_with_unpacked_weight(x, weight_fp8, weight_scale_bf16, bias=None):
- x = qdq_mxfp4(x)
-
- # dequantize weight
- w_dq = mxfp4_fp8_weight_to_bf16(weight_fp8, weight_scale_bf16)
- # matmul
- out = torch.matmul(x, w_dq.t())
- if bias is not None:
- out += bias
- return out
-
-
-def fp4_121_positive(x: torch.Tensor) -> torch.Tensor:
- step1 = torch.round(2.0 * x) / 2.0
- step2 = torch.round(x)
- step3 = 2.0 * torch.round(x / 2.0)
-
- mask1 = x < 2.0
- mask2 = x < 4.0
-
- return step1 * mask1 + step2 * (~mask1) * mask2 + step3 * (~mask1) * (~mask2)
-
-
-def fp4_121_scaled_even_rounding(x: torch.Tensor) -> torch.Tensor:
- sign = x.sign()
- x_abs = x.abs()
- amax_x = x_abs.max(dim=-1, keepdim=True)[0]
- scale_tmp = torch.floor(torch.log2(amax_x)) - 2.0
- scale_clamp = torch.clamp(scale_tmp, min=-127, max=127)
- scale = torch.pow(2.0, scale_clamp)
-
- scale = torch.where((0 < scale) * (scale < torch.inf), scale, 1.0)
-
- x_fp4_abs = fp4_121_positive(x_abs / scale) * scale
- return sign * x_fp4_abs
-
-
-# https://github.com/Anonymous1252022/fp4-all-the-way/blob/main/experimental/fp4.py
-def qdq_mxfp4(
- x: torch.Tensor,
-) -> torch.Tensor:
- # TODO: select the rounding mode based on config
- block_size = 32
- shape = x.shape
- x = x.reshape(-1, block_size)
-
- x = fp4_121_scaled_even_rounding(x)
-
- x = x.reshape(shape)
-
- return x
diff --git a/auto_round_extension/vllm_ext/mxfp8_qdq_utils.py b/auto_round_extension/vllm_ext/mxfp8_qdq_utils.py
deleted file mode 100644
index 482d77f85..000000000
--- a/auto_round_extension/vllm_ext/mxfp8_qdq_utils.py
+++ /dev/null
@@ -1,63 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-import torch
-
-__all__ = ["get_fp_scale", "dequant_mx_fp8", "quant_mx_fp8"]
-
-
-# def get_fp_scale(scale_e8m0):
-# # https://github.com/pytorch/ao/blob/994a4ba6c869854fcaa6ca7e118fcbd75e6c28cc/torchao/prototype/mx_formats/mx_tensor.py#L337
-# assert scale_e8m0.dtype == torch.uint8, f"Expected uint8, got {scale_e8m0.dtype}"
-# E8M0_EXPONENT_BIAS = 127
-# scale_e8m0 = scale_e8m0.view(torch.uint8)
-# s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS
-# # TODO(later): it would be nice if there was a way to do the 2^x operation
-# # in PyTorch without creating a tensor of twos
-# two = torch.full(s_offset.size(), 2.0, device=scale_e8m0.device)
-# # pow(two, s_offset) can be out of range of floating point formats.
-# # TODO(later): handle this for float16 if we decide to support float16
-# # scales.
-# s_fp = torch.pow(two, s_offset)
-
-# return s_fp
-
-
-def get_fp_scale(scale_e8m0):
- # https://github.com/pytorch/ao/blob/994a4ba6c869854fcaa6ca7e118fcbd75e6c28cc/torchao/prototype/mx_formats/mx_tensor.py#L337
- E8M0_EXPONENT_BIAS = 127
-
- scale_e8m0 = scale_e8m0.view(torch.uint8)
- s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS
- # TODO(later): it would be nice if there was a way to do the 2^x operation
- # in PyTorch without creating a tensor of twos
- # two = torch.full(s_offset.size(), 2.0, device=scale_e8m0.device)
- # pow(two, s_offset) can be out of range of floating point formats.
- # TODO(later): handle this for float16 if we decide to support float16
- # scales.
- # s_fp = torch.pow(two, s_offset)
- # !!!!NOTE Critical: fixed the OoM issue when using HPU graph
- s_fp = torch.pow(2.0, s_offset.to(torch.float))
-
- return s_fp
-
-
-def dequant_mx_fp8(weight_fp8, scale_e8m0, block_size, target_dtype):
- scale_float = get_fp_scale(scale_e8m0)
- weight_bf16 = weight_fp8.to(torch.bfloat16)
- weight_original_shape = weight_bf16.shape
- weight_bf16 = weight_bf16.reshape(-1, block_size)
- scale_float = scale_float.reshape(-1, 1)
- dequant_weight = weight_bf16 * scale_float
- dequant_weight = dequant_weight.reshape(weight_original_shape)
- return dequant_weight.to(target_dtype)
-
-
-def quant_mx_fp8(tensor):
- from auto_round_extension.vllm_ext.utils import to_mx_fp8e4m3
-
- scale_e8m0_biased, data_lp = to_mx_fp8e4m3(
- data_hp=tensor,
- elem_dtype=torch.float8_e4m3fn,
- block_size=32,
- )
- return scale_e8m0_biased, data_lp
diff --git a/auto_round_extension/vllm_ext/quant_impl.py b/auto_round_extension/vllm_ext/quant_impl.py
deleted file mode 100644
index b88843462..000000000
--- a/auto_round_extension/vllm_ext/quant_impl.py
+++ /dev/null
@@ -1,45 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from abc import ABC, abstractmethod
-from typing import Optional
-
-import torch
-
-
-class AutoRoundQuantImpl(ABC):
- @classmethod
- @abstractmethod
- def get_min_capability(cls) -> int:
- """
- Get minimum device capability.
- """
- raise NotImplementedError
-
- @abstractmethod
- def create_weights(self, *args, **kwargs):
- raise NotImplementedError
-
- @abstractmethod
- def apply_weights(
- self,
- layer: torch.nn.Module,
- x: torch.Tensor,
- bias: Optional[torch.Tensor],
- ):
- raise NotImplementedError
-
- @abstractmethod
- def process_weights_after_loading(self, layer: torch.nn.Module):
- raise NotImplementedError
diff --git a/auto_round_extension/vllm_ext/quant_method_linear.py b/auto_round_extension/vllm_ext/quant_method_linear.py
deleted file mode 100644
index eecc97902..000000000
--- a/auto_round_extension/vllm_ext/quant_method_linear.py
+++ /dev/null
@@ -1,112 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Optional
-
-import torch
-from vllm.logger import init_logger
-from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase, UnquantizedLinearMethod
-from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig
-
-from auto_round.schemes import QuantizationScheme
-from auto_round_extension.vllm_ext.utils import _is_mxfp4_w4a4, _is_mxfp8_w8a8, get_scheme, need_quantize
-
-logger = init_logger(__name__)
-
-
-QLINEAR_METHODS_DISPATCH_TABLE = {}
-
-
-class AutoRoundQuantLinearMethod(LinearMethodBase):
-
- def __init__(self, impl, config=None, scheme=None):
- self.config = config
- self.impl = impl
- self.scheme = scheme
-
- @staticmethod
- def get_method(
- quant_config: AutoRoundConfig,
- layer: torch.nn.Module,
- prefix: str,
- ) -> "AutoRoundQuantLinearMethod":
-
- def get_impl(scheme: QuantizationScheme):
- if _is_mxfp8_w8a8(scheme):
- from auto_round_extension.vllm_ext.linear_impl_mxfp8 import AutoRoundMXFP8LinearImpl
-
- return AutoRoundMXFP8LinearImpl(quant_config)
-
- elif _is_mxfp4_w4a4(scheme):
- from auto_round_extension.vllm_ext.linear_impl_mxfp4 import AutoRoundMXFP4LinearImpl
-
- return AutoRoundMXFP4LinearImpl(quant_config)
-
- raise ValueError(f"Unsupported Linear scheme: {scheme}, layer: {prefix}")
-
- packed_modules_mapping = quant_config.packed_modules_mapping
- for packed_name, child_names in packed_modules_mapping.items():
- if prefix.endswith(packed_name):
- prefix = prefix.replace(packed_name, child_names[0])
- break
-
- # TODO: use a more robust way to map layer names
- if prefix.endswith("gate_up_proj"):
- # update gate_up_proj to gate_proj, assume both gate and up share the same quantization scheme
- prefix = prefix.replace("gate_up_proj", "up_proj")
- if prefix.endswith("qkv_proj"):
- # update qkv_proj to q_proj, assume all qkv share the same quantization scheme
- prefix = prefix.replace("qkv_proj", "q_proj")
- layer_scheme = get_scheme(quant_config, prefix)
- if not need_quantize(layer_scheme.bits):
- return UnquantizedLinearMethod()
- impl = get_impl(layer_scheme)
- logger.info("Apply %s to %s", impl.__class__.__name__, prefix)
- return AutoRoundQuantLinearMethod(impl=impl)
-
- @classmethod
- def get_min_capability(cls) -> int:
- return cls.impl.get_min_capability()
-
- def create_weights(
- self,
- layer: torch.nn.Module,
- input_size_per_partition: int,
- output_partition_sizes: list[int],
- input_size: int,
- output_size: int,
- params_dtype: torch.dtype,
- **extra_weight_attrs,
- ):
- weight_loader = extra_weight_attrs.get("weight_loader")
- return self.impl.create_weights(
- layer=layer,
- input_size=input_size,
- input_size_per_partition=input_size_per_partition,
- output_partition_sizes=output_partition_sizes,
- output_size=output_size,
- params_dtype=params_dtype,
- weight_loader=weight_loader,
- )
-
- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
- return self.impl.process_weights_after_loading(layer)
-
- def apply(
- self,
- layer: torch.nn.Module,
- x: torch.Tensor,
- bias: Optional[torch.Tensor] = None,
- ):
- return self.impl.apply_weights(layer, x, bias=bias)
diff --git a/auto_round_extension/vllm_ext/quant_method_moe.py b/auto_round_extension/vllm_ext/quant_method_moe.py
deleted file mode 100644
index ef5c75291..000000000
--- a/auto_round_extension/vllm_ext/quant_method_moe.py
+++ /dev/null
@@ -1,73 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Optional
-
-import torch
-from vllm.logger import init_logger
-from vllm.model_executor.layers.fused_moe import (
- FusedMoEConfig,
- FusedMoEMethodBase,
-)
-from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
-from vllm.model_executor.layers.quantization.auto_round import AutoRoundConfig
-
-from auto_round.schemes import QuantizationScheme
-from auto_round_extension.vllm_ext.utils import _is_mxfp4_w4a4, _is_mxfp8_w8a8, get_scheme, need_quantize
-
-logger = init_logger(__name__)
-
-
-QMOE_METHODS_DISPATCH_TABLE = {}
-
-
-class AutoRoundMoEMethod(FusedMoEMethodBase):
- def __init__(self, moe: FusedMoEConfig):
- super().__init__(moe)
-
- @staticmethod
- def get_moe_method(
- quant_config: AutoRoundConfig,
- layer: torch.nn.Module,
- prefix: str,
- ) -> "AutoRoundMoEMethod":
-
- def get_impl(scheme: QuantizationScheme):
- if not need_quantize(scheme.bits):
- from vllm.model_executor.layers.fused_moe.layer import (
- UnquantizedFusedMoEMethod,
- )
-
- return UnquantizedFusedMoEMethod(layer.moe_config)
-
- elif _is_mxfp4_w4a4(scheme):
- from auto_round_extension.vllm_ext.moe_impl_mxfp4 import AutoRoundMoEMethodMXFp4Impl
-
- return AutoRoundMoEMethodMXFp4Impl(quant_config, layer.moe_config)
-
- elif _is_mxfp8_w8a8(scheme):
- from auto_round_extension.vllm_ext.moe_impl_mxfp8 import AutoRoundMoEMethodMXFp8Impl
-
- return AutoRoundMoEMethodMXFp8Impl(quant_config, layer.moe_config)
-
- raise ValueError(f"Unsupported FusedMoe scheme: {scheme}")
-
- layer_scheme = get_scheme(quant_config, prefix)
- impl = get_impl(layer_scheme)
- layer._prefix = prefix
- logger.debug("Apply %s to %s", impl.__class__.__name__, prefix)
- return impl
-
- def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> Optional[FusedMoEQuantConfig]:
- return self.impl.get_fused_moe_quant_config(layer)
diff --git a/auto_round_extension/vllm_ext/sitecustomize.py b/auto_round_extension/vllm_ext/sitecustomize.py
deleted file mode 100644
index 8c22d1ab2..000000000
--- a/auto_round_extension/vllm_ext/sitecustomize.py
+++ /dev/null
@@ -1,41 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-VLLM_ENABLE_AR_EXT = os.environ.get("VLLM_ENABLE_AR_EXT", "") in [
- "1",
- "true",
- "True",
-]
-
-if VLLM_ENABLE_AR_EXT:
- print("*****************************************************************************")
- print(f"* !!! VLLM_ENABLE_AR_EXT is set to {VLLM_ENABLE_AR_EXT}, applying auto_round_vllm_extension *")
- print("*****************************************************************************")
-
- import vllm.model_executor.layers.quantization.auto_round as auto_round_module
-
- from auto_round_extension.vllm_ext.auto_round_ext import AutoRoundExtensionConfig
-
- auto_round_module.AutoRoundConfig = AutoRoundExtensionConfig
- from auto_round_extension.vllm_ext.envs_ext import extra_environment_variables
-
-
-else:
- print("*****************************************************************************")
- print(
- f"* Sitecustomize is loaded, but VLLM_ENABLE_AR_EXT is set to {VLLM_ENABLE_AR_EXT}, skipping auto_round_vllm_extension *"
- )
- print("*****************************************************************************")
diff --git a/auto_round_extension/vllm_ext/tests/conftest.py b/auto_round_extension/vllm_ext/tests/conftest.py
deleted file mode 100644
index fcdd97a49..000000000
--- a/auto_round_extension/vllm_ext/tests/conftest.py
+++ /dev/null
@@ -1,1396 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-import contextlib
-import pathlib
-from copy import deepcopy
-
-from tblib import pickling_support
-
-# ruff: noqa
-
-# Install support for pickling exceptions so that we can nicely propagate
-# failures from tests running in a subprocess.
-# This should be run before any custom exception subclasses are defined.
-pickling_support.install()
-
-import http.server
-import json
-import math
-import mimetypes
-import os
-import socket
-import tempfile
-import threading
-from collections.abc import Generator
-from contextlib import nullcontext
-from enum import Enum
-from typing import Any, Callable, TypedDict, TypeVar, cast
-
-import numpy as np
-import pytest
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from huggingface_hub import snapshot_download
-from PIL import Image
-from transformers import (
- AutoConfig,
- AutoModelForCausalLM,
- AutoTokenizer,
- BatchEncoding,
- BatchFeature,
-)
-from transformers.models.auto.auto_factory import _BaseAutoModelClass
-
-# from tests.models.utils import TokensTextLogprobs, TokensTextLogprobsPromptLogprobs
-from vllm import LLM, SamplingParams, envs
-from vllm.assets.audio import AudioAsset
-from vllm.assets.image import ImageAsset
-from vllm.assets.video import VideoAsset
-from vllm.config.model import ConvertOption, RunnerOption, _get_and_verify_dtype
-from vllm.connections import global_http_connection
-from vllm.distributed import (
- cleanup_dist_env_and_memory,
- init_distributed_environment,
- initialize_model_parallel,
-)
-from vllm.logger import init_logger
-from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
-from vllm.multimodal.utils import fetch_image
-from vllm.outputs import RequestOutput
-from vllm.sampling_params import BeamSearchParams
-from vllm.transformers_utils.utils import maybe_model_redirect
-from vllm.utils.collection_utils import is_list_of
-from vllm.utils.torch_utils import set_default_torch_num_threads
-
-# Representation of generated sequence as a tuple of
-# * Token ID list
-# * String
-# * List of top sample logprobs for each sampled token
-#
-# Assumes prompt logprobs were not requested.
-TokensTextLogprobs = tuple[list[int], str, list[dict[int, float]] | SampleLogprobs | None]
-
-# Allow for tokens to be represented as str's rather than IDs;
-# tuple of
-# * Token string representations list
-# * String
-# * Optional list of top sample logprobs for each sampled token
-#
-# Assumes prompt logprobs were not requested.
-TextTextLogprobs = tuple[list[str], str, list[dict[str, float]] | list[dict[str, Logprob]] | None]
-
-# Representation of generated sequence as a tuple of
-# * Token ID list
-# * String
-# * Optional list of top sample logprobs for each sampled token
-# * Optional list of top prompt logprobs for each prompt token
-#
-# Allows prompt logprobs to be requested.
-TokensTextLogprobsPromptLogprobs = tuple[
- list[int],
- str,
- list[dict[int, float]] | SampleLogprobs | None,
- list[dict[int, float] | None] | PromptLogprobs | None,
-]
-
-
-logger = init_logger(__name__)
-
-_TEST_DIR = os.path.dirname(__file__)
-_TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
-_LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
-_SYS_MSG = os.path.join(_TEST_DIR, "system_messages", "sonnet3.5_nov2024.txt")
-
-_M = TypeVar("_M")
-
-_PromptMultiModalInput = list[_M] | list[list[_M]]
-
-PromptImageInput = _PromptMultiModalInput[Image.Image]
-PromptAudioInput = _PromptMultiModalInput[tuple[np.ndarray, int]]
-PromptVideoInput = _PromptMultiModalInput[np.ndarray]
-
-
-def _read_prompts(filename: str) -> list[str]:
- with open(filename) as f:
- prompts = f.readlines()
- return prompts
-
-
-class ImageAssetPrompts(TypedDict):
- stop_sign: str
- cherry_blossom: str
-
-
-class ImageTestAssets(list[ImageAsset]):
- def __init__(self) -> None:
- super().__init__(
- [
- ImageAsset("stop_sign"),
- ImageAsset("cherry_blossom"),
- ]
- )
-
- def prompts(self, prompts: ImageAssetPrompts) -> list[str]:
- """
- Convenience method to define the prompt for each test image.
-
- The order of the returned prompts matches the order of the
- assets when iterating through this object.
- """
- return [prompts["stop_sign"], prompts["cherry_blossom"]]
-
-
-class VideoAssetPrompts(TypedDict):
- baby_reading: str
-
-
-class VideoTestAssets(list[VideoAsset]):
- def __init__(self) -> None:
- super().__init__(
- [
- VideoAsset("baby_reading"),
- ]
- )
-
- def prompts(self, prompts: VideoAssetPrompts) -> list[str]:
- return [prompts["baby_reading"]]
-
-
-class AudioAssetPrompts(TypedDict):
- mary_had_lamb: str
- winning_call: str
-
-
-class AudioTestAssets(list[AudioAsset]):
- def __init__(self) -> None:
- super().__init__(
- [
- AudioAsset("mary_had_lamb"),
- AudioAsset("winning_call"),
- ]
- )
-
- def prompts(self, prompts: AudioAssetPrompts) -> list[str]:
- return [prompts["mary_had_lamb"], prompts["winning_call"]]
-
-
-IMAGE_ASSETS = ImageTestAssets()
-"""Singleton instance of {class}`ImageTestAssets`."""
-VIDEO_ASSETS = VideoTestAssets()
-"""Singleton instance of {class}`VideoTestAssets`."""
-AUDIO_ASSETS = AudioTestAssets()
-"""Singleton instance of {class}`AudioTestAssets`."""
-
-
[email protected](scope="function", autouse=True)
-def cleanup_VLLM_USE_V1(monkeypatch):
- """
- The V1 oracle sets "VLLM_USE_V1" during loading. This means
- that each invocation of a test change the env variable.
-
- If we touch "VLLM_USE_V1" with monkeypatch, then any changes
- made during the test run by vLLM will be cleaned up.
-
- This fixture is used by every test.
- """
-
- # If VLLM_USE_V1 is not set, set then delete. This will
- # cause monkeypatch to clean up VLLM_USE_V1 upon exit
- # if VLLM modifies the value of envs.VLLM_USE_V1.
- if "VLLM_USE_V1" not in os.environ:
- monkeypatch.setenv("VLLM_USE_V1", "")
- monkeypatch.delenv("VLLM_USE_V1")
-
-
[email protected](autouse=True)
-def init_test_http_connection():
- # pytest_asyncio may use a different event loop per test
- # so we need to make sure the async client is created anew
- global_http_connection.reuse_client = False
-
-
-def dist_init():
- temp_file = tempfile.mkstemp()[1]
- init_distributed_environment(
- world_size=1,
- rank=0,
- distributed_init_method=f"file://{temp_file}",
- local_rank=0,
- backend="nccl",
- )
- initialize_model_parallel(1, 1)
- yield
- cleanup_dist_env_and_memory()
-
-
-def should_do_global_cleanup_after_test(request) -> bool:
- """Allow subdirectories to skip global cleanup by overriding this fixture.
- This can provide a ~10x speedup for non-GPU unit tests since they don't need
- to initialize torch.
- """
-
- return not request.node.get_closest_marker("skip_global_cleanup")
-
-
[email protected](autouse=True)
-def cleanup_fixture(should_do_global_cleanup_after_test: bool):
- yield
- if should_do_global_cleanup_after_test:
- cleanup_dist_env_and_memory()
-
-
[email protected](autouse=True)
-def dynamo_reset():
- yield
- torch._dynamo.reset()
-
-
-def example_prompts() -> list[str]:
- prompts = []
- for filename in _TEST_PROMPTS:
- prompts += _read_prompts(filename)
- return prompts
-
-
-def example_system_message() -> str:
- with open(_SYS_MSG) as f:
- return f.read()
-
-
-class DecoderPromptType(Enum):
- """For encoder/decoder models only."""
-
- CUSTOM = 1
- NONE = 2
- EMPTY_STR = 3
-
-
-def example_long_prompts() -> list[str]:
- prompts = []
- for filename in _LONG_PROMPTS:
- prompts += _read_prompts(filename)
- return prompts
-
-
[email protected](scope="session")
-def image_assets() -> ImageTestAssets:
- return IMAGE_ASSETS
-
-
[email protected](scope="session")
-def video_assets() -> VideoTestAssets:
- return VIDEO_ASSETS
-
-
[email protected](scope="session")
-def audio_assets() -> AudioTestAssets:
- return AUDIO_ASSETS
-
-
-_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature, dict)
-_R = TypeVar("_R")
-
-
-class HfRunner:
- def get_default_device(self):
- from vllm.platforms import current_platform
-
- return "cpu" if current_platform.is_cpu() else current_platform.device_type
-
- def wrap_device(self, x: _T, device: str | None = None) -> _T:
- if x is None or isinstance(x, (bool,)):
- return x
-
- if device is None:
- device = self.device
-
- if isinstance(x, dict):
- return {k: self.wrap_device(v, device) for k, v in x.items()}
-
- if hasattr(x, "device") and x.device.type == device:
- return x
-
- return x.to(device)
-
- def __init__(
- self,
- model_name: str,
- dtype: str = "auto",
- *,
- model_kwargs: dict[str, Any] | None = None,
- trust_remote_code: bool = True,
- is_sentence_transformer: bool = False,
- is_cross_encoder: bool = False,
- skip_tokenizer_init: bool = False,
- auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,
- # Set this to avoid hanging issue
- default_torch_num_threads: int | None = None,
- ) -> None:
- init_ctx = (
- nullcontext()
- if default_torch_num_threads is None
- else set_default_torch_num_threads(default_torch_num_threads)
- )
-
- with init_ctx:
- self._init(
- model_name=model_name,
- dtype=dtype,
- model_kwargs=model_kwargs,
- trust_remote_code=trust_remote_code,
- is_sentence_transformer=is_sentence_transformer,
- is_cross_encoder=is_cross_encoder,
- skip_tokenizer_init=skip_tokenizer_init,
- auto_cls=auto_cls,
- )
-
- def _init(
- self,
- model_name: str,
- dtype: str = "auto",
- *,
- model_kwargs: dict[str, Any] | None = None,
- trust_remote_code: bool = True,
- is_sentence_transformer: bool = False,
- is_cross_encoder: bool = False,
- skip_tokenizer_init: bool = False,
- auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,
- ) -> None:
- model_name = maybe_model_redirect(model_name)
- self.model_name = model_name
-
- self.config = AutoConfig.from_pretrained(
- model_name,
- trust_remote_code=trust_remote_code,
- )
- self.device = self.get_default_device()
- self.dtype = dtype = _get_and_verify_dtype(
- self.model_name,
- self.config,
- dtype=dtype,
- is_pooling_model=is_sentence_transformer or is_cross_encoder,
- )
-
- model_kwargs = model_kwargs if model_kwargs is not None else {}
- model_kwargs.setdefault("dtype", dtype)
-
- if is_sentence_transformer:
- # Lazy init required for AMD CI
- from sentence_transformers import SentenceTransformer
-
- self.model = SentenceTransformer(
- model_name,
- device=self.device,
- model_kwargs=model_kwargs,
- trust_remote_code=trust_remote_code,
- )
- elif is_cross_encoder:
- # Lazy init required for AMD CI
- from sentence_transformers import CrossEncoder
-
- self.model = CrossEncoder(
- model_name,
- device=self.device,
- automodel_args=model_kwargs,
- trust_remote_code=trust_remote_code,
- )
- else:
- model = auto_cls.from_pretrained(
- model_name,
- trust_remote_code=trust_remote_code,
- **model_kwargs,
- )
-
- # in case some unquantized custom models are not in same dtype
- if getattr(model, "quantization_method", None) is None and any(
- p.dtype != self.dtype for p in model.parameters()
- ):
- model = model.to(dtype=self.dtype)
-
- if (
- getattr(model, "quantization_method", None) != "bitsandbytes"
- and len({p.device for p in model.parameters()}) < 2
- ):
- model = model.to(device=self.device)
-
- self.model = model
-
- if not skip_tokenizer_init:
- self.tokenizer = AutoTokenizer.from_pretrained(
- model_name,
- dtype=dtype,
- trust_remote_code=trust_remote_code,
- )
-
- # don't put this import at the top level
- # it will call torch.cuda.device_count()
- from transformers import AutoProcessor # noqa: F401
-
- self.processor = AutoProcessor.from_pretrained(
- model_name,
- dtype=dtype,
- trust_remote_code=trust_remote_code,
- )
- if skip_tokenizer_init:
- self.tokenizer = self.processor.tokenizer
-
- def get_inputs(
- self,
- prompts: list[str] | list[list[int]],
- images: PromptImageInput | None = None,
- videos: PromptVideoInput | None = None,
- audios: PromptAudioInput | None = None,
- ) -> list[BatchFeature | BatchEncoding | dict[str, torch.Tensor]]:
- if images is not None:
- assert len(prompts) == len(images)
-
- if videos is not None:
- assert len(prompts) == len(videos)
-
- if audios is not None:
- assert len(prompts) == len(audios)
-
- all_inputs: list[BatchFeature | BatchEncoding | dict[str, torch.Tensor]] = []
- for i, prompt in enumerate(prompts):
- if isinstance(prompt, str):
- processor_kwargs: dict[str, Any] = {
- "text": prompt,
- "return_tensors": "pt",
- }
- if images is not None and (image := images[i]) is not None:
- processor_kwargs["images"] = image
- if videos is not None and (video := videos[i]) is not None:
- processor_kwargs["videos"] = video
- if audios is not None and (audio_inputs := audios[i]) is not None:
- # HACK - not all processors take sampling_rate; we should
- # clean this up in the future.
- if len(audio_inputs) == 2:
- audio, sr = audio_inputs
- processor_kwargs["audio"] = audio
- processor_kwargs["sampling_rate"] = sr
- else:
- processor_kwargs["audio"] = audio_inputs
-
- inputs = self.processor(**processor_kwargs)
- if isinstance(inputs, BatchFeature):
- inputs = inputs.to(dtype=self.dtype)
- all_inputs.append(inputs)
- else:
- # check that prompt is (batched) list of integers (token ids)
- if not is_list_of(prompt, typ=int, check="all"):
- raise ValueError("Prompt must be a list of ints corresponding to the prompt token ids.")
- # check that no multimodal input is provided
- if images or videos or audios:
- raise ValueError("When providing prompt token ids multimodal inputs are not supported.")
- input_dict = {
- "input_ids": torch.tensor(prompt, dtype=torch.long).unsqueeze(0),
- }
- all_inputs.append(input_dict)
-
- return all_inputs
-
- def get_prompt_embeddings(self, prompts: list[str]) -> list[torch.Tensor]:
- all_inputs = self.get_inputs(prompts)
- embeddings = []
- for inputs in all_inputs:
- input_ids = self.wrap_device(inputs)["input_ids"]
- embedding = self.model.get_input_embeddings()(input_ids).squeeze(0)
- embeddings.append(embedding)
- return embeddings
-
- def classify(self, prompts: list[str]) -> list[str]:
- # output is final logits
- all_inputs = self.get_inputs(prompts)
- outputs = []
- problem_type = getattr(self.config, "problem_type", "")
-
- for inputs in all_inputs:
- output = self.model(**self.wrap_device(inputs))
- if problem_type == "regression":
- logits = output.logits[0].tolist()
- elif problem_type == "multi_label_classification":
- logits = output.logits.sigmoid()[0].tolist()
- else:
- logits = output.logits.softmax(dim=-1)[0].tolist()
- outputs.append(logits)
-
- return outputs
-
- def generate(
- self,
- prompts: list[str] | list[list[int]],
- images: PromptImageInput | None = None,
- videos: PromptVideoInput | None = None,
- audios: PromptAudioInput | None = None,
- **kwargs: Any,
- ) -> list[tuple[list[list[int]], list[str]]]:
- all_inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)
-
- outputs: list[tuple[list[list[int]], list[str]]] = []
- for inputs in all_inputs:
- output_ids = self.model.generate(
- **self.wrap_device(inputs),
- use_cache=True,
- **kwargs,
- )
- output_str = self.processor.batch_decode(
- output_ids,
- skip_special_tokens=True,
- clean_up_tokenization_spaces=False,
- )
- output_ids = output_ids.cpu().tolist()
- outputs.append((output_ids, output_str))
- return outputs
-
- def generate_greedy(
- self,
- prompts: list[str] | list[list[int]],
- max_tokens: int,
- images: PromptImageInput | None = None,
- videos: PromptVideoInput | None = None,
- audios: PromptAudioInput | None = None,
- **kwargs: Any,
- ) -> list[tuple[list[int], str]]:
- outputs = self.generate(
- prompts,
- do_sample=False,
- max_new_tokens=max_tokens,
- images=images,
- videos=videos,
- audios=audios,
- **kwargs,
- )
-
- return [(output_ids[0], output_str[0]) for output_ids, output_str in outputs]
-
- def generate_beam_search(
- self,
- prompts: list[str],
- beam_width: int,
- max_tokens: int,
- images: PromptImageInput | None = None,
- videos: PromptVideoInput | None = None,
- audios: PromptAudioInput | None = None,
- ) -> list[tuple[list[list[int]], list[str]]]:
- outputs = self.generate(
- prompts,
- do_sample=False,
- max_new_tokens=max_tokens,
- num_beams=beam_width,
- num_return_sequences=beam_width,
- images=images,
- videos=videos,
- audios=audios,
- )
-
- for i in range(len(outputs)):
- output_ids, output_str = outputs[i]
- for j in range(len(output_ids)):
- output_ids[j] = [x for x in output_ids[j] if x != self.tokenizer.pad_token_id]
- outputs[i] = (output_ids, output_str)
- return outputs
-
- def generate_greedy_logprobs(
- self,
- prompts: list[str],
- max_tokens: int,
- images: PromptImageInput | None = None,
- videos: PromptVideoInput | None = None,
- audios: PromptAudioInput | None = None,
- **kwargs: Any,
- ) -> list[list[torch.Tensor]]:
- all_inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)
-
- all_logprobs: list[list[torch.Tensor]] = []
- for inputs in all_inputs:
- output = self.model.generate(
- **self.wrap_device(inputs),
- use_cache=True,
- do_sample=False,
- max_new_tokens=max_tokens,
- output_hidden_states=True,
- return_dict_in_generate=True,
- **kwargs,
- )
- seq_logprobs = self._hidden_states_to_seq_logprobs(output.hidden_states)
- all_logprobs.append(seq_logprobs)
- return all_logprobs
-
- def _hidden_states_to_seq_logprobs(
- self,
- hidden_states: tuple[tuple[torch.Tensor, ...], ...],
- ) -> list[torch.Tensor]:
- output_embeddings = self.model.get_output_embeddings()
-
- seq_logprobs: list[torch.Tensor] = []
- for _, hidden_state in enumerate(hidden_states):
- last_hidden_states = hidden_state[-1][0]
- logits = torch.matmul(
- last_hidden_states.to(
- device=output_embeddings.weight.device,
- dtype=output_embeddings.weight.dtype,
- ),
- output_embeddings.weight.t(),
- )
- if getattr(output_embeddings, "bias", None) is not None:
- logits += output_embeddings.bias.unsqueeze(0)
- logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
- seq_logprobs.append(logprobs)
-
- return seq_logprobs
-
- def _hidden_states_to_logprobs(
- self,
- hidden_states: tuple[tuple[torch.Tensor, ...], ...],
- num_logprobs: int | None,
- ) -> tuple[list[dict[int, float]], int]:
- seq_logprobs = self._hidden_states_to_seq_logprobs(hidden_states)
- output_len = len(hidden_states)
-
- # convert to dict
- seq_logprobs_lst: list[dict[int, float]] = []
- for tok_idx, tok_logprobs in enumerate(seq_logprobs):
- # drop prompt logprobs
- if tok_idx == 0:
- tok_logprobs = tok_logprobs[-1, :].reshape(1, -1)
- topk = tok_logprobs.topk(num_logprobs)
-
- tok_logprobs_dct = {}
- for token_id, logprob in zip(topk.indices[0], topk.values[0]):
- tok_logprobs_dct[token_id.item()] = logprob.item()
-
- seq_logprobs_lst.append(tok_logprobs_dct)
-
- return (
- seq_logprobs_lst,
- output_len,
- )
-
- def generate_greedy_logprobs_limit(
- self,
- prompts: list[str],
- max_tokens: int,
- num_logprobs: int | None,
- images: PromptImageInput | None = None,
- audios: PromptAudioInput | None = None,
- videos: PromptVideoInput | None = None,
- **kwargs: Any,
- ) -> list[TokensTextLogprobs]:
- all_inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)
-
- all_logprobs: list[list[dict[int, float]]] = []
- all_output_ids: list[list[int]] = []
- all_output_strs: list[str] = []
-
- for inputs in all_inputs:
- output = self.model.generate(
- **self.wrap_device(inputs),
- use_cache=True,
- do_sample=False,
- max_new_tokens=max_tokens,
- output_hidden_states=True,
- return_dict_in_generate=True,
- **kwargs,
- )
-
- (
- seq_logprobs_lst,
- output_len,
- ) = self._hidden_states_to_logprobs(output.hidden_states, num_logprobs)
-
- all_logprobs.append(seq_logprobs_lst)
- seq_ids = output.sequences[0]
- output_len = len(seq_logprobs_lst)
- output_ids = seq_ids[-output_len:]
- all_output_ids.append(output_ids.tolist())
- all_output_strs.append(self.tokenizer.decode(output_ids))
-
- outputs = zip(all_output_ids, all_output_strs, all_logprobs)
- return [(output_ids, output_str, output_logprobs) for output_ids, output_str, output_logprobs in outputs]
-
- def encode(self, prompts: list[str], *args, **kwargs) -> list[list[torch.Tensor]]:
- return self.model.encode(prompts, *args, **kwargs)
-
- def predict(self, prompts: list[list[str]], *args, **kwargs) -> torch.Tensor:
- return self.model.predict(prompts, *args, convert_to_tensor=True, **kwargs)
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_value, traceback):
- del self.model
- cleanup_dist_env_and_memory()
-
-
[email protected](scope="session")
-def hf_runner():
- return HfRunner
-
-
-class VllmRunner:
- """
- The default value of some arguments have been modified from
- {class}`~vllm.LLM` as follows:
-
- - `trust_remote_code`: Set to `True` instead of `False` for convenience.
- - `seed`: Set to `0` instead of `None` for test reproducibility.
- - `max_model_len`: Set to `1024` instead of `None` to reduce memory usage.
- - `block_size`: To reduce memory usage, set default to `64` if on XPU
- devices, otherwise default to `16`.
- - `enable_chunked_prefill`: Set to `False` instead of `None` for
- test reproducibility.
- - `enforce_eager`: Set to `False` to test CUDA graph.
- """
-
- def __init__(
- self,
- model_name: str,
- runner: RunnerOption = "auto",
- convert: ConvertOption = "auto",
- tokenizer_name: str | None = None,
- tokenizer_mode: str = "auto",
- trust_remote_code: bool = True,
- seed: int | None = 0,
- max_model_len: int | None = 1024,
- dtype: str = "auto",
- disable_log_stats: bool = True,
- tensor_parallel_size: int = 1,
- block_size: int = 16 if not torch.xpu.is_available() else 64,
- enable_chunked_prefill: bool | None = False,
- swap_space: int = 4,
- enforce_eager: bool | None = False,
- # Set this to avoid hanging issue
- default_torch_num_threads: int | None = None,
- **kwargs,
- ) -> None:
- init_ctx = (
- nullcontext()
- if default_torch_num_threads is None
- else set_default_torch_num_threads(default_torch_num_threads)
- )
-
- if not kwargs.get("compilation_config", None):
- # Note(@tdoublep): This is set to 4 because some tests (e.g., hybrid
- # model tests) may set max_num_seqs=4. If min cudagraph_capture_size is
- # set to larger than max_num_seqs, then it will lead to *no* graphs
- # being captured which can trigger edge cases that we don't handle yet.
- kwargs["compilation_config"] = {"cudagraph_capture_sizes": [4]}
-
- with init_ctx:
- self.llm = LLM(
- model=model_name,
- runner=runner,
- convert=convert,
- tokenizer=tokenizer_name,
- tokenizer_mode=tokenizer_mode,
- trust_remote_code=trust_remote_code,
- dtype=dtype,
- seed=seed,
- swap_space=swap_space,
- enforce_eager=enforce_eager,
- disable_log_stats=disable_log_stats,
- tensor_parallel_size=tensor_parallel_size,
- max_model_len=max_model_len,
- block_size=block_size,
- enable_chunked_prefill=enable_chunked_prefill,
- **kwargs,
- )
-
- def get_inputs(
- self,
- prompts: list[str] | list[torch.Tensor] | list[list[int]],
- images: PromptImageInput | None = None,
- videos: PromptVideoInput | None = None,
- audios: PromptAudioInput | None = None,
- ) -> list[dict[str, Any]]:
- if any(x is not None and len(x) != len(prompts) for x in [images, videos, audios]):
- raise ValueError("All non-None multimodal inputs must have the same length as prompts")
-
- inputs = list[dict[str, Any]]()
- for i, prompt in enumerate(prompts):
- prompt_dict = dict[str, Any]()
- if isinstance(prompt, str):
- prompt_dict["prompt"] = prompt
- elif isinstance(prompt, list):
- prompt_dict["prompt_token_ids"] = prompt
- else:
- prompt_dict["prompt_embeds"] = prompt
-
- multi_modal_data = dict[str, Any]()
- if images is not None and (image := images[i]) is not None:
- multi_modal_data["image"] = image
- if videos is not None and (video := videos[i]) is not None:
- multi_modal_data["video"] = video
- if audios is not None and (audio := audios[i]) is not None:
- multi_modal_data["audio"] = audio
-
- if multi_modal_data:
- prompt_dict["multi_modal_data"] = multi_modal_data
-
- inputs.append(prompt_dict)
-
- return inputs
-
- def generate(
- self,
- prompts: list[str] | list[torch.Tensor] | list[list[int]],
- sampling_params: SamplingParams,
- images: PromptImageInput | None = None,
- videos: PromptVideoInput | None = None,
- audios: PromptAudioInput | None = None,
- **kwargs: Any,
- ) -> list[tuple[list[list[int]], list[str]]]:
- inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)
-
- req_outputs = self.llm.generate(inputs, sampling_params=sampling_params, **kwargs)
-
- outputs: list[tuple[list[list[int]], list[str]]] = []
- for req_output in req_outputs:
- prompt_str = req_output.prompt
- prompt_ids = req_output.prompt_token_ids
- req_sample_output_ids: list[list[int]] = []
- req_sample_output_strs: list[str] = []
- for sample in req_output.outputs:
- output_str = sample.text
- output_ids = list(sample.token_ids)
- req_sample_output_ids.append(prompt_ids + output_ids)
- req_sample_output_strs.append((prompt_str or "") + output_str)
- outputs.append((req_sample_output_ids, req_sample_output_strs))
- return outputs
-
- @staticmethod
- def _final_steps_generate_w_logprobs(
- req_outputs: list[RequestOutput],
- ) -> list[TokensTextLogprobsPromptLogprobs]:
- outputs: list[TokensTextLogprobsPromptLogprobs] = []
- for req_output in req_outputs:
- assert len(req_output.outputs) > 0
- for sample in req_output.outputs:
- output_str = sample.text
- output_ids = list(sample.token_ids)
- output_logprobs = sample.logprobs
- outputs.append((output_ids, output_str, output_logprobs, req_output.prompt_logprobs))
- return outputs
-
- def generate_w_logprobs(
- self,
- prompts: list[str],
- sampling_params: SamplingParams,
- images: PromptImageInput | None = None,
- audios: PromptAudioInput | None = None,
- videos: PromptVideoInput | None = None,
- **kwargs: Any,
- ) -> list[TokensTextLogprobs] | list[TokensTextLogprobsPromptLogprobs]:
- inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)
-
- req_outputs = self.llm.generate(inputs, sampling_params=sampling_params, **kwargs)
-
- toks_str_logsprobs_prompt_logprobs = self._final_steps_generate_w_logprobs(req_outputs)
- # Omit prompt logprobs if not required by sampling params
- return (
- [x[0:-1] for x in toks_str_logsprobs_prompt_logprobs]
- if sampling_params.prompt_logprobs is None
- else toks_str_logsprobs_prompt_logprobs
- )
-
- def generate_greedy(
- self,
- prompts: list[str] | list[torch.Tensor] | list[list[int]],
- max_tokens: int,
- images: PromptImageInput | None = None,
- videos: PromptVideoInput | None = None,
- audios: PromptAudioInput | None = None,
- **kwargs: Any,
- ) -> list[tuple[list[int], str]]:
- greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
- outputs = self.generate(
- prompts,
- greedy_params,
- images=images,
- videos=videos,
- audios=audios,
- **kwargs,
- )
- return [(output_ids[0], output_str[0]) for output_ids, output_str in outputs]
-
- def generate_greedy_logprobs(
- self,
- prompts: list[str],
- max_tokens: int,
- num_logprobs: int | None,
- num_prompt_logprobs: int | None = None,
- images: PromptImageInput | None = None,
- audios: PromptAudioInput | None = None,
- videos: PromptVideoInput | None = None,
- stop_token_ids: list[int] | None = None,
- stop: list[str] | None = None,
- **kwargs: Any,
- ) -> list[TokensTextLogprobs] | list[TokensTextLogprobsPromptLogprobs]:
- greedy_logprobs_params = SamplingParams(
- temperature=0.0,
- max_tokens=max_tokens,
- logprobs=num_logprobs,
- prompt_logprobs=num_prompt_logprobs,
- stop_token_ids=stop_token_ids,
- stop=stop,
- )
-
- return self.generate_w_logprobs(
- prompts,
- greedy_logprobs_params,
- images=images,
- audios=audios,
- videos=videos,
- **kwargs,
- )
-
- def generate_prompt_perplexity(self, prompts: list[str]) -> list[float]:
- """
- Return the perplexity score associated with generating the prompts
-
- :param prompts: list of prompts to score
- :return: perplexity score of each prompt
- """
- outputs = self.generate_greedy_logprobs(prompts, max_tokens=1, num_logprobs=None, num_prompt_logprobs=0)
-
- perplexities = []
- for output in outputs:
- output = cast(TokensTextLogprobsPromptLogprobs, output)
- token_data = cast(list[dict[int, Logprob] | None], output[3])
- assert token_data[0] is None
- token_log_probs = []
- for token_data in token_data[1:]:
- assert token_data is not None
- assert len(token_data) == 1
- token_log_prob = list(token_data.values())[0].logprob
- token_log_probs.append(token_log_prob)
-
- perplexity = math.exp(-sum(token_log_probs) / len(token_log_probs))
- perplexities.append(perplexity)
-
- return perplexities
-
- def generate_beam_search(
- self,
- prompts: list[str],
- beam_width: int,
- max_tokens: int,
- images: PromptImageInput | None = None,
- videos: PromptVideoInput | None = None,
- audios: PromptAudioInput | None = None,
- concurrency_limit: int | None = None,
- ) -> list[tuple[list[list[int]], list[str]]]:
- inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)
-
- outputs = self.llm.beam_search(
- inputs,
- BeamSearchParams(beam_width=beam_width, max_tokens=max_tokens),
- concurrency_limit=concurrency_limit,
- )
- returned_outputs = []
- for output in outputs:
- token_ids = [x.tokens for x in output.sequences]
- texts = [x.text for x in output.sequences]
- returned_outputs.append((token_ids, texts))
- return returned_outputs
-
- def classify(self, prompts: list[str]) -> list[list[float]]:
- req_outputs = self.llm.classify(prompts)
- return [req_output.outputs.probs for req_output in req_outputs]
-
- def embed(
- self,
- prompts: list[str],
- images: PromptImageInput | None = None,
- videos: PromptVideoInput | None = None,
- audios: PromptAudioInput | None = None,
- *args,
- **kwargs,
- ) -> list[list[float]]:
- inputs = self.get_inputs(prompts, images=images, videos=videos, audios=audios)
-
- req_outputs = self.llm.embed(inputs, *args, **kwargs)
- return [req_output.outputs.embedding for req_output in req_outputs]
-
- def token_embed(self, prompts: list[str]) -> list[list[float]]:
- req_outputs = self.llm.encode(prompts, pooling_task="token_embed")
- return [req_output.outputs.data for req_output in req_outputs]
-
- def token_classify(self, prompts: list[str]) -> list[list[float]]:
- req_outputs = self.llm.encode(prompts, pooling_task="token_classify")
- return [req_output.outputs.data for req_output in req_outputs]
-
- def reward(self, prompts: list[str]) -> list[list[float]]:
- req_outputs = self.llm.reward(prompts)
- return [req_output.outputs.data for req_output in req_outputs]
-
- def score(
- self,
- text_1: list[str] | str,
- text_2: list[str] | str,
- *args,
- **kwargs,
- ) -> list[float]:
- req_outputs = self.llm.score(text_1, text_2, *args, **kwargs)
- return [req_output.outputs.score for req_output in req_outputs]
-
- def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
- return self.llm.apply_model(func)
-
- def get_llm(self) -> LLM:
- return self.llm
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_value, traceback):
- del self.llm
- cleanup_dist_env_and_memory()
-
-
[email protected](scope="session")
-def vllm_runner():
- return VllmRunner
-
-
-def temporary_enable_log_propagate():
- import logging
-
- logger = logging.getLogger("vllm")
- logger.propagate = True
- yield
- logger.propagate = False
-
-
-def caplog_vllm(temporary_enable_log_propagate, caplog):
- # To capture vllm log, we should enable propagate=True temporarily
- # because caplog depends on logs propagated to the root logger.
- yield caplog
-
-
-def caplog_mp_fork():
- """
- This fixture enables capturing logs from a forked MP subprocess.
- It should be used in conjunction with caplog_vllm.
-
- By default, subprocess logs do not go through the parent process.
- We instead create a queue listener in the parent process which
- forwards logs to the logger's other handlers, and add a QueueHandler
- to the root logger. Forked subprocesses will inherit the root logger
- and pass their messages to the queue, which the listener will forward
- to the root logger, which can be captured by caplog.
-
- Note that this workaround only works for fork; with spawn, the subprocess
- reinitializes logging and does not automatically inherit the queue.
- We'd have to manually pass the queue to the subprocess at the spawn point.
- See caplog_mp_spawn below.
- """
-
- @contextlib.contextmanager
- def ctx():
- import logging.handlers
- import multiprocessing as mp
-
- logger_queue: mp.Queue[logging.LogRecord] = mp.Queue()
- logger = logging.getLogger()
- handlers = logger.handlers
-
- # The listener works on a background thread, not inherited by the child.
- queue_listener = logging.handlers.QueueListener(logger_queue, *handlers)
- queue_listener.start()
-
- # Add queue handler after creating the listener to avoid cycle
- logger.addHandler(logging.handlers.QueueHandler(logger_queue))
- yield
- queue_listener.stop()
-
- return ctx
-
-
-class LogHolder:
- def __init__(self):
- self.text = None
-
-
-def caplog_mp_spawn(tmp_path, monkeypatch):
- """
- This fixture enables capturing logs from a forked MP subprocess.
- It does not require caplog_vllm (but it only contains logs from the child).
-
- By default, subprocess logs do not go through the parent process.
- We instead add a FileHandler to the config so the spawned child process
- writes its logs to a temp file.
- In the parent, we read the file and return the contents.
-
- Note: this method could be extended to fork by either reconfiguring logging
- in the parent or using a SocketHandler:
- https://docs.python.org/3/howto/logging-cookbook.html#sending-and-receiving-logging-events-across-a-network # noqa: E501
- """
-
- @contextlib.contextmanager
- def ctx(level: int | str):
- from vllm.logger import DEFAULT_LOGGING_CONFIG
-
- config_path = tmp_path / "vllm_logging_config.json"
- log_path = tmp_path / "vllm.log"
- log_holder = LogHolder()
-
- config = deepcopy(DEFAULT_LOGGING_CONFIG)
- if envs.VLLM_LOGGING_CONFIG_PATH:
- path = pathlib.Path(envs.VLLM_LOGGING_CONFIG_PATH)
- assert path.exists()
- config = json.loads(path.read_text())
-
- config["loggers"]["vllm"]["handlers"] += ["vllm_file"]
- config["handlers"]["vllm_file"] = {
- "class": "logging.FileHandler",
- "formatter": "vllm",
- "level": level,
- "filename": log_path.as_posix(),
- }
-
- config_path.write_text(json.dumps(config))
-
- with monkeypatch.context() as monkeypatch_ctx:
- monkeypatch_ctx.setenv("VLLM_LOGGING_CONFIG_PATH", config_path.as_posix())
- monkeypatch_ctx.setenv("VLLM_CONFIGURE_LOGGING", "1")
- yield log_holder
-
- log_holder.text = log_path.read_text()
-
- return ctx
-
-
[email protected](scope="session")
-def num_gpus_available():
- """Get number of GPUs without initializing the CUDA context
- in current process."""
-
- from vllm.platforms import current_platform
-
- return current_platform.device_count()
-
-
-temp_dir = tempfile.gettempdir()
-_dummy_opt_path = os.path.join(temp_dir, "dummy_opt")
-_dummy_llava_path = os.path.join(temp_dir, "dummy_llava")
-_dummy_gemma2_embedding_path = os.path.join(temp_dir, "dummy_gemma2_embedding")
-
-
-def dummy_opt_path():
- json_path = os.path.join(_dummy_opt_path, "config.json")
- if not os.path.exists(_dummy_opt_path):
- snapshot_download(
- repo_id="facebook/opt-125m",
- local_dir=_dummy_opt_path,
- ignore_patterns=["*.bin", "*.bin.index.json", "*.pt", "*.h5", "*.msgpack"],
- )
- assert os.path.exists(json_path)
- with open(json_path) as f:
- config = json.load(f)
- config["architectures"] = ["MyOPTForCausalLM"]
- with open(json_path, "w") as f:
- json.dump(config, f)
- return _dummy_opt_path
-
-
-def dummy_llava_path():
- json_path = os.path.join(_dummy_llava_path, "config.json")
- if not os.path.exists(_dummy_llava_path):
- snapshot_download(
- repo_id="llava-hf/llava-1.5-7b-hf",
- local_dir=_dummy_llava_path,
- ignore_patterns=[
- "*.bin",
- "*.bin.index.json",
- "*.pt",
- "*.h5",
- "*.msgpack",
- "*.safetensors",
- ],
- )
- assert os.path.exists(json_path)
- with open(json_path) as f:
- config = json.load(f)
- config["architectures"] = ["MyLlava"]
- with open(json_path, "w") as f:
- json.dump(config, f)
- return _dummy_llava_path
-
-
-def dummy_gemma2_embedding_path():
- json_path = os.path.join(_dummy_gemma2_embedding_path, "config.json")
- if not os.path.exists(_dummy_gemma2_embedding_path):
- snapshot_download(
- repo_id="BAAI/bge-multilingual-gemma2",
- local_dir=_dummy_gemma2_embedding_path,
- ignore_patterns=[
- "*.bin",
- "*.bin.index.json",
- "*.pt",
- "*.h5",
- "*.msgpack",
- "*.safetensors",
- ],
- )
- assert os.path.exists(json_path)
- with open(json_path) as f:
- config = json.load(f)
- config["architectures"] = ["MyGemma2Embedding"]
- with open(json_path, "w") as f:
- json.dump(config, f)
- return _dummy_gemma2_embedding_path
-
-
-# Add the flag `--optional` to allow run tests
-# that are marked with @pytest.mark.optional
-def pytest_addoption(parser):
- parser.addoption("--optional", action="store_true", default=False, help="run optional test")
-
-
-def pytest_collection_modifyitems(config, items):
- if config.getoption("--optional"):
- # --optional given in cli: do not skip optional tests
- return
- skip_optional = pytest.mark.skip(reason="need --optional option to run")
- for item in items:
- if "optional" in item.keywords:
- item.add_marker(skip_optional)
-
-
[email protected](scope="session")
-def cli_config_file():
- """Return the path to the CLI config file."""
- return os.path.join(_TEST_DIR, "config", "test_config.yaml")
-
-
[email protected](scope="session")
-def cli_config_file_with_model():
- """Return the path to the CLI config file with model."""
- return os.path.join(_TEST_DIR, "config", "test_config_with_model.yaml")
-
-
-class AssetHandler(http.server.BaseHTTPRequestHandler):
- # _IMAGE_CACHE : Dict[str, bytes] = {}
-
- def log_message(self, *args, **kwargs):
- pass
-
- def do_GET(self):
- # Accepts paths like: /1280px-Venn_diagram_rgb.jpg
- filename = self.path.lstrip("/")
- if not filename or "." not in filename:
- self.send_error(404, "Missing filename (expected /<name>.<ext>)")
- return
-
- base, ext = filename.rsplit(".", 1)
- ext = ext.lower()
-
- if ext not in ["jpg", "png"]:
- self.send_error(404, f"Unsupported extension: .{ext}")
- return
-
- try:
- data = ImageAsset(base).read_bytes(ext=ext)
- except Exception as e:
- self.send_error(500, f"Failed to load asset: {ext} {base} {e} ")
- return
-
- ctype, _ = mimetypes.guess_type(filename)
- if ctype is None:
- ctype = {"jpg": "image/jpg", "png": "image/png"}[ext]
- self.send_response(200)
- self.send_header("Content-Type", ctype)
- self.send_header("Content-Length", str(len(data)))
- self.end_headers()
- self.wfile.write(data)
-
-
-def _find_free_port() -> int:
- with socket.socket() as s:
- s.bind(("127.0.0.1", 0))
- return s.getsockname()[1]
-
-
-class LocalAssetServer:
- address: str
- port: int
- server: http.server.ThreadingHTTPServer | None
- thread: threading.Thread | None
-
- def __init__(self, address: str = "127.0.0.1") -> None:
- self.address = address
- self.port = -1
- self.server = None
- self.thread = None
-
- def __enter__(self):
- self.port = _find_free_port()
- self.server = http.server.ThreadingHTTPServer((self.address, self.port), AssetHandler)
- self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
- self.thread.start()
- return self
-
- def __exit__(self, exc_type, exc_value, traceback):
- if self.server:
- self.server.shutdown()
- del self.server
-
- if self.thread:
- self.thread.join()
- del self.thread
-
- if exc_type is None:
- return None
-
- return False
-
- @property
- def base_url(self) -> str:
- assert self.port is not None
- return f"http://{self.address}:{self.port}"
-
- def url_for(self, name: str) -> str:
- """e.g., name='RGBA_comp.png' -> 'http://127.0.0.1:PORT/RGBA_comp.png'"""
- return f"{self.base_url}/{name}"
-
- def get_image_asset(self, name: str) -> Image.Image:
- return fetch_image(self.url_for(name))
-
-
[email protected](scope="session")
-def local_asset_server() -> Generator[LocalAssetServer, None, None]:
- """
- Starts a thread based HTTP server bound to 127.0.0.1 on a random free port.
- The server currently servers images at:
- http://127.0.0.1:<port>/<name>.<ext>
- """
- with LocalAssetServer() as srv:
- yield srv
-
-
-def image_url(request, local_asset_server) -> str:
- # request.param is one of the IMAGE_ASSETS filenames
- name = request.param
- return local_asset_server.url_for(name)
-
-
-def image_urls(request, local_asset_server) -> list[str]:
- """Indirect fixture: takes a list of names, returns list of full URLs."""
- names: list[str] = request.param
- return [local_asset_server.url_for(name) for name in names]
diff --git a/auto_round_extension/vllm_ext/tests/test_fp8kv.py b/auto_round_extension/vllm_ext/tests/test_fp8kv.py
deleted file mode 100644
index 709308f9a..000000000
--- a/auto_round_extension/vllm_ext/tests/test_fp8kv.py
+++ /dev/null
@@ -1,61 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import pytest
-import torch
-from vllm.platforms import current_platform
-
-
-def cuda_capability_at_least(major, minor):
- device_capability = torch.cuda.get_device_capability()
- return device_capability[0] >= major or (device_capability[0] == major and device_capability[1] >= minor)
-
-
-MODELS = ["/home/yiliu7/workspace/auto-round/examples/Qwen2.5-0.5B-Instruct-ar-MXFP4-fp8"]
-
-
[email protected](autouse=True)
-def set_vllm_ar_env(monkeypatch):
- monkeypatch.setenv("VLLM_AR_MXFP4_MODULAR_MOE", "1")
- monkeypatch.setenv("VLLM_MXFP4_PRE_UNPACK_TO_FP8", "1")
- monkeypatch.setenv("VLLM_MXFP4_PRE_UNPACK_WEIGHTS", "0")
- monkeypatch.setenv("VLLM_ENABLE_STATIC_MOE", "0")
- monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "0")
- monkeypatch.setenv("VLLM_ENABLE_AR_EXT", "1")
- monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
- monkeypatch.setenv("VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION", "1")
- monkeypatch.setenv("VLLM_ATTENTION_BACKEND", "FLASHINFER")
-
-
- not current_platform.is_cuda(),
- reason="only supports CUDA backend.",
-)
- not cuda_capability_at_least(10, 0), reason="FP8 KV cache only supported on CUDA with compute capability >= 10.0"
-)
[email protected]("model", MODELS)
-def test_auto_fp8_kv(vllm_runner, model):
- with vllm_runner(
- model,
- # enforce_eager=True,
- kv_cache_dtype="fp8",
- gpu_memory_utilization=0.1,
- ) as llm:
- output = llm.generate_greedy(["The capital of France is"], max_tokens=8)
- assert (
- llm.llm.llm_engine.engine_core.engine_core.model_executor.driver_worker.worker.model_runner.kv_cache_dtype
- == torch.uint8
- ), f"Expected kv_cache_dtype to be torch.uint8, but got {llm.llm.llm_engine.engine_core.engine_core.model_executor.driver_worker.worker.model_runner.kv_cache_dtype}"
- assert output
- print(f"output is: {output[0][1]}")
diff --git a/auto_round_extension/vllm_ext/tests/test_models.py b/auto_round_extension/vllm_ext/tests/test_models.py
deleted file mode 100644
index faa783cde..000000000
--- a/auto_round_extension/vllm_ext/tests/test_models.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import pytest
-from vllm.platforms import current_platform
-
-MODELS = [
- # "/data5/yliu7/HF_HOME/unsloth-gpt-oss-20b-BF16-ar-MXFP4/"
- # "/data5/yliu7/HF_HOME/Qwen2.5-0.5B-Instruct-test-FP8_STATIC-fp8kv/"
- # "/data6/yiliu4/Qwen3-15B-A2B-Base-MXFP4",
- # "/data6/yiliu4/Llama-3.2-1B-Instruct-MXFP4-fp8attention",
- # "/data6/yiliu4/Llama-3.2-1B-Instruct-MXFP8"
- "/storage/yiliu7/ar_vllm_ext/quantized_model_qwen_mxfp4",
- "/storage/yiliu7/ar_vllm_ext/quantized_model_qwen_mxfp8",
-]
-
-
[email protected](autouse=True)
-def set_vllm_ar_env(monkeypatch):
- monkeypatch.setenv("VLLM_AR_MXFP4_MODULAR_MOE", "1")
- monkeypatch.setenv("VLLM_MXFP4_PRE_UNPACK_TO_FP8", "1")
- monkeypatch.setenv("VLLM_MXFP4_PRE_UNPACK_WEIGHTS", "0")
- monkeypatch.setenv("VLLM_ENABLE_STATIC_MOE", "0")
- monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "0")
- monkeypatch.setenv("VLLM_ENABLE_AR_EXT", "1")
-
-
- not current_platform.is_cuda(),
- reason="only supports CUDA backend.",
-)
[email protected]("model", MODELS)
-def test_auto_round(vllm_runner, model):
- with vllm_runner(model, enforce_eager=True) as llm:
- output = llm.generate_greedy(["The capital of France is"], max_tokens=8)
- assert output
- print(f"output is: {output[0][1]}")
diff --git a/auto_round_extension/vllm_ext/utils.py b/auto_round_extension/vllm_ext/utils.py
deleted file mode 100644
index db5a0dff1..000000000
--- a/auto_round_extension/vllm_ext/utils.py
+++ /dev/null
@@ -1,167 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Union
-
-import torch
-
-from auto_round.schemes import QuantizationScheme
-
-E8M0_EXPONENT_BIAS = 127
-E8M0_EXPONENT_NAN_VAL = 255
-
-
-def get_scheme(quant_config, prefix: str):
- # Check extra_config first
- layer_schemes = quant_config.layer_schemes
- # FIXME: make more robust
- for name, scheme in layer_schemes.items():
- if prefix.startswith(name):
- return scheme
- # If not found, use default
- return quant_config.quant_scheme
-
-
-def need_quantize(weight_bits: int) -> bool:
- return weight_bits < 16
-
-
-def _is_mxfp4_w4a4(scheme: QuantizationScheme):
- # FIXME: below impl is incomplete
- return scheme.bits == 4 and scheme.group_size == 32
-
-
-def _is_mxfp8_w8a8(scheme: QuantizationScheme):
- # FIXME: below impl is incomplete
- return scheme.bits == 8 and scheme.group_size == 32
-
-
-def get_fp_scale(scale_e8m0):
- scale_e8m0 = scale_e8m0.view(torch.uint8)
- s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS
- # TODO(later): it would be nice if there was a way to do the 2^x operation
- # in PyTorch without creating a tensor of twos
- two = torch.full(s_offset.size(), 2.0, device=scale_e8m0.device)
- # pow(two, s_offset) can be out of range of floating point formats.
- # TODO(later): handle this for float16 if we decide to support float16
- # scales.
- s_fp = torch.pow(two, s_offset)
-
- # If a block exponent was 255, set values of that block to NaN
- s_fp = torch.where(scale_e8m0 != E8M0_EXPONENT_NAN_VAL, s_fp, float("nan"))
-
- return s_fp
-
-
-def _to_mx_rceil(
- data_hp: torch.Tensor,
- max_abs: torch.Tensor,
- max_pos: float,
-) -> tuple[torch.Tensor, torch.Tensor]:
- """
- A prototype implementation of MXFP scale factor derivation method described in
- https://docs.nvidia.com/cuda/cublas/#d-block-quantization
-
- For Nvidia GPU with Blackwell+ architecture, the scale factor derivation method
- could be accelerated by the `cvt.rp.satfinite.ue8m0x2.f32` instruction.
-
- Args:
- data_hp: High precision data.
- max_abs: Maximum absolute value for data_hp along specified dimension/block_size.
- max_pos: The maximum value of the low precision data type.
-
- Returns:
- exponent: The biased exponent with dtype E8M0 in uint8 container.
- data_lp: The targeted low precision data, in high precision container
- (requires cast to low precision data type).
- """
- descale = max_abs / max_pos
- # TODO: nan/inf needs to be set for any value
- # of nan/inf in input not just amax.
- exponent = torch.where(
- torch.isnan(descale),
- 0xFF, # Handle biased exponent for nan
- # NOTE: descale < (torch.finfo(torch.float32).smallest_normal / 2) is handled through clamping
- (
- torch.clamp(
- torch.ceil(torch.log2(descale)),
- min=-E8M0_EXPONENT_BIAS,
- max=E8M0_EXPONENT_BIAS,
- )
- + E8M0_EXPONENT_BIAS
- ).to(torch.uint8),
- )
-
- descale_fp = torch.where(exponent == 0, 1.0, torch.exp2(E8M0_EXPONENT_BIAS - exponent.to(torch.float32)))
-
- # scale and saturated cast the data elements to max of target dtype
- data_lp = torch.clamp(data_hp * descale_fp.unsqueeze(1), min=-1 * max_pos, max=max_pos)
- return exponent, data_lp
-
-
-def to_mx_fp8e4m3(
- data_hp: torch.Tensor,
- elem_dtype: Union[torch.dtype, str],
- block_size: int,
-):
- """
- Takes a high precision tensor and converts to MX scale and raw data, in
- naive layout (scale and raw data are separate tensors).
- """
-
- assert data_hp.dtype in (
- torch.bfloat16,
- torch.float,
- ), f"{data_hp.dtype} is not supported yet"
- # TODO(future PR): consider supporting padding
- data_hp = data_hp.contiguous()
-
- # calculate the scale in e8m0 format
- orig_shape = data_hp.shape
- data_hp = data_hp.reshape(-1, block_size)
-
- # find max value of the data
- # Note: this only implements the `minimally supported` version of
- # https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf
- # section 6.3.
- max_abs = torch.amax(torch.abs(data_hp), 1)
-
- # Set X to be the largest power-of-two less than or equal to
- # max_abs(v), divided by the largest power of two representable
- # in the element data type, and get the mbits at the same time
- assert elem_dtype == torch.float8_e4m3fn, f"only float8_e4m3fn is supported now, got {elem_dtype}"
-
- max_pos = torch.finfo(torch.float8_e4m3fn).max # 448.0
- # TODO: add support for other rounding modes
- scale_e8m0_biased, data_lp = _to_mx_rceil(data_hp, max_abs, max_pos)
-
- data_lp = data_lp.to(elem_dtype)
- # need to reshape at the end to help inductor fuse things
- data_lp = data_lp.reshape(orig_shape)
-
- # scale_e8m0_biased = scale_e8m0_biased.view(torch.float8_e8m0fnu)
- return scale_e8m0_biased, data_lp
-
-
-def down_size(size):
- assert size[-1] % 2 == 0, f"{size} last dim not divisible by two"
- return (*size[:-1], size[-1] // 2)
-
-
-def pack_uint4(uint8_data: torch.Tensor) -> torch.Tensor:
- # converting to uint8 for operations
- shape = uint8_data.shape
- assert shape[-1] % 2 == 0
- uint8_data = uint8_data.contiguous().view(-1)
- return (uint8_data[::2] << 4 | uint8_data[1::2]).view(down_size(shape))
diff --git a/auto_round_extension/vllm_ext/vllm_oot_patches.py b/auto_round_extension/vllm_ext/vllm_oot_patches.py
deleted file mode 100644
index cf229e596..000000000
--- a/auto_round_extension/vllm_ext/vllm_oot_patches.py
+++ /dev/null
@@ -1,131 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import vllm
-import vllm.envs as envs
-from vllm.logger import init_logger
-
-logger = init_logger(__name__)
-
-
-def update_flashinfer_workspace_buffer_size(new_size: int):
- import vllm.v1.attention.backends.flashinfer as flashinfer_backend
-
- flashinfer_backend.WORKSPACE_BUFFER_SIZE = new_size
- logger.info(f"Updated FlashInfer WORKSPACE_BUFFER_SIZE to {new_size}")
-
-
-if envs.VLLM_AR_FLASHINFER_WORKSPACE_BUFFER_SIZE > 0:
- update_flashinfer_workspace_buffer_size(envs.VLLM_AR_FLASHINFER_WORKSPACE_BUFFER_SIZE)
-
-
-def oot_maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None:
- """Remap the name of FP8 k/v_scale parameters.
-
- This function handles the remapping of FP8 k/v_scale parameter names.
- It detects if the given name ends with a suffix and attempts to remap
- it to the expected name format in the model. If the remapped name is not
- found in the params_dict, a warning is printed and None is returned.
-
- Args:
- name (str): The original loaded checkpoint parameter name.
- params_dict (dict): Dictionary containing the model's named parameters.
-
- Returns:
- str: The remapped parameter name if successful, or the original name
- if no remapping is needed.
- None: If the remapped name is not found in params_dict.
- """
-
- if name.endswith(".kv_scale"):
- logger.warning_once(
- "DEPRECATED. Found kv_scale in the checkpoint. "
- "This format is deprecated in favor of separate k_scale and "
- "v_scale tensors and will be removed in a future release. "
- "Functionally, we will remap kv_scale to k_scale and duplicate "
- "k_scale to v_scale"
- )
- # NOTE: we remap the deprecated kv_scale to k_scale
- remapped_name = name.replace(".kv_scale", ".attn.k_scale")
- if remapped_name not in params_dict:
- logger.warning_once(
- "Found kv_scale in the checkpoint (e.g. %s), but not found the expected name in the model (e.g. %s). kv_scale is not loaded.", # noqa: E501
- name,
- remapped_name,
- )
- return None
- return remapped_name
-
- if any("mla_attn" in key for key in params_dict):
- attn_str = "mla_attn.mla_attn"
- logger.debug_once(
- f"Found mla_attn with k_scale and v_scale in " f"the checkpoint, using {attn_str} as attn_str"
- )
- else:
- attn_str = "attn"
- # Define scale name mapping patterns in order of precedence
- scale_mapping_patterns = [
- # AR format:
- # .self_attn.{q,k,v}_scale ->
- # .attn.{attn_str}.{q,k,v}_scale
- (
- r"\.self_attn\.([qkv])_scale$",
- rf".self_attn.{attn_str}.\1_scale",
- ),
- (
- r"\.self_attn\.([kv])_proj\.([kv])_scale$",
- rf".self_attn.{attn_str}.\2_scale",
- ),
- # ModelOpt format: .self_attn.{k,v}_proj.{k,v}_scale ->
- # .self_attn.attn.{k,v}_scale
- (
- r"\.self_attn\.([kv])_proj\.([kv])_scale$",
- rf".self_attn.{attn_str}.\2_scale",
- ),
- # QKV proj format: .self_attn.qkv_proj.{k,v}_scale ->
- # .self_attn.attn.{k,v}_scale
- (r"\.self_attn\.qkv_proj\.([kv])_scale$", r".self_attn.attn.\1_scale"),
- # Qwen3 MoE format: .self_attn.qkqkv_proj.{k,v}_scale ->
- # .self_attn.attn.{k,v}_scale
- (r"\.self_attn\.qkqkv_proj\.([kv])_scale$", r".self_attn.attn.\1_scale"),
- # Default format: .{k,v}_scale -> .attn.{k,v}_scale
- (r"\.([kv])_scale$", r".attn.\1_scale"),
- ]
-
- # Check if name ends with k_scale or v_scale
- if name.endswith((".k_scale", ".v_scale", ".q_scale")):
- import regex as re
-
- for pattern, replacement in scale_mapping_patterns:
- if re.search(pattern, name):
- remapped_name = re.sub(pattern, replacement, name)
- if remapped_name not in params_dict:
- scale_type = name.split(".")[-1]
- logger.warning_once(
- "Found %s in the checkpoint (e.g. %s), but not found the expected name in the model (e.g. %s). %s is not loaded.", # noqa: E501
- scale_type,
- name,
- remapped_name,
- scale_type,
- )
- return None
- return remapped_name
-
- # If there were no matches, return the untouched param name
- return name
-
-
-import vllm.model_executor.model_loader.weight_utils as vllm_weight_utils
-
-vllm_weight_utils.maybe_remap_kv_scale_name = oot_maybe_remap_kv_scale_name
diff --git a/test/unit/test_cpu/utils/test_model_free_utils.py b/test/unit/test_cpu/utils/test_model_free_utils.py
index 6d4cf5577..f4e36613a 100644
--- a/test/unit/test_cpu/utils/test_model_free_utils.py
+++ b/test/unit/test_cpu/utils/test_model_free_utils.py
@@ -50,6 +50,9 @@ from auto_round.utils.model_free_utils import (
_quantize_single_tensor,
_quantize_weight_mxfp,
_validate_auto_scheme_options,
+ dequant_mx_fp8,
+ dequant_mxfp4,
+ get_fp_scale,
)
from auto_round.utils.model_free_utils import (
handle_model_type_low_precision_source_tensors as _handle_model_type_low_precision_source_tensors,
@@ -906,6 +909,24 @@ class TestHandleMXFPSourceTensors:
assert "layer.weight_scale" not in out
+class TestMXFPDequantization:
+ def test_get_fp_scale(self):
+ scale = torch.tensor([126, 127, 128], dtype=torch.uint8)
+ assert torch.equal(get_fp_scale(scale), torch.tensor([0.5, 1.0, 2.0]))
+
+ def test_dequant_mxfp4(self):
+ packed = torch.tensor([[0x10, 0x32]], dtype=torch.uint8)
+ scale = torch.tensor([[128]], dtype=torch.uint8)
+ result = dequant_mxfp4(packed, scale, "fp4_e2m1", block_size=4, target_dtype=torch.float32)
+ assert torch.equal(result, torch.tensor([[0.0, 1.0, 2.0, 3.0]]))
+
+ def test_dequant_mx_fp8(self):
+ weight = torch.tensor([[1.0, 2.0, 3.0, 4.0]], dtype=torch.float8_e4m3fn)
+ scale = torch.tensor([[126, 128]], dtype=torch.uint8)
+ result = dequant_mx_fp8(weight, scale, block_size=2, target_dtype=torch.float32)
+ assert torch.equal(result, torch.tensor([[0.5, 1.0, 6.0, 8.0]]))
+
+
# ===========================================================================
# AutoScheme helpers
# ===========================================================================