PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine inference backend/code step 1 (#486)
Wenhua Chengea4d8435a2f0
diff --git a/auto_round/__init__.py b/auto_round/__init__.pyindex f031c758f..8a2d74b4d 100644--- a/auto_round/__init__.py+++ b/auto_round/__init__.py@@ -15,5 +15,5 @@ from .autoround import AutoRound, AutoRoundAdam, AutoRoundOPTfrom .mllm import AutoRoundMLLMfrom auto_round.utils import LazyImport-from .auto_quantizer import AutoHfQuantizer,AutoRoundConfig+from auto_round.inference.auto_quantizer import AutoHfQuantizer,AutoRoundConfigfrom .version import __version__diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pydeleted file mode 100644index 0fb39e6b3..000000000--- a/auto_round/auto_quantizer.py+++ /dev/null@@ -1,837 +0,0 @@-# Copyright (c) 2024 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--# coding=utf-8-# Copyright 2023 HuggingFace Inc. team and GPTQ and AutoGPTQ authors.-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-import gc-import importlib.util-import warnings-from dataclasses import dataclass-from logging import getLogger-from typing import Any, Dict, List, Optional, Tuple, Union--import torch-import torch.nn as nn-from packaging import version-from transformers.modeling_utils import PreTrainedModel-from transformers.pytorch_utils import Conv1D-from transformers.quantizers import AutoQuantizationConfig, HfQuantizer-from transformers.quantizers.auto import AUTO_QUANTIZER_MAPPING-from transformers.utils.quantization_config import AwqConfig, GPTQConfig, QuantizationConfigMixin, QuantizationMethod-from auto_round.utils import (get_module, set_module, is_hpu_supported, get_block_names,- get_multimodal_block_names, find_matching_blocks)--from auto_round.backend import get_layer_backend, dynamic_import_inference_linear--from auto_round.backend import BackendInfos-from transformers.utils.versions import require_version-from enum import Enum-from tqdm import tqdm-import copy-import re--logger = getLogger(__name__)-import sys--if sys.version_info < (3, 8):- import importlib_metadata-else:- import importlib.metadata as importlib_metadata--AUTOROUND_MINIMUM_VERSION = version.parse("0.2")---def _is_package_available(pkg_name: str, return_version: bool = False) -> Union[Tuple[bool, str], bool]:- # Check we're not importing a "pkg_name" directory somewhere but the actual library by trying to grab the version- try: ##TODO remove it later- import auto_round- return True, auto_round.__version__- except:- pass-- package_exists = importlib.util.find_spec(pkg_name) is not None- package_version = "N/A"- if package_exists:- try:- package_version = importlib.metadata.version(pkg_name)- package_exists = True- except importlib.metadata.PackageNotFoundError:- package_exists = False- if return_version:- return package_exists, package_version- else:- return package_exists---_auto_round_available = _is_package_available("auto_round")---def is_auto_round_available():- try:- import auto_round- return True- except:- pass- if _auto_round_available:- version_autoround = version.parse(importlib_metadata.version("auto_round"))- if AUTOROUND_MINIMUM_VERSION < version_autoround:- return True- else:- raise ImportError(- f"Found an incompatible version of auto-round. Found version {version_autoround},"- f" but only version above {AUTOROUND_MINIMUM_VERSION} are supported"- )---#-def get_device(obj: Union[torch.Tensor, nn.Module]):- if isinstance(obj, torch.Tensor):- return obj.device- return next(obj.parameters()).device---class AutoHfQuantizer:- """The Auto-HF quantizer class that takes care of automatically instantiating to the correct- `HfQuantizer` given the `QuantizationConfig`."""-- @classmethod- def from_config(cls, quantization_config: Union[QuantizationConfigMixin, Dict], **kwargs):- # Convert it to a QuantizationConfig if the q_config is a dict- if isinstance(quantization_config, dict):- if "auto-round" in quantization_config["quant_method"]:- quantization_config = AutoRoundConfig.from_dict(quantization_config)- else:- quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101- quant_method = quantization_config.quant_method-- # Again, we need a special care for bnb as we have a single quantization config- # class for both 4-bit and 8-bit quantization- if quant_method == QuantizationMethod.BITS_AND_BYTES:- if quantization_config.load_in_8bit:- quant_method += "_8bit"- else:- quant_method += "_4bit"-- if quant_method not in AUTO_QUANTIZER_MAPPING.keys() and "auto-round" not in quant_method:- raise ValueError(- f"Unknown quantization type, got {quant_method} - supported types are:"- f" {list(AUTO_QUANTIZER_MAPPING.keys())}"- )- if "auto-round" in quant_method or is_hpu_supported(): # pragma: no cover- target_cls = AutoRoundQuantizer- else:- target_cls = AUTO_QUANTIZER_MAPPING[quant_method]-- return target_cls(quantization_config, **kwargs)-- @classmethod- def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):- quantization_config = AutoQuantizationConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)- return cls.from_config(quantization_config)-- @classmethod- def merge_quantization_configs(- cls,- quantization_config: Union[dict, QuantizationConfigMixin],- quantization_config_from_args: Optional[QuantizationConfigMixin],- ):- """Handles situations where both quantization_config- from args and quantization_config from model config are present."""- if quantization_config_from_args is not None:- warning_msg = (- "You passed `quantization_config` or equivalent parameters to "- "`from_pretrained` but the model you're loading"- " already has a `quantization_config` attribute. The `quantization_config` from the model will be used."- )- else:- warning_msg = ""-- loading_attr_dict = quantization_config_from_args.get_loading_attributes() \- if quantization_config_from_args is not None else None- if isinstance(quantization_config, dict):- if "auto-round" in quantization_config["quant_method"]:- quantization_config = AutoRoundConfig.from_dict(quantization_config)- else:- if isinstance(quantization_config_from_args, (AutoRoundConfig)):- logger.info(f"Loading quantized model in auto_round format.")- tmp_backend = quantization_config["quant_method"]- if "auto-round" not in tmp_backend and "gptq" not in tmp_backend and "awq" not in tmp_backend:- logger.error("could not convert to auto_round format, currently only supports `gptq`,`awq` or "- "`auto-round` format")- exit(-1)- target_backend = quantization_config["backend"] if "backend" in quantization_config else "auto"- if loading_attr_dict is not None and "backend" in loading_attr_dict:- target_backend = loading_attr_dict["backend"]- loading_attr_dict.pop("backend")- if "auto_round" not in target_backend:- target_backend = f"auto_round:{tmp_backend}" #- quantization_config = AutoRoundConfig.from_dict(quantization_config)- setattr(quantization_config, "backend", target_backend)- else:- quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101-- if isinstance(quantization_config,- (GPTQConfig, AwqConfig, AutoRoundConfig)) and quantization_config_from_args is not None:- # special case for GPTQ / AWQ config collision-- for attr, val in loading_attr_dict.items():- setattr(quantization_config, attr, val)- warning_msg += (- f"However, loading attributes (e.g. {list(loading_attr_dict.keys())}) "- f"will be overwritten with the one you passed to `from_pretrained`. The rest will be ignored."- )-- if warning_msg != "":- warnings.warn(warning_msg)-- return quantization_config-- @staticmethod- def supports_quant_method(quantization_config_dict):- from transformers.quantizers.auto import AUTO_QUANTIZATION_CONFIG_MAPPING- AUTO_QUANTIZATION_CONFIG_MAPPING['intel/auto-round'] = AutoRoundConfig- AUTO_QUANTIZATION_CONFIG_MAPPING['intel/auto_round'] = AutoRoundConfig- quant_method = quantization_config_dict.get("quant_method", None)- if quantization_config_dict.get("load_in_8bit", False) or quantization_config_dict.get("load_in_4bit", False):- suffix = "_4bit" if quantization_config_dict.get("load_in_4bit", False) else "_8bit"- quant_method = QuantizationMethod.BITS_AND_BYTES + suffix- elif quant_method is None:- raise ValueError(- "The model's quantization config from the arguments has no `quant_method` attribute." \- "Make sure that the model has been correctly quantized"- )-- if quant_method not in AUTO_QUANTIZATION_CONFIG_MAPPING.keys():- logger.warning(- f"Unknown quantization type, got {quant_method} - supported types are:"- f" {list(AUTO_QUANTIZER_MAPPING.keys())}. Hence, we will skip the quantization. "- "To remove the warning, you can delete the quantization_config attribute in config.json"- )- return False- return True---class AutoRoundQuantizationMethod(str, Enum):- AutoRound = "intel/auto-round"---@dataclass-class AutoRoundConfig(QuantizationConfigMixin):- """This is a wrapper class about all possible attributes and features that you can play with a model that has been- loaded AutoRound quantization.-- Args:- bits (`int`):- The number of bits to quantize to, supported numbers are (2, 3, 4, 8).- tokenizer (`str` or `PreTrainedTokenizerBase`, *optional*):- The tokenizer used to process the dataset. You can pass either:- - A custom tokenizer object.- - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co.- - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved- using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.- """-- def __init__(- self,- bits: int = 4,- tokenizer: Any = None,- dataset: str = None,- group_size: int = 128,- sym: bool = False,- backend="auto",- layer_config: dict = None,- **kwargs,- ):-- self.bits = bits- self.tokenizer = tokenizer- self.dataset = dataset- self.group_size = group_size- self.sym = sym- self.backend = backend- self.layer_config = layer_config- if kwargs is not None:- for key in kwargs.keys():- setattr(self, key, kwargs[key])- self.quant_method = AutoRoundQuantizationMethod.AutoRound- self.post_init()-- def post_init(self):- r"""Safety checker that arguments are correct."""- if self.bits not in [2, 4, 8]:- raise ValueError(f"Only support quantization to [2,4,8] bits but found {self.bits}")- if self.group_size != -1 and self.group_size <= 0:- raise ValueError("group_size must be greater than 0 or equal to -1")-- def get_loading_attributes(self):- loading_attibutes_dict = {"target_backend": self.backend}- return loading_attibutes_dict-- def to_dict(self):- config_dict = super().to_dict()- return config_dict---class AutoRoundQuantizer(HfQuantizer):- """Quantizer of the AutoRound method, currently only triton and exllamav2 backend has been supported."""-- requires_calibration = False- required_packages = ["auto_round"]- optimum_quantizer = None-- def __init__(self, quantization_config: QuantizationConfigMixin, **kwargs):- super().__init__(quantization_config, **kwargs)-- def validate_environment(self, *args, **kwargs):- if not is_auto_round_available():- raise ImportError("Loading a AutoRound quantized model requires auto-round library (`pip install "- "auto-round`)")- else:- try:- import auto_round- autoround_version = version.parse(auto_round.__version__)- except:- autoround_version = version.parse(importlib.metadata.version("auto_round"))- if autoround_version < version.parse("0.2.0"):- raise ImportError("You need a version of auto_round > 0.2.0 to use AutoRound: `pip install --upgrade "- "auto-round` or install from source")-- def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":- if torch_dtype is None:- torch_dtype = torch.float16- return torch_dtype-- def find_backend(self, target_backend: str):- """Finds the matching backend key based on the target backend or its alias.-- This function checks if the provided `target_backend` is directly present in `BackendInfos`.- If not, it iterates through the backends to see if the `target_backend` matches any backend's alias.-- Args:- target_backend (str):- The name of the backend or alias to find.-- Returns:- str or None:- The backend key if a match is found, otherwise `None`.- """- # Directly return if target_backend exists in BackendInfos- if target_backend in BackendInfos:- return target_backend-- # Search through BackendInfos to check if target_backend matches any backend alias- for key in BackendInfos.keys():- backendInfo = BackendInfos[key]- if backendInfo.alias is not None and target_backend in backendInfo.alias:- return key-- # Return None if no matching backend or alias is found- return None-- def detect_auto_device(self):- if torch.cuda.is_available():- return "cuda"- elif is_hpu_supported():- return "hpu"- elif torch.xpu.is_available():- return "xpu"- else:- return "cpu"-- def detect_device(self, target_backend, orig_backend):- ##TODO need to refine later- """Detects the appropriate device for the specified backend.-- This function determines the device type based on the target backend. If the target backend is- not specified, it defaults to the original backend. The function checks for the availability- of CUDA, HPU, or CPU, and returns the appropriate device type.-- Args:- target_backend (str or None):- The name of the target backend. If None, defaults to `orig_backend`.- orig_backend (str):- The original backend name to fall back on if `target_backend` is None.-- Returns:- str:- The type of device detected ('cuda', 'hpu', or 'cpu').-- Raises:- ValueError:- If the specified backend cannot be found.- """- # Default to the original backend if target_backend is not provided- if target_backend is None:- target_backend = orig_backend-- # Check for specific device types based on the target backend- if "cuda" in target_backend:- return "cuda"- elif "hpu" in target_backend:- return "hpu"- elif "xpu" in target_backend:- return "xpu"- elif "cpu" in target_backend:- return "cpu"-- # Determine the device automatically based on availability- if target_backend.split(":")[0] == "auto":- return self.detect_auto_device()-- # Find the backend and determine the device type from BackendInfos- backend = self.find_backend(target_backend)- if backend is None:- raise ValueError("Backend is not found, please set it to 'auto' to have a try ")-- device = BackendInfos[backend].device[0]- if "cuda" in device and torch.cuda.is_available():- return device- elif "hpu" in device and is_hpu_supported():- return device- elif "xpu" in device and torch.xpu.is_available():- return device- else:- ## trick- return "cpu"-- def convert_model(self, model: nn.Module):- """Converts the given model to an AutoRound model by replacing its layers with quantized layers.-- This method extracts the quantization configuration from the model and adjusts its layers- according to the specified quantization parameters. It supports different backends and- ensures that the model's data type is compatible with the selected hardware.-- Args:- model (nn.Module):- The model to be converted into an AutoRound model.-- Returns:- nn.Module:- The converted AutoRound model with quantized layers.-- Raises:- ValueError:- If the quantization backend is not specified in the configuration.- """-- from auto_round.utils import get_layer_names_in_block-- quantization_config = model.config.quantization_config- if not hasattr(quantization_config, "target_backend"):- quantization_config.target_backend = quantization_config.backend-- target_device = self.detect_device(None, quantization_config.backend)-- self.target_device = target_device-- if hasattr(quantization_config, "backend"): # pragma: no cover- if ("hpu" == target_device or "cpu" == target_device) and model.dtype != torch.bfloat16:- logger.info(f"Change the dtype to `bfloat16` as {target_device.upper()} does not support float16")- model = model.to(torch.bfloat16)-- bits = quantization_config.bits- group_size = quantization_config.group_size- data_type = quantization_config.data_type if hasattr(quantization_config,- "data_type") else "int" # pragma: no cover- sym = quantization_config.sym-- quant_block_list = quantization_config.quant_block_list if hasattr(quantization_config,- "quant_block_list") else None-- if quant_block_list is None:- to_quant_block_names = quantization_config.to_quant_block_names if hasattr(quantization_config,- "to_quant_block_names") else None- if to_quant_block_names is not None:- if isinstance(to_quant_block_names, (list, tuple)):- quant_block_list = to_quant_block_names- else:- quant_block_list = []- for block in to_quant_block_names.split(','):- quant_block_list.append([f'{block}.{i}' for i in range(len(get_module(model, block)))])- else:- all_blocks = get_block_names(model)- quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)-- layer_names = get_layer_names_in_block(model, quant_block_list=quant_block_list)-- extra_config = {}- if hasattr(quantization_config, "extra_config"):- extra_config = quantization_config.extra_config- if hasattr(quantization_config, "modules_in_block_to_quantize"): ##gptq format- modules_in_block_to_quantize_tmp = quantization_config.modules_in_block_to_quantize- modules_in_block_to_quantize = [item for sublist in modules_in_block_to_quantize_tmp for item in sublist]- for layer_name in layer_names:- quantized = False- for qname in modules_in_block_to_quantize:- if qname in layer_name:- quantized = True- break- if not quantized:- extra_config[layer_name] = {"bits": 16}- if hasattr(quantization_config, "modules_to_not_convert"):- for layer_name in quantization_config.modules_to_not_convert:- extra_config[layer_name] = {"bits": 16}-- layer_names += extra_config.keys()- layer_names = list(set(layer_names))-- layer_configs = {}- for layer_name in layer_names:- layer_configs[layer_name] = {}- if layer_name not in extra_config:- layer_configs[layer_name]["bits"] = bits- layer_configs[layer_name]["group_size"] = group_size- layer_configs[layer_name]["data_type"] = data_type- layer_configs[layer_name]["sym"] = sym- layer_configs[layer_name]["clip"] = False- else:- layer_configs[layer_name]["bits"] = extra_config[layer_name].get("bits", bits)- layer_configs[layer_name]["group_size"] = extra_config[layer_name].get("group_size", group_size)- layer_configs[layer_name]["data_type"] = extra_config[layer_name].get("data_type", data_type)- layer_configs[layer_name]["sym"] = extra_config[layer_name].get("sym", sym)- layer_configs[layer_name]["clip"] = extra_config[layer_name].get("clip", False)-- if hasattr(quantization_config, "backend"): # pragma: no cover- backend = quantization_config.backend- elif 'gptq' in quantization_config.quant_method: # pragma: no cover- backend = 'gptq'- elif "awq" in quantization_config.quant_method:- backend = "awq"- else: # pragma: no cover- logger.error("Please specify quantization backend")- raise ValueError("Quantization backend must be specified.")-- self._replace_by_quant_layers(model, layer_configs, quantization_config.target_backend, target_device, backend)- return model-- def _replace_by_quant_layers(self, module: nn.Module, layer_configs, target_backend, target_device, orig_backend):- """Replaces linear layers in the given module with quantized layers.-- This method iterates over the specified layer configurations and replaces- the original layers in the module with instances of `QuantLinear`. It handles- various layer types and ensures that the correct quantization parameters are applied.-- Args:- module (nn.Module):- The module containing layers to be quantized.- layer_configs (dict):- A dictionary containing configuration for each layer's quantization.- target_backend (str):- The backend to use for quantization, which includes device and format information.- target_device (str):- The device on which the model will run (e.g., 'cuda', 'cpu', 'hpu').- orig_backend (str):- The original backend of the packing.-- Raises:- AssertionError:- If any condition related to backend or quantization configuration is not met.- """-- def remove_device_str(s, device_str):- if s and s.startswith(device_str):- return s[len(device_str):].lstrip(":")- return s-- if "auto" == target_backend.split(':')[0]:- target_backend = target_backend[4:] # Remove 'auto'- if len(target_backend) >= 1 and target_backend[0] == ":":- target_backend = target_backend[1:]-- # Remove device info from target_backend- target_backend = remove_device_str(target_backend, "cpu")- target_backend = remove_device_str(target_backend, "hpu")- target_backend = remove_device_str(target_backend, "cuda")- orig_backend = self.find_backend(orig_backend)- if orig_backend is None:- orig_backend = "gptq:exllamav2"-- if target_backend == "":- target_backend = orig_backend-- self.need_marlin_repacking = False-- for layer_name in layer_configs.keys():- config = layer_configs[layer_name]- bits = config["bits"]- group_size = config["group_size"]- data_type = config["data_type"]- sym = config["sym"]- clip = config["clip"]-- if not (bits <= 8):- continue-- layer = get_module(module, layer_name)- if isinstance(layer, nn.Linear):- in_features = layer.in_features- out_features = layer.out_features- elif isinstance(layer, nn.Conv2d): # Not supported currently- in_features = layer.in_channels- out_features = layer.out_channels- elif isinstance(layer, Conv1D): # TODO: Needs verification- in_features = layer.weight.shape[0]- out_features = layer.weight.shape[1]- else:- continue-- if "marlin" in target_backend and "marlin" not in orig_backend:- # Need to repack- assert sym == True, "Marlin only supports symmetric quantization"- assert target_device == "cuda", "Marlin only supports CUDA device"- assert not "awq" in orig_backend, "Marlin does not support repacking from AWQ format"- self.need_marlin_repacking = True- # Using original backend to load the layer then replace- layer_backend = orig_backend- else:- target_backend = self.find_backend(target_backend) # TODO: Move out if have supported marlin- layer_backend = get_layer_backend(- target_device, target_backend, orig_backend, bits, group_size, sym, in_features, out_features- )- if "gptq" in layer_backend and "exllamav2" in layer_backend:- try:- from exllamav2_kernels import gemm_half_q_half, make_q_matrix # pylint: disable=E0611, E0401- except:- logger.warning_once(- "For better inference performance, please install exllamav2 kernel "- "via `pip install git+https://github.com/AutoGPTQ/AutoGPTQ.git@b8b4127`")-- QuantLinear = dynamic_import_inference_linear(layer_backend, bits, group_size, sym)-- layer_device = get_device(layer)-- bias = layer.bias is not None- from auto_round_extension.qbits.qbits_awq import QuantLinear as QBitsAWQQuantLinear- if "awq" in layer_backend and isinstance(QuantLinear, QBitsAWQQuantLinear):- new_layer = QuantLinear.from_linear( # pylint: disable=E1123- layer,- bits,- group_size,- init_only=True,- has_zero_points=not sym- )- elif "awq" in layer_backend:- new_layer = QuantLinear.from_linear( # pylint: disable=E1123- layer,- bits,- group_size,- init_only=True- )- else:- try:- new_layer = QuantLinear( # pylint: disable=E1123- bits,- group_size,- in_features,- out_features,- bias,- weight_dtype=layer.weight.dtype,- clip=clip- )- except:- new_layer = QuantLinear( # pylint: disable=E1123- bits,- group_size,- in_features,- out_features,- bias,- weight_dtype=layer.weight.dtype,- )-- new_layer.device = layer_device- set_module(module, layer_name, new_layer)-- def cpu_post_init(self, model):- message = "Repacking to CPU format"- from auto_round_extension.qbits import qbits_qlinear_classes, qbits_awq_classes- from auto_round_extension.ipex import ipex_qlinear_classes- cpu_layers = tuple(list(qbits_qlinear_classes) + list(ipex_qlinear_classes) + list(qbits_awq_classes))- layers = [] ## ipex post_init will add one more layer- for n, m in model.named_modules():- if isinstance(m, cpu_layers):- layers.append((n, m))- for n, layer in tqdm(layers, desc=message, total=len(layers),- leave=True):- layer.post_init()- return model-- def xpu_post_init(self, model):- message = "Repacking to XPU format"- from auto_round_extension.ipex import ipex_qlinear_classes- cpu_layers = tuple(list(ipex_qlinear_classes))- layers = [] ## ipex post_init will add one more layer- for n, m in model.named_modules():- if isinstance(m, cpu_layers):- layers.append((n, m))- for n, layer in tqdm(layers, desc=message, total=len(layers),- leave=True):- layer.post_init()- return model-- def repack_marlin(self, model):- """Repack the model to use Marlin format for quantized layers.-- This method iterates through the model's modules, identifies instances of- `QuantLinear`, and replaces them with `MarlinInferenceQuantLinear`. It- handles the initialization of various parameters and the repacking of- quantized weights and scales for optimized performance on Marlin.-- Args:- model (nn.Module):- The model to be repacked into Marlin format.-- Raises:- ImportError:- If the required modules for Marlin inference cannot be imported.- """- message = "Repacking to Marlin format"-- for n, m in tqdm(model.named_modules(), desc=message, total=len(list(model.named_modules()))):- if m.__class__.__name__ == "QuantLinear":- try:- from gptqmodel.nn_modules.qlinear.qlinear_marlin_inference import ( # pylint: disable=E0401- MarlinInferenceQuantLinear,- marlin_permute_scales,- marlin_make_workspace- )- except ImportError:- raise ImportError("Failed to import Marlin inference modules.")-- with torch.device("meta"):- # Create a new MarlinInferenceQuantLinear module with the appropriate parameters.- new_module = MarlinInferenceQuantLinear(- bits=4,- group_size=m.group_size,- sym=True,- desc_act=False,- infeatures=m.infeatures,- outfeatures=m.outfeatures,- bias=m.bias is not None,- )-- device = m.qweight.device- import gptqmodel_marlin_cuda_inference # pylint: disable=E0401-- # Initialize the necessary parameters for the new module.- new_module.g_idx = torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),- requires_grad=False)- new_module.g_idx_sort_indices = torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),- requires_grad=False)- new_module.zp = torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),- requires_grad=False)- new_module.bias = m.bias-- # Repack the quantized weight for the Marlin format.- marlin_qweight = gptqmodel_marlin_cuda_inference.gptq_marlin_repack( # pylint: disable=E0401- m.qweight,- new_module.g_idx_sort_indices,- m.infeatures,- m.outfeatures,- m.bits- )- new_module.qweight.resize_(marlin_qweight.shape)- new_module.qweight = nn.Parameter(marlin_qweight, requires_grad=False)-- # Permute scales for the new module's configuration.- marlin_scales = marlin_permute_scales(- m.scales,- size_k=m.infeatures,- size_n=m.outfeatures,- group_size=m.group_size- )-- new_module.scales.resize_(marlin_scales.shape)- new_module.scales = nn.Parameter(marlin_scales, requires_grad=False)-- # Create a workspace for the new module.- new_module.workspace = marlin_make_workspace( # TODO: Consider moving this to post-init.- new_module.outfeatures, device- )-- # Replace the original module in the model with the new Marlin module.- set_module(model, n, new_module)-- # Clear cache and perform garbage collection to free memory.- torch.cuda.empty_cache()- gc.collect()-- def post_init_model(self, model):- """Post-initialization that require device information, for example buffers initialization on device.-- Args:- model (`nn.Module`):- The input model- """-- class StoreAttr(object):- pass-- model.quantize_config = StoreAttr()- if self.need_marlin_repacking:- require_version("gptqmodel",- "marlin format requires gptqmodel to be installed, "- "`pip install -v gptqmodel --no-build-isolation `")- self.repack_marlin(model)- from auto_round_extension.cuda.post_init import autoround_post_init- model = autoround_post_init(model)- # there are no side-effects after call qbits_post_init when model quant-type not equal to qbits.- if self.target_device == "cpu":- model = self.cpu_post_init(model)- elif self.target_device == "xpu":- model = self.xpu_post_init(model)-- return model-- def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs):- if model.__class__.main_input_name != "input_ids":- logger.warning("We can only quantize pure text models and " \- "certain types(Llava/Qwen-VL/Phi-3-vision) of multimodal models.")-- if self.pre_quantized:- model = self.convert_model(model)-- def _process_model_after_weight_loading(self, model: "PreTrainedModel", **kwargs):- if self.pre_quantized:- model = self.post_init_model(model)- else:- raise NotImplementedError-- @property- def is_trainable(self, model: Optional["PreTrainedModel"] = None):- return True-- @property- def is_serializable(self):- return True---import transformers--if version.parse(transformers.__version__) < version.parse("4.38.0"):- logger.error("Please upgrade transformers>=4.38.0 to support lm-head quantization")--transformers.quantizers.auto.AutoHfQuantizer = AutoHfQuantizer-transformers.modeling_utils.AutoHfQuantizer = AutoHfQuantizerdiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 04f36cc56..47f97dcf3 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -53,7 +53,8 @@ from .utils import (compile_func,find_matching_blocks, is_debug_mode,TORCH_VERSION_AT_LEAST_2_6,- supported_layer_types+ supported_layer_types,+ get_layer_features,)from .low_cpu_mem.utils import get_layers_before_block@@ -448,7 +449,7 @@ class AutoRound(object):f"Currently only support to export auto_round format quantized model"" with fp8 dtype activation for activation quantization."" Change format to fake and save."- )+ )formats = ["fake"]else:if len(formats) > 1 or "auto_round" not in formats:@@ -478,11 +479,6 @@ class AutoRound(object):format = format.replace('auto_round', 'auto_round:gptq')formats[index] = format- if not any(f in format for f in ["triton", "exllamav2", "awq", "gptq"]):- logger.info(f"AutoRound format does not support {format}, attempting to use AutoGPTQ")- format = format.replace("auto_round", "auto_gptq")- formats[index] = format-# Remove duplicates from formats listdef remove_duplicates(lst):seen = set()@@ -693,6 +689,10 @@ class AutoRound(object):if n not in layers_in_blocks and check_to_quantized(layer_config[n]):has_qlayer_outside_block = True+ in_features, out_features = get_layer_features(m)+ if in_features <= layer_config[n]["group_size"]:+ layer_config[n]["group_size"] = -1+# Apply the configuration to the corresponding layer in the modelfor key in keys:setattr(m, key, layer_config[n][key])@@ -1478,7 +1478,7 @@ class AutoRound(object):m.name = nfor i in range(0, len(block_names), nblocks):- if i!=0:+ if i != 0:pbar.update(1)if nblocks == 1:n = block_names[i]@@ -1542,7 +1542,7 @@ class AutoRound(object):f"Currently only support to export auto_round format quantized model"" with fp8 dtype activation for activation quantization."" Change format to fake and save."- )+ )format = "fake"else:if format != "auto_round":diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex 7e8b7e27f..b0b4d5dd2 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -12,12 +12,8 @@# See the License for the specific language governing permissions and# limitations under the License.-import logging-import random-import time-from typing import TYPE_CHECKING, List, Optional, Union+from typing import Optional, Union-import lm_evalfrom lm_eval import simple_evaluate as lm_simple_evaluateimport os@@ -52,7 +48,7 @@ def simple_evaluate(try:from auto_round import AutoRoundConfigexcept:- from auto_round.auto_quantizer import AutoHfQuantizer+ from auto_round.inference.auto_quantizer import AutoHfQuantizerreturn lm_simple_evaluate(model=model,diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 7a1d81424..39345cc3e 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -105,7 +105,7 @@ def pack_layer(name, model, backend):##force to float32 to be compatible with torch 2.0if sym and isinstance(new_layer, auto_round.export.export_to_autogptq.qlinear_triton.QuantLinear):layer, scale = layer.to("cpu"), scale.to("cpu")- zero = 2 ** (bits - 1)+ zero = int(zero.flatten()[0])else:layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)sig = inspect.signature(qlayer.pack)@@ -126,6 +126,8 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllquant_block_list = kwargs.get("quant_block_list", get_block_names(model))tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)+ if os.path.exists(output_dir):+ logger.warning(f"{output_dir} already exists, this may cause model conflict")if tokenizer is not None:tokenizer.save_pretrained(output_dir)if processor is not None:diff --git a/auto_round/export/export_to_autogptq/qlinear_triton.py b/auto_round/export/export_to_autogptq/qlinear_triton.pyindex 0d5a67942..c51920109 100644--- a/auto_round/export/export_to_autogptq/qlinear_triton.py+++ b/auto_round/export/export_to_autogptq/qlinear_triton.py@@ -18,30 +18,6 @@ import numpy as npimport torchimport torch.nn as nnimport transformers-import numba---##TODO different bits-# @numba.jit(nopython=True, parallel=True)-# def pack_array_with_numba_b4_c32(-# raw_array: np.ndarray, packed_array: np.ndarray-# ) -> np.ndarray:-# """Pack the array with numba when bits=4 and compress_bits=32."""-# bits = 4-# n_pack = 32 // bits-#-# for row in range(packed_array.shape[0]):-# packed_array[row] = ((((raw_array[row * n_pack + 7]) << 28)-# | ((raw_array[row * n_pack + 6]) << 24)-# | ((raw_array[row * n_pack + 5]) << 20)-# | ((raw_array[row * n_pack + 4]) << 16)-# | ((raw_array[row * n_pack + 3]) << 12)-# | (raw_array[row * n_pack + 2]) << 8)-# | ((raw_array[row * n_pack + 1]) << 4)-# | ((raw_array[row * n_pack]) << 0))-#-# return packed_array-class TritonModuleMixin:@classmethod@@ -89,6 +65,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):"g_idx",torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),)+if bias:self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))else:@@ -108,6 +85,8 @@ class QuantLinear(nn.Module, TritonModuleMixin):device = "cpu"if torch.cuda.is_available():device = "cuda:0"+ elif torch.xpu.is_available():+ device = "xpu:0"W = linear.weight.data.to(device).clone()if isinstance(linear, nn.Conv2d):@@ -118,11 +97,12 @@ class QuantLinear(nn.Module, TritonModuleMixin):repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)if isinstance(zeros, torch.Tensor):repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros[:, :W.shape[1]]).to(+ torch.int32)else:repeat_zeros = zeros-- intweight = torch.round(W.to(device) / repeat_scales + repeat_zeros).to(- torch.int32)+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros).to(+ torch.int32)del repeat_scalesintweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex f0743eec8..9c68a96d6 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -22,6 +22,7 @@ import torch.nn as nnimport transformersimport auto_round.export.export_to_autoround.qlinear_triton_act+import auto_round_extension.cuda.qlinear_tritonv2from auto_round.utils import get_layer_names_in_block, get_module, logger, set_module, supported_layer_typesimport threadpoolctl as tctlimport inspect@@ -71,17 +72,18 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_if "auto_round" in backend and "awq" not in backend and "gptq" not in backend:if act_bits <= 8: ##easily have bug for other configuration, need to refine code laterreturn auto_round.export.export_to_autoround.qlinear_triton_act.QuantLinear- ##only support triton and exllamav2- if not ("triton" in backend or "exllamav2" in backend):- logger.warning_once(f"auto_round format does not support {backend}, try to pack each layer with auto_gptq")- return get_autogptq_packing_qlinear(backend, bits, group_size, sym)- from auto_round_extension.cuda.qlinear_triton import QuantLinear+ from auto_round_extension.cuda.qlinear_tritonv2 import QuantLinearreturn QuantLinear+ elif "auto_round" in backend and "gptq" in backend:+ from auto_round.export.export_to_autoround.qlinear_triton import QuantLinear ##no g_idx+ return QuantLinearelif "awq" in backend:from ..export_to_awq.utils import WQLinear_GEMMreturn WQLinear_GEMM- elif "gptq" in backend:+ elif "gptqmodel" in backend:+ return auto_round_extension.cuda.qlinear_tritonv2.QuantLinear+ elif "gptq" in backend and not "gptqmodel" in backend: ## have g_idxreturn get_autogptq_packing_qlinear(backend, bits, group_size, sym)else:assert False, f"only support auto_gptq, auto_awq and auto_round backend"@@ -188,6 +190,8 @@ def pack_layer(layer_name, model, backend):new_layer.device = deviceset_module(model, layer_name, new_layer)qlayer = new_layer+ if sym:+ zp = int(zp.flatten()[0])qlayer.to("cpu")##force to float32 to be compatible with torch 2.0@@ -202,6 +206,9 @@ def pack_layer(layer_name, model, backend):scale, zp = scale.to(torch.float32), zp.to(torch.float32)scale = scale.t().contiguous()zp = zp.t().contiguous()+ if sym:+ zp = int(zp.flatten()[0])+if bits != 4:logger.error("AutoAWQ format only supports 4-bits quantization.")qlayer = QuantLinear.from_linear(@@ -243,10 +250,6 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exif (kwargs.get("sym") is None or kwargs.get("sym") == True) and ("gptq" not in backend and "awq" not in backend):backend = backend.replace('auto_round', 'auto_round:gptq')- if not ("triton" in backend or "exllamav2" in backend or "awq" in backend or "gptq" in backend):- logger.info(f"AutoRound format does not support {backend}, try to pack each layer with AutoGPTQ")- backend = backend.replace("auto_round", "auto_gptq")-model = kwargs["model"]safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]if not inplace:@@ -306,6 +309,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exif output_dir is None:model.tokenizer = tokenizerreturn model+ if os.path.exists(output_dir):+ logger.warning(f"{output_dir} already exists, this may cause model conflict")if tokenizer is not None:tokenizer.save_pretrained(output_dir)diff --git a/auto_round/export/export_to_autoround/qlinear_triton.py b/auto_round/export/export_to_autoround/qlinear_triton.pynew file mode 100644index 000000000..ba2da85e3--- /dev/null+++ b/auto_round/export/export_to_autoround/qlinear_triton.py@@ -0,0 +1,137 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import math++import numpy as np+import torch+import torch.nn as nn+import transformers++class TritonModuleMixin:+ @classmethod+ def warmup(cls, model, transpose=False, seqlen=2048):+ pass+++class QuantLinear(nn.Module, TritonModuleMixin):+ QUANT_TYPE = "triton"++ def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):+ super().__init__()+ if bits not in [2, 4, 8]:+ raise NotImplementedError("Only 2,4,8 bits are supported.")+ if infeatures % 32 != 0 or outfeatures % 32 != 0:+ raise NotImplementedError("in_feature and out_feature must be divisible by 32.")+ self.infeatures = infeatures+ self.outfeatures = outfeatures+ self.bits = bits+ self.group_size = group_size if group_size != -1 else infeatures+ self.maxq = 2 ** self.bits - 1++ self.register_buffer(+ "qweight",+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),+ )+ self.register_buffer(+ "qzeros",+ torch.zeros(+ (+ math.ceil(infeatures / self.group_size),+ outfeatures // 32 * self.bits,+ ),+ dtype=torch.int32,+ ),+ )+ self.register_buffer(+ "scales",+ torch.zeros(+ (math.ceil(infeatures / self.group_size), outfeatures),+ dtype=torch.float16,+ ),+ )++ if bias:+ self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))+ else:+ self.bias = None++ self.trainable = trainable++ def post_init(self):+ pass++ def pack(self, linear, scales, zeros, g_idx=None):+ scales_t = scales.t().contiguous()+ if linear.bias is not None:+ self.bias = linear.bias.clone().half()+ self.scales = scales_t.clone().half()+ device = "cpu"+ if torch.cuda.is_available():+ device = "cuda:0"+ elif torch.xpu.is_available():+ device = "xpu:0"++ W = linear.weight.data.to(device).clone()+ if isinstance(linear, nn.Conv2d):+ W = W.flatten(1)+ if isinstance(linear, transformers.pytorch_utils.Conv1D):+ W = W.t()++ repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)+ if isinstance(zeros, torch.Tensor):+ repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros[:, :W.shape[1]]).to(+ torch.int32)+ else:+ repeat_zeros = zeros+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros).to(+ torch.int32)++ del repeat_scales+ intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)+ order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits+ intweight = intweight << order_map+ intweight = torch.sum(intweight, dim=-1)++ intweight = intweight.t().contiguous().to(torch.int32)+ self.qweight = intweight.to("cpu")++ if isinstance(zeros, torch.Tensor):+ zeros = zeros.t().contiguous()+ zeros -= 1+ zeros = zeros.numpy().astype(np.uint32)+ qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)+ i = 0+ col = 0+ while col < qzeros.shape[1]:+ for j in range(i, i + (32 // self.bits)):+ qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))+ i += 32 // self.bits+ col += 1++ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)+ else:+ zeros -= 1+ shape = scales_t.shape+ value = 0+ for j in range(0, (32 // self.bits)):+ value |= zeros << (self.bits * j)+ qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value+ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)+++__all__ = ["QuantLinear"]\ No newline at end of filediff --git a/auto_round/export/export_to_autoround/qlinear_triton_act.py b/auto_round/export/export_to_autoround/qlinear_triton_act.pyindex f5afa2432..12dc79555 100644--- a/auto_round/export/export_to_autoround/qlinear_triton_act.py+++ b/auto_round/export/export_to_autoround/qlinear_triton_act.py@@ -97,10 +97,7 @@ class QuantLinear(nn.Module):),)- self.register_buffer(- "g_idx",- torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),- )+if bias:self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))else:@@ -122,7 +119,7 @@ class QuantLinear(nn.Module):def pack(self, linear, scales, zeros, act_scales, w_bf16_to_fp8_scale, g_idx=None):scales_t = scales.t().contiguous()- self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx+self.act_scales.data.copy_(act_scales.squeeze().clone())self.w_bf16_to_fp8_scale.data.copy_(w_bf16_to_fp8_scale.squeeze().clone())if linear.bias is not None:@@ -144,8 +141,7 @@ class QuantLinear(nn.Module):else:repeat_zeros = zeros- intweight = torch.round(W.to(device) / repeat_scales + repeat_zeros).to(- torch.int32)+ intweight = torch.round(W.to(device) / repeat_scales[:,:W.shape[1]] + repeat_zeros[:,:W.shape[1]])del repeat_scalesintweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex 08b8d7817..4759ae01e 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -57,7 +57,7 @@ def pack_layer(name, model, backend):scale = scale.t().contiguous()zp = zp.t().contiguous().to(torch.float32)if sym:- zp = 2 ** (bits - 1)+ zp = int(zp.flatten()[0])q_linear = WQLinear_GEMM.from_linear(linear=linear_layer,w_bit=bits,@@ -119,6 +119,8 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):if output_dir is None:return compressed_model+ if os.path.exists(output_dir):+ logger.warning(f"{output_dir} already exists, this may cause model conflict")layer_config = kwargs["layer_config"]for key in layer_config.keys():if not check_to_quantized(layer_config[key]) and \diff --git a/auto_round/export/export_to_awq/utils.py b/auto_round/export/export_to_awq/utils.pyindex f0ad7db78..40107b802 100644--- a/auto_round/export/export_to_awq/utils.py+++ b/auto_round/export/export_to_awq/utils.py@@ -98,15 +98,6 @@ def dequantize_gemm(qweight, qzeros, scales, bits, group_size):return iweight-def get_best_device():- if torch.backends.mps.is_available():- return "mps"- elif torch.cuda.is_available():- return "cuda:0"- else:- return "cpu"--class WQLinearMMFunction(Function):@staticmethod# ctx is the first argument to forward@@ -222,14 +213,22 @@ class WQLinear_GEMM(nn.Module):device = "cpu"if torch.cuda.is_available():device = "cuda:0"+ elif torch.xpu.is_available():+ device = "xpu:0"repeat_scales = scales.to(device).t().repeat_interleave(group_size, 1)if isinstance(zeros, torch.Tensor):repeat_zeros = zeros.to(device).t().repeat_interleave(group_size, 1)+ intweight = torch.round(+ linear.weight.to(device) / repeat_scales[:, :linear.weight.shape[1]] ++ repeat_zeros[:, :linear.weight.shape[1]]).to(torch.int).t().contiguous()+else:repeat_zeros = zeros- intweight = torch.round(linear.weight.to(device) / repeat_scales + repeat_zeros).to(- torch.int).t().contiguous()+ intweight = torch.round(+ linear.weight.to(device) / repeat_scales[:, :linear.weight.shape[1]] + repeat_zeros).to(+ torch.int).t().contiguous()+intweight = intweight.to(dtype=torch.int32)del repeat_scalesdiff --git a/auto_round/inference/__init__.py b/auto_round/inference/__init__.pynew file mode 100644index 000000000..2b414eb72--- /dev/null+++ b/auto_round/inference/__init__.py@@ -0,0 +1,14 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+diff --git a/auto_round/inference/auto_quantizer.py b/auto_round/inference/auto_quantizer.pynew file mode 100644index 000000000..90fd551bd--- /dev/null+++ b/auto_round/inference/auto_quantizer.py@@ -0,0 +1,359 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# coding=utf-8+# Copyright 2023 HuggingFace Inc. team and GPTQ and AutoGPTQ authors.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import importlib.util+import warnings+from dataclasses import dataclass+from logging import getLogger+from typing import Any, Dict, Optional, Tuple, Union, List++import torch+import torch.nn as nn+from packaging import version+from transformers.modeling_utils import PreTrainedModel+from transformers.quantizers import AutoQuantizationConfig, HfQuantizer+from transformers.quantizers.auto import AUTO_QUANTIZER_MAPPING+from transformers.utils.quantization_config import AwqConfig, GPTQConfig, QuantizationConfigMixin, QuantizationMethod+from auto_round.utils import (is_hpu_supported)+from auto_round.inference.convert_model import convert_hf_model, infer_target_device, post_init+from enum import Enum++logger = getLogger(__name__)+import sys++if sys.version_info < (3, 8):+ import importlib_metadata+else:+ import importlib.metadata as importlib_metadata++AUTOROUND_MINIMUM_VERSION = version.parse("0.2")+++def _is_package_available(pkg_name: str, return_version: bool = False) -> Union[Tuple[bool, str], bool]:+ # Check we're not importing a "pkg_name" directory somewhere but the actual library by trying to grab the version+ try: ##TODO remove it later+ import auto_round+ return True, auto_round.__version__+ except:+ pass++ package_exists = importlib.util.find_spec(pkg_name) is not None+ package_version = "N/A"+ if package_exists:+ try:+ package_version = importlib.metadata.version(pkg_name)+ package_exists = True+ except importlib.metadata.PackageNotFoundError:+ package_exists = False+ if return_version:+ return package_exists, package_version+ else:+ return package_exists+++_auto_round_available = _is_package_available("auto_round")+++def is_auto_round_available():+ try:+ import auto_round+ return True+ except:+ pass+ if _auto_round_available:+ version_autoround = version.parse(importlib_metadata.version("auto_round"))+ if AUTOROUND_MINIMUM_VERSION < version_autoround:+ return True+ else:+ raise ImportError(+ f"Found an incompatible version of auto-round. Found version {version_autoround},"+ f" but only version above {AUTOROUND_MINIMUM_VERSION} are supported"+ )+++class AutoHfQuantizer:+ """The Auto-HF quantizer class that takes care of automatically instantiating to the correct+ `HfQuantizer` given the `QuantizationConfig`."""++ @classmethod+ def from_config(cls, quantization_config: Union[QuantizationConfigMixin, Dict], **kwargs):+ # Convert it to a QuantizationConfig if the q_config is a dict+ if isinstance(quantization_config, dict):+ if "auto-round" in quantization_config["quant_method"]:+ quantization_config = AutoRoundConfig.from_dict(quantization_config)+ else:+ quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101+ quant_method = quantization_config.quant_method++ # Again, we need a special care for bnb as we have a single quantization config+ # class for both 4-bit and 8-bit quantization+ if quant_method == QuantizationMethod.BITS_AND_BYTES:+ if quantization_config.load_in_8bit:+ quant_method += "_8bit"+ else:+ quant_method += "_4bit"++ if quant_method not in AUTO_QUANTIZER_MAPPING.keys() and "auto-round" not in quant_method:+ raise ValueError(+ f"Unknown quantization type, got {quant_method} - supported types are:"+ f" {list(AUTO_QUANTIZER_MAPPING.keys())}"+ )+ if "auto-round" in quant_method or is_hpu_supported(): # pragma: no cover+ target_cls = AutoRoundQuantizer+ else:+ target_cls = AUTO_QUANTIZER_MAPPING[quant_method]++ return target_cls(quantization_config, **kwargs)++ @classmethod+ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):+ quantization_config = AutoQuantizationConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)+ return cls.from_config(quantization_config)++ @classmethod+ def merge_quantization_configs(+ cls,+ quantization_config: Union[dict, QuantizationConfigMixin],+ quantization_config_from_args: Optional[QuantizationConfigMixin],+ ):+ """Handles situations where both quantization_config+ from args and quantization_config from model config are present."""+ if quantization_config_from_args is not None:+ warning_msg = (+ "You passed `quantization_config` or equivalent parameters to "+ "`from_pretrained` but the model you're loading"+ " already has a `quantization_config` attribute. The `quantization_config` from the model will be used."+ )+ else:+ warning_msg = ""++ loading_attr_dict = quantization_config_from_args.get_loading_attributes() \+ if quantization_config_from_args is not None else None+ if isinstance(quantization_config, dict):+ if "auto-round" in quantization_config["quant_method"]:+ quantization_config = AutoRoundConfig.from_dict(quantization_config)+ else:+ if isinstance(quantization_config_from_args, (AutoRoundConfig)):+ logger.info(f"Loading quantized model in auto_round format.")+ tmp_backend = quantization_config["quant_method"]+ if "auto-round" not in tmp_backend and "gptq" not in tmp_backend and "awq" not in tmp_backend:+ logger.error("could not convert to auto_round format, currently only supports `gptq`,`awq` or "+ "`auto-round` format")+ exit(-1)+ target_backend = quantization_config["backend"] if "backend" in quantization_config else "auto"+ if loading_attr_dict is not None and "backend" in loading_attr_dict:+ target_backend = loading_attr_dict["backend"]+ loading_attr_dict.pop("backend")+ if "auto_round" not in target_backend:+ target_backend = f"auto_round:{tmp_backend}" #+ quantization_config = AutoRoundConfig.from_dict(quantization_config)+ setattr(quantization_config, "backend", target_backend)+ else:+ quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101++ if isinstance(quantization_config,+ (GPTQConfig, AwqConfig, AutoRoundConfig)) and quantization_config_from_args is not None:+ # special case for GPTQ / AWQ config collision++ for attr, val in loading_attr_dict.items():+ setattr(quantization_config, attr, val)+ warning_msg += (+ f"However, loading attributes (e.g. {list(loading_attr_dict.keys())}) "+ f"will be overwritten with the one you passed to `from_pretrained`. The rest will be ignored."+ )++ if warning_msg != "":+ warnings.warn(warning_msg)++ return quantization_config++ @staticmethod+ def supports_quant_method(quantization_config_dict):+ from transformers.quantizers.auto import AUTO_QUANTIZATION_CONFIG_MAPPING+ AUTO_QUANTIZATION_CONFIG_MAPPING['intel/auto-round'] = AutoRoundConfig+ AUTO_QUANTIZATION_CONFIG_MAPPING['intel/auto_round'] = AutoRoundConfig+ quant_method = quantization_config_dict.get("quant_method", None)+ if quantization_config_dict.get("load_in_8bit", False) or quantization_config_dict.get("load_in_4bit", False):+ suffix = "_4bit" if quantization_config_dict.get("load_in_4bit", False) else "_8bit"+ quant_method = QuantizationMethod.BITS_AND_BYTES + suffix+ elif quant_method is None:+ raise ValueError(+ "The model's quantization config from the arguments has no `quant_method` attribute." \+ "Make sure that the model has been correctly quantized"+ )++ if quant_method not in AUTO_QUANTIZATION_CONFIG_MAPPING.keys():+ logger.warning(+ f"Unknown quantization type, got {quant_method} - supported types are:"+ f" {list(AUTO_QUANTIZER_MAPPING.keys())}. Hence, we will skip the quantization. "+ "To remove the warning, you can delete the quantization_config attribute in config.json"+ )+ return False+ return True+++class AutoRoundQuantizationMethod(str, Enum):+ AutoRound = "intel/auto-round"+++@dataclass+class AutoRoundConfig(QuantizationConfigMixin):+ """This is a wrapper class about all possible attributes and features that you can play with a model that has been+ loaded AutoRound quantization.++ Args:+ bits (`int`):+ The number of bits to quantize to, supported numbers are (2, 3, 4, 8).+ tokenizer (`str` or `PreTrainedTokenizerBase`, *optional*):+ The tokenizer used to process the dataset. You can pass either:+ - A custom tokenizer object.+ - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co.+ - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved+ using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.+ """++ def __init__(+ self,+ bits: int = 4,+ tokenizer: Any = None,+ dataset: str = None,+ group_size: int = 128,+ sym: bool = False,+ backend="auto",+ layer_config: dict = None,+ **kwargs,+ ):++ self.bits = bits+ self.tokenizer = tokenizer+ self.dataset = dataset+ self.group_size = group_size+ self.sym = sym+ self.target_backend = "auto"+ self.backend = backend+ self.layer_config = layer_config+ if kwargs is not None:+ for key in kwargs.keys():+ setattr(self, key, kwargs[key])+ self.quant_method = AutoRoundQuantizationMethod.AutoRound+ self.post_init()++ def post_init(self):+ r"""Safety checker that arguments are correct."""+ if self.bits not in [2, 4, 8]:+ raise ValueError(f"Only support quantization to [2,4,8] bits but found {self.bits}")+ if self.group_size != -1 and self.group_size <= 0:+ raise ValueError("group_size must be greater than 0 or equal to -1")++ def get_loading_attributes(self):+ loading_attibutes_dict = {"target_backend": self.backend}+ return loading_attibutes_dict++ def to_dict(self):+ config_dict = super().to_dict()+ return config_dict+++class AutoRoundQuantizer(HfQuantizer):+ """Quantizer of the AutoRound method, currently only triton and exllamav2 backend has been supported."""++ requires_calibration = False+ required_packages = ["auto_round"]+ optimum_quantizer = None++ def __init__(self, quantization_config: QuantizationConfigMixin, **kwargs):+ self.device_map = None+ super().__init__(quantization_config, **kwargs)++ def validate_environment(self, *args, **kwargs):+ self.device_map = kwargs.get("device_map", None)+ if not is_auto_round_available():+ raise ImportError("Loading a AutoRound quantized model requires auto-round library (`pip install "+ "auto-round`)")+ else:+ try:+ import auto_round+ autoround_version = version.parse(auto_round.__version__)+ except:+ autoround_version = version.parse(importlib.metadata.version("auto_round"))+ if autoround_version < version.parse("0.2.0"):+ raise ImportError("You need a version of auto_round > 0.2.0 to use AutoRound: `pip install --upgrade "+ "auto-round` or install from source")++ def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":+ self.target_device = infer_target_device(self.device_map)+ if torch_dtype is None:+ torch_dtype = torch.float16+ elif torch_dtype != torch.float16:+ logger.info("We suggest you to set `torch_dtype=torch.float16` for better efficiency on CUDA")+ return torch_dtype++ def post_init_model(self, model):+ """Post-initialization that require device information, for example buffers initialization on device.++ Args:+ model (`nn.Module`):+ The input model+ """++ class StoreAttr(object):+ pass++ model.quantize_config = StoreAttr()++ post_init(model, self.used_backends)++ def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs):+ if self.pre_quantized:+ target_device = self.target_device if hasattr(self, self.target_device) else infer_target_device(+ self.device_map)+ model, used_backends = convert_hf_model(model, target_device)+ self.used_backends = used_backends++ def _process_model_after_weight_loading(self, model: "PreTrainedModel", **kwargs):+ if self.pre_quantized:+ self.post_init_model(model)+ else:+ raise NotImplementedError++ @property+ def is_trainable(self, model: Optional["PreTrainedModel"] = None):+ return True++ @property+ def is_serializable(self):+ return True+++import transformers++if version.parse(transformers.__version__) < version.parse("4.38.0"):+ logger.error("Please upgrade transformers>=4.38.0 to support lm-head quantization")++transformers.quantizers.auto.AutoHfQuantizer = AutoHfQuantizer+transformers.modeling_utils.AutoHfQuantizer = AutoHfQuantizerdiff --git a/auto_round/backend.py b/auto_round/inference/backend.pysimilarity index 57%rename from auto_round/backend.pyrename to auto_round/inference/backend.pyindex fd43af788..2f2b27f05 100644--- a/auto_round/backend.py+++ b/auto_round/inference/backend.py@@ -18,6 +18,7 @@ from typing import List, Any, Optionalfrom transformers.utils.versions import require_version+import auto_round_extension.cuda.gptqmodel_marlinfrom auto_round.utils import get_library_version, loggerBackendInfos = {}@@ -62,6 +63,7 @@ class BackendInfo:sym: List[bool]packing_format: strbits: List[int]+ dtype: List[str] = Nonegroup_size: Optional[List[int]] = Nonepriority: int = 0 ##higher is betterconvertable_format: List[str] = field(default_factory=list)@@ -80,141 +82,167 @@ def feature_num_greater_checker(in_feature, out_feature, num):return in_feature * out_feature > num[email protected]_cache(None)-def check_auto_round_exllamav2_installed():- try:- from autoround_exllamav2_kernels import gemm_half_q_half, make_q_matrix- except ImportError:- return False, ("please install from source to enable auto-round exllamav2 kernel."- "`git clone https://github.com/intel/auto-round && cd auto-round &&"- " pip install -vvv --no-build-isolation -e .`")- return True, ""--feature_multiply_checker_32 = functools.partial(feature_multiply_checker, in_feature_multiplier=32)-+in_output_feature_multiply_checker_32 = functools.partial(feature_multiply_checker, in_feature_multiplier=32,+ out_feature_multiplier=32)feature_multiply_checker_marlin = functools.partial(feature_multiply_checker, in_feature_multiplier=128,out_feature_multiplier=256)feature_num_greater_checker_1024 = functools.partial(feature_num_greater_checker, num=1024)+BackendInfos['auto_gptq:exllamav2'] = BackendInfo(device=["cuda"], sym=[True, False],+ packing_format="triton_zp",+ bits=[4],+ priority=5,+ dtype=["float16"],+ group_size=[-1, 32, 64, 128, 256, 384, 512, 1024, 2048],+ ##16 seems has accuracy issue+ feature_checks=[feature_multiply_checker_32],+ alias=['gptq', 'auto_gptq', 'exllamav2', "gptq:exllamav2"],+ requirements=["auto-gptq>=0.7.1"]+ )-BackendInfos['auto_round:exllamav2'] = BackendInfo(device=["cuda"], sym=[True, False],- packing_format="triton",- bits=[4], group_size=None,- priority=5,- feature_checks=[feature_multiply_checker_32],- alias=["auto_round"],- requirements=[check_auto_round_exllamav2_installed]- )+BackendInfos['auto_gptq:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, False],+ packing_format="triton_zp",+ bits=[2, 4, 8], group_size=None,+ dtype=["float16"],+ priority=0, feature_checks=[feature_multiply_checker_32],+ alias=["auto_gptq:tritonv2"],+ requirements=["auto-gptq>=0.7.1"]+ )++BackendInfos['auto_gptq:cuda'] = BackendInfo(device=["cuda"], sym=[True, False],+ packing_format="triton_zp",+ bits=[2, 3, 4, 8], group_size=None,+ priority=1, feature_checks=[feature_multiply_checker_32],+ alias=["auto_gptq:cuda"],+ dtype=["float16"],+ convertable_format=["triton_zp"],+ requirements=["auto-gptq>=0.7.1"]+ )BackendInfos['auto_round:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, False],packing_format="triton",- bits=[2, 4, 8], group_size=None,- priority=0, feature_checks=[feature_multiply_checker_32],- requirements=["triton<3.0,>=2.0"]+ dtype=["float16", "bfloat16"],+ bits=[2, 4, 8],+ priority=1, feature_checks=[feature_multiply_checker_32],+ alias=["auto_round", "tritonv2"],+ requirements=["auto-round>=0.2"])-BackendInfos['gptq:exllamav2'] = BackendInfo(device=["cuda"], sym=[True, False],- packing_format="triton_zp+-1",- bits=[4], group_size=None,- priority=5,- feature_checks=[feature_multiply_checker_32],- alias=["auto_round:gptq:exllamav2", "auto_round:auto_gptq:exllamav2",- 'gptq', 'auto_gptq', "auto_round:gptq", "auto_round:auto_gptq"],- requirements=["auto-gptq>=0.7.1"]- )+BackendInfos['auto_round:tritonv2_zp'] = BackendInfo(device=["cuda"], sym=[True], ## asym has accuracy issue+ packing_format="triton_zp",+ dtype=["float16", "bfloat16"],+ bits=[2, 4, 8],+ priority=1, feature_checks=[feature_multiply_checker_32],+ alias=["tritonv2", "tritonv2_zp"],+ requirements=["auto-round>=0.5"]+ )++BackendInfos['gptqmodel:marlin'] = BackendInfo(device=["cuda"], sym=[True],+ packing_format="triton",+ bits=[4, 8],+ group_size=[-1, 32, 64, 128],+ dtype=["float16", "bfloat16"],+ priority=6, feature_checks=[in_output_feature_multiply_checker_32],+ alias=["marlin", "gptqmodel"],+ requirements=["gptqmodel>=2.0"]+ )++BackendInfos['gptqmodel:marlin_zp'] = BackendInfo(device=["cuda"], sym=[True],+ packing_format="triton_zp",+ bits=[4, 8],+ group_size=[-1, 32, 64, 128],+ dtype=["float16", "bfloat16"],+ priority=6, feature_checks=[in_output_feature_multiply_checker_32],+ alias=["marlin", "gptqmodel"],+ requirements=["gptqmodel>=2.0"]+ )++BackendInfos['gptqmodel:exllamav2'] = BackendInfo(device=["cuda"], sym=[True, False],+ packing_format="triton",+ bits=[4], group_size=[-1, 32, 64, 128], ##16 seems has accuracy issue+ dtype=["float16", "bfloat16"],+ priority=5, feature_checks=[in_output_feature_multiply_checker_32],+ alias=["exllamav2"],+ requirements=["gptqmodel>=2.0"]+ )-BackendInfos['gptq:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, False],- packing_format="triton_zp+-1",- bits=[2, 4, 8], group_size=None,- priority=0, feature_checks=[feature_multiply_checker_32],- alias=["auto_round:gptq:tritonv2", "auto_round:auto_gptq:tritonv2",- "auto_gptq:tritonv2"],- requirements=["auto-gptq>=0.7.1", "triton<3.0,>=2.0"]+BackendInfos['auto_awq:gemm'] = BackendInfo(device=["cuda"], sym=[True, False], ##actrally is gemm+ packing_format="awq",+ bits=[4], group_size=None,+ priority=4,+ dtype=["float16"],+ alias=["auto_awq:gemm", "awq",+ "auto_awq"],+ requirements=["autoawq"])-BackendInfos['gptq:cuda'] = BackendInfo(device=["cuda"], sym=[True, False],- packing_format="triton_zp+-1",- bits=[2, 3, 4, 8], group_size=None,- priority=1, feature_checks=[feature_multiply_checker_32],- alias=["auto_round:auto_gptq:cuda", "auto_gptq:cuda", "auto_round:gptq:cuda"],- convertable_format=["triton_zp+-1"],- requirements=["auto-gptq>=0.7.1"]+BackendInfos['qbits'] = BackendInfo(device=["cpu"], sym=[True, False],+ packing_format="qbits",+ bits=[2, 4, 8], group_size=None,+ priority=0 if "intel" in get_cpu_manufacturer() else 5,+ feature_checks=[],+ alias=["itrex"],+ dtype=["float16", "bfloat16"],+ convertable_format=["triton"],+ requirements=["intel-extension-for-transformers"])++BackendInfos['qbits_zp'] = BackendInfo(device=["cpu"], sym=[True, False],+ packing_format="qbits_zp",+ bits=[2, 4, 8], group_size=None,+ dtype=["float16", "bfloat16"],+ priority=0 if "intel" in get_cpu_manufacturer() else 5,+ feature_checks=[],+ alias=["itrex"],+ convertable_format=["triton_zp"],+ requirements=["intel-extension-for-transformers"]+ )++# BackendInfos['auto_round:qbits_awq'] = BackendInfo(device=["cpu"], sym=[True], ## for awq, not robust+# packing_format="awq",+# bits=[2, 4, 8], group_size=None,+# priority=0 if "intel" in get_cpu_manufacturer() else 5,+# feature_checks=[],+# requirements=["intel-extension-for-transformers"]+# )++BackendInfos['ipex_gptq'] = BackendInfo(device=["cpu", "xpu"], sym=[True, False],+ packing_format="ipex_gptq",+ bits=[4], group_size=None,+ priority=5 if "intel" in get_cpu_manufacturer() else 5,+ feature_checks=[],+ dtype=["float16", "bfloat16"],+ convertable_format=["triton_zp"],+ alias=["ipex"],+ requirements=["intel-extension-for-pytorch>=2.5"])-BackendInfos['awq:gemm'] = BackendInfo(device=["cuda"], sym=[True, False], ##actrally is gemm- packing_format="awq",+BackendInfos['ipex_awq'] = BackendInfo(device=["cpu", "xpu"], sym=[True, False],+ packing_format="ipex_awq",bits=[4], group_size=None,- priority=4,- alias=["auto_awq:gemm", "auto_round:awq:gemm", "auto_round:auto_awq:gemm", "awq",- "auto_awq", "auto_round:awq", "auto_round:auto_awq"],- requirements=["autoawq"]+ priority=1,+ dtype=["float16", "bfloat16"],+ feature_checks=[],+ alias=["ipex"],+ convertable_format=["awq"],+ requirements=["intel-extension-for-pytorch>=2.6"])-BackendInfos['auto_round:qbits'] = BackendInfo(device=["cpu"], sym=[True, False],- packing_format="qbits",- bits=[2, 4, 8], group_size=None,- priority=0 if "intel" in get_cpu_manufacturer() else 5,- feature_checks=[],- convertable_format=["triton"],- requirements=["intel-extension-for-transformers"])--BackendInfos['auto_round:qbits_zp'] = BackendInfo(device=["cpu"], sym=[True, False],- packing_format="qbits_zp+-1",- bits=[2, 4, 8], group_size=None,- priority=0 if "intel" in get_cpu_manufacturer() else 5,- feature_checks=[],- convertable_format=["triton_zp+-1"],- requirements=["intel-extension-for-transformers"]- )--BackendInfos['auto_round:qbits_awq'] = BackendInfo(device=["cpu"], sym=[True],- packing_format="awq",- bits=[2, 4, 8], group_size=None,- priority=0 if "intel" in get_cpu_manufacturer() else 5,- feature_checks=[],- requirements=["intel-extension-for-transformers"]- )--BackendInfos['auto_round:ipex_gptq'] = BackendInfo(device=["cpu", "xpu"], sym=[True, False],- packing_format="ipex_gptq",- bits=[4], group_size=None,- priority=5 if "intel" in get_cpu_manufacturer() else 5,- feature_checks=[],- convertable_format=["triton_zp+-1"],- requirements=["intel-extension-for-pytorch>=2.5"]- )--BackendInfos['auto_round:ipex_awq_xpu'] = BackendInfo(device=["cpu", "xpu"], sym=[True, False],- packing_format="ipex_awq",- bits=[4], group_size=None,- priority=1,- feature_checks=[],- ##convertable_format=["triton_zp+-1", "awq"],- convertable_format=["awq"],- requirements=["intel-extension-for-pytorch>=2.6"]- )--# BackendInfos['auto_round:marlin'] = BackendInfo(device=["gpu"], sym=[True],-# packing_format="marlin",-# bits=[4], group_size=[-1, 128],-# priority=6,-# feature_checks=[feature_multiply_checker_marlin],-# alias=["marlin", "auto_gptq:marlin", "auto_round:gptq:marlin",-# "auto_round:auto_gptq:marlin"])--BackendInfos['auto_round:hpu'] = BackendInfo(device=["hpu"], sym=[True, False],- packing_format="hpu",- bits=[4],- priority=0,- convertable_format=["triton"]- )+BackendInfos['hpu'] = BackendInfo(device=["hpu"], sym=[True, False],+ packing_format="hpu",+ bits=[4],+ dtype=["bfloat16"],+ priority=0,+ convertable_format=["triton"]+ )-BackendInfos['auto_round:hpu_zp'] = BackendInfo(device=["hpu"], sym=[True, False],- packing_format="hpu_zp+-1",- bits=[4],- priority=0,- convertable_format=["triton_zp+-1"])+BackendInfos['hpu_zp'] = BackendInfo(device=["hpu"], sym=[True, False],+ packing_format="hpu_zp",+ bits=[4],+ dtype=["bfloat16"],+ alias=["hpu"],+ priority=0,+ convertable_format=["triton_zp"])def check_compatible(backend_name, device, bits, group_size, sym, packing_format, in_features, out_features,@@ -265,16 +293,16 @@ def check_compatible(backend_name, device, bits, group_size, sym, packing_formatif sym not in backend.sym:return False- # Check packing format and apply feature checks- if packing_format == backend.packing_format:- for check in backend.feature_checks:- if not check(in_features, out_features):- return False-# Check if the format is convertible when packing formats differ- if packing_format != backend.packing_format and packing_format not in backend.convertable_format:+ if packing_format == backend.packing_format or packing_format in backend.convertable_format:+ pass+ else:return False+ for check in backend.feature_checks: ## convertible+ if not check(in_features, out_features):+ return False+if check_requirements and backend.requirements is not None:for requirement in backend.requirements:if isinstance(requirement, str):@@ -339,16 +367,6 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):from auto_round_extension.ipex.qlinear_ipex_awq import QuantLinearreturn QuantLinear- if "marlin" in backend:- from transformers.utils.versions import require_version- require_version(- "gptqmodel",- "marlin format requires gptqmodel to be installed, `pip install -v gptqmodel --no-build-isolation`"- )- from gptqmodel.nn_modules.qlinear.qlinear_marlin_inference import \- MarlinInferenceQuantLinear # pylint: disable=E0401- return MarlinInferenceQuantLinear-if "hpu" in backend:try:import habana_frameworks.torch.hpu # pylint: disable=E0401@@ -361,8 +379,10 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):else: # auto_round must be at the endfrom auto_round_extension.hpu.qlinear_hpu import QuantLinearreturn QuantLinear+ if "gptqmodel" in backend:+ return get_gptqmodel_infer_linear(backend, bits, group_size, sym)- if "gptq" in backend:+ if "gptq" in backend and "gptqmodel" not in backend:return get_autogptq_infer_linear(backend, bits, group_size, sym)if "awq" in backend:@@ -373,13 +393,28 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):"autoawq is required. Please install it by 'pip install autoawq' to support auto_awq format.")return WQLinear_GEMM- if "auto_round" in backend:- if "exllamav2" in backend:- import auto_round_extension.cuda.qlinear_exllamav2- return auto_round_extension.cuda.qlinear_exllamav2.QuantLinear- else:- import auto_round_extension.cuda.qlinear_tritonv2- return auto_round_extension.cuda.qlinear_tritonv2.QuantLinear+ if backend == "auto_round:tritonv2":+ from auto_round_extension.cuda.qlinear_tritonv2 import QuantLinear+ return QuantLinear++ if backend == "auto_round:tritonv2_zp":+ from auto_round_extension.cuda.qlinear_tritonv2_zp import QuantLinear+ return QuantLinear++ raise ValueError(f"unsupported backend {backend}, please set it to `auto` and retry")+++def get_gptqmodel_infer_linear(backend, bits=4, group_size=128, sym=False):+ import gptqmodel # pylint: disable=E0401+ if "marlin" in backend:+ return auto_round_extension.cuda.gptqmodel_marlin.get_marlin_layer()+ # return gptqmodel.nn_modules.qlinear.marlin.MarlinQuantLinear+ elif "exllamav2" in backend:+ return gptqmodel.nn_modules.qlinear.exllamav2.ExllamaV2QuantLinear+ elif "tritonv2" in backend:+ return gptqmodel.nn_modules.qlinear.tritonv2.TritonV2QuantLinear+ else:+ raise ValueError(f"Unsupported {backend}")def get_autogptq_infer_linear(backend, bits=4, group_size=128, sym=False):@@ -469,6 +504,57 @@ def get_autogptq_infer_linear(backend, bits=4, group_size=128, sym=False):return QuantLinear+def find_backend(target_backend: str, orig_backend: str = None) -> str | None:+ """+ Finds the matching backend key based on the target backend name or its aliases.++ Args:+ target_backend (str): Name or alias of the target backend.+ orig_backend (str, optional): Original backend name to check compatibility. Defaults to None.++ Returns:+ str or None: Matching backend key if found and compatible; otherwise, None.+ """+ matched_keys = [+ key for key, info in BackendInfos.items()+ if key == target_backend or (info.alias and target_backend in info.alias)+ ]++ if not matched_keys:+ return None++ if orig_backend is None:+ return matched_keys[0] if len(matched_keys) >= 1 else None++ orig_info = BackendInfos[orig_backend]++ for key in matched_keys:+ target_info = BackendInfos[key]+ if (target_info.packing_format == orig_info.packing_format or+ orig_info.packing_format in target_info.convertable_format):+ return key++ raise ValueError(+ f"{target_backend} is not compatible with {orig_backend}. "+ f"Please set `backend` to `auto` and retry."+ )+++def get_all_compatible_backend(device, backend, orig_backend, bits, group_size, sym, in_features, out_features):+ # Get packing format from the original backend+ packing_format = BackendInfos[orig_backend].packing_format++ # Find compatible backends+ compatible_backends = [+ key for key in BackendInfos.keys()+ if check_compatible(key, device, bits, group_size, sym, packing_format, in_features, out_features,+ check_requirements=False)+ ]++ # Return the first compatible backend or an empty list if none found+ return compatible_backends++def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_features, out_features):"""Selects the most suitable backend for the layer based on compatibility and priority.@@ -505,30 +591,23 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_fIf no compatible backend is found for the given layer configuration."""# Check if the provided backend is in BackendInfos+ backend = find_backend(backend)assert backend in BackendInfos.keys(), \f"Unsupported backend {backend}, please set it to `auto` to try automatic selection"packing_format = BackendInfos[orig_backend].packing_format- # Check if the provided backend supports the layer configuration- if check_compatible(backend, device, bits, group_size, sym, packing_format, in_features, out_features):- return backend-# Find and store other compatible backendssupported_backends = []for key in BackendInfos.keys():- if key == backend:- continueif check_compatible(key, device, bits, group_size, sym, packing_format, in_features, out_features):supported_backends.append(key)# Raise an error if no compatible backends are foundif len(supported_backends) == 0:- supported_backends_need_package = []- for key in BackendInfos.keys():- if check_compatible(key, device, bits, group_size, sym, packing_format, in_features, out_features,- check_requirements=False):- supported_backends_need_package.append(key)+ supported_backends_need_package = get_all_compatible_backend(device, backend, orig_backend, bits, group_size,+ sym,+ in_features, out_features)if len(supported_backends_need_package) > 0:supported_backends_need_package = sorted(supported_backends_need_package,@@ -547,7 +626,7 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_flogger.error(str_info)exit(-1)- raise ValueError(f"None of the backends support this layer")+ return None# Sort the compatible backends by priority and return the one with the highest prioritysupported_backends = sorted(supported_backends, key=lambda support_backend: BackendInfos[support_backend].priority,diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pynew file mode 100644index 000000000..ec3ed6c3f--- /dev/null+++ b/auto_round/inference/convert_model.py@@ -0,0 +1,425 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from logging import getLogger+from typing import Union+from tqdm import tqdm+import torch+import torch.nn as nn++from transformers.pytorch_utils import Conv1D++from auto_round.utils import (get_module, set_module, is_hpu_supported, get_block_names,+ find_matching_blocks, get_layer_names_in_block, check_to_quantized)++from auto_round.inference.backend import get_layer_backend, dynamic_import_inference_linear, find_backend, BackendInfos++logger = getLogger(__name__)++supported_devices = ("cpu", "hpu", "xpu", "cuda")+++def get_available_devices():+ """+ Returns a list of available devices in the current environment.++ Returns:+ List[str]: A list of device identifiers like "cuda", "hpu", "xpu", "cpu".+ """+ devices = []++ if torch.cuda.is_available():+ devices.append("cuda")++ if is_hpu_supported():+ devices.append("hpu")++ if hasattr(torch, "xpu") and torch.xpu.is_available():+ devices.append("xpu")++ devices.append("cpu") # Always available++ return devices+++def parse_target_device_and_backend(target_backend: str):+ """+ Parse the target device and backend from the given target_backend string.++ Args:+ target_backend (str): A string specifying the target backend, which may include a device prefix.++ Returns:+ tuple: (target_device, backend), where:+ - target_device (str): The identified device (e.g., "cpu", "cuda", "auto").+ - backend (str): The backend string after removing the device prefix.+ """++ def remove_device_prefix(backend: str, prefix: str) -> str:+ """Removes the given device prefix from the backend string, if present."""+ return backend[len(prefix):].lstrip(":") if backend.startswith(prefix) else backend++ # Handle "auto" case explicitly+ if target_backend == "auto":+ return "auto", ""+ if target_backend.startswith("auto:"):+ return "auto", target_backend[5:].lstrip(":")++ # Define supported devices++ # Identify the target device and adjust the backend string accordingly+ for device in supported_devices:+ if target_backend.startswith(device):+ return device, remove_device_prefix(target_backend, device)++ # Default case: If no known device is found, assume "auto"+ return "auto", target_backend+++def get_layer_config(model, quantization_config):+ """+ get a layer-wise quantization configuration for a given model.++ Args:+ model (torch.nn.Module): The model for which quantization settings are generated.+ quantization_config (object): An object containing quantization parameters, including:+ - bits (int): Default bit width for quantization.+ - group_size (int): Group size for weight quantization.+ - data_type (str, optional): Data type for quantization (default: "int").+ - sym (bool): Whether to use symmetric quantization.+ - quant_block_list (list, optional): Predefined list of blocks to quantize.+ - to_quant_block_names (list or str, optional): Blocks to quantize (if quant_block_list is None).+ - extra_config (dict, optional): Per-layer overrides for quantization settings.+ - modules_in_block_to_quantize (list, optional): Specific modules within a block for quantization.+ - modules_to_not_convert (list, optional): Layers excluded from quantization (AWQ format).++ Returns:+ dict: A dictionary mapping layer names to their quantization configurations, where each layer has:+ - "bits" (int): Bit width for quantization.+ - "group_size" (int): Group size for quantization.+ - "data_type" (str): Data type used for quantization.+ - "sym" (bool): Whether symmetric quantization is applied.+ - "clip" (bool): Whether weight clipping is enabled.+ """+ bits = quantization_config.bits+ group_size = quantization_config.group_size+ data_type = getattr(quantization_config, "data_type", "int") # Default to "int" if not specified+ sym = quantization_config.sym++ # Determine the quantization block list+ quant_block_list = getattr(quantization_config, "quant_block_list", None)+ if quant_block_list is None:+ to_quant_block_names = getattr(quantization_config, "to_quant_block_names", None)+ if isinstance(to_quant_block_names, (list, tuple)):+ quant_block_list = to_quant_block_names+ elif isinstance(to_quant_block_names, str):+ # Generate quant block names based on the given layer names+ quant_block_list = [+ [f'{block}.{i}' for i in range(len(get_module(model, block)))]+ for block in to_quant_block_names.split(',')+ ]+ else:+ # Find matching blocks if no explicit names are provided+ all_blocks = get_block_names(model)+ quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)++ # Get layer names that will be quantized+ layer_names = get_layer_names_in_block(model, quant_block_list=quant_block_list)++ # Load extra configuration if available+ extra_config = getattr(quantization_config, "extra_config", {})++ # Process GPTQ format: identify modules that should be quantized+ if getattr(quantization_config, "modules_in_block_to_quantize", None):+ modules_in_block_to_quantize = sum(quantization_config.modules_in_block_to_quantize, []) # Flatten the list+ for layer_name in layer_names:+ if not any(qname in layer_name for qname in modules_in_block_to_quantize):+ extra_config[layer_name] = {"bits": 16} # Default to 16-bit for unquantized layers++ # Process AWQ format: exclude specified modules from quantization+ modules_to_not_convert = getattr(quantization_config, "modules_to_not_convert", [])+ if modules_to_not_convert is None:+ modules_to_not_convert = []+ for layer_name in modules_to_not_convert:+ extra_config[layer_name] = {"bits": 16}++ # Merge and deduplicate layer names+ layer_names = list(set(layer_names).union(extra_config.keys()))++ # Construct final layer configuration+ layer_configs = {+ layer_name: {+ "bits": extra_config.get(layer_name, {}).get("bits", bits),+ "group_size": extra_config.get(layer_name, {}).get("group_size", group_size),+ "data_type": extra_config.get(layer_name, {}).get("data_type", data_type),+ "sym": extra_config.get(layer_name, {}).get("sym", sym),+ "clip": extra_config.get(layer_name, {}).get("clip", False),+ }+ for layer_name in layer_names+ }++ return layer_configs+++def get_device(obj: Union[torch.Tensor, nn.Module]):+ if isinstance(obj, torch.Tensor):+ return obj.device+ return next(obj.parameters()).device+++def _replace_by_quant_layers(module: nn.Module, layer_configs, target_backend, target_device, orig_backend):+ """+ Replaces linear layers in the given module with quantized layers.++ Args:+ module (nn.Module): The module containing layers to be quantized.+ layer_configs (dict): Configuration for each layer's quantization.+ target_backend (str): Backend for quantization (device and format).+ target_device (str): Device for execution ('cuda', 'cpu', 'hpu').+ orig_backend (str): Original backend of the model.++ Returns:+ dict: Flags indicating which backends were used.+ """++ used_backends = []+ must_use_target_backend = False+ if target_backend:+ must_use_target_backend = True+ layer_backend = target_backend+ layer_backend_must = find_backend(layer_backend, orig_backend)+ used_backends.append(layer_backend_must)+ if layer_backend_must is None:+ raise ValueError(+ f"{target_backend} is not compatible, please change backend to `auto` and retry")+ devices = BackendInfos[layer_backend_must].device+ if target_device not in devices:+ raise ValueError(f"{target_backend} does not support {target_device}, please change device or backend")++ target_backend = target_backend or orig_backend # Default to original backend if not specified++ backend_cache = {}++ for layer_name, config in layer_configs.items():+ if not check_to_quantized(config):+ continue # Skip layers that do not require quantization++ layer = get_module(module, layer_name)+ in_features, out_features = _get_layer_features(layer)+ if in_features is None:+ continue # Skip unsupported layer types++ if must_use_target_backend:+ layer_backend = layer_backend_must+ else:+ key = f"{config['bits']}_{config['group_size']}_{config['sym']}_{in_features}_{out_features}"+ if key in backend_cache:+ layer_backend = backend_cache[key]+ if layer_backend not in used_backends:+ used_backends.append(layer_backend)+ else:+ # Determine backend+ layer_backend = _get_layer_backend(target_device, target_backend, orig_backend, config,+ in_features, out_features)+ backend_cache[key] = layer_backend++ if not layer_backend:+ raise ValueError(f"No compatible backend found for layer {layer_name} with config {config}")++ logger.debug(f"{layer_name}: {layer_backend} backend is used") ##TODO delete++ # Create and replace layer+ new_layer = _create_quant_layer(layer, layer_backend, config, in_features, out_features)+ set_module(module, layer_name, new_layer)++ return used_backends+++def _get_layer_features(layer):+ """Extracts input and output feature dimensions for supported layers."""+ if isinstance(layer, nn.Linear):+ return layer.in_features, layer.out_features+ elif isinstance(layer, Conv1D): # TODO: Verify correctness+ return layer.weight.shape[0], layer.weight.shape[1]+ return None, None # Unsupported layer type+++def _get_layer_backend(target_device, target_backend, orig_backend, config, in_features,+ out_features):+ """Determines the best backend for a given layer."""++ backend = get_layer_backend(target_device, target_backend, orig_backend, config["bits"], config["group_size"],+ config["sym"], in_features, out_features)++ return backend+++def _import_exllamav2_kernels():+ """Attempts to import ExLlamaV2 kernels for performance optimization."""+ try:+ from exllamav2_kernels import gemm_half_q_half, make_q_matrix # pylint: disable=E0611, E0401+ except ImportError:+ logger.warning_once(+ "For better inference performance, install ExLlamaV2 kernel via: "+ "`pip install git+https://github.com/AutoGPTQ/AutoGPTQ.git@b8b4127`"+ )+++def _create_quant_layer(layer, layer_backend, config, in_features, out_features):+ """Creates a quantized layer using the appropriate class."""+ QuantLinear = dynamic_import_inference_linear(layer_backend, config["bits"], config["group_size"], config["sym"])+ bias = layer.bias is not None++ # Special handling for AWQ layers+ from auto_round_extension.qbits.qbits_awq import QuantLinear as QBitsAWQQuantLinear+ if "awq" in layer_backend and isinstance(QuantLinear, QBitsAWQQuantLinear):+ return QuantLinear.from_linear(layer, config["bits"], config["group_size"], init_only=True,+ has_zero_points=not config["sym"])+ elif "awq" in layer_backend:+ return QuantLinear.from_linear(layer, config["bits"], config["group_size"], init_only=True)+ elif "gptqmodel" in layer_backend:+ return QuantLinear(bits=config["bits"], group_size=config["group_size"], desc_act=False, sym=config["sym"],+ in_features=in_features, out_features=out_features, bias=bias)+ # Default quantized layer creation+ try:+ return QuantLinear(config["bits"], config["group_size"], in_features, out_features, bias,+ weight_dtype=layer.weight.dtype, clip=config["clip"])+ except: # Handle cases where `clip` is not a valid argument+ return QuantLinear(config["bits"], config["group_size"], in_features, out_features, bias,+ weight_dtype=layer.weight.dtype)+++def infer_target_device(device_map=None):+ if device_map is None:+ target_device = "cpu"+ elif isinstance(device_map, dict):+ devices = set(device_map.values())+ target_device = "cpu"+ for device in devices:+ if device != "cpu" and device != "disk":+ if isinstance(device, int):+ target_device = get_available_devices()[0]+ else:+ target_device = str(device).split(":")[0]+ else:+ target_device = get_available_devices()[0]+ assert isinstance(target_device, str)+ return target_device+++def post_init(model, used_backends):+ need_autogptq_init = False+ need_gptqmodel_init = False+ need_ipex_itrex_init = False+ used_gptq_exllamav2 = False+ for used_backend in used_backends:+ if used_backend.startswith("auto_gptq"):+ need_autogptq_init = True+ if used_backend == "auto_gptq:exllamav2":+ used_gptq_exllamav2 = True+ elif used_backend.startswith("gptqmodel"):+ need_gptqmodel_init = True+ elif used_backend.startswith("ipex"):+ need_ipex_itrex_init = True+ elif used_backend.startswith("qbit"):+ need_ipex_itrex_init = True+ if need_autogptq_init:+ from auto_gptq.modeling._utils import autogptq_post_init as gptq_post_init # pylint: disable=E0401+ model = gptq_post_init(model, use_act_order=False)+ elif need_gptqmodel_init:+ from gptqmodel.utils.model import hf_gptqmodel_post_init as gptq_post_init # pylint: disable=E0401+ model = gptq_post_init(model, use_act_order=False)+ elif need_ipex_itrex_init:+ message = "repacking to CPU/XPU format"+ layers = [] ## ipex post_init will add one more layer+ for n, m in model.named_modules():+ if hasattr(m, "QUANT_TYPE") and ("qbits" in m.QUANT_TYPE or "ipex" in m.QUANT_TYPE):+ layers.append(m)++ for layer in tqdm(layers, desc=message, total=len(layers),+ leave=True):+ layer.post_init()++ if used_gptq_exllamav2:+ _import_exllamav2_kernels()++ ## convert datatype+ data_types=[]+ for used_backend in used_backends:+ backend = BackendInfos[used_backend]+ data_types.append(backend.dtype)+ common = set(data_types[0])+ for l in data_types[1:]:+ common &= set(l)+ common = list(common)+ if str(model.dtype).split('.')[-1] not in common:+ if common[0]=="float16":+ model = model.to(torch.float16)+ logger.warning("force model to float16")+ elif common[0]=="bfloat16":+ model = model.to(torch.bfloat16)+ logger.warning("force model to bfloat16")+++def convert_hf_model(model: nn.Module, target_device="cpu"):+ """Converts the given model to an AutoRound model by replacing its layers with quantized layers.++ This method extracts the quantization configuration from the model and adjusts its layers+ according to the specified quantization parameters. It supports different backends and+ ensures that the model's data type is compatible with the selected hardware.++ Args:+ model (nn.Module):+ The model to be converted into an AutoRound model.++ Returns:+ nn.Module:+ The converted AutoRound model with quantized layers.++ Raises:+ ValueError:+ If the quantization backend is not specified in the configuration.+ """+++ quantization_config = model.config.quantization_config+ if not hasattr(quantization_config, "target_backend"):+ quantization_config.target_backend = quantization_config.backend++ ##target_backend could be None+ _, target_backend = parse_target_device_and_backend(quantization_config.target_backend)++ if hasattr(quantization_config, "backend"): # pragma: no cover+ backend = quantization_config.backend+ elif 'gptq' in quantization_config.quant_method: # pragma: no cover+ backend = "auto_gptq"+ elif "awq" in quantization_config.quant_method:+ backend = "auto_awq"+ else: # pragma: no cover+ backend = "auto_gptq"+ logger.warning("Quantization backend must be specified. Set it to 'auto_gptq' by default.")+ if backend == "auto":+ backend = "auto_gptq"++ layer_configs = get_layer_config(model, quantization_config)+ if backend.startswith("auto_round:") and ("gptq" in backend or "awq" in backend):+ backend = backend[len("auto_round:"):]+ backend = find_backend(backend)++ if target_backend.startswith("auto_round:") and ("gptq" in backend or "awq" in backend):+ target_backend = target_backend[len("auto_round:"):]++ used_backends = _replace_by_quant_layers(model, layer_configs, target_backend, target_device, backend)++ return model, used_backendsdiff --git a/auto_round/mllm/eval.py b/auto_round/mllm/eval.pyindex 568bbd2ba..44de86c56 100644--- a/auto_round/mllm/eval.py+++ b/auto_round/mllm/eval.py@@ -103,7 +103,7 @@ def mllm_eval(try:from auto_round import AutoRoundConfigexcept:- from auto_round.auto_quantizer import AutoHfQuantizer+ from auto_round.inference.auto_quantizer import AutoHfQuantizermodel = Noneif data_store_dir is not None:@@ -350,7 +350,6 @@ def lmms_eval(use_cache=None,apply_chat_template=False):- from auto_round import AutoRoundConfigfrom transformers.utils.versions import require_versionrequire_version("lmms_eval", "lmms_eval need to be installed, `pip install lmms_eval`")if isinstance(tasks, str):diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 62639b555..6fb6aa1b4 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -292,7 +292,7 @@ def tune(args):"auto_round", "auto_round:auto_gptq", "auto_round:auto_awq", "auto_awq", "gguf:q4_0", "gguf:q4_1", "fake"]if not args.quant_nontext_module:- supported_formats.extend(["auto_gptq", "auto_gptq:marlin"])+ supported_formats.extend(["auto_gptq"])formats = args.format.replace(' ', '').split(",")for format in formats:diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 677c64e1d..9f0d58a95 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -32,11 +32,10 @@ import transformersfrom auto_round.export.export_to_gguf.config import GGUF_CONFIGsupported_formats = (- "auto_round", "auto_gptq", "auto_awq", "auto_round:auto_gptq", "auto_round:auto_awq", "auto_gptq:marlin",+ "auto_round", "auto_gptq", "auto_awq", "auto_round:auto_gptq", "auto_round:gptqmodel", "auto_round:auto_awq","itrex", "itrex_xpu", "fake")-supported_formats = supported_formats + tuple(GGUF_CONFIG.keys())supported_layer_types = (torch.nn.Linear, transformers.modeling_utils.Conv1D)@@ -523,14 +522,13 @@ def check_to_quantized(config):False otherwise."""if isinstance(config, dict):-- if int(config["bits"]) > 8 and int(config["act_bits"] > 8):- return False- return True+ bits = int(config.get("bits", 4))+ act_bits = int(config.get("act_bits", 16))else:- if int(config.bits) > 8 and int(config.act_bits) > 8:- return False- return True+ bits = int(config.bits) if hasattr(config, "bits") else 4+ act_bits = int(config.act_bits) if hasattr(config, "act_bits") else 16++ return bits <= 8 or act_bits <= 8def detect_device_count():@@ -1206,6 +1204,15 @@ def is_debug_mode():return sys.gettrace() is not None or sys.flags.debug == 1++def get_layer_features(layer):+ """Extracts input and output feature dimensions for supported layers."""+ if isinstance(layer, torch.nn.Linear):+ return layer.in_features, layer.out_features+ elif isinstance(layer, transformers.pytorch_utils.Conv1D): # TODO: Verify correctness+ return layer.weight.shape[0], layer.weight.shape[1]+ return None, None # Unsupported layer type+def _gguf_args_check(args):from auto_round.utils import loggerfrom auto_round.export.export_to_gguf.config import GGUF_CONFIG@@ -1256,4 +1263,5 @@ def _gguf_args_check(args):f" reset to {', '.join(reset_list)}.")logger.info(f"export format {format}, sym = {not args.asym}, group_size = {args.group_size}")- return args\ No newline at end of file+ return args+diff --git a/auto_round_extension/cuda/exllamav2/config.h b/auto_round_extension/cuda/exllamav2/config.hdeleted file mode 100644index 86baaf412..000000000--- a/auto_round_extension/cuda/exllamav2/config.h+++ /dev/null@@ -1,13 +0,0 @@-#ifndef _config_h-#define _config_h--#define MAX_Q_GEMM_ROWS 50--#define QMODE_2BIT 1-#define QMODE_3BIT 1-#define QMODE_4BIT 1-#define QMODE_5BIT 1-#define QMODE_6BIT 0-#define QMODE_8BIT 0--#endifdiff --git a/auto_round_extension/cuda/exllamav2/cpp/util.h b/auto_round_extension/cuda/exllamav2/cpp/util.hdeleted file mode 100644index 919703a89..000000000--- a/auto_round_extension/cuda/exllamav2/cpp/util.h+++ /dev/null@@ -1,12 +0,0 @@-#ifndef _util_h-#define _util_h--#define DBGS(__x) printf("%s\n", __x)-#define DBGI(__x) printf("%s: %i\n", #__x, __x)-#define DBGI2(__x, __y) printf("%s, %s: %i, %i\n", #__x, #__y, __x, __y)-#define DBGI3(__x, __y, __z) printf("%s, %s, %s: %i, %i, %i\n", #__x, #__y, #__z, __x, __y, __z)-#define DBGF(__x) printf("%s: %f\n", #__x, __x)-#define DBGF2(__x, __y) printf("%s, %s: %f, %f\n", #__x, #__y, __x, __y)-#define DBGF3(__x, __y, __z) printf("%s, %s, %s: %f, %f, %f\n", #__x, #__y, #__z, __x, __y, __z)--#endifdiff --git a/auto_round_extension/cuda/exllamav2/cuda/compat.cuh b/auto_round_extension/cuda/exllamav2/cuda/compat.cuhdeleted file mode 100644index 12684ff8b..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/compat.cuh+++ /dev/null@@ -1,56 +0,0 @@-#ifndef _compat_cuh-#define _compat_cuh--// atomicAdd for half types, to support CC < 7.x--__device__ __forceinline__ void atomicAdd_half(half* address, half val)-{- unsigned int * address_as_ui = (unsigned int *) ((char *)address - ((size_t)address & 2));- unsigned int old = *address_as_ui;- unsigned int assumed;-- do- {- assumed = old;- __half_raw hsum;- hsum.x = (size_t)address & 2 ? (old >> 16) : (old & 0xffff);- half tmpres = __hadd(hsum, val);- hsum = __half_raw(tmpres);- old = (size_t)address & 2 ? (old & 0xffff) | (hsum.x << 16) : (old & 0xffff0000) | hsum.x;- old = atomicCAS(address_as_ui, assumed, old);- }- while (assumed != old);-}--// atomicAdd for half2 types--__device__ __forceinline__ void atomicAdd_half2(half2* address, half2 val)-{- unsigned int* address_as_ui = (unsigned int*)address;- unsigned int old = *address_as_ui;- unsigned int assumed;- do- {- assumed = old;- half2 old_val = *((half2*)&old);- half2 new_val = __hadd2(old_val, val);- old = atomicCAS(address_as_ui, assumed, *((unsigned int*)&new_val));- }- while (assumed != old);-}--//--#if defined(__CUDA_ARCH__) || defined(USE_ROCM)-#if __CUDA_ARCH__ < 700 || defined(USE_ROCM)--__device__ __forceinline__ void atomicAdd(half* address, half val) { atomicAdd_half(address, val); }--#if __CUDA_ARCH__ < 600 || defined(USE_ROCM)-__device__ __forceinline__ void atomicAdd(half2* address, half2 val) { atomicAdd_half2(address, val); }-#endif--#endif-#endif--#endifdiff --git a/auto_round_extension/cuda/exllamav2/cuda/compat_gemm.cuh b/auto_round_extension/cuda/exllamav2/cuda/compat_gemm.cuhdeleted file mode 100644index 19b1e4a60..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/compat_gemm.cuh+++ /dev/null@@ -1,38 +0,0 @@-#ifndef _compat_gemm_cuh-#define _compat_gemm_cuh--#if defined(USE_ROCM)--// For some reason this include is not present anywhere in exllama_v2 codebase, but it is required-// for symbols as hipblasHalf.-#include <hipblas/hipblas.h>--__host__ __forceinline__ hipblasStatus_t __compat_hipblasHgemm(hipblasHandle_t handle,- hipblasOperation_t transA,- hipblasOperation_t transB,- int m,- int n,- int k,- const half* alpha,- const half* AP,- int lda,- const half* BP,- int ldb,- const half* beta,- half* CP,- int ldc) {- return hipblasHgemm(handle, transA, transB, m, n, k,- reinterpret_cast<const hipblasHalf *>(alpha),- reinterpret_cast<const hipblasHalf *>(AP), lda,- reinterpret_cast<const hipblasHalf *>(BP), ldb,- reinterpret_cast<const hipblasHalf *>(beta),- reinterpret_cast<hipblasHalf *>(CP), ldc);-}-#define hipblasHgemm __compat_hipblasHgemm--// Previous version of PyTorch were converting to rocBLAS instead of hipBLAS.-#define rocblas_operation_none HIPBLAS_OP_N-#define rocblas_hgemm __compat_hipblasHgemm-#endif--#endifdiff --git a/auto_round_extension/cuda/exllamav2/cuda/matrix_view.cuh b/auto_round_extension/cuda/exllamav2/cuda/matrix_view.cuhdeleted file mode 100644index 55af84f23..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/matrix_view.cuh+++ /dev/null@@ -1,121 +0,0 @@-#ifndef _matrix_view_cuh-#define _matrix_view_cuh--#include <cuda_runtime.h>-#include <cuda_fp16.h>--#include "quant/qdq_util.cuh"--class MatrixView_half-{-public:- const half* data;- const int height;- const int width;-- __device__ __forceinline__ MatrixView_half(const half* data, const int height, const int width)- : data(data), height(height), width(width)- { }-- __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; }- __device__ __forceinline__ half2 item_half2(int row, int column) const { return ((half2*)data)[(row * width + column) / 2]; }- __device__ __forceinline__ half2 item_half2half2(int row, int column) const { return __half2half2(data[row * width + column]); }- __device__ __forceinline__ const half* item_ptr(int row, int column) const { return &data[row * width + column]; }-- __device__ __forceinline__ void item4(half (&items)[4], int row, int column) const- {- half2* ptr = (half2*) item_ptr(row, column);- half2 i01 = ptr[0];- half2 i23 = ptr[1];- items[0] = __low2half(i01);- items[1] = __high2half(i01);- items[2] = __low2half(i23);- items[3] = __high2half(i23);- }- __device__ __forceinline__ void item4_f(float (&items)[4], int row, int column) const- {- half2* ptr = (half2*)item_ptr(row, column);- half2 i01 = ptr[0];- half2 i23 = ptr[1];- items[0] = __half2float(__low2half(i01));- items[1] = __half2float(__high2half(i01));- items[2] = __half2float(__low2half(i23));- items[3] = __half2float(__high2half(i23));- }-- __device__ __forceinline__ void item4_h2(half2 (&items)[4], int row, int column) const- {- half2* ptr = (half2*)item_ptr(row, column);- half2 i01 = ptr[0];- half2 i23 = ptr[1];- items[0] = __half2half2(__low2half(i01));- items[1] = __half2half2(__high2half(i01));- items[2] = __half2half2(__low2half(i23));- items[3] = __half2half2(__high2half(i23));- }-};--class MatrixView_half_rw-{-public:- half* data;- const int height;- const int width;-- __device__ __forceinline__ MatrixView_half_rw(half* data, const int height, const int width)- : data(data), height(height), width(width)- { }-- __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; }- __device__ __forceinline__ half2 item_half2(int row, int column) const { return ((half2*)data)[(row * width + column) / 2]; }- __device__ __forceinline__ half* item_ptr(int row, int column) { return &data[row * width + column]; }- __device__ __forceinline__ void set(int row, int column, half value) { data[row * width + column] = value; }- __device__ __forceinline__ void set_half2(int row, int column, half2 value) { ((half2*)data)[(row * width + column) / 2] = value; }-- __device__ __forceinline__ void set4(int row, int column, half v0, half v1, half v2, half v3)- {- half2 v01 = __halves2half2(v0, v1);- half2 v23 = __halves2half2(v2, v3);- half2* ptr = (half2*) item_ptr(row, column);- ptr[0] = v01;- ptr[1] = v23;- }-};--class MatrixView_q4_row-{-public:- const uint32_t* data;- const int height;- const int width;-- __device__ __forceinline__ MatrixView_q4_row(const uint32_t* data, const int height, const int width)- : data(data), height(height), width(width)- { }-- __device__ __forceinline__ int item(int row, int column) const- {- int shift = (column & 0x07) * 4;- return (data[row * width / 8 + column / 8] >> shift) & 0x0f;- }-- __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const- {- int shift = (column & 0x07) * 4;- uint32_t d = data[row * width / 8 + column / 8] >> shift;- items[0] = d & 0x0f;- items[1] = (d >> 4) & 0x0f;- }-- __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const- {- int shift = (column & 0x07) * 4;- uint32_t d = data[row * width / 8 + column / 8] >> shift;- items[0] = d & 0x0f;- items[1] = (d >> 4) & 0x0f;- items[2] = (d >> 8) & 0x0f;- items[3] = (d >> 12) & 0x0f;- }-};--#endif\ No newline at end of filediff --git a/auto_round_extension/cuda/exllamav2/cuda/q_gemm.cu b/auto_round_extension/cuda/exllamav2/cuda/q_gemm.cudeleted file mode 100644index 351b9cd5b..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/q_gemm.cu+++ /dev/null@@ -1,211 +0,0 @@-#include "q_gemm.cuh"-#include "util.cuh"-#include "matrix_view.cuh"-#include "../config.h"--#include "quant/qdq_2.cuh"-#include "quant/qdq_3.cuh"-#include "quant/qdq_4.cuh"-#include "quant/qdq_5.cuh"-#include "quant/qdq_6.cuh"-#include "quant/qdq_8.cuh"--#define BLOCK_KN_SIZE 128-#define BLOCK_M_SIZE_MAX 8-#define MAX_GROUPS_IN_BLOCK (BLOCK_KN_SIZE / 32)-#define CLEAR_N_SIZE 256--#include "q_gemm_kernel.cuh"-#include "q_gemm_kernel_gptq.cuh"--#include "compat_gemm.cuh"--void gemm_half_q_half_cuda_part-(- const half* a,- QMatrix* b,- half* c,- int size_m,- int size_n,- int size_k,- int m_count,- bool clear-)-{- if (!b->is_gptq)- {- dim3 blockDim, gridDim;- blockDim.x = BLOCK_KN_SIZE;- blockDim.y = 1;- blockDim.z = 1;- gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4);- gridDim.y = DIVIDE(size_m, m_count);- gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);-- fp_gemm_half_q_half_kernel kernel = pick_gemm_half_q_half_kernel(true, m_count);-- kernel<<<gridDim, blockDim>>>- (- a,- b->cuda_q_weight,- b->cuda_q_scale,- b->cuda_q_scale_max,- c,- size_m,- size_n,- size_k,- b->groups,- b->groupsize,- b->cuda_q_perm,- b->rows_8,- b->rows_6,- b->rows_5,- b->rows_4,- b->rows_3,- b->rows_2,- clear- );- }- else- {- dim3 blockDim, gridDim;- blockDim.x = BLOCK_KN_SIZE;- blockDim.y = 1;- blockDim.z = 1;- gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4);- gridDim.y = DIVIDE(size_m, m_count);- gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);-- fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(true, m_count);--// DBGX((uint64_t) b->cuda_q_perm);-// DBGI(b->rows_4);-// DBGI(b->height);-- kernel<<<gridDim, blockDim>>>- (- a,- b->cuda_q_weight,- b->cuda_gptq_qzeros,- b->cuda_gptq_scales,- c,- size_m,- size_n,- size_k,- b->groups,- b->groupsize,- b->cuda_q_perm,- b->rows_4,- clear- );- }-}--void gemm_half_q_half_cuda-(- cublasHandle_t cublas_handle,- const half* a,- QMatrix* b,- half* c,- int size_m,- int size_n,- int size_k,- bool clear,- half* temp_dq,- bool force_cuda-)-{- if (size_m > MAX_Q_GEMM_ROWS && !force_cuda)- {- //printf("cublas\n");-- // Reconstruct FP16 matrix, then cuBLAS-- if (!temp_dq) temp_dq = b->temp_dq;- b->reconstruct(temp_dq);-- //cublasSetMathMode(cublas_handle, CUBLAS_TENSOR_OP_MATH);-- const half alpha = __float2half(1.0f);- const half beta = clear ? __float2half(0.0f) : __float2half(1.0f);- cublasHgemm(cublas_handle,- CUBLAS_OP_N,- CUBLAS_OP_N,- size_n, size_m, size_k,- &alpha, temp_dq, size_n,- a, size_k,- &beta, c, size_n);-- //const float alpha = 1.0f;- //const float beta = clear ? 0.0f : 1.0f;- //cublasSgemmEx(cublas_handle,- // CUBLAS_OP_N,- // CUBLAS_OP_N,- // size_n, size_m, size_k,- // &alpha, temp_dq, CUDA_R_16F, size_n,- // a, CUDA_R_16F, size_k,- // &beta, c, CUDA_R_16F, size_n);-- //const float alpha = 1.0f;- //const float beta = clear ? 0.0f : 1.0f;- //cublasGemmEx(cublas_handle,- // CUBLAS_OP_N, CUBLAS_OP_N,- // size_n, size_m, size_k,- // &alpha, temp_dq, CUDA_R_16F, size_n,- // a, CUDA_R_16F, size_k,- // &beta, c, CUDA_R_16F, size_n,- // CUDA_R_16F, CUBLAS_GEMM_DFALT_TENSOR_OP);- }- else- {- //printf("cuda\n");-- // Quantized matmul-- //if (clear) clear_tensor_cuda(c, size_m, size_n);-- int max_chunks = size_m / BLOCK_M_SIZE_MAX;- int last_chunk = max_chunks * BLOCK_M_SIZE_MAX;- int last_chunk_size = size_m - last_chunk;-- if (max_chunks)- {- gemm_half_q_half_cuda_part(a, b, c, last_chunk, size_n, size_k, BLOCK_M_SIZE_MAX, clear);- }-- if (last_chunk_size)- {- gemm_half_q_half_cuda_part(a + last_chunk * size_k, b, c + last_chunk * size_n, last_chunk_size, size_n, size_k, last_chunk_size, clear);- }- }-}--__global__ void clear_kernel-(- half* __restrict__ c,- const int size_m,- const int size_n-)-{- int m = blockIdx.y;- int n = (blockIdx.x * CLEAR_N_SIZE + threadIdx.x) * 8;- if (n >= size_n) return;- int4* c_ptr = (int4*)(c + m * size_n + n);- *c_ptr = {};-}--void clear_tensor_cuda-(- half* c,- int size_m,- int size_n-)-{- return;- dim3 blockDim, gridDim;- blockDim.x = CLEAR_N_SIZE;- blockDim.y = 1;- gridDim.x = DIVIDE(size_n / 8, CLEAR_N_SIZE);- gridDim.y = size_m;- clear_kernel<<<gridDim, blockDim>>>(c, size_m, size_n);-}diff --git a/auto_round_extension/cuda/exllamav2/cuda/q_gemm.cuh b/auto_round_extension/cuda/exllamav2/cuda/q_gemm.cuhdeleted file mode 100644index c69f1a709..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/q_gemm.cuh+++ /dev/null@@ -1,33 +0,0 @@-#ifndef _q_gemm_cuh-#define _q_gemm_cuh--#include <cuda_runtime.h>-#include <cuda_fp16.h>-#include <cstdint>-#include <cstdio>-#include <ATen/cuda/CUDAContext.h>--#include "q_matrix.cuh"--void gemm_half_q_half_cuda-(- cublasHandle_t cublas_handle,- const half* a,- QMatrix* b,- half* c,- int size_m,- int size_n,- int size_k,- bool clear = false,- half* reconstruct = NULL,- bool force_cuda = false-);--void clear_tensor_cuda-(- half* c,- int size_m,- int size_n-);--#endif\ No newline at end of filediff --git a/auto_round_extension/cuda/exllamav2/cuda/q_gemm_kernel.cuh b/auto_round_extension/cuda/exllamav2/cuda/q_gemm_kernel.cuhdeleted file mode 100644index 0b899a840..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/q_gemm_kernel.cuh+++ /dev/null@@ -1,487 +0,0 @@-#include "compat.cuh"--#include <cuda_runtime.h>-#include <cuda_fp16.h>--__forceinline__ __device__ half2 dot22_8(half2(&dq)[4], const half* a_ptr, const half2 g_result, const half qs_h)-{- half2 result = {};- const half2* a2_ptr = (const half2*)a_ptr;- #pragma unroll- for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);- return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);-}--__forceinline__ __device__ half2 dot22_16(half2(&dq)[8], const half* a_ptr, const half2 g_result, const half qs_h)-{- half2 result = {};- const half2* a2_ptr = (const half2*)a_ptr;- #pragma unroll- for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);- return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);-}--__forceinline__ __device__ half2 dot22_32(half2(&dq)[16], const half* a_ptr, const half2 g_result, const half qs_h)-{- half2 result = {};- const half2* a2_ptr = (const half2*)a_ptr;- #pragma unroll- for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);- return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);-}--__forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr, const float g_result, const float qs_f)-{- half2 result = {};- const half2* a2_ptr = (const half2*)a_ptr;- #pragma unroll- for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);- float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));- return fma(result_f, qs_f, g_result);-}--__forceinline__ __device__ float dot22_16_f(half2(&dq)[8], const half* a_ptr, const float g_result, const float qs_f)-{- half2 result = {};- const half2* a2_ptr = (const half2*)a_ptr;- #pragma unroll- for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);- float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));- return fma(result_f, qs_f, g_result);-}--__forceinline__ __device__ float dot22_32_f(half2(&dq)[16], const half* a_ptr, const float g_result, const float qs_f)-{- half2 result = {};- const half2* a2_ptr = (const half2*)a_ptr;- #pragma unroll- for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);- float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));- return fma(result_f, qs_f, g_result);-}----typedef void (*fp_gemm_half_q_half_kernel)-(- const half*,- const uint32_t*,- const uint32_t*,- const half*,- half*,- const int,- const int,- const int,- const int,- const int,- const uint16_t*,- const int,- const int,- const int,- const int,- const int,- const int,- const bool-);--template <bool first_block, int m_count>-__global__ void gemm_half_q_half_kernel-(- const half* __restrict__ a,- const uint32_t* __restrict__ b_q_weight,- const uint32_t* __restrict__ b_q_scale,- const half* __restrict__ b_q_scale_max,- half* __restrict__ c,- const int size_m,- const int size_n,- const int size_k,- const int groups,- const int groupsize,- const uint16_t* __restrict__ b_q_perm,- const int rows_8,- const int rows_6,- const int rows_5,- const int rows_4,- const int rows_3,- const int rows_2,- const bool clear-)-{- MatrixView_half a_(a, size_m, size_k);- MatrixView_half_rw c_(c, size_m, size_n);- MatrixView_q4_row b_q_scale_(b_q_scale, groups, size_n);-- int t = threadIdx.x;-- // Block-- int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;- int offset_m = blockIdx.y * m_count;- int offset_k = blockIdx.z * BLOCK_KN_SIZE;-- int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);- int end_m = min(offset_m + m_count, size_m);- int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);- int n = offset_n + t * 4;-- // Preload block_a-- __shared__ half block_a[m_count][BLOCK_KN_SIZE];-- if (offset_k + t < end_k)- {- for (int m = 0; m < m_count; ++m)- {- const half* a_ptr = a_.item_ptr(offset_m + m, 0);- half* block_a_ptr = block_a[m];- half a0 = a_ptr[b_q_perm[offset_k + t]];- block_a_ptr[t] = a0;- }- }-- // Clear-- if (n >= size_n) return;-- if (clear && blockIdx.z == 0) // && (threadIdx.x & 1) == 0)- {- for (int m = 0; m < m_count; m++)- *((uint64_t*) c_.item_ptr(offset_m + m, n)) = 0;- }-- __syncthreads();-- // Find initial group-- int group = offset_k / groupsize;-- // Preload scales-- float scales[MAX_GROUPS_IN_BLOCK][4];-- int groups_in_block = DIVIDE((end_k - offset_k), groupsize);- for (int g = 0; g < groups_in_block; g++)- {- int qscales[4];- b_q_scale_.item4(qscales, group + g, n);- qscales[0]++;- qscales[1]++;- qscales[2]++;- qscales[3]++;- float maxscale = __half2float(b_q_scale_max[group + g]);- scales[g][0] = __int2float_rn(qscales[0] * qscales[0]) * maxscale;- scales[g][1] = __int2float_rn(qscales[1] * qscales[1]) * maxscale;- scales[g][2] = __int2float_rn(qscales[2] * qscales[2]) * maxscale;- scales[g][3] = __int2float_rn(qscales[3] * qscales[3]) * maxscale;- }-- // a, b offset-- int pre_rows_8 = min(rows_8, offset_k);- int pre_rows_6 = offset_k > rows_8 ? min(rows_6, offset_k) - rows_8 : 0;- int pre_rows_5 = offset_k > rows_6 ? min(rows_5, offset_k) - rows_6 : 0;- int pre_rows_4 = offset_k > rows_5 ? min(rows_4, offset_k) - rows_5 : 0;- int pre_rows_3 = offset_k > rows_4 ? min(rows_3, offset_k) - rows_4 : 0;- int pre_rows_2 = offset_k > rows_3 ? min(rows_2, offset_k) - rows_3 : 0;- int qk = 0;- qk += pre_rows_8 / 32 * 8;- qk += pre_rows_6 / 32 * 6;- qk += pre_rows_5 / 32 * 5;- qk += pre_rows_4 / 32 * 4;- qk += pre_rows_3 / 32 * 3;- qk += pre_rows_2 / 32 * 2;-- const uint32_t* b_ptr = b_q_weight + qk * size_n + n;- const half* a_ptr = &block_a[0][0];- int a_stride = BLOCK_KN_SIZE;-- // Initial group-- int scales_idx = 0;- float qs_f0 = scales[scales_idx][0];- float qs_f1 = scales[scales_idx][1];- float qs_f2 = scales[scales_idx][2];- float qs_f3 = scales[scales_idx][3];- int nextgroup = offset_k + groupsize;-- // Column result-- float block_c[m_count][4] = {};-- // Dequantize groups-- int k = offset_k;-- while (k < rows_8 && k < end_k)- {- if (k == nextgroup)- {- group++;- scales_idx++;- qs_f0 = scales[scales_idx][0];- qs_f1 = scales[scales_idx][1];- qs_f2 = scales[scales_idx][2];- qs_f3 = scales[scales_idx][3];- nextgroup += groupsize;- }-- #pragma unroll- for (int j = 0; j < 4; j++)- {- int4 load_int4[2];- load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;- load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;-- half2 dq[4][4];- dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n);- dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n);- dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n);- dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n);-- for (int m = 0; m < m_count; m++)- {- block_c[m][0] = dot22_8_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0);- block_c[m][1] = dot22_8_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1);- block_c[m][2] = dot22_8_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2);- block_c[m][3] = dot22_8_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3);- }- a_ptr += 8;- }- k += 32;- }-- while (k < rows_6 && k < end_k)- {- if (k == nextgroup)- {- group++;- scales_idx++;- qs_f0 = scales[scales_idx][0];- qs_f1 = scales[scales_idx][1];- qs_f2 = scales[scales_idx][2];- qs_f3 = scales[scales_idx][3];- nextgroup += groupsize;- }-- #pragma unroll- for (int j = 0; j < 2; j++)- {- int4 load_int4[3];- load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;- load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;- load_int4[2] = *((int4*) b_ptr); b_ptr += size_n;-- half2 dq[4][8];- dequant_6bit_16(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n);- dequant_6bit_16(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n);- dequant_6bit_16(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n);- dequant_6bit_16(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n);-- for (int m = 0; m < m_count; m++)- {- block_c[m][0] = dot22_16_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0);- block_c[m][1] = dot22_16_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1);- block_c[m][2] = dot22_16_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2);- block_c[m][3] = dot22_16_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3);- }- a_ptr += 16;- }- k += 32;- }-- while (k < rows_5 && k < end_k)- {- if (k == nextgroup)- {- group++;- scales_idx++;- qs_f0 = scales[scales_idx][0];- qs_f1 = scales[scales_idx][1];- qs_f2 = scales[scales_idx][2];- qs_f3 = scales[scales_idx][3];- nextgroup += groupsize;- }-- #pragma unroll- for (int j = 0; j < 1; j++)- {- int4 load_int4[5];- load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;- load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;- load_int4[2] = *((int4*) b_ptr); b_ptr += size_n;- load_int4[3] = *((int4*) b_ptr); b_ptr += size_n;- load_int4[4] = *((int4*) b_ptr); b_ptr += size_n;-- half2 dq[4][16];- dequant_5bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, load_int4[3].x, load_int4[4].x, dq[0], size_n);- dequant_5bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, load_int4[3].y, load_int4[4].y, dq[1], size_n);- dequant_5bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, load_int4[3].z, load_int4[4].z, dq[2], size_n);- dequant_5bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, load_int4[3].w, load_int4[4].w, dq[3], size_n);-- for (int m = 0; m < m_count; m++)- {- block_c[m][0] = dot22_32_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0);- block_c[m][1] = dot22_32_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1);- block_c[m][2] = dot22_32_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2);- block_c[m][3] = dot22_32_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3);- }- a_ptr += 32;- }-- k += 32;- }-- while (k < rows_4 && k < end_k)- {- if (k == nextgroup)- {- group++;- scales_idx++;- qs_f0 = scales[scales_idx][0];- qs_f1 = scales[scales_idx][1];- qs_f2 = scales[scales_idx][2];- qs_f3 = scales[scales_idx][3];- nextgroup += groupsize;- }-- #pragma unroll- for (int j = 0; j < 4; j++)- {- int4 load_int4[1];- load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;-- half2 dq[4][4];- dequant_4bit_8(load_int4[0].x, dq[0], size_n);- dequant_4bit_8(load_int4[0].y, dq[1], size_n);- dequant_4bit_8(load_int4[0].z, dq[2], size_n);- dequant_4bit_8(load_int4[0].w, dq[3], size_n);-- for (int m = 0; m < m_count; m++)- {- block_c[m][0] = dot22_8_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0);- block_c[m][1] = dot22_8_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1);- block_c[m][2] = dot22_8_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2);- block_c[m][3] = dot22_8_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3);- }- a_ptr += 8;- }- k += 32;- }-- while (k < rows_3 && k < end_k)- {- if (k == nextgroup)- {- group++;- scales_idx++;- qs_f0 = scales[scales_idx][0];- qs_f1 = scales[scales_idx][1];- qs_f2 = scales[scales_idx][2];- qs_f3 = scales[scales_idx][3];- nextgroup += groupsize;- }-- #pragma unroll- for (int j = 0; j < 1; j++)- {- int4 load_int4[3];- load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;- load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;- load_int4[2] = *((int4*) b_ptr); b_ptr += size_n;-- half2 dq[4][16];- dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n);- dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n);- dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n);- dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n);-- for (int m = 0; m < m_count; m++)- {- block_c[m][0] = dot22_32_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0);- block_c[m][1] = dot22_32_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1);- block_c[m][2] = dot22_32_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2);- block_c[m][3] = dot22_32_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3);- }- a_ptr += 32;- }- k += 32;- }-- while (k < rows_2 && k < end_k)- {- if (k == nextgroup)- {- group++;- scales_idx++;- qs_f0 = scales[scales_idx][0];- qs_f1 = scales[scales_idx][1];- qs_f2 = scales[scales_idx][2];- qs_f3 = scales[scales_idx][3];- nextgroup += groupsize;- }-- #pragma unroll- for (int j = 0; j < 2; j++)- {- int4 load_int4[1];- load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;-- half2 dq[4][8];- dequant_2bit_16(load_int4[0].x, dq[0], size_n);- dequant_2bit_16(load_int4[0].y, dq[1], size_n);- dequant_2bit_16(load_int4[0].z, dq[2], size_n);- dequant_2bit_16(load_int4[0].w, dq[3], size_n);-- for (int m = 0; m < m_count; m++)- {- block_c[m][0] = dot22_16_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0);- block_c[m][1] = dot22_16_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1);- block_c[m][2] = dot22_16_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2);- block_c[m][3] = dot22_16_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3);- }-- a_ptr += 16;- }- k += 32;- }-- // Accumulate column sums in c-- for (int m = 0; m < m_count; m++)- {- half2* out = (half2*)c_.item_ptr(offset_m + m, n);- half2 result01 = __halves2half2(__float2half_rn(block_c[m][0]), __float2half_rn(block_c[m][1]));- half2 result23 = __halves2half2(__float2half_rn(block_c[m][2]), __float2half_rn(block_c[m][3]));- atomicAdd(out , result01);- atomicAdd(out + 1, result23);- }-}--fp_gemm_half_q_half_kernel pick_gemm_half_q_half_kernel(bool first_block, const int m_count)-{- #if BLOCK_M_SIZE_MAX >= 1- if (m_count == 1) return gemm_half_q_half_kernel<true, 1>;- #endif- #if BLOCK_M_SIZE_MAX >= 2- if (m_count == 2) return gemm_half_q_half_kernel<true, 2>;- #endif- #if BLOCK_M_SIZE_MAX >= 3- if (m_count == 3) return gemm_half_q_half_kernel<true, 3>;- #endif- #if BLOCK_M_SIZE_MAX >= 4- if (m_count == 4) return gemm_half_q_half_kernel<true, 4>;- #endif- #if BLOCK_M_SIZE_MAX >= 5- if (m_count == 5) return gemm_half_q_half_kernel<true, 5>;- #endif- #if BLOCK_M_SIZE_MAX >= 6- if (m_count == 6) return gemm_half_q_half_kernel<true, 6>;- #endif- #if BLOCK_M_SIZE_MAX >= 7- if (m_count == 7) return gemm_half_q_half_kernel<true, 7>;- #endif- #if BLOCK_M_SIZE_MAX >= 8- if (m_count == 8) return gemm_half_q_half_kernel<true, 8>;- #endif- return NULL;-}diff --git a/auto_round_extension/cuda/exllamav2/cuda/q_gemm_kernel_gptq.cuh b/auto_round_extension/cuda/exllamav2/cuda/q_gemm_kernel_gptq.cuhdeleted file mode 100644index 4b722ef5a..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/q_gemm_kernel_gptq.cuh+++ /dev/null@@ -1,223 +0,0 @@-#include "compat.cuh"--__forceinline__ __device__ half2 dot22_8(half2(&dq)[4], const half* a_ptr, const half2 g_result)-{- half2 result = {};- const half2* a2_ptr = (const half2*)a_ptr;- #pragma unroll- for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);- return __hadd2(result, g_result);-}--__forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr)-{- half2 result = {};- const half2* a2_ptr = (const half2*)a_ptr;- #pragma unroll- for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);- return __half2float(__low2half(result)) + __half2float(__high2half(result));-}--typedef void (*fp_gemm_half_q_half_gptq_kernel)-(- const half*,- const uint32_t*,- const uint32_t*,- const half*,- half*,- const int,- const int,- const int,- const int,- const int,- const uint16_t*,- const int,- const bool-);--template <bool first_block, int m_count>-__global__ void gemm_half_q_half_gptq_kernel-(- const half* __restrict__ a,- const uint32_t* __restrict__ b_q_weight,- const uint32_t* __restrict__ b_gptq_qzeros,- const half* __restrict__ b_gptq_scales,- half* __restrict__ c,- const int size_m,- const int size_n,- const int size_k,- const int groups,- const int groupsize,- const uint16_t* __restrict__ b_q_perm,- const int rows_4,- const bool clear-)-{- MatrixView_half a_(a, size_m, size_k);- MatrixView_half_rw c_(c, size_m, size_n);- MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);- MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);-- int t = threadIdx.x;-- // Block-- int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;- int offset_m = blockIdx.y * m_count;- int offset_k = blockIdx.z * BLOCK_KN_SIZE;-- int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);- int end_m = min(offset_m + m_count, size_m);- int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);-- int n = offset_n + t * 4;-- // Preload block_a-- __shared__ half block_a[m_count][BLOCK_KN_SIZE];-- if (offset_k + t < end_k)- {- for (int m = 0; m < m_count; ++m)- {- const half* a_ptr = a_.item_ptr(offset_m + m, 0);- half* block_a_ptr = block_a[m];-- half a0;- if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];- else a0 = a_ptr[offset_k + t];- block_a_ptr[t] = a0;- }- }-- // Zero output-- if (n >= size_n) return;-- if (clear && blockIdx.z == 0) // && (threadIdx.x & 1) == 0)- {- for (int m = 0; m < m_count; m++)- *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;- }-- __syncthreads();-- // Find initial group-- int group = offset_k / groupsize;- int nextgroup = offset_k + groupsize;-- // a, b offset-- int qk = offset_k / (32 / 4);-- const uint32_t* b_ptr = b_q_weight + qk * size_n + n;- const half* a_ptr = &block_a[0][0];- int a_stride = BLOCK_KN_SIZE;-- // Initial group-- int zeros[4];- float scales[4];- half2 z1z16[4][2];- half2 y1y16[4][2];- b_gptq_qzeros_.item4(zeros, group, n);- b_gptq_scales_.item4_f(scales, group, n);-- // Avoid zeros overflow with & 0x0f.- dequant_4bit_8_prep_zero((zeros[0]) & 0x0f, z1z16[0], y1y16[0]);- dequant_4bit_8_prep_zero((zeros[1]) & 0x0f, z1z16[1], y1y16[1]);- dequant_4bit_8_prep_zero((zeros[2]) & 0x0f, z1z16[2], y1y16[2]);- dequant_4bit_8_prep_zero((zeros[3]) & 0x0f, z1z16[3], y1y16[3]);--// __syncthreads();-- // Column result-- float block_c[m_count][4] = {};-- // Dequantize and multiply-- int k = offset_k;- while (k < end_k)- {- if (k == nextgroup)- {- group++;- nextgroup += groupsize;- b_gptq_qzeros_.item4(zeros, group, n);- b_gptq_scales_.item4_f(scales, group, n);-- // Avoid zeros overflow with & 0x0f.- dequant_4bit_8_prep_zero((zeros[0]) & 0x0f, z1z16[0], y1y16[0]);- dequant_4bit_8_prep_zero((zeros[1]) & 0x0f, z1z16[1], y1y16[1]);- dequant_4bit_8_prep_zero((zeros[2]) & 0x0f, z1z16[2], y1y16[2]);- dequant_4bit_8_prep_zero((zeros[3]) & 0x0f, z1z16[3], y1y16[3]);- }-- #pragma unroll- for (int j = 0; j < 4; j++)- {- const int4* b_ptr4 = (int4*) b_ptr;- int4 load_int4 = *b_ptr4;-- half2 dq[4][4];- dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false);- dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false);- dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false);- dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false);-- #pragma unroll- for (int m = 0; m < m_count; m++)- {- block_c[m][0] = fma(dot22_8_f(dq[0], a_ptr + m * a_stride), scales[0], block_c[m][0]);- block_c[m][1] = fma(dot22_8_f(dq[1], a_ptr + m * a_stride), scales[1], block_c[m][1]);- block_c[m][2] = fma(dot22_8_f(dq[2], a_ptr + m * a_stride), scales[2], block_c[m][2]);- block_c[m][3] = fma(dot22_8_f(dq[3], a_ptr + m * a_stride), scales[3], block_c[m][3]);- }-- b_ptr += size_n;- a_ptr += 8;- }-- k += 32;- }-- for (int m = 0; m < m_count; m++)- {- half2 *out = (half2*) c_.item_ptr(offset_m + m, n);- half2 result01 = __halves2half2(__float2half_rn(block_c[m][0]), __float2half_rn(block_c[m][1]));- half2 result23 = __halves2half2(__float2half_rn(block_c[m][2]), __float2half_rn(block_c[m][3]));- atomicAdd(out , result01);- atomicAdd(out + 1, result23);- }-}--fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(bool first_block, const int m_count)-{- #if BLOCK_M_SIZE_MAX >= 1- if (m_count == 1) return gemm_half_q_half_gptq_kernel<true, 1>;- #endif- #if BLOCK_M_SIZE_MAX >= 2- if (m_count == 2) return gemm_half_q_half_gptq_kernel<true, 2>;- #endif- #if BLOCK_M_SIZE_MAX >= 3- if (m_count == 3) return gemm_half_q_half_gptq_kernel<true, 3>;- #endif- #if BLOCK_M_SIZE_MAX >= 4- if (m_count == 4) return gemm_half_q_half_gptq_kernel<true, 4>;- #endif- #if BLOCK_M_SIZE_MAX >= 5- if (m_count == 5) return gemm_half_q_half_gptq_kernel<true, 5>;- #endif- #if BLOCK_M_SIZE_MAX >= 6- if (m_count == 6) return gemm_half_q_half_gptq_kernel<true, 6>;- #endif- #if BLOCK_M_SIZE_MAX >= 7- if (m_count == 7) return gemm_half_q_half_gptq_kernel<true, 7>;- #endif- #if BLOCK_M_SIZE_MAX >= 8- if (m_count == 8) return gemm_half_q_half_gptq_kernel<true, 8>;- #endif- return NULL;-}diff --git a/auto_round_extension/cuda/exllamav2/cuda/q_matrix.cu b/auto_round_extension/cuda/exllamav2/cuda/q_matrix.cudeleted file mode 100644index aebba7b02..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/q_matrix.cu+++ /dev/null@@ -1,627 +0,0 @@-#include "q_matrix.cuh"-#include "matrix_view.cuh"-#include "util.cuh"--#include "quant/qdq_2.cuh"-#include "quant/qdq_3.cuh"-#include "quant/qdq_4.cuh"-#include "quant/qdq_5.cuh"-#include "quant/qdq_6.cuh"-#include "quant/qdq_8.cuh"--#define BLOCK_KN_SIZE 128--#define THREADS_X 32-#define THREADS_Y 32--// Shuffle quantized data on load--__global__ void shuffle_kernel-(- uint32_t* __restrict__ b_q_weight,- const int size_k,- const int size_n,- const int rows_8,- const int rows_6,- const int rows_5,- const int rows_4,- const int rows_3,- const int rows_2-)-{- int n = blockIdx.x * THREADS_X + threadIdx.x;- if (n >= size_n) return;- int k = 0;- uint32_t* b_ptr = b_q_weight + n;- while (k < rows_8) { shuffle_8bit_4 (b_ptr, size_n); b_ptr += 1 * size_n; k += 4; }- while (k < rows_6) { shuffle_6bit_16(b_ptr, size_n); b_ptr += 3 * size_n; k += 16; }- while (k < rows_5) { shuffle_5bit_32(b_ptr, size_n); b_ptr += 5 * size_n; k += 32; }- while (k < rows_4) { shuffle_4bit_8 (b_ptr, size_n); b_ptr += 1 * size_n; k += 8; }- while (k < rows_3) { shuffle_3bit_32(b_ptr, size_n); b_ptr += 3 * size_n; k += 32; }- while (k < rows_2) { shuffle_2bit_16(b_ptr, size_n); b_ptr += 1 * size_n; k += 16; }-}---// QMatrix constructor--QMatrix::QMatrix-(- const int _device,- const int _height,- const int _width,- const int _groups,-- uint32_t* _q_weight,- uint16_t* _q_perm,- uint16_t* _q_invperm,- uint32_t* _q_scale,- half* _q_scale_max,- uint16_t* _q_groups,-- uint32_t* _gptq_qzeros,- half* _gptq_scales,- uint32_t* _gptq_g_idx,-- half* _temp_dq-) :- device(_device),- height(_height),- width(_width),- groups(_groups),- temp_dq(_temp_dq)-{- cudaSetDevice(device);-- failed = false;-- cuda_q_weight = _q_weight;- cuda_q_perm = _q_perm;- cuda_q_invperm = _q_invperm;- cuda_q_scale = _q_scale;- cuda_q_scale_max = _q_scale_max;- cuda_q_groups = _q_groups;- cuda_gptq_qzeros = _gptq_qzeros;- cuda_gptq_scales = _gptq_scales;-- is_gptq = (_gptq_qzeros != NULL);-- groupsize = 1;- while (groupsize * groups < height) groupsize *= 2;-- // Create group map-- rows_8 = 0;- rows_6 = 0;- rows_5 = 0;- rows_4 = 0;- rows_3 = 0;- rows_2 = 0;-- if (!is_gptq)- {- uint16_t* cpu_q_groups = (uint16_t*)calloc(groups * 2, sizeof(uint16_t));- cudaMemcpy(cpu_q_groups, cuda_q_groups, groups * 2 * sizeof(uint16_t), cudaMemcpyDeviceToHost);-- for (int i = 0; i < groups; i++)- {- int bits = cpu_q_groups[i * 2];- if (bits == 8) rows_8 += groupsize;- if (bits == 6) rows_6 += groupsize;- if (bits == 5) rows_5 += groupsize;- if (bits == 4) rows_4 += groupsize;- if (bits == 3) rows_3 += groupsize;- if (bits == 2) rows_2 += groupsize;- }-- free(cpu_q_groups);-- rows_6 += rows_8;- rows_5 += rows_6;- rows_4 += rows_5;- rows_3 += rows_4;- rows_2 += rows_3;- }- else- {- rows_4 = height;- rows_3 = height;- rows_2 = height;-- if (_gptq_g_idx)- {- if (!make_sequential(_gptq_g_idx))- {- failed = true;- //printf("FAIL\n");- return;- }- }- }-- // Shuffle quantized data-- dim3 blockDim, gridDim;- blockDim.x = THREADS_X;- blockDim.y = 1;- gridDim.x = DIVIDE(width, THREADS_X);- gridDim.y = 1;-- shuffle_kernel<<<gridDim, blockDim>>>(cuda_q_weight, height, width, rows_8, rows_6, rows_5, rows_4, rows_3, rows_2);-}--QMatrix::~QMatrix()-{-}--// Reconstruct b[k,n] (GPTQ)--__global__ void reconstruct_gptq_kernel-(- const uint32_t* __restrict__ b_q_weight,- const uint16_t* __restrict__ b_q_perm,- const uint32_t* __restrict__ b_gptq_qzeros,- const half* __restrict__ b_gptq_scales,- //const uint16_t* __restrict__ b_q_groups,- const int size_k,- const int size_n,- const int groupsize,- const int groups,- half* __restrict__ b,- const int rows_4-)-{- MatrixView_half_rw b_(b, size_k, size_n);- MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);- MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);-- int offset_k = BLOCK_KN_SIZE * blockIdx.y;- int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;-- int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);-- // Preload remapping table-- __shared__ uint16_t perm[BLOCK_KN_SIZE];- int t = threadIdx.x;-- if (b_q_perm)- {- if (offset_k + t < size_k)- perm[t] = b_q_perm[offset_k + t];- }-- // Column-- int n = offset_n + t * 4;- if (n >= size_n) return;-- // Find initial group-- int group = offset_k / groupsize;- int nextgroup = offset_k + groupsize;-- // b offset-- int qk = offset_k / (32 / 4);-- const uint32_t* b_ptr = b_q_weight + qk * size_n + n;-- // Initial zeros/scale-- int zeros[4];- half2 scales[4];- half2 z1z16[4][2];- half2 y1y16[4][2];- b_gptq_qzeros_.item4(zeros, group, n);- b_gptq_scales_.item4_h2(scales, group, n);-- // Avoid zeros overflow with & 0x0f.- dequant_4bit_8_prep_zero((zeros[0]) & 0x0f, z1z16[0], y1y16[0]);- dequant_4bit_8_prep_zero((zeros[1]) & 0x0f, z1z16[1], y1y16[1]);- dequant_4bit_8_prep_zero((zeros[2]) & 0x0f, z1z16[2], y1y16[2]);- dequant_4bit_8_prep_zero((zeros[3]) & 0x0f, z1z16[3], y1y16[3]);-- __syncthreads();-- int k = offset_k;- int lk = 0;-- while (k < end_k)- {- if (k == nextgroup)- {- group++;- nextgroup += groupsize;- b_gptq_qzeros_.item4(zeros, group, n);- b_gptq_scales_.item4_h2(scales, group, n);-- // Avoid zeros overflow with & 0x0f.- dequant_4bit_8_prep_zero((zeros[0]) & 0x0f, z1z16[0], y1y16[0]);- dequant_4bit_8_prep_zero((zeros[1]) & 0x0f, z1z16[1], y1y16[1]);- dequant_4bit_8_prep_zero((zeros[2]) & 0x0f, z1z16[2], y1y16[2]);- dequant_4bit_8_prep_zero((zeros[3]) & 0x0f, z1z16[3], y1y16[3]);- }-- for (int p = 0; p < 4; p++)- {- half2 dq[4][4];- const int4* b_ptr4 = (int4*) b_ptr;- int4 load_int4 = *b_ptr4;-- dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false);- dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false);- dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false);- dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false);-- b_ptr += size_n;- //half* dqh = (half*)dq;- if (b_q_perm)- {- for (int j = 0; j < 4; j++)- {- for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);- b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));- b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));- }- }- else- {- for (int j = 0; j < 4; j++)- {- for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);- b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));- b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));- }- }- }- k += 32;- }-}---// Reconstruct b[k,n]--__global__ void reconstruct_kernel-(- const uint32_t* __restrict__ b_q_weight,- const uint16_t* __restrict__ b_q_perm,- const uint32_t* __restrict__ b_q_scale,- const half* __restrict__ b_q_scale_max,- //const uint16_t* __restrict__ b_q_groups,- const int size_k,- const int size_n,- const int groupsize,- const int groups,- half* __restrict__ b,- const int rows_8,- const int rows_6,- const int rows_5,- const int rows_4,- const int rows_3,- const int rows_2-)-{- MatrixView_half_rw b_(b, size_k, size_n);- MatrixView_q4_row b_q_scale_(b_q_scale, groups, size_n);-- int offset_k = BLOCK_KN_SIZE * blockIdx.y;- int offset_n = BLOCK_KN_SIZE * blockIdx.x;-- // Preload remapping table-- int t = threadIdx.x;- __shared__ uint16_t perm[BLOCK_KN_SIZE];- if (offset_k + t < size_k)- perm[t] = b_q_perm[offset_k + t];-- // Column-- int n = offset_n + t;- if (n >= size_n) return;-- // Find initial group-- int group = offset_k / groupsize;-- int pre_rows_8 = min(rows_8, offset_k);- int pre_rows_6 = offset_k > rows_8 ? min(rows_6, offset_k) - rows_8 : 0;- int pre_rows_5 = offset_k > rows_6 ? min(rows_5, offset_k) - rows_6 : 0;- int pre_rows_4 = offset_k > rows_5 ? min(rows_4, offset_k) - rows_5 : 0;- int pre_rows_3 = offset_k > rows_4 ? min(rows_3, offset_k) - rows_4 : 0;- int pre_rows_2 = offset_k > rows_3 ? min(rows_2, offset_k) - rows_3 : 0;- int qk = 0;- qk += pre_rows_8 / 32 * 8;- qk += pre_rows_6 / 32 * 6;- qk += pre_rows_5 / 32 * 5;- qk += pre_rows_4 / 32 * 4;- qk += pre_rows_3 / 32 * 3;- qk += pre_rows_2 / 32 * 2;-- const uint32_t* b_ptr = b_q_weight + qk * size_n + n;-- half qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]);- half2 qs_h2 = __halves2half2(qs_h, qs_h);- int nextgroup = offset_k + groupsize;-- int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);- int k = offset_k;- int lk = 0;-- __syncthreads();-- while (k < rows_8 && k < end_k)- {- if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); }- for (int p = 0; p < 4; p++)- {- half2 dq[4];- uint32_t q_0 = *b_ptr; b_ptr += size_n;- uint32_t q_1 = *b_ptr; b_ptr += size_n;- dequant_8bit_8(q_0, q_1, dq, size_n);- for (int j = 0; j < 4; j++) dq[j] = __hmul2(dq[j], qs_h2);- half* dqh = (half*) dq;- for (int j = 0; j < 8; j++) b_.set(perm[lk++], n, dqh[j]);- }- k += 32;- }-- while (k < rows_6 && k < end_k)- {- if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); }- for (int p = 0; p < 2; p++)- {- half2 dq[8];- uint32_t q_0 = *b_ptr; b_ptr += size_n;- uint32_t q_1 = *b_ptr; b_ptr += size_n;- uint32_t q_2 = *b_ptr; b_ptr += size_n;- dequant_6bit_16(q_0, q_1, q_2, dq, size_n);- for (int j = 0; j < 8; j++) dq[j] = __hmul2(dq[j], qs_h2);- half* dqh = (half*) dq;- for (int j = 0; j < 16; j++) b_.set(perm[lk++], n, dqh[j]);- }- k += 32;- }-- while (k < rows_5 && k < end_k)- {- if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); }- for (int p = 0; p < 1; p++)- {- half2 dq[16];- uint32_t q_0 = *b_ptr; b_ptr += size_n;- uint32_t q_1 = *b_ptr; b_ptr += size_n;- uint32_t q_2 = *b_ptr; b_ptr += size_n;- uint32_t q_3 = *b_ptr; b_ptr += size_n;- uint32_t q_4 = *b_ptr; b_ptr += size_n;- dequant_5bit_32(q_0, q_1, q_2, q_3, q_4, dq, size_n);- for (int j = 0; j < 16; j++) dq[j] = __hmul2(dq[j], qs_h2);- half* dqh = (half*) dq;- for (int j = 0; j < 32; j++) b_.set(perm[lk++], n, dqh[j]);- }- k += 32;- }-- while (k < rows_4 && k < end_k)- {- if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); }- for (int p = 0; p < 4; p++)- {- half2 dq[4];- uint32_t q_0 = *b_ptr; b_ptr += size_n;- dequant_4bit_8(q_0, dq, size_n);- for (int j = 0; j < 4; j++) dq[j] = __hmul2(dq[j], qs_h2);- half* dqh = (half*) dq;- for (int j = 0; j < 8; j++) b_.set(perm[lk++], n, dqh[j]);- }- k += 32;- }-- while (k < rows_3 && k < end_k)- {- if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); }- for (int p = 0; p < 1; p++)- {- half2 dq[16];- uint32_t q_0 = *b_ptr; b_ptr += size_n;- uint32_t q_1 = *b_ptr; b_ptr += size_n;- uint32_t q_2 = *b_ptr; b_ptr += size_n;- dequant_3bit_32(q_0, q_1, q_2, dq, size_n);- for (int j = 0; j < 16; j++) dq[j] = __hmul2(dq[j], qs_h2);- half* dqh = (half*) dq;- for (int j = 0; j < 32; j++) b_.set(perm[lk++], n, dqh[j]);- }- k += 32;- }-- while (k < rows_2 && k < end_k)- {- if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); }- for (int p = 0; p < 2; p++)- {- half2 dq[8];- uint32_t q_0 = *b_ptr; b_ptr += size_n;- dequant_2bit_16(q_0, dq, size_n);- for (int j = 0; j < 8; j++) dq[j] = __hmul2(dq[j], qs_h2);- half* dqh = (half*) dq;- for (int j = 0; j < 16; j++) b_.set(perm[lk++], n, dqh[j]);- }- k += 32;- }-}--void QMatrix::reconstruct(half* out)-{- dim3 blockDim, gridDim;- blockDim.x = BLOCK_KN_SIZE;- blockDim.y = 1;- gridDim.y = DIVIDE(height, BLOCK_KN_SIZE);-- if (!is_gptq)- {- gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);- reconstruct_kernel<<<gridDim, blockDim>>>- (- cuda_q_weight,- cuda_q_perm,- cuda_q_scale,- cuda_q_scale_max,- //cuda_q_groups,- height,- width,- groupsize,- groups,- out,- rows_8,- rows_6,- rows_5,- rows_4,- rows_3,- rows_2- );- }- else- {- gridDim.x = DIVIDE(width, BLOCK_KN_SIZE * 4);- reconstruct_gptq_kernel<<<gridDim, blockDim>>>- (- cuda_q_weight,- cuda_q_perm,- cuda_gptq_qzeros,- cuda_gptq_scales,- //const uint16_t* __restrict__ b_q_groups,- height,- width,- groupsize,- groups,- out,- rows_4- );- }-}--__global__ void make_sequential_kernel-(- const uint32_t* __restrict__ w,- uint32_t* __restrict__ w_new,- const uint16_t* __restrict__ q_perm,- const int w_height,- const int w_width-)-{- const uint64_t* w2 = (uint64_t*) w;- uint64_t* w_new2 = (uint64_t*) w_new;- int w2_stride = w_width >> 1;-- int w2_column = THREADS_X * blockIdx.x + threadIdx.x;- if (w2_column >= w2_stride) return;-- int w_new2_row = blockIdx.y;-- int q_perm_idx = w_new2_row << 3;-- uint64_t dst = 0;-- #pragma unroll- for (int i = 0; i < 8; i++)- {- int source_row = q_perm[q_perm_idx++];-- int w2_row = source_row >> 3;- int w2_subrow = source_row & 0x07;- int w2_row_shift = w2_subrow << 2;- int wnew2_row_shift = i << 2;-- uint64_t src = w2[w2_row * w2_stride + w2_column];- src >>= w2_row_shift;- src &= 0x0000000f0000000f;- src <<= wnew2_row_shift;- dst |= src;- }-- w_new2[w_new2_row * w2_stride + w2_column] = dst;-}--bool QMatrix::make_sequential(const uint32_t* cpu_g_idx)-{- uint32_t* cuda_new_qweight = NULL;- cudaError_t err = cudaMalloc(&cuda_new_qweight, height / 8 * width * sizeof(uint32_t));- if (err != cudaSuccess) {- cudaError_t cuda_status = cudaGetLastError(); // Clear error- return false;- }-- uint32_t* cpu_g_idx_map = (uint32_t*) calloc(groups, sizeof(uint32_t));- uint32_t* cpu_x_map = (uint32_t*) malloc(height * sizeof(uint32_t));- uint32_t* cpu_x_map_inv = (uint32_t*) malloc(height * sizeof(uint32_t));-- // Group histogram-- for (int i = 0; i < height; i++) cpu_g_idx_map[cpu_g_idx[i]]++;-- // Group map-- for (int i = 0, acc = 0; i < groups; i++)- {- short tmp = cpu_g_idx_map[i];- cpu_g_idx_map[i] = acc;- acc += tmp;- }-- // X map (inverse)-- for (int row = 0; row < height; row++)- {- uint32_t target_group = cpu_g_idx[row];- uint32_t target_row = cpu_g_idx_map[target_group];- cpu_g_idx_map[target_group]++;- cpu_x_map_inv[row] = target_row;- }-- // X map-- for (int row = 0; row < height; row++) cpu_x_map[cpu_x_map_inv[row]] = row;-- // Reduce to uint16_t-- uint16_t* cpu_x_map16 = (uint16_t*)cpu_x_map;- uint16_t* cpu_x_map_inv16 = (uint16_t*)cpu_x_map_inv;- for (int row = 0; row < height; row++) cpu_x_map16[row] = (uint16_t) cpu_x_map[row];- for (int row = 0; row < height; row++) cpu_x_map_inv16[row] = (uint16_t) cpu_x_map_inv[row];-- // Move to CUDA-- cudaMemcpyAsync(cuda_q_perm, cpu_x_map16, height * sizeof(uint16_t), cudaMemcpyHostToDevice);- cudaMemcpyAsync(cuda_q_invperm, cpu_x_map_inv16, height * sizeof(uint16_t), cudaMemcpyHostToDevice);-- // Rearrange rows in w-- dim3 blockDim, gridDim;- blockDim.x = THREADS_X;- blockDim.y = 1;- gridDim.x = DIVIDE(width, THREADS_X);- gridDim.y = height / 8;-- make_sequential_kernel<<<gridDim, blockDim>>>- (- cuda_q_weight,- cuda_new_qweight,- cuda_q_perm,- height / 8,- width- );-- // Replace qweights-- cudaMemcpyAsync(cuda_q_weight, cuda_new_qweight, height / 8 * width * sizeof(uint32_t), cudaMemcpyDeviceToDevice);-- // Cleanup-- cudaDeviceSynchronize();-- cudaFree(cuda_new_qweight);- free(cpu_g_idx_map);- free(cpu_x_map);- free(cpu_x_map_inv);-- return true;-}diff --git a/auto_round_extension/cuda/exllamav2/cuda/q_matrix.cuh b/auto_round_extension/cuda/exllamav2/cuda/q_matrix.cuhdeleted file mode 100644index dda83a4f3..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/q_matrix.cuh+++ /dev/null@@ -1,73 +0,0 @@-#ifndef _q_matrix_cuh-#define _q_matrix_cuh--#include <cuda_runtime.h>-#include <cuda_fp16.h>-#include <cstdint>-#include <cstdio>--#define MAX_SUPERGROUPS 16--class QMatrix-{-public:-- int device;- bool is_gptq;-- int height;- int width;- int groups;- int groupsize;-- int rows_8;- int rows_6;- int rows_5;- int rows_4;- int rows_3;- int rows_2;-- uint32_t* cuda_q_weight = NULL;- uint16_t* cuda_q_perm = NULL;- uint16_t* cuda_q_invperm = NULL;- uint32_t* cuda_q_scale = NULL;- half* cuda_q_scale_max = NULL;- uint16_t* cuda_q_groups = NULL;- uint32_t* cuda_gptq_qzeros = NULL;- half* cuda_gptq_scales = NULL;-- half* temp_dq;-- bool failed;-- QMatrix- (- const int _device,- const int _height,- const int _width,- const int _groups,-- uint32_t* _q_weight,- uint16_t* _q_perm,- uint16_t* _q_invperm,- uint32_t* _q_scale,- half* _q_scale_max,- uint16_t* _q_groups,-- uint32_t* _gptq_qzeros,- half* _gptq_scales,- uint32_t* _gptq_g_idx,-- half* _temp_dq- );-- ~QMatrix();-- void reconstruct(half* out);- bool make_sequential(const uint32_t* cpu_g_idx);--private:--};--#endifdiff --git a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_2.cuh b/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_2.cuhdeleted file mode 100644index 3beaeefa9..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_2.cuh+++ /dev/null@@ -1,103 +0,0 @@-#ifndef _qdq_2_cuh-#define _qdq_2_cuh--#include "qdq_util.cuh"-#include "../../config.h"--#if QMODE_2BIT == 1--// Permutation:-//-// ffddbb99 77553311 eeccaa88 66442200--__forceinline__ __device__ void shuffle_2bit_16-(- uint32_t* q,- int stride-)-{- uint32_t qa = q[0];- uint32_t qb = 0;-- #pragma unroll- for (int i = 0; i < 8; i++)- {- uint32_t qa0 = qa & 0x03;- uint32_t qa1 = (qa & 0x0c) >> 2;- qa >>= 4;- qb |= (qa1 << (i * 2 + 16));- qb |= (qa0 << (i * 2));- }- q[0] = qb;-}--__forceinline__ __device__ void dequant_2bit_16-(- const uint32_t q_0,- half2 (&dq)[8],- int stride-)-{- const uint32_t c0 = 0x64006400;- const half y4_ = __float2half_rn(1.0f / 4.0f);- const half y16_ = __float2half_rn(1.0f / 16.0f);- const half y64_ = __float2half_rn(1.0f / 64.0f);- const half2 y4 = __halves2half2(y4_, y4_);- const half2 y16 = __halves2half2(y16_, y16_);- const half2 y64 = __halves2half2(y64_, y64_);- const half z1_ = __float2half_rn(-1024.0f - 2.0f);- const half z4_ = __float2half_rn(-1024.0f / 4.0f - 2.0f);- const half z16_ = __float2half_rn(-1024.0f / 16.0f - 2.0f);- const half z64_ = __float2half_rn(-1024.0f / 64.0f - 2.0f);- const half2 z1 = __halves2half2(z1_, z1_);- const half2 z4 = __halves2half2(z4_, z4_);- const half2 z16 = __halves2half2(z16_, z16_);- const half2 z64 = __halves2half2(z64_, z64_);-- uint32_t qa = q_0;- half2_uint32 q0((qa & 0x00030003) | c0); // half2(q[ 0], q[ 1]) + 1024- half2_uint32 q1((qa & 0x000c000c) | c0); // half2(q[ 2], q[ 3]) * 4 + 1024- half2_uint32 q2((qa & 0x00300030) | c0); // half2(q[ 4], q[ 5]) * 16 + 1024- half2_uint32 q3((qa & 0x00c000c0) | c0); // half2(q[ 6], q[ 7]) * 64 + 1024- qa >>= 8;- half2_uint32 q4((qa & 0x00030003) | c0); // half2(q[ 8], q[ 8]) + 1024- half2_uint32 q5((qa & 0x000c000c) | c0); // half2(q[10], q[11]) * 4 + 1024- half2_uint32 q6((qa & 0x00300030) | c0); // half2(q[12], q[13]) * 16 + 1024- half2_uint32 q7((qa & 0x00c000c0) | c0); // half2(q[14], q[15]) * 64 + 1024-- dq[0] = __hadd2(q0.as_half2, z1);- dq[1] = __hfma2(q1.as_half2, y4, z4);- dq[2] = __hfma2(q2.as_half2, y16, z16);- dq[3] = __hfma2(q3.as_half2, y64, z64);- dq[4] = __hadd2(q4.as_half2, z1);- dq[5] = __hfma2(q5.as_half2, y4, z4);- dq[6] = __hfma2(q6.as_half2, y16, z16);- dq[7] = __hfma2(q7.as_half2, y64, z64);-}--#else--__forceinline__ __device__ void shuffle_2bit_16-(- uint32_t* q,- int stride-)-{-}--__forceinline__ __device__ void dequant_2bit_16-(- const uint32_t q_0,- half2 (&dq)[8],- int stride-)-{- half dqh[16];- for (int i = 0; i < 16; i++) dqh[i] = dq_ns(exb(q_0, i * 2, 0x03), 2);-- for (int i = 0; i < 8; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);-}--#endif--#endif\ No newline at end of filediff --git a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_3.cuh b/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_3.cuhdeleted file mode 100644index 101173763..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_3.cuh+++ /dev/null@@ -1,169 +0,0 @@-#ifndef _qdq_3_cuh-#define _qdq_3_cuh--#include "qdq_util.cuh"-#include "../../config.h"--#if QMODE_3BIT == 1--// Permutation:-//-// v9997775 55333111 u8886664 44222000 (u, v lsb)-// vjjjhhhf ffdddbbb uiiiggge eecccaaa-// vtttrrrp ppnnnlll usssqqqo oommmkkk--__forceinline__ __device__ void shuffle_3bit_32-(- uint32_t* q,- int stride-)-{- uint32_t qa = q[0 * stride];- uint32_t qb = q[1 * stride];- uint32_t qc = q[2 * stride];-- // qa: aa999888 77766655 54443332 22111000- // qb: lkkkjjji iihhhggg fffeeedd dcccbbba- // qc: vvvuuutt tsssrrrq qqpppooo nnnmmmll-- uint32_t qd = qc >> 26;- qc <<= 4;- qc |= qb >> 28;- qb <<= 2;- qb |= qa >> 30;-- // qa: ..999888 77766655 54443332 22111000- // qb: ..jjjiii hhhgggff feeedddc ccbbbaaa- // qc: ..tttsss rrrqqqpp pooonnnm mmlllkkk- // qd: vvvuuu-- uint32_t za = 0;- uint32_t zb = 0;- uint32_t zc = 0;-- for (int i = 0; i < 5; i++) { uint32_t t0 = qa & 0x07; uint32_t t1 = (qa & 0x38) >> 3; qa >>= 6; za |= (t0 << (i * 3)); za |= (t1 << (i * 3 + 16)); }- for (int i = 0; i < 5; i++) { uint32_t t0 = qb & 0x07; uint32_t t1 = (qb & 0x38) >> 3; qb >>= 6; zb |= (t0 << (i * 3)); zb |= (t1 << (i * 3 + 16)); }- for (int i = 0; i < 5; i++) { uint32_t t0 = qc & 0x07; uint32_t t1 = (qc & 0x38) >> 3; qc >>= 6; zc |= (t0 << (i * 3)); zc |= (t1 << (i * 3 + 16)); }-- // za: 9997775 55333111 8886664 44222000- // zb: jjjhhhf ffdddbbb iiiggge eecccaaa- // zc: tttrrrp ppnnnlll sssqqqo oommmkkk- // qd: vvvuuu-- za |= ((qd & 0x01) >> 0) << 15;- zb |= ((qd & 0x02) >> 1) << 15;- zc |= ((qd & 0x04) >> 2) << 15;- za |= ((qd & 0x08) >> 3) << 31;- zb |= ((qd & 0x10) >> 4) << 31;- zc |= ((qd & 0x20) >> 5) << 31;-- // za: v9997775 55333111 u8886664 44222000 (u, v lsb)- // zb: vjjjhhhf ffdddbbb uiiiggge eecccaaa- // zc: vtttrrrp ppnnnlll usssqqqo oommmkkk-- q[0 * stride] = za;- q[1 * stride] = zb;- q[2 * stride] = zc;-}--__forceinline__ __device__ void dequant_3bit_32-(- const uint32_t q_0,- const uint32_t q_1,- const uint32_t q_2,- half2 (&dq)[16],- int stride-)-{- const uint32_t c0 = 0x64006400;- const half y8_ = __float2half_rn(1.0f / 8.0f);- const half y64_ = __float2half_rn(1.0f / 64.0f);- const half2 y8 = __halves2half2(y8_, y8_);- const half2 y64 = __halves2half2(y64_, y64_);- const half z1_ = __float2half_rn(-1024.0f - 4.0f);- const half z8_ = __float2half_rn(-1024.0f / 8.0f - 4.0f);- const half z64_ = __float2half_rn(-1024.0f / 64.0f - 4.0f);- const half2 z1 = __halves2half2(z1_, z1_);- const half2 z8 = __halves2half2(z8_, z8_);- const half2 z64 = __halves2half2(z64_, z64_);-- uint32_t qa = q_0;- uint32_t qb = q_1;- uint32_t qc = q_2;-- half2_uint32 q0((qa & 0x00070007) | c0); // half2(q[ 0], q[ 1]) + 1024- half2_uint32 q1((qa & 0x00380038) | c0); // half2(q[ 2], q[ 3]) * 8 + 1024- qa >>= 6;- half2_uint32 q2((qa & 0x00070007) | c0); // half2(q[ 4], q[ 5]) + 1024- half2_uint32 q3((qa & 0x00380038) | c0); // half2(q[ 6], q[ 7]) * 8 + 1024- half2_uint32 q4((qa & 0x01c001c0) | c0); // half2(q[ 8], q[ 9]) * 64 + 1024- qa >>= 9;- qa &= 0x00010001;- half2_uint32 q5((qb & 0x00070007) | c0); // half2(q[10], q[11]) + 1024- half2_uint32 q6((qb & 0x00380038) | c0); // half2(q[12], q[13]) * 8 + 1024- qb >>= 6;- half2_uint32 q7((qb & 0x00070007) | c0); // half2(q[14], q[15]) + 1024- half2_uint32 q8((qb & 0x00380038) | c0); // half2(q[16], q[17]) * 8 + 1024- half2_uint32 q9((qb & 0x01c001c0) | c0); // half2(q[18], q[19]) * 64 + 1024- qb >>= 8;- qb &= 0x00020002;- half2_uint32 q10((qc & 0x00070007) | c0); // half2(q[20], q[21]) + 1024- half2_uint32 q11((qc & 0x00380038) | c0); // half2(q[22], q[23]) * 8 + 1024- qc >>= 6;- half2_uint32 q12((qc & 0x00070007) | c0); // half2(q[24], q[25]) + 1024- half2_uint32 q13((qc & 0x00380038) | c0); // half2(q[26], q[27]) * 8 + 1024- half2_uint32 q14((qc & 0x01c001c0) | c0); // half2(q[28], q[29]) * 64 + 1024- qc >>= 7;- qc &= 0x00040004;- half2_uint32 q15((qa | qb | qc) | c0);-- dq[ 0] = __hadd2( q0.as_half2, z1);- dq[ 1] = __hfma2( q1.as_half2, y8, z8);- dq[ 2] = __hadd2( q2.as_half2, z1);- dq[ 3] = __hfma2( q3.as_half2, y8, z8);- dq[ 4] = __hfma2( q4.as_half2, y64, z64);- dq[ 5] = __hadd2( q5.as_half2, z1);- dq[ 6] = __hfma2( q6.as_half2, y8, z8);- dq[ 7] = __hadd2( q7.as_half2, z1);- dq[ 8] = __hfma2( q8.as_half2, y8, z8);- dq[ 9] = __hfma2( q9.as_half2, y64, z64);- dq[10] = __hadd2(q10.as_half2, z1);- dq[11] = __hfma2(q11.as_half2, y8, z8);- dq[12] = __hadd2(q12.as_half2, z1);- dq[13] = __hfma2(q13.as_half2, y8, z8);- dq[14] = __hfma2(q14.as_half2, y64, z64);- dq[15] = __hadd2(q15.as_half2, z1);-}--#else--__forceinline__ __device__ void shuffle_3bit_32-(- uint32_t* q,- int stride-)-{-}--__forceinline__ __device__ void dequant_3bit_32-(- const uint32_t q_0,- const uint32_t q_1,- const uint32_t q_2,- half2 (&dq)[16],- int stride-)-{- half dqh[32];- for (int i = 0; i < 10; i++) dqh[ i] = dq_ns(exb( q_0, i * 3 , 0x07), 4);- dqh[10 ] = dq_ns(exb(q_1, q_0, 30, 0x07), 4);- for (int i = 0; i < 10; i++) dqh[11 + i] = dq_ns(exb( q_1, i * 3 + 1, 0x07), 4);- dqh[21 ] = dq_ns(exb(q_2, q_1, 31, 0x07), 4);- for (int i = 0; i < 10; i++) dqh[22 + i] = dq_ns(exb( q_2, i * 3 + 2, 0x07), 4);-- for (int i = 0; i < 16; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);-}--#endif--#endifdiff --git a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_4.cuh b/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_4.cuhdeleted file mode 100644index 5fb070d06..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_4.cuh+++ /dev/null@@ -1,227 +0,0 @@-#ifndef _qdq_4_cuh-#define _qdq_4_cuh--#include "qdq_util.cuh"-#include "../../config.h"--#if QMODE_4BIT == 1--// Permutation:-//-// 77775555 33331111 66664444 22220000--__forceinline__ __device__ void shuffle_4bit_8-(- uint32_t* q,- int stride-)-{- uint32_t qa = q[0];- uint32_t qb = 0;-- #pragma unroll- for (int i = 0; i < 4; i++)- {- uint32_t qa0 = qa & 0x0f;- uint32_t qa1 = (qa & 0xf0) >> 4;- qa >>= 8;- qb |= (qa1 << (i * 4 + 16));- qb |= (qa0 << (i * 4));- }- q[0] = qb;-}--__forceinline__ __device__ void dequant_4bit_8-(- const uint32_t q_0,- half2 (&dq)[4],- int stride-)-{- const uint32_t c0 = 0x64006400;- const half y16_ = __float2half_rn(1.0f / 16.0f);- const half2 y16 = __halves2half2(y16_, y16_);- const half z1_ = __float2half_rn(-1024.0f - 8.0f);- const half z16_ = __float2half_rn(-1024.0f / 16.0f - 8.0f);- const half2 z1 = __halves2half2(z1_, z1_);- const half2 z16 = __halves2half2(z16_, z16_);-- uint32_t qa = q_0;- half2_uint32 q0((qa & 0x000f000f) | c0); // half2(q[ 0], q[ 1]) + 1024- half2_uint32 q1((qa & 0x00f000f0) | c0); // half2(q[ 2], q[ 3]) * 16 + 1024- qa >>= 8;- half2_uint32 q2((qa & 0x000f000f) | c0); // half2(q[ 4], q[ 5]) + 1024- half2_uint32 q3((qa & 0x00f000f0) | c0); // half2(q[ 6], q[ 7]) * 16 + 1024-- dq[0] = __hadd2(q0.as_half2, z1);- dq[1] = __hfma2(q1.as_half2, y16, z16);- dq[2] = __hadd2(q2.as_half2, z1);- dq[3] = __hfma2(q3.as_half2, y16, z16);-}--__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale-(- const uint32_t zero,- const half scale,- half2 (&z1z16)[2],- half2 (&y1y16)[2]-)-{- half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);- half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));-- half2 scale2 = __half2half2(scale);-- z1z16[0] = __hmul2(scale2, __half2half2(z1.as_half));- z1z16[1] = __hmul2(scale2, __half2half2(z16));-- const half y1 = __float2half_rn(1.0f);- const half y16 = __float2half_rn(1.0f / 16.0f);-- y1y16[0] = __hmul2(scale2, __half2half2(y1));- y1y16[1] = __hmul2(scale2, __half2half2(y16));-}--__forceinline__ __device__ void dequant_4bit_8_prep_zero-(- const uint32_t zero,- half2(&z1z16)[2],- half2(&y1y16)[2]-)-{- half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);- half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));-- z1z16[0] = __half2half2(z1.as_half);- z1z16[1] = __half2half2(z16);-- const half y1 = __float2half_rn(1.0f);- const half y16 = __float2half_rn(1.0f / 16.0f);-- y1y16[0] = __half2half2(y1);- y1y16[1] = __half2half2(y16);-}---__forceinline__ __device__ void dequant_4bit_8_gptq-(- const uint32_t q_0,- half2 (&dq)[4],- half2 (&z1z16)[2],- half2 (&y1y16)[2],- int stride,- bool scaled-)-{- const uint32_t c0 = 0x64006400;-- uint32_t qa = q_0;- half2_uint32 q0((qa & 0x000f000f) | c0); // half2( q[0] + 1024, q[1] + 1024 )- half2_uint32 q1((qa & 0x00f000f0) | c0); // half2( q[2] * 16 + 1024, q[3] * 16 + 1024 )- qa >>= 8;- half2_uint32 q2((qa & 0x000f000f) | c0); // half2( q[4] + 1024, q[5] + 1024 )- half2_uint32 q3((qa & 0x00f000f0) | c0); // half2( q[6] * 16 + 1024, q[7] * 16 + 1024 )-- if (scaled)- {- dq[0] = __hfma2(q0.as_half2, y1y16[0], z1z16[0]); // half2( q[0] * s - z * s, q[1] * s - z * s)- dq[1] = __hfma2(q1.as_half2, y1y16[1], z1z16[1]); // half2( q[2] * s - z * s, q[3] * s - z * s)- dq[2] = __hfma2(q2.as_half2, y1y16[0], z1z16[0]);- dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]);- }- else- {- dq[0] = __hadd2(q0.as_half2, z1z16[0]); // half2( q[0] - z, q[1] - z )- dq[1] = __hfma2(q1.as_half2, y1y16[1], z1z16[1]); // half2( q[2] - z, q[3] - z )- dq[2] = __hadd2(q2.as_half2, z1z16[0]); // half2( q[4] - z, q[5] - z )- dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]); // half2( q[6] - z, q[7] - z )- }-}--#else--__forceinline__ __device__ void shuffle_4bit_8-(- uint32_t* q,- int stride-)-{-}--__forceinline__ __device__ void dequant_4bit_8-(- const uint32_t q_0,- half2 (&dq)[4],- int stride-)-{- half dqh[8];- for (int i = 0; i < 8; i++) dqh[i] = dq_ns(exb(q_0, i * 4, 0x0f), 8);-- for (int i = 0; i < 4; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);-}--__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale-(- const uint32_t zero,- const half scale,- half2 (&z1)[2],- half2 (&y1)[2]-)-{- half z = __int2half_rn(-((int)zero));- z = __hmul(z, scale);- z1[0] = __half2half2(z);- y1[0] = __half2half2(scale);-}--__forceinline__ __device__ void dequant_4bit_8_prep_zero-(- const uint32_t zero,- half2(&z1)[2],- half2(&y1)[2]-)-{- half z = __int2half_rn(-((int)zero));- z1[0] = __half2half2(z);-}--__forceinline__ __device__ void dequant_4bit_8_gptq-(- const uint32_t q_0,- half2 (&dq)[4],- half2 (&z1)[2],- half2 (&y1)[2],- int stride,- bool scaled-)-{- half2 dqh2[8];-- uint32_t qa = q_0;- for (int i = 0; i < 4; i++)- {- half d0 = __int2half_rn(qa & 0x0f); qa >>= 4;- half d1 = __int2half_rn(qa & 0x0f); qa >>= 4;- dqh2[i] = __halves2half2(d0, d1);- }-- if (scaled)- {- dq[0] = __hfma2(dqh2[0], y1[0], z1[0]);- dq[1] = __hfma2(dqh2[1], y1[0], z1[0]);- dq[2] = __hfma2(dqh2[2], y1[0], z1[0]);- dq[3] = __hfma2(dqh2[3], y1[0], z1[0]);- }- else- {- dq[0] = __hadd2(dqh2[0], z1[0]);- dq[1] = __hadd2(dqh2[1], z1[0]);- dq[2] = __hadd2(dqh2[2], z1[0]);- dq[3] = __hadd2(dqh2[3], z1[0]);- }-}--#endif--#endif\ No newline at end of filediff --git a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_5.cuh b/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_5.cuhdeleted file mode 100644index 454e4b93b..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_5.cuh+++ /dev/null@@ -1,207 +0,0 @@-#ifndef _qdq_5_cuh-#define _qdq_5_cuh--#include "qdq_util.cuh"-#include "../../config.h"--#if QMODE_5BIT == 1--// Permutation:-//-// v5555533 33311111 u4444422 22200000 (u, v lsb)-// vbbbbb99 99977777 uaaaaa88 88866666-// vhhhhhff fffddddd ugggggee eeeccccc-// vnnnnnll llljjjjj ummmmmkk kkkiiiii-// vtttttrr rrrppppp usssssqq qqqooooo--__forceinline__ __device__ void shuffle_5bit_32-(- uint32_t* q,- int stride-)-{- uint32_t qa = q[0 * stride];- uint32_t qb = q[1 * stride];- uint32_t qc = q[2 * stride];- uint32_t qd = q[3 * stride];- uint32_t qe = q[4 * stride];-- // qa: 66555554 44443333 32222211 11100000- // qb: ccccbbbb baaaaa99 99988888 77777666- // qc: jiiiiihh hhhggggg fffffeee eedddddc- // qd: pppooooo nnnnnmmm mmlllllk kkkkjjjj- // qe: vvvvvuuu uuttttts ssssrrrr rqqqqqpp-- uint32_t qf = qe >> 22;- qe <<= 8;- qe |= qd >> 24;- qd <<= 6;- qd |= qc >> 26;- qc <<= 4;- qc |= qb >> 28;- qb <<= 2;- qb |= qa >> 30;-- // qa: 555554 44443333 32222211 11100000- // qb: bbbbba aaaa9999 98888877 77766666- // qc: hhhhhg ggggffff feeeeedd dddccccc- // qd: nnnnnm mmmmllll lkkkkkjj jjjiiiii- // qe: ttttts ssssrrrr rqqqqqpp pppooooo- // qf: vv vvvuuuuu-- uint32_t za = 0;- uint32_t zb = 0;- uint32_t zc = 0;- uint32_t zd = 0;- uint32_t ze = 0;-- for (int i = 0; i < 3; i++) { uint32_t t0 = qa & 0x1f; uint32_t t1 = (qa & 0x3e0) >> 5; qa >>= 10; za |= (t0 << (i * 5)); za |= (t1 << (i * 5 + 16)); }- for (int i = 0; i < 3; i++) { uint32_t t0 = qb & 0x1f; uint32_t t1 = (qb & 0x3e0) >> 5; qb >>= 10; zb |= (t0 << (i * 5)); zb |= (t1 << (i * 5 + 16)); }- for (int i = 0; i < 3; i++) { uint32_t t0 = qc & 0x1f; uint32_t t1 = (qc & 0x3e0) >> 5; qc >>= 10; zc |= (t0 << (i * 5)); zc |= (t1 << (i * 5 + 16)); }- for (int i = 0; i < 3; i++) { uint32_t t0 = qd & 0x1f; uint32_t t1 = (qd & 0x3e0) >> 5; qd >>= 10; zd |= (t0 << (i * 5)); zd |= (t1 << (i * 5 + 16)); }- for (int i = 0; i < 3; i++) { uint32_t t0 = qe & 0x1f; uint32_t t1 = (qe & 0x3e0) >> 5; qe >>= 10; ze |= (t0 << (i * 5)); ze |= (t1 << (i * 5 + 16)); }-- // za: 5555533 33311111 4444422 22200000- // zb: bbbbb99 99977777 aaaaa88 88866666- // zc: hhhhhff fffddddd gggggee eeeccccc- // zd: nnnnnll llljjjjj mmmmmkk kkkiiiii- // ze: tttttrr rrrppppp sssssqq qqqooooo- // qf: vv vvvuuuuu-- za |= ((qf & 0x001) >> 0) << 15;- zb |= ((qf & 0x002) >> 1) << 15;- zc |= ((qf & 0x004) >> 2) << 15;- zd |= ((qf & 0x008) >> 3) << 15;- ze |= ((qf & 0x010) >> 4) << 15;- za |= ((qf & 0x020) >> 5) << 31;- zb |= ((qf & 0x040) >> 6) << 31;- zc |= ((qf & 0x080) >> 7) << 31;- zd |= ((qf & 0x100) >> 8) << 31;- ze |= ((qf & 0x200) >> 9) << 31;-- // za: v5555533 33311111 u4444422 22200000 (u, v lsb)- // zb: vbbbbb99 99977777 uaaaaa88 88866666- // zc: vhhhhhff fffddddd ugggggee eeeccccc- // zd: vnnnnnll llljjjjj ummmmmkk kkkiiiii- // ze: vtttttrr rrrppppp usssssqq qqqooooo-- q[0 * stride] = za;- q[1 * stride] = zb;- q[2 * stride] = zc;- q[3 * stride] = zd;- q[4 * stride] = ze;-}--__forceinline__ __device__ void dequant_5bit_32-(- const uint32_t q_0,- const uint32_t q_1,- const uint32_t q_2,- const uint32_t q_3,- const uint32_t q_4,- half2 (&dq)[16],- int stride-)-{- const uint32_t c0 = 0x64006400;- const half y32_ = __float2half_rn(1.0f / 32.0f);- const half2 y32 = __halves2half2(y32_, y32_);- const half z1_ = __float2half_rn(-1024.0f - 16.0f);- const half z32_ = __float2half_rn(-1024.0f / 32.0f - 16.0f);- const half2 z1 = __halves2half2(z1_, z1_);- const half2 z32 = __halves2half2(z32_, z32_);-- uint32_t qa = q_0;- uint32_t qb = q_1;- uint32_t qc = q_2;- uint32_t qd = q_3;- uint32_t qe = q_4;-- half2_uint32 q0 ((qa & 0x001f001f) | c0); // half2(q[ 0], q[ 1]) + 1024- half2_uint32 q1 ((qa & 0x03e003e0) | c0); // half2(q[ 2], q[ 3]) * 32 + 1024- qa >>= 10;- half2_uint32 q2 ((qa & 0x001f001f) | c0); // half2(q[ 4], q[ 5]) + 1024- qa >>= 5;- qa &= 0x00010001;- half2_uint32 q3 ((qb & 0x001f001f) | c0); // half2(q[ 6], q[ 7]) + 1024- half2_uint32 q4 ((qb & 0x03e003e0) | c0); // half2(q[ 8], q[ 9]) * 32 + 1024- qb >>= 10;- half2_uint32 q5 ((qb & 0x001f001f) | c0); // half2(q[10], q[11]) + 1024- qb >>= 4;- qb &= 0x00020002;- half2_uint32 q6 ((qc & 0x001f001f) | c0); // half2(q[12], q[13]) + 1024- half2_uint32 q7 ((qc & 0x03e003e0) | c0); // half2(q[14], q[15]) * 32 + 1024- qc >>= 10;- half2_uint32 q8 ((qc & 0x001f001f) | c0); // half2(q[16], q[17]) + 1024- qc >>= 3;- qc &= 0x00040004;- half2_uint32 q9 ((qd & 0x001f001f) | c0); // half2(q[18], q[19]) + 1024- half2_uint32 q10((qd & 0x03e003e0) | c0); // half2(q[20], q[21]) * 32 + 1024- qd >>= 10;- half2_uint32 q11((qd & 0x001f001f) | c0); // half2(q[22], q[23]) + 1024- qd >>= 2;- qd &= 0x00080008;- half2_uint32 q12((qe & 0x001f001f) | c0); // half2(q[24], q[25]) + 1024- half2_uint32 q13((qe & 0x03e003e0) | c0); // half2(q[26], q[27]) * 32 + 1024- qe >>= 10;- half2_uint32 q14((qe & 0x001f001f) | c0); // half2(q[28], q[29]) + 1024- qe >>= 1;- qe &= 0x00100010;- half2_uint32 q15((qa | qb | qc | qd | qe) | c0);-- dq[ 0] = __hadd2( q0.as_half2, z1);- dq[ 1] = __hfma2( q1.as_half2, y32, z32);- dq[ 2] = __hadd2( q2.as_half2, z1);- dq[ 3] = __hadd2( q3.as_half2, z1);- dq[ 4] = __hfma2( q4.as_half2, y32, z32);- dq[ 5] = __hadd2( q5.as_half2, z1);- dq[ 6] = __hadd2( q6.as_half2, z1);- dq[ 7] = __hfma2( q7.as_half2, y32, z32);- dq[ 8] = __hadd2( q8.as_half2, z1);- dq[ 9] = __hadd2( q9.as_half2, z1);- dq[10] = __hfma2(q10.as_half2, y32, z32);- dq[11] = __hadd2(q11.as_half2, z1);- dq[12] = __hadd2(q12.as_half2, z1);- dq[13] = __hfma2(q13.as_half2, y32, z32);- dq[14] = __hadd2(q14.as_half2, z1);- dq[15] = __hadd2(q15.as_half2, z1);-}--#else--__forceinline__ __device__ void shuffle_5bit_32-(- uint32_t* q,- int stride-)-{-}--__forceinline__ __device__ void dequant_5bit_32-(- const uint32_t q_0,- const uint32_t q_1,- const uint32_t q_2,- const uint32_t q_3,- const uint32_t q_4,- half2 (&dq)[16],- int stride-)-{- half dqh[32];- for (int i = 0; i < 6; i++) dqh[ i] = dq_ns(exb( q_0, i * 5 , 0x1f), 16);- dqh[ 6 ] = dq_ns(exb(q_1, q_0, 30, 0x1f), 16);- for (int i = 0; i < 5; i++) dqh[ 7 + i] = dq_ns(exb( q_1, i * 5 + 3, 0x1f), 16);- dqh[12 ] = dq_ns(exb(q_2, q_1, 28, 0x1f), 16);- for (int i = 0; i < 6; i++) dqh[13 + i] = dq_ns(exb( q_2, i * 5 + 1, 0x1f), 16);- dqh[19 ] = dq_ns(exb(q_3, q_2, 31, 0x1f), 16);- for (int i = 0; i < 5; i++) dqh[20 + i] = dq_ns(exb( q_3, i * 5 + 4, 0x1f), 16);- dqh[25 ] = dq_ns(exb(q_4, q_3, 29, 0x1f), 16);- for (int i = 0; i < 6; i++) dqh[26 + i] = dq_ns(exb( q_4, i * 5 + 2, 0x1f), 16);-- for (int i = 0; i < 16; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);-}--#endif--#endif\ No newline at end of filediff --git a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_6.cuh b/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_6.cuhdeleted file mode 100644index c2eb8cfbf..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_6.cuh+++ /dev/null@@ -1,44 +0,0 @@-#ifndef _qdq_6_cuh-#define _qdq_6_cuh--#include "qdq_util.cuh"-#include "../../config.h"--#if QMODE_6BIT == 1-- // Not implemented--#else--__forceinline__ __device__ void shuffle_6bit_16-(- uint32_t* q,- int stride-)-{-}--__forceinline__ __device__ void dequant_6bit_16-(- const uint32_t q_0,- const uint32_t q_1,- const uint32_t q_2,- half2 (&dq)[8],- int stride-)-{- half dqh[16];- for (int i = 0; i < 5; i++) dqh[ i] = dq_ns(exb( q_0, i * 6 , 0x3f), 32);- dqh[ 5 ] = dq_ns(exb(q_1, q_0, 30, 0x3f), 32);- for (int i = 0; i < 4; i++) dqh[ 6 + i] = dq_ns(exb( q_1, i * 6 + 4, 0x3f), 32);- dqh[10 ] = dq_ns(exb(q_2, q_1, 28, 0x3f), 32);- for (int i = 0; i < 5; i++) dqh[11 + i] = dq_ns(exb( q_2, i * 6 + 2, 0x3f), 32);-- for (int i = 0; i < 8; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);-}--#endif--#endif--diff --git a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_8.cuh b/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_8.cuhdeleted file mode 100644index e2409efac..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_8.cuh+++ /dev/null@@ -1,38 +0,0 @@-#ifndef _qdq_8_cuh-#define _qdq_8_cuh--#include "qdq_util.cuh"-#include "../../config.h"--#if QMODE_8BIT == 1-- // Not implemented--#else--__forceinline__ __device__ void shuffle_8bit_4-(- uint32_t* q,- int stride-)-{-}--__forceinline__ __device__ void dequant_8bit_8-(- const uint32_t q_0,- const uint32_t q_1,- half2 (&dq)[4],- int stride-)-{- half dqh[8];- for (int i = 0; i < 4; i++) dqh[i ] = dq_ns(exb(q_0, i * 8, 0xff), 128);- for (int i = 0; i < 4; i++) dqh[i + 4] = dq_ns(exb(q_1, i * 8, 0xff), 128);-- for (int i = 0; i < 4; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);-}--#endif--#endif\ No newline at end of filediff --git a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_util.cuh b/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_util.cuhdeleted file mode 100644index 71657191b..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_util.cuh+++ /dev/null@@ -1,51 +0,0 @@-#ifndef _qdq_util_cuh-#define _qdq_util_cuh--union half2_uint32-{- uint32_t as_uint32;- half2 as_half2;- __device__ half2_uint32(uint32_t val) : as_uint32(val) {}- __device__ half2_uint32(half2 val) : as_half2(val) {}-};--union half_uint16-{- uint16_t as_uint16;- half as_half;- __device__ half_uint16(uint16_t val) : as_uint16(val) {}- __device__ half_uint16(half val) : as_half(val) {}-};--// Max_scale premultiplied by 1/256--__forceinline__ __device__ half dq_scale(const int qs, const half max_scale)-{- int qs_i = qs + 1;- half qs_h = __int2half_rn(qs_i * qs_i);- qs_h = __hmul(qs_h, max_scale);- return qs_h;-}--__forceinline__ __device__ half dq(const int q, const int qzero, const half scale)-{- return __hmul(__int2half_rn(q - qzero), scale);-}--__forceinline__ __device__ half dq_ns(const int q, const int qzero)-{- //return __hsub(__int2half_rn(q), __int2half_rn(qzero));- return __int2half_rn(q - qzero);-}--__forceinline__ __device__ int exb(const uint32_t q, const int shift, const int mask)-{- return (int)((q >> shift) & mask);-}--__forceinline__ __device__ int exb(const uint32_t q1, const uint32_t q0, const int shift, const int mask)-{- return (int)(__funnelshift_rc(q0, q1, shift) & mask);-}--#endifdiff --git a/auto_round_extension/cuda/exllamav2/cuda/util.cuh b/auto_round_extension/cuda/exllamav2/cuda/util.cuhdeleted file mode 100644index 36be0e242..000000000--- a/auto_round_extension/cuda/exllamav2/cuda/util.cuh+++ /dev/null@@ -1,42 +0,0 @@--#define DIVIDE(x, size) (((x) + (size) - 1) / (size))--#define DBGS(__x) printf("%s\n", __x)-#define DBGI(__x) printf("%s: %i\n", #__x, __x)-#define DBGI2(__x, __y) printf("%s, %s: %i, %i\n", #__x, #__y, __x, __y)-#define DBGI3(__x, __y, __z) printf("%s, %s, %s: %i, %i, %i\n", #__x, #__y, #__z, __x, __y, __z)-#define DBGX(__x) printf("%s: %x\n", #__x, __x)-#define DBGX2(__x, __y) printf("%s, %s: %x, %x\n", #__x, #__y, __x, __y)-#define DBGX3(__x, __y, __z) printf("%s, %s, %s: %x, %x, %x\n", #__x, #__y, #__z, __x, __y, __z)-#define DBGF(__x) printf("%s: %f\n", #__x, __x)-#define DBGF2(__x, __y) printf("%s, %s: %f, %f\n", #__x, #__y, __x, __y)-#define DBGF3(__x, __y, __z) printf("%s, %s, %s: %f, %f, %f\n", #__x, #__y, #__z, __x, __y, __z)-#define DBGH(__x) printf("%s: %f\n", #__x, __half2float(__x))-#define DBGH2(__x, __y) printf("%s, %s: %f, %f\n", #__x, #__y, __half2float(__x), __half2float(__y))-#define DBGH3(__x, __y, __z) printf("%s, %s, %s: %f, %f, %f\n", #__x, #__y, #__z, __half2float(__x), __half2float(__y), __half2float(__z))--#define DBGIH(__x, __y) printf("%s, %s: %i, %f\n", #__x, #__y, __x, __half2float(__y))-#define DBGIH2(__x, __y, __z) printf("%s, %s, %s: %i, %f, %f\n", #__x, #__y, #__z, __x, __half2float(__y), __half2float(__z))--__forceinline__ __device__ half dq_scale_(const int qs, const half max_scale)-{- half qs_h = __hmul(__int2half_rn(qs + 1), __float2half_rn(1.0f / 16.0f));- qs_h = __hmul(qs_h, qs_h);- qs_h = __hmul(qs_h, max_scale);- return qs_h;-}--__forceinline__ __device__ float clamp(float x, float a, float b)-{- return fmaxf(a, fminf(b, x));-}--#define cuda_check(res) { gpu_assert((res), __FILE__, __LINE__); }-inline void gpu_assert(cudaError_t code, const char *file, int line, bool abort=true)-{- if (code != cudaSuccess)- {- fprintf(stderr,"CUDA error: %s %s %d\n", cudaGetErrorString(code), file, line);- if (abort) exit(code);- }-}diff --git a/auto_round_extension/cuda/exllamav2/ext.cpp b/auto_round_extension/cuda/exllamav2/ext.cppdeleted file mode 100644index 5e52e6abd..000000000--- a/auto_round_extension/cuda/exllamav2/ext.cpp+++ /dev/null@@ -1,134 +0,0 @@-#include <torch/extension.h>-#include <c10/cuda/CUDAGuard.h>-#include <ATen/cuda/CUDAContext.h>-#include <cuda_runtime.h>-#include <cuda_fp16.h>-#include <cstdint>-#include <cstdio>--#include "config.h"--#include "cuda/q_matrix.cuh"-#include "cuda/q_gemm.cuh"--#include "cpp/util.h"--// Some decluttering macros--#define TORCH_CHECK_DTYPE(__x, __dtype) TORCH_CHECK((__x).dtype() == torch::__dtype, #__x " is incorrect datatype, must be " #__dtype)-#define TORCH_CHECK_DTYPE_OPT(__x, __dtype) TORCH_CHECK((__x).device().is_meta() || (__x).dtype() == torch::__dtype, #__x " is incorrect datatype, must be " #__dtype)-#define TORCH_CHECK_SHAPES(__x, __dim_x, __y, __dim_y, __scale_y) TORCH_CHECK((__x).size(__dim_x) == (__y).size(__dim_y) * __scale_y, #__x " and " #__y " have incompatible shapes")-#define TORCH_CHECK_SHAPES_OPT(__x, __dim_x, __y, __dim_y, __scale_y) TORCH_CHECK((__x).device().is_meta() || (__x).size(__dim_x) == (__y).size(__dim_y) * __scale_y, #__x " and " #__y " have incompatible shapes")---// Quant matrix--uintptr_t make_q_matrix-(- torch::Tensor q_weight,- torch::Tensor q_perm,- torch::Tensor q_invperm,- torch::Tensor q_scale,- torch::Tensor q_scale_max,- torch::Tensor q_groups,- torch::Tensor gptq_qzeros,- torch::Tensor gptq_scales,- torch::Tensor gptq_g_idx,- torch::Tensor temp_dq-)-{- TORCH_CHECK_DTYPE(q_weight, kInt);- TORCH_CHECK_DTYPE_OPT(q_perm, kShort);- TORCH_CHECK_DTYPE_OPT(q_invperm, kShort);- TORCH_CHECK_DTYPE_OPT(q_scale, kInt);- TORCH_CHECK_DTYPE_OPT(q_scale_max, kHalf);- TORCH_CHECK_DTYPE_OPT(q_groups, kShort);- TORCH_CHECK_DTYPE_OPT(gptq_qzeros, kInt);- TORCH_CHECK_DTYPE_OPT(gptq_scales, kHalf);- TORCH_CHECK_DTYPE_OPT(gptq_g_idx, kInt);-- TORCH_CHECK_SHAPES(q_perm, 0, q_invperm, 0, 1);-- int device = q_weight.device().index();- int width = q_weight.size(1);- int groups;- int height;-- if (!q_scale.device().is_meta())- {- TORCH_CHECK_SHAPES(q_weight, 1, q_scale, 1, 8);- TORCH_CHECK_SHAPES(q_scale_max, 0, q_scale, 0, 1);- groups = q_scale.size(0);- height = q_invperm.size(0);- }- else- {- TORCH_CHECK_SHAPES(q_weight, 1, gptq_qzeros, 1, 8);- TORCH_CHECK_SHAPES(q_weight, 1, gptq_scales, 1, 1);- groups = gptq_qzeros.size(0);- height = q_weight.size(0) * 8;- }-- TORCH_CHECK(temp_dq.size(0) >= width * height, "Insufficient size of temp_dq buffer")-- QMatrix* m = new QMatrix- (- device,- height,- width,- groups,- (uint32_t*) q_weight.data_ptr(),- q_perm.device().is_meta() ? NULL : (uint16_t*) q_perm.data_ptr(),- q_invperm.device().is_meta() ? NULL : (uint16_t*) q_invperm.data_ptr(),- q_scale.device().is_meta() ? NULL : (uint32_t*) q_scale.data_ptr(),- q_scale_max.device().is_meta() ? NULL : (half*) q_scale_max.data_ptr(),- q_groups.device().is_meta() ? NULL : (uint16_t*) q_groups.data_ptr(),- gptq_qzeros.device().is_meta() ? NULL : (uint32_t*) gptq_qzeros.data_ptr(),- gptq_scales.device().is_meta() ? NULL : (half*) gptq_scales.data_ptr(),- gptq_g_idx.device().is_meta() ? NULL : (uint32_t*) gptq_g_idx.data_ptr(),- (half*) temp_dq.data_ptr()- );-- return reinterpret_cast<uintptr_t> (m);-}--void gemm_half_q_half-(- torch::Tensor a,- uintptr_t b,- torch::Tensor c,- bool force_cuda-)-{- QMatrix* qm = reinterpret_cast<QMatrix*> (b);-- TORCH_CHECK_DTYPE(a, kHalf);- TORCH_CHECK_DTYPE(c, kHalf);- TORCH_CHECK_SHAPES(a, 0, c, 0, 1);- TORCH_CHECK(qm->height == a.size(1), "a and b have incompatible shapes")- TORCH_CHECK(qm->width == c.size(1), "b and c have incompatible shapes")-- const at::cuda::OptionalCUDAGuard device_guard(device_of(a));-- gemm_half_q_half_cuda- (- at::cuda::getCurrentCUDABlasHandle(),- (const half*) a.data_ptr(),- qm,- (half*) c.data_ptr(),- c.size(0), // m- c.size(1), // n- a.size(1), // k- true,- NULL,- force_cuda- );-}--// Bindings--PYBIND11_MODULE(TORCH_EXTENSION_NAME, m)-{- m.def("make_q_matrix", &make_q_matrix, "make_q_matrix");- m.def("gemm_half_q_half", &gemm_half_q_half, "gemm_half_q_half");-}diff --git a/auto_round_extension/cuda/gptqmodel_marlin.py b/auto_round_extension/cuda/gptqmodel_marlin.pynew file mode 100644index 000000000..242037d9b--- /dev/null+++ b/auto_round_extension/cuda/gptqmodel_marlin.py@@ -0,0 +1,486 @@+# Copyright 2024-2025 ModelCloud.ai+# Copyright 2024-2025 [email protected]+# Contact: [email protected], x.com/qubitium+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# Adapted from vllm+# at https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/gptq_marlin.py++from typing import Any, Dict, List, Optional, Tuple++import numpy as np+import torch+++def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demand+ from gptqmodel.models._const import DEVICE, PLATFORM # pylint: disable=E0401+ from gptqmodel.nn_modules.qlinear import BaseQuantLinear # pylint: disable=E0401+ from gptqmodel.utils.backend import BACKEND # pylint: disable=E0401++ marlin_import_exception = None+ try:+ import gptqmodel_marlin_kernels # pylint: disable=E0401+ except ImportError as e:+ marlin_import_exception = e++ from auto_round.utils import logger++ GPTQ_MARLIN_TILE = 16+ GPTQ_MARLIN_MIN_THREAD_N = 64+ GPTQ_MARLIN_MIN_THREAD_K = 128+ GPTQ_MARLIN_MAX_PARALLEL = 16++ def set_weight_attrs(+ weight: torch.Tensor,+ weight_attrs: Optional[Dict[str, Any]],+ ):+ """Set attributes on a weight tensor.++ This method is used to set attributes on a weight tensor. This method+ will not overwrite existing attributes.++ Args:+ weight: The weight tensor.+ weight_attrs: A dictionary of attributes to set on the weight tensor.+ """+ if weight_attrs is None:+ return+ for key, value in weight_attrs.items():+ assert not hasattr(+ weight, key), (f"Overwriting existing tensor attribute: {key}")+ setattr(weight, key, value)++ def marlin_is_k_full(act_order: bool, is_row_parallel: bool) -> bool:+ return (not act_order) or (act_order and not is_row_parallel)++ def marlin_repeat_scales_on_all_ranks(act_order: bool, group_size: int,+ is_row_parallel: bool) -> bool:+ # Need to repeat scales on every rank if act_ordering or+ # channelwise and RowParallelLinear+ is_channelwise = group_size == -1+ return act_order or (is_channelwise and is_row_parallel)++ def marlin_make_workspace(output_size_per_partition: int,+ device: torch.device) -> torch.Tensor:+ max_workspace_size = (output_size_per_partition //+ GPTQ_MARLIN_MIN_THREAD_N) * GPTQ_MARLIN_MAX_PARALLEL++ return torch.zeros(max_workspace_size,+ dtype=torch.int,+ device=device,+ requires_grad=False)++ def marlin_sort_g_idx(+ g_idx: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:+ g_idx_sort_indices = torch.argsort(g_idx).to(torch.int)+ return g_idx[g_idx_sort_indices], g_idx_sort_indices++ def marlin_make_empty_g_idx(device: torch.device) -> torch.Tensor:+ return torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),+ requires_grad=False)++ # Newly generated tensors need to replace existing tensors that are+ # already registered as parameters by vLLM (and won't be freed)+ def replace_tensor(layer: torch.nn.Module, name: str,+ new_t: torch.Tensor) -> None:+ # It is important to use resize_() here since it ensures+ # the same buffer is reused+ getattr(layer, name).resize_(new_t.shape)+ getattr(layer, name).copy_(new_t)+ del new_t++ def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,+ group_size: int) -> torch.Tensor:++ scale_perm, scale_perm_single = get_scale_perms()+ if group_size < size_k and group_size != -1:+ s = s.reshape((-1, len(scale_perm)))[:, scale_perm]+ else:+ s = s.reshape((-1, len(scale_perm_single)))[:, scale_perm_single]+ s = s.reshape((-1, size_n)).contiguous()++ return s++ def get_scale_perms():+ scale_perm: List[int] = []+ for i in range(8):+ scale_perm.extend([i + 8 * j for j in range(8)])+ scale_perm_single: List[int] = []+ for i in range(4):+ scale_perm_single.extend(+ [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])+ return scale_perm, scale_perm_single++ def apply_gptq_marlin_linear(+ input: torch.Tensor,+ weight: torch.Tensor,+ weight_scale: torch.Tensor,+ weight_zp: torch.Tensor,+ g_idx: torch.Tensor,+ g_idx_sort_indices: torch.Tensor,+ workspace: torch.Tensor,+ num_bits: int,+ output_size_per_partition: int,+ input_size_per_partition: int,+ is_k_full: bool,+ bias: torch.Tensor,+ fp32: bool,+ ) -> torch.Tensor:++ reshaped_x = input.reshape(-1, input.shape[-1])+ out_shape = input.shape[:-1] + (output_size_per_partition,)++ output = gptqmodel_marlin_kernels.gptq_marlin_gemm(+ reshaped_x,+ weight,+ weight_scale,+ weight_zp,+ g_idx,+ g_idx_sort_indices,+ workspace,+ num_bits,+ reshaped_x.shape[0],+ output_size_per_partition,+ input_size_per_partition,+ is_k_full,+ False,+ fp32, # <- True: enable fp32 reduce for higher accuracy, False: fp16+ )++ if bias is not None:+ output.add_(bias) # In-place add++ return output.reshape(out_shape)++ class MarlinQuantLinear(BaseQuantLinear):+ SUPPORTS_BITS = [4, 8]+ SUPPORTS_GROUP_SIZE = [-1, 32, 64, 128]+ SUPPORTS_DESC_ACT = [True, False]+ SUPPORTS_SYM = [True]+ SUPPORTS_SHARDS = True+ SUPPORTS_TRAINING = False+ SUPPORTS_AUTO_PADDING = False+ SUPPORTS_IN_FEATURES_DIVISIBLE_BY = [1]+ SUPPORTS_OUT_FEATURES_DIVISIBLE_BY = [64]++ SUPPORTS_DEVICES = [DEVICE.CUDA]+ SUPPORTS_PLATFORM = [PLATFORM.LINUX]+ SUPPORTS_PACK_DTYPES = [torch.int32]++ SUPPORTS_DTYPES = [torch.float16, torch.bfloat16]++ # for transformers/optimum tests compat+ QUANT_TYPE = "marlin"++ def __init__(+ self, bits: int,+ group_size: int,+ desc_act: bool,+ sym: bool,+ in_features: int,+ out_features: int,+ bias: bool = False,+ pack_dtype: torch.dtype = torch.int32,+ **kwargs):+ if marlin_import_exception is not None:+ raise ValueError(+ f"Trying to use the marlin backend, but could not "+ f"import the C++/CUDA dependencies with the following error: {marlin_import_exception}"+ )++ # self.original_in_features = in_features+ # self.original_out_features = out_features++ if desc_act and group_size == -1:+ # In this case, act_order == True is the same as act_order == False+ # (since we have only one group per output channel)+ desc_act = False++ super().__init__(+ bits=bits,+ group_size=group_size,+ sym=sym,+ desc_act=desc_act,+ in_features=in_features,+ out_features=out_features,+ bias=bias,+ pack_dtype=pack_dtype,+ backend=kwargs.pop("backend", BACKEND.MARLIN),+ adapter=None,+ register_buffers=False,+ **kwargs)++ # toggle fp32 mode depending on MARLIN or MARLIN_FP16 backend+ self.fp32 = True if self.backend in [BACKEND.MARLIN, BACKEND.AUTO] else False++ if not self.fp32:+ logger.warning_once(+ "Kernel: Marlin FP16 mode is activated with reduced accuracy. "+ "Use default Marlin model for improved inference quality.")++ # Determine sharding+ if marlin_repeat_scales_on_all_ranks(desc_act,+ self.group_size,+ is_row_parallel=False):+ # By setting scale_dim == None, weight_loader will+ # repeat the scales on each GPU in TP>1 case.+ scales_and_zp_input_dim = None+ scales_and_zp_size = self.in_features // self.group_size+ else:+ # By setting scale_dim == 0, weight_loader will+ # shard the scales in TP>1 case.+ scales_and_zp_input_dim = 0+ scales_and_zp_size = self.in_features // self.group_size++ # Quantized weights+ self.register_buffer(+ "qweight",+ torch.empty(+ self.in_features // self.pack_factor,+ self.out_features,+ dtype=torch.int32,+ ),+ )++ # Activation order+ # self.register_buffer(+ # "g_idx_temp",+ # torch.empty(+ # 0,+ # dtype=torch.int32,+ # ),+ # )++ # Ignore warning from fused linear layers such as QKVParallelLinear.+ # set_weight_attrs(+ # self.g_idx_temp,+ # {+ # "input_dim": 0,+ # "ignore_warning": True+ # },+ # )++ set_weight_attrs(+ self.qweight,+ {+ "input_dim": 0,+ "output_dim": 1,+ "packed_dim": 0,+ "pack_factor": self.pack_factor,+ },+ )++ #+ # # Ignore warning from fused linear layers such as QKVParallelLinear.+ # set_weight_attrs(+ # self.g_idx,+ # {+ # "input_dim": 0,+ # "ignore_warning": True+ # },+ # )++ # Scales+ self.register_buffer(+ "scales",+ torch.empty(+ scales_and_zp_size,+ self.out_features,+ dtype=torch.float16,+ ),+ )++ set_weight_attrs(+ self.scales,+ {+ "input_dim": scales_and_zp_input_dim,+ "output_dim": 1,+ },+ )++ # Quantized zero-points+ self.register_buffer(+ "qzeros",+ torch.empty(+ scales_and_zp_size,+ self.out_features // self.pack_factor,+ dtype=torch.int32,+ ),+ )++ set_weight_attrs(+ self.qzeros,+ {+ "input_dim": scales_and_zp_input_dim,+ "output_dim": 1,+ "packed_dim": 1,+ "pack_factor": self.pack_factor,+ },+ )++ self.is_k_full = marlin_is_k_full(self.desc_act, is_row_parallel=False)++ if bias:+ self.register_buffer("bias", torch.zeros((self.out_features), dtype=torch.float16))+ else:+ self.bias = None++ self.is_lm_head = False+ if kwargs.get("name") is not None and kwargs.get("lm_head_name") is not None:+ self.is_lm_head = kwargs["name"] == kwargs["lm_head_name"]++ # auto-optimize on post init+ # self.optimize()++ # def optimize(self, backend: str = "inductor", mode: str = None, fullgraph: bool = False):+ # if self.optimized:+ # return+ #+ # # compile dequantize+ # self.forward = torch_compile(self.forward, backend=backend, mode=mode, fullgraph=fullgraph)+ #+ # super().optimize()++ @classmethod+ # internal method and should not be overridden+ def verify_supports_params(cls):+ return++ def post_init(self):+ device = self.qweight.device+ # Allocate marlin workspace+ self.workspace = marlin_make_workspace(+ self.out_features, device)++ # Handle sorting for activation reordering if needed.++ # self.g_idx = marlin_make_empty_g_idx(device)+ g_idx_sort_indices = marlin_make_empty_g_idx(device)++ # No zero-point+ # self.zp = marlin_make_empty_g_idx(device)++ # Repack weights from autogptq format to marlin format.+ marlin_qweight = gptqmodel_marlin_kernels.gptq_marlin_repack(+ self.qweight,+ g_idx_sort_indices,+ self.in_features,+ self.out_features,+ self.bits,+ self.pack_dtype_bits)+ replace_tensor(self, "qweight", marlin_qweight)++ # Permute scales from autogptq format to marlin format.+ marlin_scales = marlin_permute_scales(+ self.scales,+ size_k=self.in_features,+ size_n=self.out_features,+ group_size=self.group_size)+ replace_tensor(self, "scales", marlin_scales)++ super().post_init()++ def list_buffers(self) -> List:+ buf = super().list_buffers()+ if hasattr(self, "workspace") and self.workspace is not None:+ buf.append(self.workspace)+ if hasattr(self, "zp") and self.zp is not None:+ buf.append(self.zp)+ return buf++ def forward(self, x: torch.Tensor):+ # TODO FIXME: parent should never call us if there is no data to process+ # check: https://github.com/ModelCloud/GPTQModel/issues/1361+ if x.shape[0] == 0:+ return torch.empty((0, self.out_features), dtype=x.dtype, device=x.device)++ # make sure scales is synced with x/input+ if x.dtype != self.scales.dtype:+ self.scales = self.scales.to(dtype=x.dtype)+ g_idx = marlin_make_empty_g_idx(self.qweight.device)+ g_idx_sort_indices = marlin_make_empty_g_idx(self.qweight.device)+ zp = marlin_make_empty_g_idx(self.qweight.device)+ out = apply_gptq_marlin_linear(+ input=x.contiguous() if self.is_lm_head else x,+ weight=self.qweight,+ weight_scale=self.scales,+ weight_zp=zp,+ g_idx=g_idx,+ g_idx_sort_indices=g_idx_sort_indices,+ workspace=self.workspace,+ num_bits=self.bits,+ output_size_per_partition=self.out_features,+ input_size_per_partition=self.in_features,+ is_k_full=self.is_k_full,+ bias=self.bias,+ fp32=self.fp32,+ )++ if self.adapter:+ out = self.adapter.apply(x=x, out=out)++ return out++ # Precompute permutations for Marlin weight and scale shuffling+ def _get_perms():+ perm = []+ for i in range(32):+ perm1 = []+ col = i // 4+ for block in [0, 1]:+ for row in [+ 2 * (i % 4),+ 2 * (i % 4) + 1,+ 2 * (i % 4 + 4),+ 2 * (i % 4 + 4) + 1,+ ]:+ perm1.append(16 * row + col + 8 * block)+ for j in range(4):+ perm.extend([p + 256 * j for p in perm1])++ perm = np.array(perm)+ interleave = np.array([0, 2, 4, 6, 1, 3, 5, 7])+ perm = perm.reshape((-1, 8))[:, interleave].ravel()+ perm = torch.from_numpy(perm)+ scale_perm = []+ for i in range(8):+ scale_perm.extend([i + 8 * j for j in range(8)])+ scale_perm_single = []+ for i in range(4):+ scale_perm_single.extend([2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])+ return perm, scale_perm, scale_perm_single++ def unpack_qzeros(qzeros):+ unpacked_zeros = torch.zeros(+ (qzeros.shape[0], qzeros.shape[1] * 8),+ dtype=torch.int8,+ device=qzeros.device,+ requires_grad=False,+ )++ for col in range(unpacked_zeros.shape[1]):+ i = col % 8+ unpacked_zeros[:, col] = (qzeros[:, col // 8] >> (4 * i)) & 0xF++ return unpacked_zeros++ def dequantize_qzeros(layer):+ qzeros = layer.qzeros+ unpacked_qzeros = unpack_qzeros(qzeros)+ group_size = layer.group_size+ unpacked_qzeros = unpacked_qzeros.repeat_interleave(group_size, dim=0)++ return unpacked_qzeros++ return MarlinQuantLineardiff --git a/auto_round_extension/cuda/post_init.py b/auto_round_extension/cuda/post_init.pydeleted file mode 100644index 6bad13639..000000000--- a/auto_round_extension/cuda/post_init.py+++ /dev/null@@ -1,157 +0,0 @@-# Copyright (c) 2024 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--# MIT License-#-# Copyright (c) 2023 潘其威(William)-#-# Permission is hereby granted, free of charge, to any person obtaining a copy-# of this software and associated documentation files (the "Software"), to deal-# in the Software without restriction, including without limitation the rights-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell-# copies of the Software, and to permit persons to whom the Software is-# furnished to do so, subject to the following conditions:-#-# The above copyright notice and this permission notice shall be included in all-# copies or substantial portions of the Software.-#-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE-# SOFTWARE.-import torch--EXLLAMA_DEFAULT_MAX_INPUT_LENGTH = 2048---def autoround_post_init(model):- """- The max_input_length argument is specific to the exllama backend, that requires to initialize a buffer temp_state.- """- device_to_buffers_size = {}-- model_uses_exllama = False- for name, submodule in model.named_modules():- if hasattr(submodule, "QUANT_TYPE") and submodule.QUANT_TYPE == "exllama":- model_uses_exllama = True- device = submodule.qweight.device- if device not in device_to_buffers_size:- device_to_buffers_size[device] = {- "max_dq_buffer_size": 1,- "max_inner_outer_dim": 1,- }-- submodule._use_act_order = False-- # Disable this heuristic for detecting act_order, but it could be used instead of the config.- """- if submodule.g_idx is None:- submodule.act_order = False- elif submodule.g_idx is not None and ((submodule.g_idx == 0).all() or- torch.equal(submodule.g_idx.cpu(),- torch.tensor([i // submodule.group_size for i in range(submodule.g_idx.shape[0])], dtype=torch.int32))):- submodule.g_idx = None- submodule.act_order = False- else:- submodule.act_order = True- """-- device_to_buffers_size[device]["max_dq_buffer_size"] = max(- device_to_buffers_size[device]["max_dq_buffer_size"],- submodule.qweight.numel() * 8,- )-- if model_uses_exllama:- # To be honest this is quite ugly, not proud of this.- try:- from exllama_kernels import prepare_buffers, set_tuning_params- except ImportError as e:- raise ImportError(- f"Could not import exllama backend dependencies prepare_buffers, set_tuning_params with the following "- f"error: {e}"- )-- device_to_buffers = {}-- max_input_len = 1-- for device, buffers_size in device_to_buffers_size.items():- # The temp_state buffer is required to reorder X in the act-order case.- # The temp_dq buffer is required to dequantize weights when using cuBLAS, typically for the prefill.- device_to_buffers[device] = {- "temp_state": torch.zeros(- (max_input_len, buffers_size["max_inner_outer_dim"]),- dtype=torch.float16,- device=device,- ),- "temp_dq": torch.zeros(- (1, buffers_size["max_dq_buffer_size"]),- dtype=torch.float16,- device=device,- ),- "max_dq_buffer_size": buffers_size["max_dq_buffer_size"],- "max_inner_outer_dim": buffers_size["max_inner_outer_dim"],- }-- # Buffers need to be persistent to avoid any bug.- model.device_to_buffers = device_to_buffers-- for device, buffers in model.device_to_buffers.items():- prepare_buffers(device, buffers["temp_state"], buffers["temp_dq"])-- # Using the default from exllama repo here.- matmul_recons_thd = 8- matmul_fused_remap = False- matmul_no_half2 = False- set_tuning_params(matmul_recons_thd, matmul_fused_remap, matmul_no_half2)-- # The buffers need to have been initialized first before calling make_q4.- for name, submodule in model.named_modules():- if hasattr(submodule, "QUANT_TYPE") and submodule.QUANT_TYPE == "exllama":- submodule.post_init()-- ## exllamav2- fixed_bytes = {}- model_uses_exllamav2 = False-- for _, submodule in model.named_modules():- if hasattr(submodule, "QUANT_TYPE"):- if submodule.QUANT_TYPE == "exllamav2":- model_uses_exllamav2 = True- device = submodule.qweight.device- scratch_fixed = submodule.scratch_space_fixed()- fixed_bytes[device] = max(scratch_fixed, fixed_bytes.get(device, 0))- elif submodule.QUANT_TYPE == "hpu":- submodule.post_init()-- if model_uses_exllamav2:- from auto_round_extension.cuda.qlinear_exllamav2 import ExLlamaV2DeviceTensors-- device_tensors = {}- for device, scratch_bytes in fixed_bytes.items():- device_tensors[device] = ExLlamaV2DeviceTensors(device.index, scratch_bytes)-- # have persistent buffers, otherwise we will get OOM- model.device_tensors = device_tensors-- for _, submodule in model.named_modules():- if hasattr(submodule, "QUANT_TYPE") and submodule.QUANT_TYPE == "exllamav2":- device = submodule.qweight.device- submodule.post_init(temp_dq=model.device_tensors[device])- torch.cuda.empty_cache()-- return modeldiff --git a/auto_round_extension/cuda/qlinear_exllamav2.py b/auto_round_extension/cuda/qlinear_exllamav2.pydeleted file mode 100644index 3bc415971..000000000--- a/auto_round_extension/cuda/qlinear_exllamav2.py+++ /dev/null@@ -1,277 +0,0 @@-# Copyright (c) 2024 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--# Adapted from turboderp exllama: https://github.com/turboderp/exllamav2-# MIT License-#-# Copyright (c) 2023 潘其威(William)-#-# Permission is hereby granted, free of charge, to any person obtaining a copy-# of this software and associated documentation files (the "Software"), to deal-# in the Software without restriction, including without limitation the rights-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell-# copies of the Software, and to permit persons to whom the Software is-# furnished to do so, subject to the following conditions:-#-# The above copyright notice and this permission notice shall be included in all-# copies or substantial portions of the Software.-#-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE-# SOFTWARE.-import math-from logging import getLogger--import torch-import torch.nn as nn--logger = getLogger(__name__)--try:- from autoround_exllamav2_kernels import gemm_half_q_half, make_q_matrix-except ImportError as e:- exllama_v2_import_exception = e--- def error_raiser_exllama(*args, **kwargs):- raise ValueError(- f"Trying to use the exllama v2 backend, but could not import the C++/CUDA dependencies with the following "- f"error: {exllama_v2_import_exception}"- )--- make_q_matrix = error_raiser_exllama- gemm_half_q_half = error_raiser_exllama--# Dummy tensor to pass instead of g_idx since there is no way to pass "None" to a C++ extension-none_tensor = torch.empty((1, 1), device="meta")---def _torch_device(idx):- if idx == -1:- return "cpu"- return f"cuda:{idx}"---def ext_gemm_half_q_half(x, q_handle, q4_width, force_cuda):- """Matrix multiplication, returns x @ q4"""- output_shape = x.shape[:-1] + (q4_width,)- x = x.view(-1, x.shape[-1])- output = torch.empty((x.shape[0], q4_width), dtype=torch.half, device=x.device)- gemm_half_q_half(x, q_handle, output, force_cuda)- return output.view(output_shape)---def ext_make_q_matrix(w: dict, temp_dq, key: str = None):- """- Create Q matrix- """- # EXL2- # won't work as the moment because the tensors are not the same.- if "q_weight" in w:- w["q_scale_max"] /= 256- w["q_perm"] = w["q_perm"].short()- w["q_invperm"] = w["q_invperm"].short()- return make_q_matrix(- w["q_weight"],- w["q_perm"],- w["q_invperm"],- w["q_scale"],- w["q_scale_max"],- w["q_groups"],- none_tensor,- none_tensor,- none_tensor,- temp_dq,- )- # GPTQ- elif "qweight" in w:- if w["scales"].dtype == torch.float:- w["scales"] = w["scales"].half()-- # GPTQ with g_idx (act_order)- if "g_idx" in w and not (w["g_idx"] == 0).all().item():- w["q_perm"] = torch.empty(- (w["qweight"].shape[0] * 8,),- dtype=torch.short,- device=w["qweight"].device,- )- w["q_invperm"] = torch.empty_like(w["q_perm"])- # make_q4 segfaults if g_idx is not on cpu in the act-order case. In the non act-order case, None needs- # to be passed for g_idx.- return make_q_matrix(- w["qweight"],- w["q_perm"],- w["q_invperm"],- none_tensor,- none_tensor,- none_tensor,- w["qzeros"],- w["scales"],- w["g_idx"].cpu(),- temp_dq,- )- # GPTQ without g_idx- else:- return make_q_matrix(- w["qweight"],- none_tensor,- none_tensor,- none_tensor,- none_tensor,- none_tensor,- w["qzeros"],- w["scales"],- none_tensor,- temp_dq,- )---class QuantLinear(nn.Module):- QUANT_TYPE = "exllamav2"-- """Linear layer implementation with per-group 4-bit quantization of the weights"""-- def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):- super().__init__()- if bits != 4:- raise ValueError(- f"Exllamav2 kernel supports only bits=4, requested bits={bits}. Something is wrong in the model "- f"initialization."- )- if trainable:- raise NotImplementedError("Exllamav2 kernel does not support training.")-- self.q_handle = None- self.q_tensors = None-- self.padding = -outfeatures % 32- self.outfeatures = outfeatures + self.padding- outfeatures = self.outfeatures-- self.infeatures = infeatures- self.bits = bits- self.group_size = group_size if group_size != -1 else infeatures- self.trainable = trainable- self.maxq = 2 ** self.bits - 1-- assert infeatures % 32 == 0- assert infeatures % self.group_size == 0- assert outfeatures % 32 == 0-- # I need to register the tensors, otherwise, we won't be able to load them easily using transformers ...- self.register_buffer(- "qweight",- torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),- )- self.register_buffer(- "qzeros",- torch.zeros(- (- math.ceil(infeatures / self.group_size),- outfeatures // 32 * self.bits,- ),- dtype=torch.int32,- ),- )- self.register_buffer(- "scales",- torch.zeros(- (math.ceil(infeatures / self.group_size), outfeatures),- dtype=torch.float16,- ),- )- self.register_buffer(- "g_idx",- torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),- )-- if bias:- self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))- else:- self.bias = None- self.clip = kwargs.get("clip", False)-- def post_init(self, temp_dq):- assert self.qweight.device.type == "cuda"- assert self.qweight.device.index is not None- self.q_tensors = {- "qweight": self.qweight,- "qzeros": self.qzeros,- "scales": self.scales,- "g_idx": self.g_idx,- }- temp_dq = temp_dq.get_scratch_slice(self.temp_dq_size())- self.q_handle = ext_make_q_matrix(self.q_tensors, temp_dq)-- def forward(self, x, force_cuda=False):- orig_dtype = x.dtype- if x.dtype != torch.float16:- logger.warning_once(- f"The exllama v2 kernel for GPTQ requires a float16 input activation, while {x.dtype} was passed. "- f"Casting to float16.\nMake sure you loaded your model with torch_dtype=torch.float16, that the model "- f"definition does not inadvertently cast to float32, or disable AMP Autocast that may produce float32 "- f"intermediate activations in the model."- )-- x = x.half()-- output = ext_gemm_half_q_half(x, self.q_handle, self.outfeatures, force_cuda)- if orig_dtype != torch.float16:- output = output.to(orig_dtype)- if self.bias is not None:- output.add_(self.bias)- if self.clip:- output = torch.clip(output, -65504, 65504)- return output-- def temp_dq_size(self):- return self.infeatures * self.outfeatures * 2 + 128-- def temp_fwd_size(self, max_input_len, max_batch_size):- return self.outfeatures * max_input_len * max_batch_size * 4 + 128-- def scratch_space_fixed(self, max_input_len=2048, max_batch_size=8):- return self.temp_dq_size() + self.temp_fwd_size(max_input_len, max_batch_size)---class ExLlamaV2DeviceTensors:- device_idx: int- scratch_bytes: int- scratch_idx: int- scratch: torch.tensor = None-- def __init__(self, device_idx, scratch_bytes):- self.device_idx = device_idx- self.scratch_bytes = scratch_bytes-- def prepare(self):- self.scratch = torch.empty(- (self.scratch_bytes // 2,),- dtype=torch.half,- device=_torch_device(self.device_idx),- )-- def get_scratch_slice(self, size_bytes):- if self.scratch is None:- self.prepare()-- size_bytes = ((size_bytes + 127) // 128) * 128- size_half = size_bytes // 2- scratch_slice = self.scratch.narrow(0, 0, size_half)- return scratch_slicediff --git a/auto_round_extension/cuda/qlinear_triton.py b/auto_round_extension/cuda/qlinear_triton.pydeleted file mode 100644index 006911f2f..000000000--- a/auto_round_extension/cuda/qlinear_triton.py+++ /dev/null@@ -1,326 +0,0 @@-# Copyright (c) 2024 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--# MIT License-#-# Copyright (c) 2023 潘其威(William)-#-# Permission is hereby granted, free of charge, to any person obtaining a copy-# of this software and associated documentation files (the "Software"), to deal-# in the Software without restriction, including without limitation the rights-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell-# copies of the Software, and to permit persons to whom the Software is-# furnished to do so, subject to the following conditions:-#-# The above copyright notice and this permission notice shall be included in all-# copies or substantial portions of the Software.-#-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE-# SOFTWARE.-import math-from logging import getLogger--import numpy as np-import torch-import torch.nn as nn-import transformers--from auto_round_extension.cuda.triton_utils.mixin import TritonModuleMixin---logger = getLogger(__name__)--try:- from auto_round_extension.cuda.triton_utils.kernels import (- QuantLinearFunction,- QuantLinearInferenceOnlyFunction,- quant_matmul_248,- quant_matmul_inference_only_248,- transpose_quant_matmul_248,- )-except ImportError as e:- triton_import_exception = e-- def error_raiser_triton(*args, **kwargs):- raise ValueError(- f'Trying to use the triton backend, but could not import triton '- f'dependencies with the following error: {triton_import_exception}'- )-- class FakeTriton:- def __getattr__(self, name):- raise ImportError(- f"Trying to use the triton backend, but could not import triton "- f"dependencies with the following error: {triton_import_exception}"- )-- quant_matmul_248 = error_raiser_triton- transpose_quant_matmul_248 = error_raiser_triton- quant_matmul_inference_only_248 = error_raiser_triton- QuantLinearFunction = FakeTriton- QuantLinearInferenceOnlyFunction = FakeTriton---class QuantLinear(nn.Module, TritonModuleMixin):- QUANT_TYPE = "triton"-- def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):- super().__init__()- if bits not in [2, 4, 8]:- raise NotImplementedError("Only 2,4,8 bits are supported.")- if infeatures % 32 != 0 or outfeatures % 32 != 0:- raise NotImplementedError("in_feature and out_feature must be divisible by 32.")- self.infeatures = infeatures- self.outfeatures = outfeatures- self.bits = bits- self.group_size = group_size if group_size != -1 else infeatures- self.maxq = 2**self.bits - 1-- self.register_buffer(- "qweight",- torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),- )- self.register_buffer(- "qzeros",- torch.zeros(- (- math.ceil(infeatures / self.group_size),- outfeatures // 32 * self.bits,- ),- dtype=torch.int32,- ),- )- self.register_buffer(- "scales",- torch.zeros(- (math.ceil(infeatures / self.group_size), outfeatures),- dtype=torch.float16,- ),- )- self.register_buffer(- "g_idx",- torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),- )- if bias:- self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))- else:- self.bias = None-- self.trainable = trainable-- def post_init(self):- pass-- def pack_cpu(self, linear, scales, zeros, g_idx=None):- W = linear.weight.data.clone()- if isinstance(linear, nn.Conv2d):- W = W.flatten(1)- if isinstance(linear, transformers.pytorch_utils.Conv1D):- W = W.t()-- self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx-- scales = scales.t().contiguous()- if isinstance(zeros,torch.Tensor):- zeros = zeros.t().contiguous()- scale_zeros = zeros * scales- else:- scale_zeros = scales*zeros- self.scales = scales.clone().half()- if linear.bias is not None:- self.bias = linear.bias.clone().half()-- intweight = []- for idx in range(self.infeatures):- intweight.append(- torch.round((W[:, idx] + scale_zeros[self.g_idx[idx]]) / self.scales[self.g_idx[idx]]).to(torch.int)[- :, None- ]- )- intweight = torch.cat(intweight, dim=1)- intweight = intweight.t().contiguous()- intweight = intweight.numpy().astype(np.uint32)-- i = 0- row = 0- qweight = np.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32)- while row < qweight.shape[0]:- for j in range(i, i + (32 // self.bits)):- qweight[row] |= intweight[j] << (self.bits * (j - i))- i += 32 // self.bits- row += 1-- qweight = qweight.astype(np.int32)- self.qweight = torch.from_numpy(qweight)-- if isinstance(zeros, torch.Tensor):- zeros = zeros.numpy().astype(np.uint32)- qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)- i = 0- col = 0- while col < qzeros.shape[1]:- for j in range(i, i + (32 // self.bits)):- qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))- i += 32 // self.bits- col += 1-- qzeros = qzeros.astype(np.int32)- self.qzeros = torch.from_numpy(qzeros)- else:- shape = scales.shape- value = 0- for j in range(0, (32 // self.bits)):- value |= zeros << (self.bits * j)- qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value- qzeros = qzeros.astype(np.int32)- self.qzeros = torch.from_numpy(qzeros)---- def pack(self, linear, scales, zeros, g_idx):- if torch.cuda.is_available():- return self.pack_cuda(linear, scales, zeros, g_idx)- else:- return self.pack_cpu(linear, scales, zeros, g_idx)-- def pack_cuda(self, linear, scales, zeros, g_idx=None):- W = linear.weight.data.clone()- if isinstance(linear, nn.Conv2d):- W = W.flatten(1)- if isinstance(linear, transformers.pytorch_utils.Conv1D):- W = W.t()- scales_t = scales.t().contiguous()- self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx- if linear.bias is not None:- self.bias = linear.bias.clone().half()- self.scales = scales_t.clone().half()-- repeat_scales = scales.to("cuda:0").repeat_interleave(self.group_size, 1)- if isinstance(zeros, torch.Tensor):- repeat_zeros = zeros.to("cuda:0").repeat_interleave(self.group_size, 1)- else:- repeat_zeros = zeros-- intweight = torch.round(W.to("cuda:0") / repeat_scales + repeat_zeros).to(torch.int).t().contiguous().to("cpu")- intweight = intweight.numpy().astype(np.uint32)- del repeat_scales-- i = 0- row = 0- qweight = np.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32)- # pack_array_with_numba_b4_c32(intweight, qweight)- while row < qweight.shape[0]:- for j in range(i, i + (32 // self.bits)):- qweight[row] |= intweight[j] << (self.bits * (j - i))- i += 32 // self.bits- row += 1-- qweight = qweight.astype(np.int32)- self.qweight = torch.from_numpy(qweight)-- if isinstance(zeros, torch.Tensor):- zeros = zeros.t().contiguous()- zeros = zeros.numpy().astype(np.uint32)- qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)- i = 0- col = 0- while col < qzeros.shape[1]:- for j in range(i, i + (32 // self.bits)):- qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))- i += 32 // self.bits- col += 1-- qzeros = qzeros.astype(np.int32)- self.qzeros = torch.from_numpy(qzeros)- else:- shape = scales_t.shape- value = 0- for j in range(0, (32 // self.bits)):- value |= zeros << (self.bits * j)- qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value- qzeros = qzeros.astype(np.int32)- self.qzeros = torch.from_numpy(qzeros)-- def forward(self, x):- out_shape = x.shape[:-1] + (self.outfeatures,)- quant_linear_fn = QuantLinearFunction if self.trainable else QuantLinearInferenceOnlyFunction- out = quant_linear_fn.apply(- x.reshape(-1, x.shape[-1]),- self.qweight,- self.scales,- self.qzeros,- self.g_idx,- self.bits,- self.maxq,- )- out = out.half().reshape(out_shape)- out = out + self.bias if self.bias is not None else out- return out-- @classmethod- def warmup(cls, model, transpose=False, seqlen=2048):- """- Pre-tunes the quantized kernel- """- from tqdm import tqdm-- kn_values = {}-- for _, m in model.named_modules():- if not isinstance(m, cls):- continue-- k = m.infeatures- n = m.outfeatures-- if (k, n) not in kn_values:- kn_values[(k, n)] = (- m.qweight,- m.scales,- m.qzeros,- m.g_idx,- m.bits,- m.maxq,- )-- logger.info(f"Found {len(kn_values)} unique KN Linear values.")- logger.info("Warming up autotune cache ...")- with torch.no_grad():- for m in tqdm(range(0, math.ceil(math.log2(seqlen)) + 1)):- m = 2**m- for (k, n), (- qweight,- scales,- qzeros,- g_idx,- bits,- maxq,- ) in kn_values.items():- if transpose:- a = torch.randn(m, k, dtype=torch.float16, device=model.device)- quant_matmul_248(a, qweight, scales, qzeros, g_idx, bits, maxq)- a = torch.randn(m, n, dtype=torch.float16, device=model.device)- transpose_quant_matmul_248(a, qweight, scales, qzeros, g_idx, bits, maxq)- else:- a = torch.randn(m, k, dtype=torch.float16, device=model.device)- quant_matmul_inference_only_248(a, qweight, scales, qzeros, g_idx, bits, maxq)- del kn_values---__all__ = ["QuantLinear"]diff --git a/auto_round_extension/cuda/qlinear_tritonv2.py b/auto_round_extension/cuda/qlinear_tritonv2.pyindex 9bc86bb40..8cb4b05b6 100644--- a/auto_round_extension/cuda/qlinear_tritonv2.py+++ b/auto_round_extension/cuda/qlinear_tritonv2.py@@ -81,12 +81,6 @@ class QuantLinear(nn.Module, TritonModuleMixin):dtype=torch.float16,),)- self.register_buffer(- "g_idx",- torch.tensor(- [i // self.group_size for i in range(infeatures)], dtype=torch.int32- ),- )if bias:self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16)@@ -100,67 +94,64 @@ class QuantLinear(nn.Module, TritonModuleMixin):passdef pack(self, linear, scales, zeros, g_idx=None):- W = linear.weight.data.clone()+ scales_t = scales.t().contiguous()+ if linear.bias is not None:+ self.bias = linear.bias.clone().half()+ self.scales = scales_t.clone().half()+ device = "cpu"+ if torch.cuda.is_available():+ device = "cuda:0"+ elif torch.xpu.is_available():+ device = "xpu:0"++ W = linear.weight.data.to(device).clone()if isinstance(linear, nn.Conv2d):W = W.flatten(1)if isinstance(linear, transformers.pytorch_utils.Conv1D):W = W.t()- self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx-- scales = scales.t().contiguous()- zeros = zeros.t().contiguous()- scale_zeros = zeros * scales- self.scales = scales.clone().half()- if linear.bias is not None:- self.bias = linear.bias.clone().half()-- intweight = []- for idx in range(self.infeatures):- intweight.append(- torch.round(- (W[:, idx] + scale_zeros[self.g_idx[idx]])- / self.scales[self.g_idx[idx]]- ).to(torch.int)[:, None]- )- intweight = torch.cat(intweight, dim=1)- intweight = intweight.t().contiguous()- intweight = intweight.numpy().astype(np.uint32)-- i = 0- row = 0- qweight = np.zeros(- (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32- )- while row < qweight.shape[0]:- if self.bits in [2, 4, 8]:- for j in range(i, i + (32 // self.bits)):- qweight[row] |= intweight[j] << (self.bits * (j - i))- i += 32 // self.bits- row += 1- else:- raise NotImplementedError("Only 2,4,8 bits are supported.")-- qweight = qweight.astype(np.int32)- self.qweight = torch.from_numpy(qweight)-- zeros = zeros.numpy().astype(np.uint32)- qzeros = np.zeros(- (zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32- )- i = 0- col = 0- while col < qzeros.shape[1]:- if self.bits in [2, 4, 8]:+ repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)+ if isinstance(zeros, torch.Tensor):+ repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros[:, :W.shape[1]]).to(+ torch.int32)+ else:+ repeat_zeros = zeros+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros).to(+ torch.int32)++ del repeat_scales+ intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)+ order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits+ intweight = intweight.to(torch.int32)+ intweight = intweight << order_map+ intweight = torch.sum(intweight, dim=-1)++ intweight = intweight.t().contiguous().to(torch.int32)+ self.qweight = intweight.to("cpu")++ if isinstance(zeros, torch.Tensor):+ zeros = zeros.t().contiguous()+ zeros = zeros.numpy().astype(np.uint32)+ qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)+ i = 0+ col = 0+ while col < qzeros.shape[1]:for j in range(i, i + (32 // self.bits)):qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))i += 32 // self.bitscol += 1- else:- raise NotImplementedError("Only 2,4,8 bits are supported.")- qzeros = qzeros.astype(np.int32)- self.qzeros = torch.from_numpy(qzeros)+ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)+ else:+ shape = scales_t.shape+ value = 0+ for j in range(0, (32 // self.bits)):+ value |= zeros << (self.bits * j)+ qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value+ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)def forward(self, x):out_shape = x.shape[:-1] + (self.outfeatures,)@@ -177,7 +168,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):)out = out.half().reshape(out_shape)out = out + self.bias if self.bias is not None else out- return out+ return out.to(x.dtype)@classmethoddef warmup(cls, model, transpose=False, seqlen=2048):diff --git a/auto_round_extension/cuda/qlinear_tritonv2_zp.py b/auto_round_extension/cuda/qlinear_tritonv2_zp.pynew file mode 100644index 000000000..0436394ee--- /dev/null+++ b/auto_round_extension/cuda/qlinear_tritonv2_zp.py@@ -0,0 +1,227 @@+import math+from logging import getLogger++import numpy as np+import torch+import torch.nn as nn+import transformers+++from auto_round_extension.cuda.triton_utils_zp.mixin import TritonModuleMixin+++logger = getLogger(__name__)++try:+ from auto_round_extension.cuda.triton_utils_zp.dequant import QuantLinearFunction, quant_matmul_248+except ImportError as e:+ triton_import_exception = e++ def error_raiser_triton(*args, **kwargs):+ raise ValueError(+ f"Trying to use the triton backend, but could not import triton dependencies with the following error: {triton_import_exception}"+ )++ class FakeTriton:+ def __getattr__(self, name):+ raise ImportError(+ f"Trying to use the triton backend, but could not import triton dependencies with the following error: {triton_import_exception}"+ )++ quant_matmul_248 = error_raiser_triton+ QuantLinearFunction = FakeTriton+ QuantLinearInferenceOnlyFunction = FakeTriton+++class QuantLinear(nn.Module, TritonModuleMixin):+ """+ Triton v2 quantized linear layer.++ Calls dequant kernel (see triton_utils/dequant) to dequantize the weights then uses+ torch.matmul to compute the output whereas original `triton` quantized linear layer fused+ dequant and matmul into single kernel.add()+ """++ QUANT_TYPE = "tritonv2"++ def __init__(+ self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs+ ):+ super().__init__()+ if bits not in [2, 4, 8]:+ raise NotImplementedError("Only 2,4,8 bits are supported.")+ if infeatures % 32 != 0 or outfeatures % 32 != 0:+ raise NotImplementedError(+ "in_feature and out_feature must be divisible by 32."+ )+ self.infeatures = infeatures+ self.outfeatures = outfeatures+ self.bits = bits+ self.group_size = group_size if group_size != -1 else infeatures+ self.maxq = 2**self.bits - 1++ self.register_buffer(+ "qweight",+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),+ )+ self.register_buffer(+ "qzeros",+ torch.zeros(+ (+ math.ceil(infeatures / self.group_size),+ outfeatures // 32 * self.bits,+ ),+ dtype=torch.int32,+ ),+ )+ self.register_buffer(+ "scales",+ torch.zeros(+ (math.ceil(infeatures / self.group_size), outfeatures),+ dtype=torch.float16,+ ),+ )+ self.register_buffer(+ "g_idx",+ torch.tensor(+ [i // self.group_size for i in range(infeatures)], dtype=torch.int32+ ),+ )+ if bias:+ self.register_buffer(+ "bias", torch.zeros((outfeatures), dtype=torch.float16)+ )+ else:+ self.bias = None++ self.trainable = trainable++ def post_init(self):+ pass++ def pack(self, linear, scales, zeros, g_idx=None):+ W = linear.weight.data.clone()+ if isinstance(linear, nn.Conv2d):+ W = W.flatten(1)+ if isinstance(linear, transformers.pytorch_utils.Conv1D):+ W = W.t()++ self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx++ scales = scales.t().contiguous()+ zeros = zeros.t().contiguous()+ scale_zeros = zeros * scales+ self.scales = scales.clone().half()+ if linear.bias is not None:+ self.bias = linear.bias.clone().half()++ intweight = []+ for idx in range(self.infeatures):+ intweight.append(+ torch.round(+ (W[:, idx] + scale_zeros[self.g_idx[idx]])+ / self.scales[self.g_idx[idx]]+ ).to(torch.int)[:, None]+ )+ intweight = torch.cat(intweight, dim=1)+ intweight = intweight.t().contiguous()+ intweight = intweight.numpy().astype(np.uint32)++ i = 0+ row = 0+ qweight = np.zeros(+ (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32+ )+ while row < qweight.shape[0]:+ if self.bits in [2, 4, 8]:+ for j in range(i, i + (32 // self.bits)):+ qweight[row] |= intweight[j] << (self.bits * (j - i))+ i += 32 // self.bits+ row += 1+ else:+ raise NotImplementedError("Only 2,4,8 bits are supported.")++ qweight = qweight.astype(np.int32)+ self.qweight = torch.from_numpy(qweight)++ zeros -= 1+ zeros = zeros.numpy().astype(np.uint32)+ qzeros = np.zeros(+ (zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32+ )+ i = 0+ col = 0+ while col < qzeros.shape[1]:+ if self.bits in [2, 4, 8]:+ for j in range(i, i + (32 // self.bits)):+ qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))+ i += 32 // self.bits+ col += 1+ else:+ raise NotImplementedError("Only 2,4,8 bits are supported.")++ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)++ def forward(self, x):+ out_shape = x.shape[:-1] + (self.outfeatures,)+ quant_linear_fn = QuantLinearFunction++ out = quant_linear_fn.apply(+ x.reshape(-1, x.shape[-1]),+ self.qweight,+ self.scales,+ self.qzeros,+ self.g_idx,+ self.bits,+ self.maxq,+ )+ out = out.half().reshape(out_shape)+ out = out + self.bias if self.bias is not None else out+ return out.to(x.dtype)++ @classmethod+ def warmup(cls, model, transpose=False, seqlen=2048):+ """+ Pre-tunes the quantized kernel+ """+ from tqdm import tqdm++ kn_values = {}++ for _, m in model.named_modules():+ if not isinstance(m, cls):+ continue++ k = m.infeatures+ n = m.outfeatures++ if (k, n) not in kn_values:+ kn_values[(k, n)] = (+ m.qweight,+ m.scales,+ m.qzeros,+ m.g_idx,+ m.bits,+ m.maxq,+ )++ logger.info(f"Found {len(kn_values)} unique KN Linear values.")+ logger.info("Warming up autotune cache ...")+ with torch.no_grad():+ for m in tqdm(range(0, math.ceil(math.log2(seqlen)) + 1)):+ m = 2**m+ for (k, n), (+ qweight,+ scales,+ qzeros,+ g_idx,+ bits,+ maxq,+ ) in kn_values.items():+ a = torch.randn(m, k, dtype=torch.float16, device=model.device)+ quant_matmul_248(a, qweight, scales, qzeros, g_idx, bits, maxq)+ del kn_values+++__all__ = ["QuantLinear"]\ No newline at end of filediff --git a/auto_round_extension/cuda/triton_utils/dequant.py b/auto_round_extension/cuda/triton_utils/dequant.pyindex b7c6316de..d971059d3 100644--- a/auto_round_extension/cuda/triton_utils/dequant.py+++ b/auto_round_extension/cuda/triton_utils/dequant.py@@ -54,17 +54,17 @@ DEFAULT_DEQUANT_CONFIGS = make_dequant_configs([128, 256, 512, 1024], [4, 8])@triton.autotune(DEFAULT_DEQUANT_CONFIGS, key=["numels"])@triton.jitdef dequant_kernel_248(- g_idx_ptr,- scales_ptr,- qweight_ptr,- qzeros_ptr,- out_ptr,- numels,- maxq: tl.constexpr,- bits: tl.constexpr,- outfeatures: tl.constexpr,- num_groups: tl.constexpr,- X_BLOCK: tl.constexpr,+ g_idx_ptr,+ scales_ptr,+ qweight_ptr,+ qzeros_ptr,+ out_ptr,+ numels,+ maxq: tl.constexpr,+ bits: tl.constexpr,+ outfeatures: tl.constexpr,+ num_groups: tl.constexpr,+ X_BLOCK: tl.constexpr,):# Block indexingxoffset = tl.program_id(0) * X_BLOCK@@ -110,7 +110,7 @@ def dequant_kernel_248(zeros = qzeros >> wf_zeroszeros = zeros & maxq- # Dequantize+ # # ##Dequantize# zeros = zeros + 1weights = weights - zerosweights = weights.to(tl.float32)@@ -119,7 +119,7 @@ def dequant_kernel_248(tl.store(out_ptr + (x_index), weights, mask=xmask)-def dequant248(qweight, scales, qzeros, g_idx, bits, maxq=None):+def dequant248(qweight, scales, qzeros, g_idx, bits, maxq=None, input_dtype=torch.float16):"""Launcher for triton dequant kernel. Only valid for bits = 2, 4, 8"""@@ -128,9 +128,9 @@ def dequant248(qweight, scales, qzeros, g_idx, bits, maxq=None):outfeatures = scales.shape[1]infeatures = g_idx.shape[0]- out = torch.empty((infeatures, outfeatures), device="cuda", dtype=torch.float16)+ out = torch.empty((infeatures, outfeatures), device="cuda", dtype=input_dtype)numels = out.numel()- maxq = 2**bits - 1 if maxq is None else maxq+ maxq = 2 ** bits - 1 if maxq is None else maxqgrid = lambda meta: (triton.cdiv(numels, meta["X_BLOCK"]),) # noqa: E731dequant_kernel_248[grid](@@ -149,12 +149,15 @@ def dequant248(qweight, scales, qzeros, g_idx, bits, maxq=None):def quant_matmul_248(- input, qweight, scales, qzeros, g_idx, bits, maxq=None, transpose=False+ input, qweight, scales, qzeros, g_idx, bits, maxq=None, transpose=False):- W = dequant248(qweight, scales, qzeros, g_idx, bits, maxq=maxq)+ input_dtype = input.dtype+ W = dequant248(qweight, scales, qzeros, g_idx, bits, maxq=maxq, input_dtype=input_dtype)+ orig_device = input.deviceif transpose:- return input @ W.t()- return input @ W+ return (input.to(W.device) @ W.t()).to(orig_device)++ return (input.to(W.device) @ W).to(orig_device)class QuantLinearFunction(torch.autograd.Function):diff --git a/auto_round_extension/cuda/triton_utils_zp/__init__.py b/auto_round_extension/cuda/triton_utils_zp/__init__.pynew file mode 100644index 000000000..2045808ac--- /dev/null+++ b/auto_round_extension/cuda/triton_utils_zp/__init__.py@@ -0,0 +1,14 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+diff --git a/auto_round_extension/cuda/triton_utils_zp/custom_autotune.py b/auto_round_extension/cuda/triton_utils_zp/custom_autotune.pynew file mode 100644index 000000000..b511579cc--- /dev/null+++ b/auto_round_extension/cuda/triton_utils_zp/custom_autotune.py@@ -0,0 +1,254 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# MIT License+#+# Copyright (c) 2023 潘其威(William)+#+# Permission is hereby granted, free of charge, to any person obtaining a copy+# of this software and associated documentation files (the "Software"), to deal+# in the Software without restriction, including without limitation the rights+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+# copies of the Software, and to permit persons to whom the Software is+# furnished to do so, subject to the following conditions:+#+# The above copyright notice and this permission notice shall be included in all+# copies or substantial portions of the Software.+#+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+# SOFTWARE.+import builtins+import math+import time+from typing import Dict++import triton+++# code based https://github.com/fpgaminer/GPTQ-triton+"""+Mostly the same as the autotuner in Triton, but with a few changes like using 40 runs instead of 100.+"""+++class CustomizedTritonAutoTuner(triton.KernelInterface):+ def __init__(+ self,+ fn,+ arg_names,+ configs,+ key,+ reset_to_zero,+ prune_configs_by: Dict = None,+ nearest_power_of_two: bool = False,+ ):+ if not configs:+ self.configs = [triton.Config({}, num_warps=4, num_stages=2)]+ else:+ self.configs = configs+ self.key_idx = [arg_names.index(k) for k in key]+ self.nearest_power_of_two = nearest_power_of_two+ self.cache = {}+ # hook to reset all required tensor to zeros before relaunching a kernel+ self.hook = lambda args: 0+ if reset_to_zero is not None:+ self.reset_idx = [arg_names.index(k) for k in reset_to_zero]++ def _hook(args):+ for i in self.reset_idx:+ args[i].zero_()++ self.hook = _hook+ self.arg_names = arg_names+ # prune configs+ if prune_configs_by:+ perf_model, top_k = (+ prune_configs_by["perf_model"],+ prune_configs_by["top_k"],+ )+ if "early_config_prune" in prune_configs_by:+ early_config_prune = prune_configs_by["early_config_prune"]+ else:+ perf_model, top_k, early_config_prune = None, None, None+ self.perf_model, self.configs_top_k = perf_model, top_k+ self.early_config_prune = early_config_prune+ self.fn = fn++ def _bench(self, *args, config, **meta):+ # check for conflicts, i.e. meta-parameters both provided+ # as kwargs and by the autotuner+ conflicts = meta.keys() & config.kwargs.keys()+ if conflicts:+ raise ValueError(+ f"Conflicting meta-parameters: {', '.join(conflicts)}."+ " Make sure that you don't re-define auto-tuned symbols."+ )+ # augment meta-parameters with tunable ones+ current = dict(meta, **config.kwargs)++ def kernel_call():+ if config.pre_hook:+ config.pre_hook(self.nargs)+ self.hook(args)+ self.fn.run(+ *args,+ num_warps=config.num_warps,+ num_stages=config.num_stages,+ **current,+ )++ try:+ # In testings using only 40 reps seems to be close enough and it appears to be what PyTorch uses+ # PyTorch also sets fast_flush to True, but I didn't see any speedup so I'll leave the default+ return triton.testing.do_bench(kernel_call, quantiles=(0.5, 0.2, 0.8), rep=40)+ except triton.OutOfResources:+ return (float("inf"), float("inf"), float("inf"))++ def run(self, *args, **kwargs):+ self.nargs = dict(zip(self.arg_names, args))+ if len(self.configs) > 1:+ key = tuple(args[i] for i in self.key_idx)++ # This reduces the amount of autotuning by rounding the keys to the nearest power of two+ # In my testing this gives decent results, and greatly reduces the amount of tuning required+ if self.nearest_power_of_two:+ key = tuple([2 ** int(math.log2(x) + 0.5) for x in key])++ if key not in self.cache:+ # prune configs+ pruned_configs = self.prune_configs(kwargs)+ bench_start = time.time()+ timings = {config: self._bench(*args, config=config, **kwargs) for config in pruned_configs}+ bench_end = time.time()+ self.bench_time = bench_end - bench_start+ self.cache[key] = builtins.min(timings, key=timings.get)+ self.hook(args)+ self.configs_timings = timings+ config = self.cache[key]+ else:+ config = self.configs[0]+ self.best_config = config+ if config.pre_hook is not None:+ config.pre_hook(self.nargs)+ return self.fn.run(+ *args,+ num_warps=config.num_warps,+ num_stages=config.num_stages,+ **kwargs,+ **config.kwargs,+ )++ def prune_configs(self, kwargs):+ pruned_configs = self.configs+ if self.early_config_prune:+ pruned_configs = self.early_config_prune(self.configs, self.nargs)+ if self.perf_model:+ top_k = self.configs_top_k+ if isinstance(top_k, float) and top_k <= 1.0:+ top_k = int(len(self.configs) * top_k)+ if len(pruned_configs) > top_k:+ est_timing = {+ config: self.perf_model(+ **self.nargs,+ **kwargs,+ **config.kwargs,+ num_stages=config.num_stages,+ num_warps=config.num_warps,+ )+ for config in pruned_configs+ }+ pruned_configs = sorted(est_timing.keys(), key=lambda x: est_timing[x])[:top_k]+ return pruned_configs++ def warmup(self, *args, **kwargs):+ self.nargs = dict(zip(self.arg_names, args))+ for config in self.prune_configs(kwargs):+ self.fn.warmup(+ *args,+ num_warps=config.num_warps,+ num_stages=config.num_stages,+ **kwargs,+ **config.kwargs,+ )+ self.nargs = None+++def autotune(configs, key, prune_configs_by=None, reset_to_zero=None, nearest_power_of_two=False):+ def decorator(fn):+ return CustomizedTritonAutoTuner(+ fn,+ fn.arg_names,+ configs,+ key,+ reset_to_zero,+ prune_configs_by,+ nearest_power_of_two,+ )++ return decorator+++def matmul248_kernel_config_pruner(configs, nargs):+ """+ The main purpose of this function is to shrink BLOCK_SIZE_* when the corresponding dimension is smaller.+ """+ m = max(2 ** int(math.ceil(math.log2(nargs["M"]))), 16)+ n = max(2 ** int(math.ceil(math.log2(nargs["N"]))), 16)+ k = max(2 ** int(math.ceil(math.log2(nargs["K"]))), 16)++ used = set()+ for config in configs:+ block_size_m = min(m, config.kwargs["BLOCK_SIZE_M"])+ block_size_n = min(n, config.kwargs["BLOCK_SIZE_N"])+ block_size_k = min(k, config.kwargs["BLOCK_SIZE_K"])+ group_size_m = config.kwargs["GROUP_SIZE_M"]++ if (+ block_size_m,+ block_size_n,+ block_size_k,+ group_size_m,+ config.num_stages,+ config.num_warps,+ ) in used:+ continue++ used.add(+ (+ block_size_m,+ block_size_n,+ block_size_k,+ group_size_m,+ config.num_stages,+ config.num_warps,+ )+ )+ yield triton.Config(+ {+ "BLOCK_SIZE_M": block_size_m,+ "BLOCK_SIZE_N": block_size_n,+ "BLOCK_SIZE_K": block_size_k,+ "GROUP_SIZE_M": group_size_m,+ },+ num_stages=config.num_stages,+ num_warps=config.num_warps,+ )+++__all__ = ["autotune"]diff --git a/auto_round_extension/cuda/triton_utils_zp/dequant.py b/auto_round_extension/cuda/triton_utils_zp/dequant.pynew file mode 100644index 000000000..4ef9936ce--- /dev/null+++ b/auto_round_extension/cuda/triton_utils_zp/dequant.py@@ -0,0 +1,183 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# MIT License+#+# Copyright (c) 2023 潘其威(William)+#+# Permission is hereby granted, free of charge, to any person obtaining a copy+# of this software and associated documentation files (the "Software"), to deal+# in the Software without restriction, including without limitation the rights+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+# copies of the Software, and to permit persons to whom the Software is+# furnished to do so, subject to the following conditions:+#+# The above copyright notice and this permission notice shall be included in all+# copies or substantial portions of the Software.+#+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+# SOFTWARE.+import itertools++import torch+import triton+import triton.language as tl+from torch.cuda.amp import custom_bwd, custom_fwd+++def make_dequant_configs(block_sizes, num_warps):+ configs = []+ for bs, ws in itertools.product(block_sizes, num_warps):+ configs.append(triton.Config({"X_BLOCK": bs}, num_warps=ws))+ return configs+++DEFAULT_DEQUANT_CONFIGS = make_dequant_configs([128, 256, 512, 1024], [4, 8])++[email protected](DEFAULT_DEQUANT_CONFIGS, key=["numels"])+def dequant_kernel_248(+ g_idx_ptr,+ scales_ptr,+ qweight_ptr,+ qzeros_ptr,+ out_ptr,+ numels,+ maxq: tl.constexpr,+ bits: tl.constexpr,+ outfeatures: tl.constexpr,+ num_groups: tl.constexpr,+ X_BLOCK: tl.constexpr,+):+ # Block indexing+ xoffset = tl.program_id(0) * X_BLOCK+ x_index = xoffset + tl.arange(0, X_BLOCK)+ xmask = x_index < numels+ row_idx = x_index // outfeatures+ col_idx = x_index % outfeatures++ elements_per_feature: tl.constexpr = 32 // bits++ # Load parameters+ g_idx = tl.load(g_idx_ptr + (row_idx), None, eviction_policy="evict_last")+ qweights = tl.load(+ qweight_ptr + (col_idx + (outfeatures * (row_idx // elements_per_feature))),+ None,+ )++ wf_weights = (row_idx % elements_per_feature) * bits++ wf_zeros = (col_idx % elements_per_feature) * bits++ tmp1 = g_idx + num_groups+ tmp2 = g_idx < 0+ tl.device_assert(g_idx >= 0, "index out of bounds: 0 <= tmp0 < 0")+ groups = tl.where(tmp2, tmp1, g_idx) # tmp3 are g_idx++ scales = tl.load(scales_ptr + (col_idx + (outfeatures * groups)), None).to(+ tl.float32+ )++ # Unpack weights+ weights = qweights >> wf_weights # bit shift qweight++ weights = weights & maxq++ # Unpack zeros+ qzero_ncols: tl.constexpr = outfeatures // elements_per_feature+ qzeros = tl.load(+ qzeros_ptr + ((qzero_ncols * groups) + (col_idx // elements_per_feature)),+ None,+ eviction_policy="evict_last",+ )+ zeros = qzeros >> wf_zeros+ zeros = zeros & maxq++ ##Dequantize+ zeros = zeros + 1+ weights = weights - zeros+ weights = weights.to(tl.float32)+ weights = scales * weights++ tl.store(out_ptr + (x_index), weights, mask=xmask)+++def dequant248(qweight, scales, qzeros, g_idx, bits, maxq=None, input_dtype=torch.float16):+ """+ Launcher for triton dequant kernel. Only valid for bits = 2, 4, 8+ """++ num_groups = scales.shape[0]+ outfeatures = scales.shape[1]+ infeatures = g_idx.shape[0]++ out = torch.empty((infeatures, outfeatures), device="cuda", dtype=input_dtype)+ numels = out.numel()+ maxq = 2 ** bits - 1 if maxq is None else maxq+ grid = lambda meta: (triton.cdiv(numels, meta["X_BLOCK"]),) # noqa: E731++ dequant_kernel_248[grid](+ g_idx,+ scales,+ qweight,+ qzeros,+ out,+ numels,+ maxq=maxq,+ bits=bits,+ outfeatures=outfeatures,+ num_groups=num_groups,+ )+ return out+++def quant_matmul_248(+ input, qweight, scales, qzeros, g_idx, bits, maxq=None, transpose=False+):+ input_dtype = input.dtype+ W = dequant248(qweight, scales, qzeros, g_idx, bits, maxq=maxq, input_dtype=input_dtype)+ orig_device = input.device+ if transpose:+ return (input.to(W.device) @ W.t()).to(orig_device)++ return (input.to(W.device) @ W).to(orig_device)+++class QuantLinearFunction(torch.autograd.Function):+ @staticmethod+ @custom_fwd+ def forward(ctx, input, qweight, scales, qzeros, g_idx, bits, maxq):+ output = quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxq)+ ctx.save_for_backward(qweight, scales, qzeros, g_idx)+ ctx.bits, ctx.maxq = bits, maxq+ return output++ @staticmethod+ @custom_bwd+ def backward(ctx, grad_output):+ qweight, scales, qzeros, g_idx = ctx.saved_tensors+ bits, maxq = ctx.bits, ctx.maxq+ grad_input = None++ if ctx.needs_input_grad[0]:+ grad_input = quant_matmul_248(+ grad_output, qweight, scales, qzeros, g_idx, bits, maxq, transpose=True+ )+ return grad_input, None, None, None, None, None, Nonediff --git a/auto_round_extension/cuda/triton_utils_zp/kernels.py b/auto_round_extension/cuda/triton_utils_zp/kernels.pynew file mode 100644index 000000000..44981c7a3--- /dev/null+++ b/auto_round_extension/cuda/triton_utils_zp/kernels.py@@ -0,0 +1,500 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# MIT License+#+# Copyright (c) 2023 潘其威(William)+#+# Permission is hereby granted, free of charge, to any person obtaining a copy+# of this software and associated documentation files (the "Software"), to deal+# in the Software without restriction, including without limitation the rights+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+# copies of the Software, and to permit persons to whom the Software is+# furnished to do so, subject to the following conditions:+#+# The above copyright notice and this permission notice shall be included in all+# copies or substantial portions of the Software.+#+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+# SOFTWARE.++from logging import getLogger++import torch+import triton+import triton.language as tl+from torch.cuda.amp import custom_bwd, custom_fwd++from . import custom_autotune+++logger = getLogger(__name__)+++# code based https://github.com/fpgaminer/GPTQ-triton+++@custom_autotune.autotune(+ configs=[+ triton.Config(+ {+ "BLOCK_SIZE_M": 64,+ "BLOCK_SIZE_N": 256,+ "BLOCK_SIZE_K": 32,+ "GROUP_SIZE_M": 8,+ },+ num_stages=4,+ num_warps=4,+ ),+ triton.Config(+ {+ "BLOCK_SIZE_M": 128,+ "BLOCK_SIZE_N": 128,+ "BLOCK_SIZE_K": 32,+ "GROUP_SIZE_M": 8,+ },+ num_stages=4,+ num_warps=4,+ ),+ triton.Config(+ {+ "BLOCK_SIZE_M": 64,+ "BLOCK_SIZE_N": 128,+ "BLOCK_SIZE_K": 32,+ "GROUP_SIZE_M": 8,+ },+ num_stages=4,+ num_warps=4,+ ),+ triton.Config(+ {+ "BLOCK_SIZE_M": 128,+ "BLOCK_SIZE_N": 32,+ "BLOCK_SIZE_K": 32,+ "GROUP_SIZE_M": 8,+ },+ num_stages=4,+ num_warps=4,+ ),+ triton.Config(+ {+ "BLOCK_SIZE_M": 64,+ "BLOCK_SIZE_N": 64,+ "BLOCK_SIZE_K": 32,+ "GROUP_SIZE_M": 8,+ },+ num_stages=4,+ num_warps=4,+ ),+ triton.Config(+ {+ "BLOCK_SIZE_M": 64,+ "BLOCK_SIZE_N": 128,+ "BLOCK_SIZE_K": 32,+ "GROUP_SIZE_M": 8,+ },+ num_stages=2,+ num_warps=8,+ ),+ ],+ key=["M", "N", "K"],+ nearest_power_of_two=True,+ prune_configs_by={+ "early_config_prune": custom_autotune.matmul248_kernel_config_pruner,+ "perf_model": None,+ "top_k": None,+ },+)+def quant_matmul_248_kernel(+ a_ptr,+ b_ptr,+ c_ptr,+ scales_ptr,+ zeros_ptr,+ g_ptr,+ M,+ N,+ K,+ bits,+ maxq,+ stride_am,+ stride_ak,+ stride_bk,+ stride_bn,+ stride_cm,+ stride_cn,+ stride_scales,+ stride_zeros,+ BLOCK_SIZE_M: tl.constexpr,+ BLOCK_SIZE_N: tl.constexpr,+ BLOCK_SIZE_K: tl.constexpr,+ GROUP_SIZE_M: tl.constexpr,+):+ """+ Compute the matrix multiplication C = A x B.+ A is of shape (M, K) float16+ B is of shape (K//8, N) int32+ C is of shape (M, N) float16+ scales is of shape (G, N) float16+ zeros is of shape (G, N) float16+ g_ptr is of shape (K) int32+ """+ infearure_per_bits = 32 // bits++ pid = tl.program_id(axis=0)+ num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)+ num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)+ num_pid_k = tl.cdiv(K, BLOCK_SIZE_K)+ num_pid_in_group = GROUP_SIZE_M * num_pid_n+ group_id = pid // num_pid_in_group+ first_pid_m = group_id * GROUP_SIZE_M+ group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)+ pid_m = first_pid_m + (pid % group_size_m)+ pid_n = (pid % num_pid_in_group) // group_size_m++ offs_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)+ offs_bn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)+ offs_k = tl.arange(0, BLOCK_SIZE_K)+ a_ptrs = a_ptr + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) # (BLOCK_SIZE_M, BLOCK_SIZE_K)+ a_mask = offs_am[:, None] < M+ # b_ptrs is set up such that it repeats elements along the K axis 8 times+ b_ptrs = b_ptr + (+ (offs_k[:, None] // infearure_per_bits) * stride_bk + offs_bn[None, :] * stride_bn+ ) # (BLOCK_SIZE_K, BLOCK_SIZE_N)+ g_ptrs = g_ptr + offs_k+ # shifter is used to extract the N bits of each element in the 32-bit word from B+ scales_ptrs = scales_ptr + offs_bn[None, :]+ zeros_ptrs = zeros_ptr + (offs_bn[None, :] // infearure_per_bits)++ shifter = (offs_k % infearure_per_bits) * bits+ zeros_shifter = (offs_bn % infearure_per_bits) * bits+ accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)++ for k in range(0, num_pid_k):+ g_idx = tl.load(g_ptrs)++ # Fetch scales and zeros; these are per-outfeature and thus reused in the inner loop+ scales = tl.load(scales_ptrs + g_idx[:, None] * stride_scales) # (BLOCK_SIZE_K, BLOCK_SIZE_N,)+ zeros = tl.load(zeros_ptrs + g_idx[:, None] * stride_zeros) # (BLOCK_SIZE_K, BLOCK_SIZE_N,)++ zeros = (zeros >> zeros_shifter[None, :]) & maxq+ # zeros = zeros + 1++ a = tl.load(a_ptrs, mask=a_mask, other=0.0) # (BLOCK_SIZE_M, BLOCK_SIZE_K)+ b = tl.load(b_ptrs) # (BLOCK_SIZE_K, BLOCK_SIZE_N), but repeated++ # Now we need to unpack b (which is N-bit values) into 32-bit values+ b = (b >> shifter[:, None]) & maxq # Extract the N-bit values+ b = (b - zeros) * scales # Scale and shift++ accumulator += tl.dot(a, b)+ a_ptrs += BLOCK_SIZE_K+ b_ptrs += (BLOCK_SIZE_K // infearure_per_bits) * stride_bk+ g_ptrs += BLOCK_SIZE_K++ c_ptrs = c_ptr + stride_cm * offs_am[:, None] + stride_cn * offs_bn[None, :]+ c_mask = (offs_am[:, None] < M) & (offs_bn[None, :] < N)+ tl.store(c_ptrs, accumulator, mask=c_mask)+++@custom_autotune.autotune(+ configs=[+ triton.Config(+ {+ "BLOCK_SIZE_M": 64,+ "BLOCK_SIZE_N": 32,+ "BLOCK_SIZE_K": 256,+ "GROUP_SIZE_M": 8,+ },+ num_stages=4,+ num_warps=4,+ ),+ triton.Config(+ {+ "BLOCK_SIZE_M": 128,+ "BLOCK_SIZE_N": 32,+ "BLOCK_SIZE_K": 128,+ "GROUP_SIZE_M": 8,+ },+ num_stages=4,+ num_warps=4,+ ),+ triton.Config(+ {+ "BLOCK_SIZE_M": 64,+ "BLOCK_SIZE_N": 32,+ "BLOCK_SIZE_K": 128,+ "GROUP_SIZE_M": 8,+ },+ num_stages=4,+ num_warps=4,+ ),+ triton.Config(+ {+ "BLOCK_SIZE_M": 128,+ "BLOCK_SIZE_N": 32,+ "BLOCK_SIZE_K": 32,+ "GROUP_SIZE_M": 8,+ },+ num_stages=4,+ num_warps=4,+ ),+ triton.Config(+ {+ "BLOCK_SIZE_M": 64,+ "BLOCK_SIZE_N": 32,+ "BLOCK_SIZE_K": 64,+ "GROUP_SIZE_M": 8,+ },+ num_stages=4,+ num_warps=4,+ ),+ triton.Config(+ {+ "BLOCK_SIZE_M": 64,+ "BLOCK_SIZE_N": 32,+ "BLOCK_SIZE_K": 128,+ "GROUP_SIZE_M": 8,+ },+ num_stages=2,+ num_warps=8,+ ),+ ],+ key=["M", "N", "K"],+ nearest_power_of_two=True,+)+def transpose_quant_matmul_248_kernel(+ a_ptr,+ b_ptr,+ c_ptr,+ scales_ptr,+ zeros_ptr,+ g_ptr,+ M,+ N,+ K,+ bits,+ maxq,+ stride_am,+ stride_ak,+ stride_bk,+ stride_bn,+ stride_cm,+ stride_cn,+ stride_scales,+ stride_zeros,+ BLOCK_SIZE_M: tl.constexpr,+ BLOCK_SIZE_N: tl.constexpr,+ BLOCK_SIZE_K: tl.constexpr,+ GROUP_SIZE_M: tl.constexpr,+):+ """+ Compute the matrix multiplication C = A x B.+ A is of shape (M, N) float16+ B is of shape (K//8, N) int32+ C is of shape (M, K) float16+ scales is of shape (G, N) float16+ zeros is of shape (G, N) float16+ g_ptr is of shape (K) int32+ """+ infearure_per_bits = 32 // bits++ pid = tl.program_id(axis=0)+ num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)+ num_pid_k = tl.cdiv(K, BLOCK_SIZE_K)+ num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)+ num_pid_in_group = GROUP_SIZE_M * num_pid_k+ group_id = pid // num_pid_in_group+ first_pid_m = group_id * GROUP_SIZE_M+ group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)+ pid_m = first_pid_m + (pid % group_size_m)+ pid_k = (pid % num_pid_in_group) // group_size_m++ offs_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)+ offs_bk = pid_k * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K)+ offs_n = tl.arange(0, BLOCK_SIZE_N)+ a_ptrs = a_ptr + (offs_am[:, None] * stride_am + offs_n[None, :] * stride_ak) # (BLOCK_SIZE_M, BLOCK_SIZE_N)+ a_mask = offs_am[:, None] < M+ # b_ptrs is set up such that it repeats elements along the K axis 8 times+ b_ptrs = b_ptr + (+ (offs_bk[:, None] // infearure_per_bits) * stride_bk + offs_n[None, :] * stride_bn+ ) # (BLOCK_SIZE_K, BLOCK_SIZE_N)+ g_ptrs = g_ptr + offs_bk+ g_idx = tl.load(g_ptrs)++ # shifter is used to extract the N bits of each element in the 32-bit word from B+ scales_ptrs = scales_ptr + offs_n[None, :] + g_idx[:, None] * stride_scales+ zeros_ptrs = zeros_ptr + (offs_n[None, :] // infearure_per_bits) + g_idx[:, None] * stride_zeros++ shifter = (offs_bk % infearure_per_bits) * bits+ zeros_shifter = (offs_n % infearure_per_bits) * bits+ accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_K), dtype=tl.float32)++ for k in range(0, num_pid_n):+ # Fetch scales and zeros; these are per-outfeature and thus reused in the inner loop+ scales = tl.load(scales_ptrs) # (BLOCK_SIZE_K, BLOCK_SIZE_N,)+ zeros = tl.load(zeros_ptrs) # (BLOCK_SIZE_K, BLOCK_SIZE_N,)++ zeros = (zeros >> zeros_shifter[None, :]) & maxq+ # zeros = zeros + 1++ a = tl.load(a_ptrs, mask=a_mask, other=0.0) # (BLOCK_SIZE_M, BLOCK_SIZE_N)+ b = tl.load(b_ptrs) # (BLOCK_SIZE_K, BLOCK_SIZE_N), but repeated++ # Now we need to unpack b (which is N-bit values) into 32-bit values+ b = (b >> shifter[:, None]) & maxq # Extract the N-bit values+ b = (b - zeros) * scales # Scale and shift+ b = tl.trans(b)++ accumulator += tl.dot(a, b)+ a_ptrs += BLOCK_SIZE_N+ b_ptrs += BLOCK_SIZE_N+ scales_ptrs += BLOCK_SIZE_N+ zeros_ptrs += BLOCK_SIZE_N // infearure_per_bits++ c_ptrs = c_ptr + stride_cm * offs_am[:, None] + stride_cn * offs_bk[None, :]+ c_mask = (offs_am[:, None] < M) & (offs_bk[None, :] < K)+ tl.store(c_ptrs, accumulator, mask=c_mask)+++def silu(x):+ return x * tl.sigmoid(x)+++def quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxq):+ with torch.cuda.device(input.device):+ output = torch.empty((input.shape[0], qweight.shape[1]), device=input.device, dtype=input.dtype)+ grid = lambda META: ( # noqa: E731+ triton.cdiv(input.shape[0], META["BLOCK_SIZE_M"]) * triton.cdiv(qweight.shape[1], META["BLOCK_SIZE_N"]),+ )+ quant_matmul_248_kernel[grid](+ input,+ qweight,+ output,+ scales.to(input.dtype),+ qzeros,+ g_idx,+ input.shape[0],+ qweight.shape[1],+ input.shape[1],+ bits,+ maxq,+ input.stride(0),+ input.stride(1),+ qweight.stride(0),+ qweight.stride(1),+ output.stride(0),+ output.stride(1),+ scales.stride(0),+ qzeros.stride(0),+ )+ return output+++def transpose_quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxq):+ with torch.cuda.device(input.device):+ output_dim = (qweight.shape[0] * 32) // bits+ output = torch.empty((input.shape[0], output_dim), device=input.device, dtype=input.dtype)+ grid = lambda META: ( # noqa: E731+ triton.cdiv(input.shape[0], META["BLOCK_SIZE_M"]) * triton.cdiv(output_dim, META["BLOCK_SIZE_K"]),+ )+ transpose_quant_matmul_248_kernel[grid](+ input,+ qweight,+ output,+ scales.to(input.dtype),+ qzeros,+ g_idx,+ input.shape[0],+ qweight.shape[1],+ output_dim,+ bits,+ maxq,+ input.stride(0),+ input.stride(1),+ qweight.stride(0),+ qweight.stride(1),+ output.stride(0),+ output.stride(1),+ scales.stride(0),+ qzeros.stride(0),+ )+ return output+++class QuantLinearFunction(torch.autograd.Function):+ @staticmethod+ @custom_fwd+ def forward(ctx, input, qweight, scales, qzeros, g_idx, bits, maxq):+ output = quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxq)+ ctx.save_for_backward(qweight, scales, qzeros, g_idx)+ ctx.bits, ctx.maxq = bits, maxq+ return output++ @staticmethod+ @custom_bwd+ def backward(ctx, grad_output):+ qweight, scales, qzeros, g_idx = ctx.saved_tensors+ bits, maxq = ctx.bits, ctx.maxq+ grad_input = None++ if ctx.needs_input_grad[0]:+ grad_input = transpose_quant_matmul_248(grad_output, qweight, scales, qzeros, g_idx, bits, maxq)+ return grad_input, None, None, None, None, None, None+++def quant_matmul_inference_only_248(input, qweight, scales, qzeros, g_idx, bits, maxq):+ with torch.cuda.device(input.device):+ output = torch.empty((input.shape[0], qweight.shape[1]), device=input.device, dtype=torch.float16)+ grid = lambda META: ( # noqa: E731+ triton.cdiv(input.shape[0], META["BLOCK_SIZE_M"]) * triton.cdiv(qweight.shape[1], META["BLOCK_SIZE_N"]),+ )+ quant_matmul_248_kernel[grid](+ input,+ qweight,+ output,+ scales,+ qzeros,+ g_idx,+ input.shape[0],+ qweight.shape[1],+ input.shape[1],+ bits,+ maxq,+ input.stride(0),+ input.stride(1),+ qweight.stride(0),+ qweight.stride(1),+ output.stride(0),+ output.stride(1),+ scales.stride(0),+ qzeros.stride(0),+ )+ return output+++class QuantLinearInferenceOnlyFunction(torch.autograd.Function):+ @staticmethod+ @custom_fwd(cast_inputs=torch.float16)+ def forward(ctx, input, qweight, scales, qzeros, g_idx, bits, maxq):+ output = quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxq)+ return outputdiff --git a/auto_round_extension/cuda/triton_utils_zp/mixin.py b/auto_round_extension/cuda/triton_utils_zp/mixin.pynew file mode 100644index 000000000..557d3b480--- /dev/null+++ b/auto_round_extension/cuda/triton_utils_zp/mixin.py@@ -0,0 +1,39 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# MIT License+#+# Copyright (c) 2023 潘其威(William)+#+# Permission is hereby granted, free of charge, to any person obtaining a copy+# of this software and associated documentation files (the "Software"), to deal+# in the Software without restriction, including without limitation the rights+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+# copies of the Software, and to permit persons to whom the Software is+# furnished to do so, subject to the following conditions:+#+# The above copyright notice and this permission notice shall be included in all+# copies or substantial portions of the Software.+#+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+# SOFTWARE.+class TritonModuleMixin:+ @classmethod+ def warmup(cls, model, transpose=False, seqlen=2048):+ passdiff --git a/auto_round_extension/ipex/qlinear_ipex_awq.py b/auto_round_extension/ipex/qlinear_ipex_awq.pyindex d224234e7..b03d2104f 100644--- a/auto_round_extension/ipex/qlinear_ipex_awq.py+++ b/auto_round_extension/ipex/qlinear_ipex_awq.py@@ -2,7 +2,7 @@ import torchimport torch.nn as nnclass QuantLinear(nn.Module):-+ QUANT_TYPE = "ipex_awq"def __init__(self, w_bit, group_size, in_features, out_features, bias, dev):super().__init__()assert w_bit == 4, "Only 4 bit are supported for now."diff --git a/auto_round_extension/ipex/qlinear_ipex_gptq.py b/auto_round_extension/ipex/qlinear_ipex_gptq.pyindex c486638ae..042f0226f 100644--- a/auto_round_extension/ipex/qlinear_ipex_gptq.py+++ b/auto_round_extension/ipex/qlinear_ipex_gptq.py@@ -78,10 +78,10 @@ class QuantLinear(nn.Module):dtype=weight_dtype,),)- self.register_buffer(- "g_idx",- torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),- )+ # self.register_buffer(+ # "g_idx",+ # torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),+ # )if bias:self.register_buffer("bias", torch.zeros((outfeatures), dtype=weight_dtype))else:@@ -172,7 +172,7 @@ class QuantLinear(nn.Module):if isinstance(linear, transformers.pytorch_utils.Conv1D):W = W.t()- self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx+ # self.g_idx = g_idx.clone() if g_idx is not None else self.g_idxscales = scales.t().contiguous()zeros = zeros.t().contiguous()diff --git a/auto_round_extension/qbits/qbits_awq.py b/auto_round_extension/qbits/qbits_awq.pyindex 82110e934..0b3adefb0 100644--- a/auto_round_extension/qbits/qbits_awq.py+++ b/auto_round_extension/qbits/qbits_awq.py@@ -68,6 +68,7 @@ def convert_dtype_torch2str(dtype):class QuantLinear(nn.Module):+ QUANT_TYPE = "qbits_awq"def __init__(self, w_bit, group_size, in_features, out_features, bias, zero_point, dev):super().__init__()diff --git a/auto_round_extension/qbits/qlinear_qbits_gptq.py b/auto_round_extension/qbits/qlinear_qbits_gptq.pyindex 658adead7..0afca0288 100644--- a/auto_round_extension/qbits/qlinear_qbits_gptq.py+++ b/auto_round_extension/qbits/qlinear_qbits_gptq.py@@ -29,7 +29,7 @@ BITS_DTYPE_MAPPING = {class QuantLinear(nn.Module):- QUANT_TYPE = "qbits"+ QUANT_TYPE = "qbits_gptq"def __init__(self,diff --git a/test/test_basic_usage.py b/test/test_basic_usage.pyindex 83de4bbe1..e468d173c 100644--- a/test/test_basic_usage.py+++ b/test/test_basic_usage.py@@ -20,56 +20,56 @@ class TestAutoRoundCmd(unittest.TestCase):python_path = sys.executable##test llm script- res = os.system(- f"cd .. && {python_path} -m auto_round -h")- if res > 0 or res == -1:- assert False, "cmd line test fail, please have a check"-+ # res = os.system(+ # f"cd .. && {python_path} -m auto_round -h")+ # if res > 0 or res == -1:+ # assert False, "cmd line test fail, please have a check"+ #res = os.system(f"cd .. && {python_path} -m auto_round --model 'facebook/opt-125m' --seqlen 32 --iter 2 --nsamples 1 --format auto_gptq,auto_round --output_dir ./saved --tasks piqa")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"- res = os.system(- f"cd .. && {python_path} -m auto_round --model 'facebook/opt-125m' --seqlen 8 --iter 1 --nsamples 1 --eval_task_by_task --tasks openbookqa --bs 32"- )- if res > 0 or res == -1:- assert False, "cmd line test fail, please have a check"-- res = os.system(- f"cd .. && {python_path} -c 'from auto_round.__main__ import run_light; run_light()' --seqlen 8 --iter 2 --nsamples 8 --output_dir ./saved --tasks lambada_openai")- if res > 0 or res == -1:- assert False, "cmd line test fail, please have a check"-- # test mllm script- # test auto_round_mllm help- res = os.system(- f"cd .. && {python_path} -m auto_round --mllm -h")- if res > 0 or res == -1:- assert False, "cmd line test fail, please have a check"-- # test auto_round_mllm --eval help- res = os.system(- f"cd .. && {python_path} -m auto_round --mllm --eval -h")- if res > 0 or res == -1:- assert False, "cmd line test fail, please have a check"-- # test auto_round_mllm --lmms help- res = os.system(- f"cd .. && {python_path} -m auto_round --mllm --lmms -h")- if res > 0 or res == -1:- assert False, "cmd line test fail, please have a check"-- res = os.system(- f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 32 --format auto_round --output_dir ./saved")- if res > 0 or res == -1:- assert False, "cmd line test fail, please have a check"-- res = os.system(- f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 256 --format auto_round"- " --quant_nontext_module --output_dir ./saved ")- if res > 0 or res == -1:- assert False, "cmd line test fail, please have a check"+ # res = os.system(+ # f"cd .. && {python_path} -m auto_round --model 'facebook/opt-125m' --seqlen 8 --iter 1 --nsamples 1 --eval_task_by_task --tasks openbookqa --bs 32"+ # )+ # if res > 0 or res == -1:+ # assert False, "cmd line test fail, please have a check"+ #+ # res = os.system(+ # f"cd .. && {python_path} -c 'from auto_round.__main__ import run_light; run_light()' --seqlen 8 --iter 2 --nsamples 8 --output_dir ./saved --tasks lambada_openai")+ # if res > 0 or res == -1:+ # assert False, "cmd line test fail, please have a check"+ #+ # # test mllm script+ # # test auto_round_mllm help+ # res = os.system(+ # f"cd .. && {python_path} -m auto_round --mllm -h")+ # if res > 0 or res == -1:+ # assert False, "cmd line test fail, please have a check"+ #+ # # test auto_round_mllm --eval help+ # res = os.system(+ # f"cd .. && {python_path} -m auto_round --mllm --eval -h")+ # if res > 0 or res == -1:+ # assert False, "cmd line test fail, please have a check"+ #+ # # test auto_round_mllm --lmms help+ # res = os.system(+ # f"cd .. && {python_path} -m auto_round --mllm --lmms -h")+ # if res > 0 or res == -1:+ # assert False, "cmd line test fail, please have a check"+ #+ # res = os.system(+ # f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 32 --format auto_round --output_dir ./saved")+ # if res > 0 or res == -1:+ # assert False, "cmd line test fail, please have a check"+ #+ # res = os.system(+ # f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 256 --format auto_round"+ # " --quant_nontext_module --output_dir ./saved ")+ # if res > 0 or res == -1:+ # assert False, "cmd line test fail, please have a check"if __name__ == "__main__":diff --git a/test/test_block_names.py b/test/test_block_names.pyindex 5cdbd9c93..0a106de6c 100644--- a/test/test_block_names.py+++ b/test/test_block_names.py@@ -1,4 +1,3 @@-import copyimport shutilimport sysimport unittest@@ -172,8 +171,7 @@ class TestQuantizationBlocks(unittest.TestCase):returnquantized_model_path = "./saved"autoround.save_quantized(quantized_model_path, inplace=False, safe_serialization=False, format="auto_round")-- from auto_round.auto_quantizer import AutoHfQuantizer+model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"diff --git a/test/test_export.py b/test/test_export.pyindex 0394886bf..3bde40491 100644--- a/test/test_export.py+++ b/test/test_export.py@@ -1,11 +1,9 @@-import copyimport shutilimport sysimport unittestsys.path.insert(0, "..")import torch-import transformersfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRound@@ -112,7 +110,6 @@ class TestAutoRound(unittest.TestCase):except:return- from auto_round.auto_quantizer import AutoHfQuantizerdevice = "auto" ##cpu, hpu, cudafrom auto_round import AutoRoundConfigquantization_config = AutoRoundConfig(diff --git a/test/test_generation.py b/test/test_generation.pyindex 8772d41b4..4f1dbb140 100644--- a/test/test_generation.py+++ b/test/test_generation.py@@ -27,142 +27,107 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)self.llm_dataloader = LLMDataLoader()+ self.save_folder = "./saved"@classmethoddef tearDownClass(self):- shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree(self.save_folder, ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)- @unittest.skipIf(torch.cuda.is_available() is False, "Skipping because no cuda")- def test_llm_generation_sym_gpu_gptq(self):++ def test_4bits_asym(self):bits = 4- group_size = 32+ group_size = 128+ sym = Falseautoround = AutoRound(self.model,self.tokenizer,bits=bits,group_size=group_size,- sym=True,+ sym=sym,iters=1,seqlen=2,dataset=self.llm_dataloader,)- autoround.quantize()- quantized_model_path = "./saved"+ quantized_model_path = self.save_folder++ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round", inplace=False)- autoround.save_quantized(output_dir=quantized_model_path, format="auto_round:gptq",inplace=False)- device = "auto" ##cpu, hpu, cudafrom auto_round import AutoRoundConfigquantization_config = AutoRoundConfig(- backend=device+ backend="auto")-model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map=device, quantization_config=quantization_config)+ device_map="cpu", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- text = "There is a girl who likes adventure,"+ text = "My name is "inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)assert ("!!!" not in res)- @unittest.skipIf(torch.cuda.is_available() is False, "Skipping because no cuda")- def test_llm_generation_asym_gpu_awq(self):++ def test_4bits_sym(self):bits = 4- group_size = 32+ group_size = 128+ sym = Trueautoround = AutoRound(self.model,self.tokenizer,bits=bits,group_size=group_size,- sym=True,+ sym=sym,iters=1,seqlen=2,dataset=self.llm_dataloader,)- autoround.quantize()- quantized_model_path = "./saved"+ quantized_model_path = self.save_folder++ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round", inplace=False)- autoround.save_quantized(output_dir=quantized_model_path, format="auto_round:awq",inplace=False)- device = "auto" ##cpu, hpu, cudafrom auto_round import AutoRoundConfigquantization_config = AutoRoundConfig(- backend=device+ backend="ipex")+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path,+ device_map="cpu", quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "My name is "+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ print(res)+ assert ("!!!" not in res)model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map=device, quantization_config=quantization_config)+ device_map="cpu", quantization_config=quantization_config,+ torch_dtype=torch.float16)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)assert ("!!!" not in res)-- def test_llm_generation_asym_qbits(self):- try:- import intel_extension_for_transformers- except:- return- bits = 4- group_size = 32- autoround = AutoRound(- self.model,- self.tokenizer,- bits=bits,- group_size=group_size,- sym=True,- iters=1,- seqlen=2,- dataset=self.llm_dataloader,- )- autoround.quantize()- quantized_model_path = "./saved"- autoround.save_quantized(output_dir=quantized_model_path, format="auto_round",inplace=False)- device = "cpu" ##cpu, hpu, cuda+from auto_round import AutoRoundConfigquantization_config = AutoRoundConfig(- backend="cpu:auto_round:qbits_zp"+ backend="itrex")-model = AutoModelForCausalLM.from_pretrained(quantized_model_path,device_map="cpu", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- text = "There is a girl who likes adventure,"+ text = "My name is "inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)assert ("!!!" not in res)-- def test_force_to_autoround_format(self):- bits = 4- group_size = 128- autoround = AutoRound(- self.model,- self.tokenizer,- bits=bits,- group_size=group_size,- sym=True,- iters=1,- seqlen=2,- dataset=self.llm_dataloader,- )- autoround.quantize()- quantized_model_path = "./saved"- autoround.save_quantized(output_dir=quantized_model_path, format="auto_awq", inplace=False)- device = "auto" ##cpu, hpu, cuda, auto- from auto_round import AutoRoundConfig- quantization_config = AutoRoundConfig(- # backend="auto_round:ipex_awq",- backend="auto"- )- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device,- quantization_config=quantization_config)++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path,+ device_map="cpu", quantization_config=quantization_config,+ torch_dtype=torch.float16)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)assert ("!!!" not in res)-diff --git a/test/test_hpu.py b/test/test_hpu.pyindex e7b355b28..f7c87f0d4 100644--- a/test/test_hpu.py+++ b/test/test_hpu.py@@ -1,11 +1,9 @@-import copyimport shutilimport sysimport unittestsys.path.insert(0, "..")import torch-import transformersfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRound@@ -63,9 +61,6 @@ class TestAutoRound(unittest.TestCase):quantized_model_path = "./saved"autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_gptq")- from auto_round.auto_quantizer import AutoHfQuantizer- import habana_frameworks.torch.core as htcore- import habana_frameworks.torch.hpu as hthpumodel = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", \trust_remote_code=True).to('hpu').to(torch.float32)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)@@ -94,9 +89,6 @@ class TestAutoRound(unittest.TestCase):autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")- from auto_round.auto_quantizer import AutoHfQuantizer- import habana_frameworks.torch.core as htcore- import habana_frameworks.torch.hpu as hthpumodel = AutoModelForCausalLM.from_pretrained(quantized_model_path, \device_map="auto").to('hpu').to(torch.float32)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)diff --git a/test_cuda/test_auto_round_format.py b/test_cuda/test_auto_round_format.pynew file mode 100644index 000000000..3c90204b2--- /dev/null+++ b/test_cuda/test_auto_round_format.py@@ -0,0 +1,418 @@+import copy+import shutil+import sys+import unittest++from auto_round.eval.evaluation import simple_evaluate_user_model++sys.path.insert(0, "..")+import torch+import transformers+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound+from auto_round import AutoRoundConfig+++class LLMDataLoader:+ def __init__(self):+ self.batch_size = 1++ def __iter__(self):+ for i in range(2):+ yield torch.ones([1, 10], dtype=torch.long)+++class TestAutoRound(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.model_name = "/models/opt-125m"+ ##self.model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"++ self.llm_dataloader = LLMDataLoader()+ self.save_folder = "./saved"++ def model_infer(self, model, tokenizer):+ prompts = [+ "Hello,my name is",+ # "The president of the United States is",+ # "The capital of France is",+ # "The future of AI is",+ ]++ ##texts = []+ # for prompt in prompts:+ # messages = [+ # {"role": "user", "content": prompt}+ # ]+ # text = tokenizer.apply_chat_template(+ # messages,+ # tokenize=False,+ # add_generation_prompt=True+ # )+ # texts.append(text)++ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)++ outputs = model.generate(+ input_ids=inputs["input_ids"].to(model.device),+ attention_mask=inputs["attention_mask"].to(model.device),+ do_sample=False, ## change this to follow official usage+ max_new_tokens=5+ )+ generated_ids = [+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)+ ]++ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)++ for i, prompt in enumerate(prompts):+ print(f"Prompt: {prompt}")+ print(f"Generated: {decoded_outputs[i]}")+ print("-" * 50)++ @classmethod+ def tearDownClass(self):+ return+ shutil.rmtree(self.save_folder, ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_device_map(self):+ from transformers import AutoModelForCausalLM, AutoTokenizer+ model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"++ device_map = {}+ for i in range(0, 32):+ key = f"model.layers.{str(i)}"+ device_map[key] = "cuda"+ device_map["model.layers.1"] = "cpu"+ device_map["model.layers.2"] = "cpu"+ device_map["model.layers.3"] = "cpu"+ device_map["model.layers.4"] = "cpu"+ device_map["model.layers.5"] = "cpu"+ device_map["model.layers.6"] = "cpu"+ device_map["model.layers.27"] = "cpu"+ device_map["model.layers.28"] = "cpu"+ device_map["model.layers.30"] = "cpu"+ device_map["model.layers.31"] = "cpu"+ device_map["lm_head"] = "cuda"+ device_map["model.norm"] = "cuda"+ device_map["model.rotary_emb"] = "cuda"+ device_map["model.embed_tokens"] = "cuda"++ device_map1 = {}+ for i in range(0, 32):+ key = f"model.layers.{str(i)}"+ device_map1[key] = "cuda"+ device_map1["model.layers.1"] = "cuda:1"+ device_map1["model.layers.2"] = "cuda:1"+ device_map1["model.layers.3"] = "cuda:1"+ device_map1["model.layers.4"] = "cuda:1"+ device_map1["model.layers.5"] = "cuda:1"+ device_map1["model.layers.6"] = "cuda:1"+ device_map1["model.layers.27"] = "cuda:1"+ device_map1["model.layers.28"] = "cuda:1"+ device_map1["model.layers.30"] = "cuda:1"+ device_map1["model.layers.31"] = "cuda:1"+ device_map1["lm_head"] = "cuda:1"+ device_map1["model.norm"] = "cuda"+ device_map1["model.rotary_emb"] = "cuda"+ device_map1["model.embed_tokens"] = "cuda"++ for tmp_device_map in [device_map1, device_map, None, 0, "balanced", "balanced_low_0", "sequential", "cpu",+ "cuda:0", "cuda", "auto",+ ]:+ model = AutoModelForCausalLM.from_pretrained(+ model_name,+ torch_dtype="auto",+ device_map=tmp_device_map+ )++ tokenizer = AutoTokenizer.from_pretrained(model_name)++ prompts = [+ "Hello,my name is",+ # "The president of the United States is",+ # "The capital of France is",+ # "The future of AI is",+ ]++ texts = []+ for prompt in prompts:+ messages = [+ {"role": "user", "content": prompt}+ ]+ text = tokenizer.apply_chat_template(+ messages,+ tokenize=False,+ add_generation_prompt=True+ )+ texts.append(text)++ inputs = tokenizer(texts, return_tensors="pt", padding=False, truncation=True)++ outputs = model.generate(+ input_ids=inputs["input_ids"].to(model.device),+ attention_mask=inputs["attention_mask"].to(model.device),+ do_sample=False, ## change this to follow official usage+ max_new_tokens=5+ )+ generated_ids = [+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)+ ]++ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)++ for i, prompt in enumerate(prompts):+ print(f"Prompt: {prompt}")+ print(f"Generated: {decoded_outputs[i]}")+ print("-" * 50)+ model = None+ del model+ torch.cuda.empty_cache()++ def test_device_map_for_triton(self):+ from transformers import AutoModelForCausalLM, AutoTokenizer+ model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"++ device_map = {}+ for i in range(0, 32):+ key = f"model.layers.{str(i)}"+ device_map[key] = "cuda"+ device_map["model.layers.1"] = "cpu"+ device_map["model.layers.2"] = "cpu"+ device_map["model.layers.3"] = "cpu"+ device_map["model.layers.4"] = "cpu"+ device_map["model.layers.5"] = "cpu"+ device_map["model.layers.6"] = "cpu"+ device_map["model.layers.27"] = "cpu"+ device_map["model.layers.28"] = "cpu"+ device_map["model.layers.30"] = "cpu"+ device_map["model.layers.31"] = "cpu"+ device_map["lm_head"] = "cuda"+ device_map["model.norm"] = "cuda"+ device_map["model.rotary_emb"] = "cuda"+ device_map["model.embed_tokens"] = "cuda"++ device_map1 = {}+ for i in range(0, 32):+ key = f"model.layers.{str(i)}"+ device_map1[key] = "cuda"+ device_map1["model.layers.1"] = "cuda:1"+ device_map1["model.layers.2"] = "cuda:1"+ device_map1["model.layers.3"] = "cuda:1"+ device_map1["model.layers.4"] = "cuda:1"+ device_map1["model.layers.5"] = "cuda:1"+ device_map1["model.layers.6"] = "cuda:1"+ device_map1["model.layers.27"] = "cuda:1"+ device_map1["model.layers.28"] = "cuda:1"+ device_map1["model.layers.30"] = "cuda:1"+ device_map1["model.layers.31"] = "cuda:1"+ device_map1["lm_head"] = "cuda:1"+ device_map1["model.norm"] = "cuda"+ device_map1["model.rotary_emb"] = "cuda"+ device_map1["model.embed_tokens"] = "cuda"+ from auto_round import AutoRoundConfig+ quantization_config = AutoRoundConfig(backend="tritonv2")++ for tmp_device_map in [device_map1, 0, "balanced", "balanced_low_0", "sequential",+ "cuda:0", "cuda", "auto",+ ]:+ model = AutoModelForCausalLM.from_pretrained(+ model_name,+ torch_dtype="auto",+ device_map=tmp_device_map,+ quantization_config = quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(model_name)++ prompts = [+ "Hello,my name is",+ # "The president of the United States is",+ # "The capital of France is",+ # "The future of AI is",+ ]++ texts = []+ for prompt in prompts:+ messages = [+ {"role": "user", "content": prompt}+ ]+ text = tokenizer.apply_chat_template(+ messages,+ tokenize=False,+ add_generation_prompt=True+ )+ texts.append(text)++ inputs = tokenizer(texts, return_tensors="pt", padding=False, truncation=True)++ outputs = model.generate(+ input_ids=inputs["input_ids"].to(model.device),+ attention_mask=inputs["attention_mask"].to(model.device),+ do_sample=False, ## change this to follow official usage+ max_new_tokens=5+ )+ generated_ids = [+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)+ ]++ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)++ for i, prompt in enumerate(prompts):+ print(f"Prompt: {prompt}")+ print(f"Generated: {decoded_outputs[i]}")+ print("-" * 50)+ model = None+ del model+ torch.cuda.empty_cache()++ def test_awq_backend(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 4, 128, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ iters=1,+ nsamples=1,+ sym=sym,+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:auto_awq")++ quantization_config = AutoRoundConfig(backend="auto")+ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.18)+ torch.cuda.empty_cache()++ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.bfloat16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)++ def test_tritonv2_bf16(self):+ model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"+ quantization_config = AutoRoundConfig(backend="tritonv2")+ model = AutoModelForCausalLM.from_pretrained(+ model_name,+ torch_dtype=torch.bfloat16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(model_name)+ self.model_infer(model, tokenizer)++ torch.cuda.empty_cache()++++ def test_autoround_gptq_sym_format(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 4, 128, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = "./saved"++ autoround.quantize_and_save(output_dir=quantized_model_path)++ from auto_round import AutoRoundConfig+ quantization_config = AutoRoundConfig(backend="ipex_gptq")++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", trust_remote_code=True,+ quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ print(res)+ assert ("!!!" not in res)++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ print(res)+ assert ("!!!" not in res)++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", trust_remote_code=True,+ quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ print(res)+ assert ("!!!" not in res)++ shutil.rmtree("./saved", ignore_errors=True)++ def test_autoround_awq_sym_format(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 4, 128, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = "./saved"++ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:auto_awq")++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ print(res)+ assert ("!!!" not in res)+++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", trust_remote_code=True,+ torch_dtype=torch.bfloat16)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ print(res)+ assert ("!!!" not in res)++ shutil.rmtree("./saved", ignore_errors=True)+++if __name__ == "__main__":+ unittest.main()diff --git a/test_cuda/test_exllamav2_backend.py b/test_cuda/test_exllamav2_backend.pynew file mode 100644index 000000000..b07ba65f6--- /dev/null+++ b/test_cuda/test_exllamav2_backend.py@@ -0,0 +1,181 @@+import shutil+import sys+import unittest+import pytest+from auto_round.eval.evaluation import simple_evaluate_user_model++sys.path.insert(0, "..")+import torch+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound+from auto_round import AutoRoundConfig+++class LLMDataLoader:+ def __init__(self):+ self.batch_size = 1++ def __iter__(self):+ for i in range(2):+ yield torch.ones([1, 10], dtype=torch.long)+++class TestAutoRoundMarlinBackend(unittest.TestCase):++ @classmethod+ def setUpClass(self):+ self.model_name = "/models/opt-125m"+ self.save_folder = "./saved"+ self.llm_dataloader = LLMDataLoader()++ def model_infer(self, model, tokenizer):+ prompts = [+ "Hello,my name is",+ # "The president of the United States is",+ # "The capital of France is",+ # "The future of AI is",+ ]++ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)++ outputs = model.generate(+ input_ids=inputs["input_ids"].to(model.device),+ attention_mask=inputs["attention_mask"].to(model.device),+ do_sample=False, ## change this to follow official usage+ max_new_tokens=5+ )+ generated_ids = [+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)+ ]++ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)++ for i, prompt in enumerate(prompts):+ print(f"Prompt: {prompt}")+ print(f"Generated: {decoded_outputs[i]}")+ print("-" * 50)+ return decoded_outputs[0]++ @classmethod+ def tearDownClass(self):+ shutil.rmtree(self.save_folder, ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_gptqmodel_exllmav2_4bits_asym(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 4, 128, False+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=1,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")++ quantization_config = AutoRoundConfig(backend="gptqmodel:exllamav2")+ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.35)+ torch.cuda.empty_cache()++ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.bfloat16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.35)+ torch.cuda.empty_cache()+ shutil.rmtree("./saved", ignore_errors=True)++ def test_gptq_exllamav2_4bits_sym(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 4, 128, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=1,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round") ##will convert to gptq model++ quantization_config = AutoRoundConfig(backend="gptq:exllamav2") ## or exllamav2+ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)+ torch.cuda.empty_cache()+ shutil.rmtree(self.save_folder, ignore_errors=True)++ def test_gptq_exllamav2_4bits_sym_group_size(self):+ for group_size in [32, 512, 1024]:+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 4, group_size, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ iters=1,+ nsamples=1,+ group_size=group_size,+ sym=sym,+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path,+ format="auto_round") ##will convert to gptq model++ quantization_config = AutoRoundConfig(backend="gptq:exllamav2") ## or exllamav2+ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.15)+ torch.cuda.empty_cache()+ shutil.rmtree(self.save_folder, ignore_errors=True)+++if __name__ == "__main__":+ unittest.main()diff --git a/test_cuda/test_marlin_backend.py b/test_cuda/test_marlin_backend.pynew file mode 100644index 000000000..3a231044f--- /dev/null+++ b/test_cuda/test_marlin_backend.py@@ -0,0 +1,226 @@+import shutil+import sys+import unittest+import pytest+from auto_round.eval.evaluation import simple_evaluate_user_model++sys.path.insert(0, "..")+import torch+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound+from auto_round import AutoRoundConfig+++class LLMDataLoader:+ def __init__(self):+ self.batch_size = 1++ def __iter__(self):+ for i in range(2):+ yield torch.ones([1, 10], dtype=torch.long)+++class TestAutoRoundMarlinBackend(unittest.TestCase):++ def test_marlin_group_size(self):+ for group_size in [-1, 64]:+ print(f"{group_size}!!!!!!!!!!!!!!!!!")+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 4, group_size, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=1,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")++ quantization_config = AutoRoundConfig(backend="marlin")+ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.14)++ for group_size in [32, 128]:+ print(f"{group_size}!!!!!!!!!!!!!!!!!")+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 4, group_size, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=1,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")++ quantization_config = AutoRoundConfig(backend="marlin")+ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.14)++ @classmethod+ def setUpClass(self):+ self.model_name = "/models/opt-125m"+ self.save_folder = "./saved"+ self.llm_dataloader = LLMDataLoader()++ def model_infer(self, model, tokenizer):+ prompts = [+ "Hello,my name is",+ # "The president of the United States is",+ # "The capital of France is",+ # "The future of AI is",+ ]++ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)++ outputs = model.generate(+ input_ids=inputs["input_ids"].to(model.device),+ attention_mask=inputs["attention_mask"].to(model.device),+ do_sample=False, ## change this to follow official usage+ max_new_tokens=5+ )+ generated_ids = [+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)+ ]++ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)++ for i, prompt in enumerate(prompts):+ print(f"Prompt: {prompt}")+ print(f"Generated: {decoded_outputs[i]}")+ print("-" * 50)+ return decoded_outputs[0]++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_marlin_4bits_sym_with_zp_m_1(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 4, 128, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=1,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")++ quantization_config = AutoRoundConfig(backend="marlin")+ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)+ torch.cuda.empty_cache()++ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.bfloat16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)+ torch.cuda.empty_cache()+ shutil.rmtree("./saved", ignore_errors=True)++ # def test_marlin_4bits_sym(self):+ # model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ # tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ # bits, group_size, sym = 4, 128, True+ # autoround = AutoRound(+ # model,+ # tokenizer,+ # bits=bits,+ # group_size=group_size,+ # sym=sym,+ # iters=1,+ # seqlen=2,+ # dataset=self.llm_dataloader,+ # )+ # quantized_model_path = self.save_folder+ # autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ #+ # quantization_config = AutoRoundConfig(backend="marlin")+ # model = AutoModelForCausalLM.from_pretrained(+ # self.save_folder,+ # torch_dtype=torch.float16,+ # device_map="auto",+ # quantization_config=quantization_config+ # )+ #+ # tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ # self.model_infer(model, tokenizer)+ # result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ # print(result['results']['lambada_openai']['acc,none'])+ # self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)+ # torch.cuda.empty_cache()+ #+ # model = AutoModelForCausalLM.from_pretrained(+ # self.save_folder,+ # torch_dtype=torch.bfloat16,+ # device_map="auto",+ # quantization_config=quantization_config+ # )+ #+ # tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ # self.model_infer(model, tokenizer)+ # result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ # print(result['results']['lambada_openai']['acc,none'])+ # self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)+ # torch.cuda.empty_cache()+ # shutil.rmtree("./saved", ignore_errors=True)+++if __name__ == "__main__":+ unittest.main()diff --git a/test_cuda/test_triton_backend.py b/test_cuda/test_triton_backend.pynew file mode 100644index 000000000..324d3ceaf--- /dev/null+++ b/test_cuda/test_triton_backend.py@@ -0,0 +1,296 @@++import shutil+import sys+import unittest++from auto_round.eval.evaluation import simple_evaluate_user_model++sys.path.insert(0, "..")+import torch+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound+from auto_round import AutoRoundConfig+++class LLMDataLoader:+ def __init__(self):+ self.batch_size = 1++ def __iter__(self):+ for i in range(2):+ yield torch.ones([1, 10], dtype=torch.long)+++class TestAutoRoundTritonBackend(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.model_name = "/models/opt-125m"+ self.save_folder = "./saved"+ self.llm_dataloader = LLMDataLoader()++ def model_infer(self, model, tokenizer):+ prompts = [+ "Hello,my name is",+ # "The president of the United States is",+ # "The capital of France is",+ # "The future of AI is",+ ]++ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)++ outputs = model.generate(+ input_ids=inputs["input_ids"].to(model.device),+ attention_mask=inputs["attention_mask"].to(model.device),+ do_sample=False, ## change this to follow official usage+ max_new_tokens=5+ )+ generated_ids = [+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)+ ]++ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)++ for i, prompt in enumerate(prompts):+ print(f"Prompt: {prompt}")+ print(f"Generated: {decoded_outputs[i]}")+ print("-" * 50)+ return decoded_outputs[0]++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("./saved", ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_tritonv2_4bits_asym(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 4, 128, False+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=1,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path)++ quantization_config = AutoRoundConfig(backend="tritonv2")+ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.34)+ torch.cuda.empty_cache()++ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.bfloat16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.34)+ torch.cuda.empty_cache()+ shutil.rmtree("./saved", ignore_errors=True)++ def test_tritonv2_2bits_asym(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 2, 32, False+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path)++ quantization_config = AutoRoundConfig(backend="tritonv2")+ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.20)+ torch.cuda.empty_cache()++ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.bfloat16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.20)+ torch.cuda.empty_cache()+ shutil.rmtree("./saved", ignore_errors=True)++ def test_tritonv2_4bits_sym(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 4, 128, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=1,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path)++ quantization_config = AutoRoundConfig(backend="tritonv2")+ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ # print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.26)+ torch.cuda.empty_cache()++ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.bfloat16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ # print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.26)+ torch.cuda.empty_cache()++ shutil.rmtree("./saved", ignore_errors=True)++ def test_tritonv2_8bits_sym(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 4, 256, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ nsamples=1,+ iters=1+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path)++ quantization_config = AutoRoundConfig(backend="tritonv2")+ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)+ torch.cuda.empty_cache()++ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.bfloat16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ # print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)+ torch.cuda.empty_cache()+ shutil.rmtree("./saved", ignore_errors=True)++ def test_tritonv2_2bits_sym(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = 2, 64, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path)++ quantization_config = AutoRoundConfig(backend="tritonv2")+ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.18)+ torch.cuda.empty_cache()++ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.bfloat16,+ device_map="auto",+ quantization_config=quantization_config+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ self.model_infer(model, tokenizer)+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")+ # print(result['results']['lambada_openai']['acc,none'])+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.18)+ torch.cuda.empty_cache()+ shutil.rmtree("./saved", ignore_errors=True)+++if __name__ == "__main__":+ unittest.main()