Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refine inference backend/code step 1 (#486)

Wenhua Chengea4d8435a2f0
diff --git a/auto_round/__init__.py b/auto_round/__init__.py
index f031c758f..8a2d74b4d 100644
--- a/auto_round/__init__.py
+++ b/auto_round/__init__.py
@@ -15,5 +15,5 @@ from .autoround import AutoRound, AutoRoundAdam, AutoRoundOPT
from .mllm import AutoRoundMLLM
from auto_round.utils import LazyImport
-from .auto_quantizer import AutoHfQuantizer,AutoRoundConfig
+from auto_round.inference.auto_quantizer import AutoHfQuantizer,AutoRoundConfig
from .version import __version__
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.py
deleted file mode 100644
index 0fb39e6b3..000000000
--- a/auto_round/auto_quantizer.py
+++ /dev/null
@@ -1,837 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# coding=utf-8
-# Copyright 2023 HuggingFace Inc. team and GPTQ and AutoGPTQ authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import gc
-import importlib.util
-import warnings
-from dataclasses import dataclass
-from logging import getLogger
-from typing import Any, Dict, List, Optional, Tuple, Union
-
-import torch
-import torch.nn as nn
-from packaging import version
-from transformers.modeling_utils import PreTrainedModel
-from transformers.pytorch_utils import Conv1D
-from transformers.quantizers import AutoQuantizationConfig, HfQuantizer
-from transformers.quantizers.auto import AUTO_QUANTIZER_MAPPING
-from transformers.utils.quantization_config import AwqConfig, GPTQConfig, QuantizationConfigMixin, QuantizationMethod
-from auto_round.utils import (get_module, set_module, is_hpu_supported, get_block_names,
- get_multimodal_block_names, find_matching_blocks)
-
-from auto_round.backend import get_layer_backend, dynamic_import_inference_linear
-
-from auto_round.backend import BackendInfos
-from transformers.utils.versions import require_version
-from enum import Enum
-from tqdm import tqdm
-import copy
-import re
-
-logger = getLogger(__name__)
-import sys
-
-if sys.version_info < (3, 8):
- import importlib_metadata
-else:
- import importlib.metadata as importlib_metadata
-
-AUTOROUND_MINIMUM_VERSION = version.parse("0.2")
-
-
-def _is_package_available(pkg_name: str, return_version: bool = False) -> Union[Tuple[bool, str], bool]:
- # Check we're not importing a "pkg_name" directory somewhere but the actual library by trying to grab the version
- try: ##TODO remove it later
- import auto_round
- return True, auto_round.__version__
- except:
- pass
-
- package_exists = importlib.util.find_spec(pkg_name) is not None
- package_version = "N/A"
- if package_exists:
- try:
- package_version = importlib.metadata.version(pkg_name)
- package_exists = True
- except importlib.metadata.PackageNotFoundError:
- package_exists = False
- if return_version:
- return package_exists, package_version
- else:
- return package_exists
-
-
-_auto_round_available = _is_package_available("auto_round")
-
-
-def is_auto_round_available():
- try:
- import auto_round
- return True
- except:
- pass
- if _auto_round_available:
- version_autoround = version.parse(importlib_metadata.version("auto_round"))
- if AUTOROUND_MINIMUM_VERSION < version_autoround:
- return True
- else:
- raise ImportError(
- f"Found an incompatible version of auto-round. Found version {version_autoround},"
- f" but only version above {AUTOROUND_MINIMUM_VERSION} are supported"
- )
-
-
-#
-def get_device(obj: Union[torch.Tensor, nn.Module]):
- if isinstance(obj, torch.Tensor):
- return obj.device
- return next(obj.parameters()).device
-
-
-class AutoHfQuantizer:
- """The Auto-HF quantizer class that takes care of automatically instantiating to the correct
- `HfQuantizer` given the `QuantizationConfig`."""
-
- @classmethod
- def from_config(cls, quantization_config: Union[QuantizationConfigMixin, Dict], **kwargs):
- # Convert it to a QuantizationConfig if the q_config is a dict
- if isinstance(quantization_config, dict):
- if "auto-round" in quantization_config["quant_method"]:
- quantization_config = AutoRoundConfig.from_dict(quantization_config)
- else:
- quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101
- quant_method = quantization_config.quant_method
-
- # Again, we need a special care for bnb as we have a single quantization config
- # class for both 4-bit and 8-bit quantization
- if quant_method == QuantizationMethod.BITS_AND_BYTES:
- if quantization_config.load_in_8bit:
- quant_method += "_8bit"
- else:
- quant_method += "_4bit"
-
- if quant_method not in AUTO_QUANTIZER_MAPPING.keys() and "auto-round" not in quant_method:
- raise ValueError(
- f"Unknown quantization type, got {quant_method} - supported types are:"
- f" {list(AUTO_QUANTIZER_MAPPING.keys())}"
- )
- if "auto-round" in quant_method or is_hpu_supported(): # pragma: no cover
- target_cls = AutoRoundQuantizer
- else:
- target_cls = AUTO_QUANTIZER_MAPPING[quant_method]
-
- return target_cls(quantization_config, **kwargs)
-
- @classmethod
- def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
- quantization_config = AutoQuantizationConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
- return cls.from_config(quantization_config)
-
- @classmethod
- def merge_quantization_configs(
- cls,
- quantization_config: Union[dict, QuantizationConfigMixin],
- quantization_config_from_args: Optional[QuantizationConfigMixin],
- ):
- """Handles situations where both quantization_config
- from args and quantization_config from model config are present."""
- if quantization_config_from_args is not None:
- warning_msg = (
- "You passed `quantization_config` or equivalent parameters to "
- "`from_pretrained` but the model you're loading"
- " already has a `quantization_config` attribute. The `quantization_config` from the model will be used."
- )
- else:
- warning_msg = ""
-
- loading_attr_dict = quantization_config_from_args.get_loading_attributes() \
- if quantization_config_from_args is not None else None
- if isinstance(quantization_config, dict):
- if "auto-round" in quantization_config["quant_method"]:
- quantization_config = AutoRoundConfig.from_dict(quantization_config)
- else:
- if isinstance(quantization_config_from_args, (AutoRoundConfig)):
- logger.info(f"Loading quantized model in auto_round format.")
- tmp_backend = quantization_config["quant_method"]
- if "auto-round" not in tmp_backend and "gptq" not in tmp_backend and "awq" not in tmp_backend:
- logger.error("could not convert to auto_round format, currently only supports `gptq`,`awq` or "
- "`auto-round` format")
- exit(-1)
- target_backend = quantization_config["backend"] if "backend" in quantization_config else "auto"
- if loading_attr_dict is not None and "backend" in loading_attr_dict:
- target_backend = loading_attr_dict["backend"]
- loading_attr_dict.pop("backend")
- if "auto_round" not in target_backend:
- target_backend = f"auto_round:{tmp_backend}" #
- quantization_config = AutoRoundConfig.from_dict(quantization_config)
- setattr(quantization_config, "backend", target_backend)
- else:
- quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101
-
- if isinstance(quantization_config,
- (GPTQConfig, AwqConfig, AutoRoundConfig)) and quantization_config_from_args is not None:
- # special case for GPTQ / AWQ config collision
-
- for attr, val in loading_attr_dict.items():
- setattr(quantization_config, attr, val)
- warning_msg += (
- f"However, loading attributes (e.g. {list(loading_attr_dict.keys())}) "
- f"will be overwritten with the one you passed to `from_pretrained`. The rest will be ignored."
- )
-
- if warning_msg != "":
- warnings.warn(warning_msg)
-
- return quantization_config
-
- @staticmethod
- def supports_quant_method(quantization_config_dict):
- from transformers.quantizers.auto import AUTO_QUANTIZATION_CONFIG_MAPPING
- AUTO_QUANTIZATION_CONFIG_MAPPING['intel/auto-round'] = AutoRoundConfig
- AUTO_QUANTIZATION_CONFIG_MAPPING['intel/auto_round'] = AutoRoundConfig
- quant_method = quantization_config_dict.get("quant_method", None)
- if quantization_config_dict.get("load_in_8bit", False) or quantization_config_dict.get("load_in_4bit", False):
- suffix = "_4bit" if quantization_config_dict.get("load_in_4bit", False) else "_8bit"
- quant_method = QuantizationMethod.BITS_AND_BYTES + suffix
- elif quant_method is None:
- raise ValueError(
- "The model's quantization config from the arguments has no `quant_method` attribute." \
- "Make sure that the model has been correctly quantized"
- )
-
- if quant_method not in AUTO_QUANTIZATION_CONFIG_MAPPING.keys():
- logger.warning(
- f"Unknown quantization type, got {quant_method} - supported types are:"
- f" {list(AUTO_QUANTIZER_MAPPING.keys())}. Hence, we will skip the quantization. "
- "To remove the warning, you can delete the quantization_config attribute in config.json"
- )
- return False
- return True
-
-
-class AutoRoundQuantizationMethod(str, Enum):
- AutoRound = "intel/auto-round"
-
-
-@dataclass
-class AutoRoundConfig(QuantizationConfigMixin):
- """This is a wrapper class about all possible attributes and features that you can play with a model that has been
- loaded AutoRound quantization.
-
- Args:
- bits (`int`):
- The number of bits to quantize to, supported numbers are (2, 3, 4, 8).
- tokenizer (`str` or `PreTrainedTokenizerBase`, *optional*):
- The tokenizer used to process the dataset. You can pass either:
- - A custom tokenizer object.
- - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co.
- - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved
- using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.
- """
-
- def __init__(
- self,
- bits: int = 4,
- tokenizer: Any = None,
- dataset: str = None,
- group_size: int = 128,
- sym: bool = False,
- backend="auto",
- layer_config: dict = None,
- **kwargs,
- ):
-
- self.bits = bits
- self.tokenizer = tokenizer
- self.dataset = dataset
- self.group_size = group_size
- self.sym = sym
- self.backend = backend
- self.layer_config = layer_config
- if kwargs is not None:
- for key in kwargs.keys():
- setattr(self, key, kwargs[key])
- self.quant_method = AutoRoundQuantizationMethod.AutoRound
- self.post_init()
-
- def post_init(self):
- r"""Safety checker that arguments are correct."""
- if self.bits not in [2, 4, 8]:
- raise ValueError(f"Only support quantization to [2,4,8] bits but found {self.bits}")
- if self.group_size != -1 and self.group_size <= 0:
- raise ValueError("group_size must be greater than 0 or equal to -1")
-
- def get_loading_attributes(self):
- loading_attibutes_dict = {"target_backend": self.backend}
- return loading_attibutes_dict
-
- def to_dict(self):
- config_dict = super().to_dict()
- return config_dict
-
-
-class AutoRoundQuantizer(HfQuantizer):
- """Quantizer of the AutoRound method, currently only triton and exllamav2 backend has been supported."""
-
- requires_calibration = False
- required_packages = ["auto_round"]
- optimum_quantizer = None
-
- def __init__(self, quantization_config: QuantizationConfigMixin, **kwargs):
- super().__init__(quantization_config, **kwargs)
-
- def validate_environment(self, *args, **kwargs):
- if not is_auto_round_available():
- raise ImportError("Loading a AutoRound quantized model requires auto-round library (`pip install "
- "auto-round`)")
- else:
- try:
- import auto_round
- autoround_version = version.parse(auto_round.__version__)
- except:
- autoround_version = version.parse(importlib.metadata.version("auto_round"))
- if autoround_version < version.parse("0.2.0"):
- raise ImportError("You need a version of auto_round > 0.2.0 to use AutoRound: `pip install --upgrade "
- "auto-round` or install from source")
-
- def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":
- if torch_dtype is None:
- torch_dtype = torch.float16
- return torch_dtype
-
- def find_backend(self, target_backend: str):
- """Finds the matching backend key based on the target backend or its alias.
-
- This function checks if the provided `target_backend` is directly present in `BackendInfos`.
- If not, it iterates through the backends to see if the `target_backend` matches any backend's alias.
-
- Args:
- target_backend (str):
- The name of the backend or alias to find.
-
- Returns:
- str or None:
- The backend key if a match is found, otherwise `None`.
- """
- # Directly return if target_backend exists in BackendInfos
- if target_backend in BackendInfos:
- return target_backend
-
- # Search through BackendInfos to check if target_backend matches any backend alias
- for key in BackendInfos.keys():
- backendInfo = BackendInfos[key]
- if backendInfo.alias is not None and target_backend in backendInfo.alias:
- return key
-
- # Return None if no matching backend or alias is found
- return None
-
- def detect_auto_device(self):
- if torch.cuda.is_available():
- return "cuda"
- elif is_hpu_supported():
- return "hpu"
- elif torch.xpu.is_available():
- return "xpu"
- else:
- return "cpu"
-
- def detect_device(self, target_backend, orig_backend):
- ##TODO need to refine later
- """Detects the appropriate device for the specified backend.
-
- This function determines the device type based on the target backend. If the target backend is
- not specified, it defaults to the original backend. The function checks for the availability
- of CUDA, HPU, or CPU, and returns the appropriate device type.
-
- Args:
- target_backend (str or None):
- The name of the target backend. If None, defaults to `orig_backend`.
- orig_backend (str):
- The original backend name to fall back on if `target_backend` is None.
-
- Returns:
- str:
- The type of device detected ('cuda', 'hpu', or 'cpu').
-
- Raises:
- ValueError:
- If the specified backend cannot be found.
- """
- # Default to the original backend if target_backend is not provided
- if target_backend is None:
- target_backend = orig_backend
-
- # Check for specific device types based on the target backend
- if "cuda" in target_backend:
- return "cuda"
- elif "hpu" in target_backend:
- return "hpu"
- elif "xpu" in target_backend:
- return "xpu"
- elif "cpu" in target_backend:
- return "cpu"
-
- # Determine the device automatically based on availability
- if target_backend.split(":")[0] == "auto":
- return self.detect_auto_device()
-
- # Find the backend and determine the device type from BackendInfos
- backend = self.find_backend(target_backend)
- if backend is None:
- raise ValueError("Backend is not found, please set it to 'auto' to have a try ")
-
- device = BackendInfos[backend].device[0]
- if "cuda" in device and torch.cuda.is_available():
- return device
- elif "hpu" in device and is_hpu_supported():
- return device
- elif "xpu" in device and torch.xpu.is_available():
- return device
- else:
- ## trick
- return "cpu"
-
- def convert_model(self, model: nn.Module):
- """Converts the given model to an AutoRound model by replacing its layers with quantized layers.
-
- This method extracts the quantization configuration from the model and adjusts its layers
- according to the specified quantization parameters. It supports different backends and
- ensures that the model's data type is compatible with the selected hardware.
-
- Args:
- model (nn.Module):
- The model to be converted into an AutoRound model.
-
- Returns:
- nn.Module:
- The converted AutoRound model with quantized layers.
-
- Raises:
- ValueError:
- If the quantization backend is not specified in the configuration.
- """
-
- from auto_round.utils import get_layer_names_in_block
-
- quantization_config = model.config.quantization_config
- if not hasattr(quantization_config, "target_backend"):
- quantization_config.target_backend = quantization_config.backend
-
- target_device = self.detect_device(None, quantization_config.backend)
-
- self.target_device = target_device
-
- if hasattr(quantization_config, "backend"): # pragma: no cover
- if ("hpu" == target_device or "cpu" == target_device) and model.dtype != torch.bfloat16:
- logger.info(f"Change the dtype to `bfloat16` as {target_device.upper()} does not support float16")
- model = model.to(torch.bfloat16)
-
- bits = quantization_config.bits
- group_size = quantization_config.group_size
- data_type = quantization_config.data_type if hasattr(quantization_config,
- "data_type") else "int" # pragma: no cover
- sym = quantization_config.sym
-
- quant_block_list = quantization_config.quant_block_list if hasattr(quantization_config,
- "quant_block_list") else None
-
- if quant_block_list is None:
- to_quant_block_names = quantization_config.to_quant_block_names if hasattr(quantization_config,
- "to_quant_block_names") else None
- if to_quant_block_names is not None:
- if isinstance(to_quant_block_names, (list, tuple)):
- quant_block_list = to_quant_block_names
- else:
- quant_block_list = []
- for block in to_quant_block_names.split(','):
- quant_block_list.append([f'{block}.{i}' for i in range(len(get_module(model, block)))])
- else:
- all_blocks = get_block_names(model)
- quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)
-
- layer_names = get_layer_names_in_block(model, quant_block_list=quant_block_list)
-
- extra_config = {}
- if hasattr(quantization_config, "extra_config"):
- extra_config = quantization_config.extra_config
- if hasattr(quantization_config, "modules_in_block_to_quantize"): ##gptq format
- modules_in_block_to_quantize_tmp = quantization_config.modules_in_block_to_quantize
- modules_in_block_to_quantize = [item for sublist in modules_in_block_to_quantize_tmp for item in sublist]
- for layer_name in layer_names:
- quantized = False
- for qname in modules_in_block_to_quantize:
- if qname in layer_name:
- quantized = True
- break
- if not quantized:
- extra_config[layer_name] = {"bits": 16}
- if hasattr(quantization_config, "modules_to_not_convert"):
- for layer_name in quantization_config.modules_to_not_convert:
- extra_config[layer_name] = {"bits": 16}
-
- layer_names += extra_config.keys()
- layer_names = list(set(layer_names))
-
- layer_configs = {}
- for layer_name in layer_names:
- layer_configs[layer_name] = {}
- if layer_name not in extra_config:
- layer_configs[layer_name]["bits"] = bits
- layer_configs[layer_name]["group_size"] = group_size
- layer_configs[layer_name]["data_type"] = data_type
- layer_configs[layer_name]["sym"] = sym
- layer_configs[layer_name]["clip"] = False
- else:
- layer_configs[layer_name]["bits"] = extra_config[layer_name].get("bits", bits)
- layer_configs[layer_name]["group_size"] = extra_config[layer_name].get("group_size", group_size)
- layer_configs[layer_name]["data_type"] = extra_config[layer_name].get("data_type", data_type)
- layer_configs[layer_name]["sym"] = extra_config[layer_name].get("sym", sym)
- layer_configs[layer_name]["clip"] = extra_config[layer_name].get("clip", False)
-
- if hasattr(quantization_config, "backend"): # pragma: no cover
- backend = quantization_config.backend
- elif 'gptq' in quantization_config.quant_method: # pragma: no cover
- backend = 'gptq'
- elif "awq" in quantization_config.quant_method:
- backend = "awq"
- else: # pragma: no cover
- logger.error("Please specify quantization backend")
- raise ValueError("Quantization backend must be specified.")
-
- self._replace_by_quant_layers(model, layer_configs, quantization_config.target_backend, target_device, backend)
- return model
-
- def _replace_by_quant_layers(self, module: nn.Module, layer_configs, target_backend, target_device, orig_backend):
- """Replaces linear layers in the given module with quantized layers.
-
- This method iterates over the specified layer configurations and replaces
- the original layers in the module with instances of `QuantLinear`. It handles
- various layer types and ensures that the correct quantization parameters are applied.
-
- Args:
- module (nn.Module):
- The module containing layers to be quantized.
- layer_configs (dict):
- A dictionary containing configuration for each layer's quantization.
- target_backend (str):
- The backend to use for quantization, which includes device and format information.
- target_device (str):
- The device on which the model will run (e.g., 'cuda', 'cpu', 'hpu').
- orig_backend (str):
- The original backend of the packing.
-
- Raises:
- AssertionError:
- If any condition related to backend or quantization configuration is not met.
- """
-
- def remove_device_str(s, device_str):
- if s and s.startswith(device_str):
- return s[len(device_str):].lstrip(":")
- return s
-
- if "auto" == target_backend.split(':')[0]:
- target_backend = target_backend[4:] # Remove 'auto'
- if len(target_backend) >= 1 and target_backend[0] == ":":
- target_backend = target_backend[1:]
-
- # Remove device info from target_backend
- target_backend = remove_device_str(target_backend, "cpu")
- target_backend = remove_device_str(target_backend, "hpu")
- target_backend = remove_device_str(target_backend, "cuda")
- orig_backend = self.find_backend(orig_backend)
- if orig_backend is None:
- orig_backend = "gptq:exllamav2"
-
- if target_backend == "":
- target_backend = orig_backend
-
- self.need_marlin_repacking = False
-
- for layer_name in layer_configs.keys():
- config = layer_configs[layer_name]
- bits = config["bits"]
- group_size = config["group_size"]
- data_type = config["data_type"]
- sym = config["sym"]
- clip = config["clip"]
-
- if not (bits <= 8):
- continue
-
- layer = get_module(module, layer_name)
- if isinstance(layer, nn.Linear):
- in_features = layer.in_features
- out_features = layer.out_features
- elif isinstance(layer, nn.Conv2d): # Not supported currently
- in_features = layer.in_channels
- out_features = layer.out_channels
- elif isinstance(layer, Conv1D): # TODO: Needs verification
- in_features = layer.weight.shape[0]
- out_features = layer.weight.shape[1]
- else:
- continue
-
- if "marlin" in target_backend and "marlin" not in orig_backend:
- # Need to repack
- assert sym == True, "Marlin only supports symmetric quantization"
- assert target_device == "cuda", "Marlin only supports CUDA device"
- assert not "awq" in orig_backend, "Marlin does not support repacking from AWQ format"
- self.need_marlin_repacking = True
- # Using original backend to load the layer then replace
- layer_backend = orig_backend
- else:
- target_backend = self.find_backend(target_backend) # TODO: Move out if have supported marlin
- layer_backend = get_layer_backend(
- target_device, target_backend, orig_backend, bits, group_size, sym, in_features, out_features
- )
- if "gptq" in layer_backend and "exllamav2" in layer_backend:
- try:
- from exllamav2_kernels import gemm_half_q_half, make_q_matrix # pylint: disable=E0611, E0401
- except:
- logger.warning_once(
- "For better inference performance, please install exllamav2 kernel "
- "via `pip install git+https://github.com/AutoGPTQ/AutoGPTQ.git@b8b4127`")
-
- QuantLinear = dynamic_import_inference_linear(layer_backend, bits, group_size, sym)
-
- layer_device = get_device(layer)
-
- bias = layer.bias is not None
- from auto_round_extension.qbits.qbits_awq import QuantLinear as QBitsAWQQuantLinear
- if "awq" in layer_backend and isinstance(QuantLinear, QBitsAWQQuantLinear):
- new_layer = QuantLinear.from_linear( # pylint: disable=E1123
- layer,
- bits,
- group_size,
- init_only=True,
- has_zero_points=not sym
- )
- elif "awq" in layer_backend:
- new_layer = QuantLinear.from_linear( # pylint: disable=E1123
- layer,
- bits,
- group_size,
- init_only=True
- )
- else:
- try:
- new_layer = QuantLinear( # pylint: disable=E1123
- bits,
- group_size,
- in_features,
- out_features,
- bias,
- weight_dtype=layer.weight.dtype,
- clip=clip
- )
- except:
- new_layer = QuantLinear( # pylint: disable=E1123
- bits,
- group_size,
- in_features,
- out_features,
- bias,
- weight_dtype=layer.weight.dtype,
- )
-
- new_layer.device = layer_device
- set_module(module, layer_name, new_layer)
-
- def cpu_post_init(self, model):
- message = "Repacking to CPU format"
- from auto_round_extension.qbits import qbits_qlinear_classes, qbits_awq_classes
- from auto_round_extension.ipex import ipex_qlinear_classes
- cpu_layers = tuple(list(qbits_qlinear_classes) + list(ipex_qlinear_classes) + list(qbits_awq_classes))
- layers = [] ## ipex post_init will add one more layer
- for n, m in model.named_modules():
- if isinstance(m, cpu_layers):
- layers.append((n, m))
- for n, layer in tqdm(layers, desc=message, total=len(layers),
- leave=True):
- layer.post_init()
- return model
-
- def xpu_post_init(self, model):
- message = "Repacking to XPU format"
- from auto_round_extension.ipex import ipex_qlinear_classes
- cpu_layers = tuple(list(ipex_qlinear_classes))
- layers = [] ## ipex post_init will add one more layer
- for n, m in model.named_modules():
- if isinstance(m, cpu_layers):
- layers.append((n, m))
- for n, layer in tqdm(layers, desc=message, total=len(layers),
- leave=True):
- layer.post_init()
- return model
-
- def repack_marlin(self, model):
- """Repack the model to use Marlin format for quantized layers.
-
- This method iterates through the model's modules, identifies instances of
- `QuantLinear`, and replaces them with `MarlinInferenceQuantLinear`. It
- handles the initialization of various parameters and the repacking of
- quantized weights and scales for optimized performance on Marlin.
-
- Args:
- model (nn.Module):
- The model to be repacked into Marlin format.
-
- Raises:
- ImportError:
- If the required modules for Marlin inference cannot be imported.
- """
- message = "Repacking to Marlin format"
-
- for n, m in tqdm(model.named_modules(), desc=message, total=len(list(model.named_modules()))):
- if m.__class__.__name__ == "QuantLinear":
- try:
- from gptqmodel.nn_modules.qlinear.qlinear_marlin_inference import ( # pylint: disable=E0401
- MarlinInferenceQuantLinear,
- marlin_permute_scales,
- marlin_make_workspace
- )
- except ImportError:
- raise ImportError("Failed to import Marlin inference modules.")
-
- with torch.device("meta"):
- # Create a new MarlinInferenceQuantLinear module with the appropriate parameters.
- new_module = MarlinInferenceQuantLinear(
- bits=4,
- group_size=m.group_size,
- sym=True,
- desc_act=False,
- infeatures=m.infeatures,
- outfeatures=m.outfeatures,
- bias=m.bias is not None,
- )
-
- device = m.qweight.device
- import gptqmodel_marlin_cuda_inference # pylint: disable=E0401
-
- # Initialize the necessary parameters for the new module.
- new_module.g_idx = torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),
- requires_grad=False)
- new_module.g_idx_sort_indices = torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),
- requires_grad=False)
- new_module.zp = torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),
- requires_grad=False)
- new_module.bias = m.bias
-
- # Repack the quantized weight for the Marlin format.
- marlin_qweight = gptqmodel_marlin_cuda_inference.gptq_marlin_repack( # pylint: disable=E0401
- m.qweight,
- new_module.g_idx_sort_indices,
- m.infeatures,
- m.outfeatures,
- m.bits
- )
- new_module.qweight.resize_(marlin_qweight.shape)
- new_module.qweight = nn.Parameter(marlin_qweight, requires_grad=False)
-
- # Permute scales for the new module's configuration.
- marlin_scales = marlin_permute_scales(
- m.scales,
- size_k=m.infeatures,
- size_n=m.outfeatures,
- group_size=m.group_size
- )
-
- new_module.scales.resize_(marlin_scales.shape)
- new_module.scales = nn.Parameter(marlin_scales, requires_grad=False)
-
- # Create a workspace for the new module.
- new_module.workspace = marlin_make_workspace( # TODO: Consider moving this to post-init.
- new_module.outfeatures, device
- )
-
- # Replace the original module in the model with the new Marlin module.
- set_module(model, n, new_module)
-
- # Clear cache and perform garbage collection to free memory.
- torch.cuda.empty_cache()
- gc.collect()
-
- def post_init_model(self, model):
- """Post-initialization that require device information, for example buffers initialization on device.
-
- Args:
- model (`nn.Module`):
- The input model
- """
-
- class StoreAttr(object):
- pass
-
- model.quantize_config = StoreAttr()
- if self.need_marlin_repacking:
- require_version("gptqmodel",
- "marlin format requires gptqmodel to be installed, "
- "`pip install -v gptqmodel --no-build-isolation `")
- self.repack_marlin(model)
- from auto_round_extension.cuda.post_init import autoround_post_init
- model = autoround_post_init(model)
- # there are no side-effects after call qbits_post_init when model quant-type not equal to qbits.
- if self.target_device == "cpu":
- model = self.cpu_post_init(model)
- elif self.target_device == "xpu":
- model = self.xpu_post_init(model)
-
- return model
-
- def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs):
- if model.__class__.main_input_name != "input_ids":
- logger.warning("We can only quantize pure text models and " \
- "certain types(Llava/Qwen-VL/Phi-3-vision) of multimodal models.")
-
- if self.pre_quantized:
- model = self.convert_model(model)
-
- def _process_model_after_weight_loading(self, model: "PreTrainedModel", **kwargs):
- if self.pre_quantized:
- model = self.post_init_model(model)
- else:
- raise NotImplementedError
-
- @property
- def is_trainable(self, model: Optional["PreTrainedModel"] = None):
- return True
-
- @property
- def is_serializable(self):
- return True
-
-
-import transformers
-
-if version.parse(transformers.__version__) < version.parse("4.38.0"):
- logger.error("Please upgrade transformers>=4.38.0 to support lm-head quantization")
-
-transformers.quantizers.auto.AutoHfQuantizer = AutoHfQuantizer
-transformers.modeling_utils.AutoHfQuantizer = AutoHfQuantizer
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 04f36cc56..47f97dcf3 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -53,7 +53,8 @@ from .utils import (
compile_func,
find_matching_blocks, is_debug_mode,
TORCH_VERSION_AT_LEAST_2_6,
- supported_layer_types
+ supported_layer_types,
+ get_layer_features,
)
from .low_cpu_mem.utils import get_layers_before_block
@@ -448,7 +449,7 @@ class AutoRound(object):
f"Currently only support to export auto_round format quantized model"
" with fp8 dtype activation for activation quantization."
" Change format to fake and save."
- )
+ )
formats = ["fake"]
else:
if len(formats) > 1 or "auto_round" not in formats:
@@ -478,11 +479,6 @@ class AutoRound(object):
format = format.replace('auto_round', 'auto_round:gptq')
formats[index] = format
- if not any(f in format for f in ["triton", "exllamav2", "awq", "gptq"]):
- logger.info(f"AutoRound format does not support {format}, attempting to use AutoGPTQ")
- format = format.replace("auto_round", "auto_gptq")
- formats[index] = format
-
# Remove duplicates from formats list
def remove_duplicates(lst):
seen = set()
@@ -693,6 +689,10 @@ class AutoRound(object):
if n not in layers_in_blocks and check_to_quantized(layer_config[n]):
has_qlayer_outside_block = True
+ in_features, out_features = get_layer_features(m)
+ if in_features <= layer_config[n]["group_size"]:
+ layer_config[n]["group_size"] = -1
+
# Apply the configuration to the corresponding layer in the model
for key in keys:
setattr(m, key, layer_config[n][key])
@@ -1478,7 +1478,7 @@ class AutoRound(object):
m.name = n
for i in range(0, len(block_names), nblocks):
- if i!=0:
+ if i != 0:
pbar.update(1)
if nblocks == 1:
n = block_names[i]
@@ -1542,7 +1542,7 @@ class AutoRound(object):
f"Currently only support to export auto_round format quantized model"
" with fp8 dtype activation for activation quantization."
" Change format to fake and save."
- )
+ )
format = "fake"
else:
if format != "auto_round":
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index 7e8b7e27f..b0b4d5dd2 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -12,12 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import logging
-import random
-import time
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import Optional, Union
-import lm_eval
from lm_eval import simple_evaluate as lm_simple_evaluate
import os
@@ -52,7 +48,7 @@ def simple_evaluate(
try:
from auto_round import AutoRoundConfig
except:
- from auto_round.auto_quantizer import AutoHfQuantizer
+ from auto_round.inference.auto_quantizer import AutoHfQuantizer
return lm_simple_evaluate(
model=model,
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index 7a1d81424..39345cc3e 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -105,7 +105,7 @@ def pack_layer(name, model, backend):
##force to float32 to be compatible with torch 2.0
if sym and isinstance(new_layer, auto_round.export.export_to_autogptq.qlinear_triton.QuantLinear):
layer, scale = layer.to("cpu"), scale.to("cpu")
- zero = 2 ** (bits - 1)
+ zero = int(zero.flatten()[0])
else:
layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)
sig = inspect.signature(qlayer.pack)
@@ -126,6 +126,8 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
quant_block_list = kwargs.get("quant_block_list", get_block_names(model))
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
+ if os.path.exists(output_dir):
+ logger.warning(f"{output_dir} already exists, this may cause model conflict")
if tokenizer is not None:
tokenizer.save_pretrained(output_dir)
if processor is not None:
diff --git a/auto_round/export/export_to_autogptq/qlinear_triton.py b/auto_round/export/export_to_autogptq/qlinear_triton.py
index 0d5a67942..c51920109 100644
--- a/auto_round/export/export_to_autogptq/qlinear_triton.py
+++ b/auto_round/export/export_to_autogptq/qlinear_triton.py
@@ -18,30 +18,6 @@ import numpy as np
import torch
import torch.nn as nn
import transformers
-import numba
-
-
-##TODO different bits
-# @numba.jit(nopython=True, parallel=True)
-# def pack_array_with_numba_b4_c32(
-# raw_array: np.ndarray, packed_array: np.ndarray
-# ) -> np.ndarray:
-# """Pack the array with numba when bits=4 and compress_bits=32."""
-# bits = 4
-# n_pack = 32 // bits
-#
-# for row in range(packed_array.shape[0]):
-# packed_array[row] = ((((raw_array[row * n_pack + 7]) << 28)
-# | ((raw_array[row * n_pack + 6]) << 24)
-# | ((raw_array[row * n_pack + 5]) << 20)
-# | ((raw_array[row * n_pack + 4]) << 16)
-# | ((raw_array[row * n_pack + 3]) << 12)
-# | (raw_array[row * n_pack + 2]) << 8)
-# | ((raw_array[row * n_pack + 1]) << 4)
-# | ((raw_array[row * n_pack]) << 0))
-#
-# return packed_array
-
class TritonModuleMixin:
@classmethod
@@ -89,6 +65,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):
"g_idx",
torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),
)
+
if bias:
self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))
else:
@@ -108,6 +85,8 @@ class QuantLinear(nn.Module, TritonModuleMixin):
device = "cpu"
if torch.cuda.is_available():
device = "cuda:0"
+ elif torch.xpu.is_available():
+ device = "xpu:0"
W = linear.weight.data.to(device).clone()
if isinstance(linear, nn.Conv2d):
@@ -118,11 +97,12 @@ class QuantLinear(nn.Module, TritonModuleMixin):
repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)
if isinstance(zeros, torch.Tensor):
repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)
+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros[:, :W.shape[1]]).to(
+ torch.int32)
else:
repeat_zeros = zeros
-
- intweight = torch.round(W.to(device) / repeat_scales + repeat_zeros).to(
- torch.int32)
+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros).to(
+ torch.int32)
del repeat_scales
intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index f0743eec8..9c68a96d6 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -22,6 +22,7 @@ import torch.nn as nn
import transformers
import auto_round.export.export_to_autoround.qlinear_triton_act
+import auto_round_extension.cuda.qlinear_tritonv2
from auto_round.utils import get_layer_names_in_block, get_module, logger, set_module, supported_layer_types
import threadpoolctl as tctl
import inspect
@@ -71,17 +72,18 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_
if "auto_round" in backend and "awq" not in backend and "gptq" not in backend:
if act_bits <= 8: ##easily have bug for other configuration, need to refine code later
return auto_round.export.export_to_autoround.qlinear_triton_act.QuantLinear
- ##only support triton and exllamav2
- if not ("triton" in backend or "exllamav2" in backend):
- logger.warning_once(f"auto_round format does not support {backend}, try to pack each layer with auto_gptq")
- return get_autogptq_packing_qlinear(backend, bits, group_size, sym)
- from auto_round_extension.cuda.qlinear_triton import QuantLinear
+ from auto_round_extension.cuda.qlinear_tritonv2 import QuantLinear
return QuantLinear
+ elif "auto_round" in backend and "gptq" in backend:
+ from auto_round.export.export_to_autoround.qlinear_triton import QuantLinear ##no g_idx
+ return QuantLinear
elif "awq" in backend:
from ..export_to_awq.utils import WQLinear_GEMM
return WQLinear_GEMM
- elif "gptq" in backend:
+ elif "gptqmodel" in backend:
+ return auto_round_extension.cuda.qlinear_tritonv2.QuantLinear
+ elif "gptq" in backend and not "gptqmodel" in backend: ## have g_idx
return get_autogptq_packing_qlinear(backend, bits, group_size, sym)
else:
assert False, f"only support auto_gptq, auto_awq and auto_round backend"
@@ -188,6 +190,8 @@ def pack_layer(layer_name, model, backend):
new_layer.device = device
set_module(model, layer_name, new_layer)
qlayer = new_layer
+ if sym:
+ zp = int(zp.flatten()[0])
qlayer.to("cpu")
##force to float32 to be compatible with torch 2.0
@@ -202,6 +206,9 @@ def pack_layer(layer_name, model, backend):
scale, zp = scale.to(torch.float32), zp.to(torch.float32)
scale = scale.t().contiguous()
zp = zp.t().contiguous()
+ if sym:
+ zp = int(zp.flatten()[0])
+
if bits != 4:
logger.error("AutoAWQ format only supports 4-bits quantization.")
qlayer = QuantLinear.from_linear(
@@ -243,10 +250,6 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
if (kwargs.get("sym") is None or kwargs.get("sym") == True) and ("gptq" not in backend and "awq" not in backend):
backend = backend.replace('auto_round', 'auto_round:gptq')
- if not ("triton" in backend or "exllamav2" in backend or "awq" in backend or "gptq" in backend):
- logger.info(f"AutoRound format does not support {backend}, try to pack each layer with AutoGPTQ")
- backend = backend.replace("auto_round", "auto_gptq")
-
model = kwargs["model"]
safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]
if not inplace:
@@ -306,6 +309,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
if output_dir is None:
model.tokenizer = tokenizer
return model
+ if os.path.exists(output_dir):
+ logger.warning(f"{output_dir} already exists, this may cause model conflict")
if tokenizer is not None:
tokenizer.save_pretrained(output_dir)
diff --git a/auto_round/export/export_to_autoround/qlinear_triton.py b/auto_round/export/export_to_autoround/qlinear_triton.py
new file mode 100644
index 000000000..ba2da85e3
--- /dev/null
+++ b/auto_round/export/export_to_autoround/qlinear_triton.py
@@ -0,0 +1,137 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+
+import numpy as np
+import torch
+import torch.nn as nn
+import transformers
+
+class TritonModuleMixin:
+ @classmethod
+ def warmup(cls, model, transpose=False, seqlen=2048):
+ pass
+
+
+class QuantLinear(nn.Module, TritonModuleMixin):
+ QUANT_TYPE = "triton"
+
+ def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):
+ super().__init__()
+ if bits not in [2, 4, 8]:
+ raise NotImplementedError("Only 2,4,8 bits are supported.")
+ if infeatures % 32 != 0 or outfeatures % 32 != 0:
+ raise NotImplementedError("in_feature and out_feature must be divisible by 32.")
+ self.infeatures = infeatures
+ self.outfeatures = outfeatures
+ self.bits = bits
+ self.group_size = group_size if group_size != -1 else infeatures
+ self.maxq = 2 ** self.bits - 1
+
+ self.register_buffer(
+ "qweight",
+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),
+ )
+ self.register_buffer(
+ "qzeros",
+ torch.zeros(
+ (
+ math.ceil(infeatures / self.group_size),
+ outfeatures // 32 * self.bits,
+ ),
+ dtype=torch.int32,
+ ),
+ )
+ self.register_buffer(
+ "scales",
+ torch.zeros(
+ (math.ceil(infeatures / self.group_size), outfeatures),
+ dtype=torch.float16,
+ ),
+ )
+
+ if bias:
+ self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))
+ else:
+ self.bias = None
+
+ self.trainable = trainable
+
+ def post_init(self):
+ pass
+
+ def pack(self, linear, scales, zeros, g_idx=None):
+ scales_t = scales.t().contiguous()
+ if linear.bias is not None:
+ self.bias = linear.bias.clone().half()
+ self.scales = scales_t.clone().half()
+ device = "cpu"
+ if torch.cuda.is_available():
+ device = "cuda:0"
+ elif torch.xpu.is_available():
+ device = "xpu:0"
+
+ W = linear.weight.data.to(device).clone()
+ if isinstance(linear, nn.Conv2d):
+ W = W.flatten(1)
+ if isinstance(linear, transformers.pytorch_utils.Conv1D):
+ W = W.t()
+
+ repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)
+ if isinstance(zeros, torch.Tensor):
+ repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)
+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros[:, :W.shape[1]]).to(
+ torch.int32)
+ else:
+ repeat_zeros = zeros
+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros).to(
+ torch.int32)
+
+ del repeat_scales
+ intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)
+ order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits
+ intweight = intweight << order_map
+ intweight = torch.sum(intweight, dim=-1)
+
+ intweight = intweight.t().contiguous().to(torch.int32)
+ self.qweight = intweight.to("cpu")
+
+ if isinstance(zeros, torch.Tensor):
+ zeros = zeros.t().contiguous()
+ zeros -= 1
+ zeros = zeros.numpy().astype(np.uint32)
+ qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)
+ i = 0
+ col = 0
+ while col < qzeros.shape[1]:
+ for j in range(i, i + (32 // self.bits)):
+ qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
+ i += 32 // self.bits
+ col += 1
+
+ qzeros = qzeros.astype(np.int32)
+ self.qzeros = torch.from_numpy(qzeros)
+ else:
+ zeros -= 1
+ shape = scales_t.shape
+ value = 0
+ for j in range(0, (32 // self.bits)):
+ value |= zeros << (self.bits * j)
+ qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value
+ qzeros = qzeros.astype(np.int32)
+ self.qzeros = torch.from_numpy(qzeros)
+
+
+__all__ = ["QuantLinear"]
\ No newline at end of file
diff --git a/auto_round/export/export_to_autoround/qlinear_triton_act.py b/auto_round/export/export_to_autoround/qlinear_triton_act.py
index f5afa2432..12dc79555 100644
--- a/auto_round/export/export_to_autoround/qlinear_triton_act.py
+++ b/auto_round/export/export_to_autoround/qlinear_triton_act.py
@@ -97,10 +97,7 @@ class QuantLinear(nn.Module):
),
)
- self.register_buffer(
- "g_idx",
- torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),
- )
+
if bias:
self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))
else:
@@ -122,7 +119,7 @@ class QuantLinear(nn.Module):
def pack(self, linear, scales, zeros, act_scales, w_bf16_to_fp8_scale, g_idx=None):
scales_t = scales.t().contiguous()
- self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
+
self.act_scales.data.copy_(act_scales.squeeze().clone())
self.w_bf16_to_fp8_scale.data.copy_(w_bf16_to_fp8_scale.squeeze().clone())
if linear.bias is not None:
@@ -144,8 +141,7 @@ class QuantLinear(nn.Module):
else:
repeat_zeros = zeros
- intweight = torch.round(W.to(device) / repeat_scales + repeat_zeros).to(
- torch.int32)
+ intweight = torch.round(W.to(device) / repeat_scales[:,:W.shape[1]] + repeat_zeros[:,:W.shape[1]])
del repeat_scales
intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index 08b8d7817..4759ae01e 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -57,7 +57,7 @@ def pack_layer(name, model, backend):
scale = scale.t().contiguous()
zp = zp.t().contiguous().to(torch.float32)
if sym:
- zp = 2 ** (bits - 1)
+ zp = int(zp.flatten()[0])
q_linear = WQLinear_GEMM.from_linear(
linear=linear_layer,
w_bit=bits,
@@ -119,6 +119,8 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
if output_dir is None:
return compressed_model
+ if os.path.exists(output_dir):
+ logger.warning(f"{output_dir} already exists, this may cause model conflict")
layer_config = kwargs["layer_config"]
for key in layer_config.keys():
if not check_to_quantized(layer_config[key]) and \
diff --git a/auto_round/export/export_to_awq/utils.py b/auto_round/export/export_to_awq/utils.py
index f0ad7db78..40107b802 100644
--- a/auto_round/export/export_to_awq/utils.py
+++ b/auto_round/export/export_to_awq/utils.py
@@ -98,15 +98,6 @@ def dequantize_gemm(qweight, qzeros, scales, bits, group_size):
return iweight
-def get_best_device():
- if torch.backends.mps.is_available():
- return "mps"
- elif torch.cuda.is_available():
- return "cuda:0"
- else:
- return "cpu"
-
-
class WQLinearMMFunction(Function):
@staticmethod
# ctx is the first argument to forward
@@ -222,14 +213,22 @@ class WQLinear_GEMM(nn.Module):
device = "cpu"
if torch.cuda.is_available():
device = "cuda:0"
+ elif torch.xpu.is_available():
+ device = "xpu:0"
repeat_scales = scales.to(device).t().repeat_interleave(group_size, 1)
if isinstance(zeros, torch.Tensor):
repeat_zeros = zeros.to(device).t().repeat_interleave(group_size, 1)
+ intweight = torch.round(
+ linear.weight.to(device) / repeat_scales[:, :linear.weight.shape[1]] +
+ repeat_zeros[:, :linear.weight.shape[1]]).to(torch.int).t().contiguous()
+
else:
repeat_zeros = zeros
- intweight = torch.round(linear.weight.to(device) / repeat_scales + repeat_zeros).to(
- torch.int).t().contiguous()
+ intweight = torch.round(
+ linear.weight.to(device) / repeat_scales[:, :linear.weight.shape[1]] + repeat_zeros).to(
+ torch.int).t().contiguous()
+
intweight = intweight.to(dtype=torch.int32)
del repeat_scales
diff --git a/auto_round/inference/__init__.py b/auto_round/inference/__init__.py
new file mode 100644
index 000000000..2b414eb72
--- /dev/null
+++ b/auto_round/inference/__init__.py
@@ -0,0 +1,14 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/auto_round/inference/auto_quantizer.py b/auto_round/inference/auto_quantizer.py
new file mode 100644
index 000000000..90fd551bd
--- /dev/null
+++ b/auto_round/inference/auto_quantizer.py
@@ -0,0 +1,359 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# coding=utf-8
+# Copyright 2023 HuggingFace Inc. team and GPTQ and AutoGPTQ authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import importlib.util
+import warnings
+from dataclasses import dataclass
+from logging import getLogger
+from typing import Any, Dict, Optional, Tuple, Union, List
+
+import torch
+import torch.nn as nn
+from packaging import version
+from transformers.modeling_utils import PreTrainedModel
+from transformers.quantizers import AutoQuantizationConfig, HfQuantizer
+from transformers.quantizers.auto import AUTO_QUANTIZER_MAPPING
+from transformers.utils.quantization_config import AwqConfig, GPTQConfig, QuantizationConfigMixin, QuantizationMethod
+from auto_round.utils import (is_hpu_supported)
+from auto_round.inference.convert_model import convert_hf_model, infer_target_device, post_init
+from enum import Enum
+
+logger = getLogger(__name__)
+import sys
+
+if sys.version_info < (3, 8):
+ import importlib_metadata
+else:
+ import importlib.metadata as importlib_metadata
+
+AUTOROUND_MINIMUM_VERSION = version.parse("0.2")
+
+
+def _is_package_available(pkg_name: str, return_version: bool = False) -> Union[Tuple[bool, str], bool]:
+ # Check we're not importing a "pkg_name" directory somewhere but the actual library by trying to grab the version
+ try: ##TODO remove it later
+ import auto_round
+ return True, auto_round.__version__
+ except:
+ pass
+
+ package_exists = importlib.util.find_spec(pkg_name) is not None
+ package_version = "N/A"
+ if package_exists:
+ try:
+ package_version = importlib.metadata.version(pkg_name)
+ package_exists = True
+ except importlib.metadata.PackageNotFoundError:
+ package_exists = False
+ if return_version:
+ return package_exists, package_version
+ else:
+ return package_exists
+
+
+_auto_round_available = _is_package_available("auto_round")
+
+
+def is_auto_round_available():
+ try:
+ import auto_round
+ return True
+ except:
+ pass
+ if _auto_round_available:
+ version_autoround = version.parse(importlib_metadata.version("auto_round"))
+ if AUTOROUND_MINIMUM_VERSION < version_autoround:
+ return True
+ else:
+ raise ImportError(
+ f"Found an incompatible version of auto-round. Found version {version_autoround},"
+ f" but only version above {AUTOROUND_MINIMUM_VERSION} are supported"
+ )
+
+
+class AutoHfQuantizer:
+ """The Auto-HF quantizer class that takes care of automatically instantiating to the correct
+ `HfQuantizer` given the `QuantizationConfig`."""
+
+ @classmethod
+ def from_config(cls, quantization_config: Union[QuantizationConfigMixin, Dict], **kwargs):
+ # Convert it to a QuantizationConfig if the q_config is a dict
+ if isinstance(quantization_config, dict):
+ if "auto-round" in quantization_config["quant_method"]:
+ quantization_config = AutoRoundConfig.from_dict(quantization_config)
+ else:
+ quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101
+ quant_method = quantization_config.quant_method
+
+ # Again, we need a special care for bnb as we have a single quantization config
+ # class for both 4-bit and 8-bit quantization
+ if quant_method == QuantizationMethod.BITS_AND_BYTES:
+ if quantization_config.load_in_8bit:
+ quant_method += "_8bit"
+ else:
+ quant_method += "_4bit"
+
+ if quant_method not in AUTO_QUANTIZER_MAPPING.keys() and "auto-round" not in quant_method:
+ raise ValueError(
+ f"Unknown quantization type, got {quant_method} - supported types are:"
+ f" {list(AUTO_QUANTIZER_MAPPING.keys())}"
+ )
+ if "auto-round" in quant_method or is_hpu_supported(): # pragma: no cover
+ target_cls = AutoRoundQuantizer
+ else:
+ target_cls = AUTO_QUANTIZER_MAPPING[quant_method]
+
+ return target_cls(quantization_config, **kwargs)
+
+ @classmethod
+ def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
+ quantization_config = AutoQuantizationConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
+ return cls.from_config(quantization_config)
+
+ @classmethod
+ def merge_quantization_configs(
+ cls,
+ quantization_config: Union[dict, QuantizationConfigMixin],
+ quantization_config_from_args: Optional[QuantizationConfigMixin],
+ ):
+ """Handles situations where both quantization_config
+ from args and quantization_config from model config are present."""
+ if quantization_config_from_args is not None:
+ warning_msg = (
+ "You passed `quantization_config` or equivalent parameters to "
+ "`from_pretrained` but the model you're loading"
+ " already has a `quantization_config` attribute. The `quantization_config` from the model will be used."
+ )
+ else:
+ warning_msg = ""
+
+ loading_attr_dict = quantization_config_from_args.get_loading_attributes() \
+ if quantization_config_from_args is not None else None
+ if isinstance(quantization_config, dict):
+ if "auto-round" in quantization_config["quant_method"]:
+ quantization_config = AutoRoundConfig.from_dict(quantization_config)
+ else:
+ if isinstance(quantization_config_from_args, (AutoRoundConfig)):
+ logger.info(f"Loading quantized model in auto_round format.")
+ tmp_backend = quantization_config["quant_method"]
+ if "auto-round" not in tmp_backend and "gptq" not in tmp_backend and "awq" not in tmp_backend:
+ logger.error("could not convert to auto_round format, currently only supports `gptq`,`awq` or "
+ "`auto-round` format")
+ exit(-1)
+ target_backend = quantization_config["backend"] if "backend" in quantization_config else "auto"
+ if loading_attr_dict is not None and "backend" in loading_attr_dict:
+ target_backend = loading_attr_dict["backend"]
+ loading_attr_dict.pop("backend")
+ if "auto_round" not in target_backend:
+ target_backend = f"auto_round:{tmp_backend}" #
+ quantization_config = AutoRoundConfig.from_dict(quantization_config)
+ setattr(quantization_config, "backend", target_backend)
+ else:
+ quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101
+
+ if isinstance(quantization_config,
+ (GPTQConfig, AwqConfig, AutoRoundConfig)) and quantization_config_from_args is not None:
+ # special case for GPTQ / AWQ config collision
+
+ for attr, val in loading_attr_dict.items():
+ setattr(quantization_config, attr, val)
+ warning_msg += (
+ f"However, loading attributes (e.g. {list(loading_attr_dict.keys())}) "
+ f"will be overwritten with the one you passed to `from_pretrained`. The rest will be ignored."
+ )
+
+ if warning_msg != "":
+ warnings.warn(warning_msg)
+
+ return quantization_config
+
+ @staticmethod
+ def supports_quant_method(quantization_config_dict):
+ from transformers.quantizers.auto import AUTO_QUANTIZATION_CONFIG_MAPPING
+ AUTO_QUANTIZATION_CONFIG_MAPPING['intel/auto-round'] = AutoRoundConfig
+ AUTO_QUANTIZATION_CONFIG_MAPPING['intel/auto_round'] = AutoRoundConfig
+ quant_method = quantization_config_dict.get("quant_method", None)
+ if quantization_config_dict.get("load_in_8bit", False) or quantization_config_dict.get("load_in_4bit", False):
+ suffix = "_4bit" if quantization_config_dict.get("load_in_4bit", False) else "_8bit"
+ quant_method = QuantizationMethod.BITS_AND_BYTES + suffix
+ elif quant_method is None:
+ raise ValueError(
+ "The model's quantization config from the arguments has no `quant_method` attribute." \
+ "Make sure that the model has been correctly quantized"
+ )
+
+ if quant_method not in AUTO_QUANTIZATION_CONFIG_MAPPING.keys():
+ logger.warning(
+ f"Unknown quantization type, got {quant_method} - supported types are:"
+ f" {list(AUTO_QUANTIZER_MAPPING.keys())}. Hence, we will skip the quantization. "
+ "To remove the warning, you can delete the quantization_config attribute in config.json"
+ )
+ return False
+ return True
+
+
+class AutoRoundQuantizationMethod(str, Enum):
+ AutoRound = "intel/auto-round"
+
+
+@dataclass
+class AutoRoundConfig(QuantizationConfigMixin):
+ """This is a wrapper class about all possible attributes and features that you can play with a model that has been
+ loaded AutoRound quantization.
+
+ Args:
+ bits (`int`):
+ The number of bits to quantize to, supported numbers are (2, 3, 4, 8).
+ tokenizer (`str` or `PreTrainedTokenizerBase`, *optional*):
+ The tokenizer used to process the dataset. You can pass either:
+ - A custom tokenizer object.
+ - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co.
+ - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved
+ using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.
+ """
+
+ def __init__(
+ self,
+ bits: int = 4,
+ tokenizer: Any = None,
+ dataset: str = None,
+ group_size: int = 128,
+ sym: bool = False,
+ backend="auto",
+ layer_config: dict = None,
+ **kwargs,
+ ):
+
+ self.bits = bits
+ self.tokenizer = tokenizer
+ self.dataset = dataset
+ self.group_size = group_size
+ self.sym = sym
+ self.target_backend = "auto"
+ self.backend = backend
+ self.layer_config = layer_config
+ if kwargs is not None:
+ for key in kwargs.keys():
+ setattr(self, key, kwargs[key])
+ self.quant_method = AutoRoundQuantizationMethod.AutoRound
+ self.post_init()
+
+ def post_init(self):
+ r"""Safety checker that arguments are correct."""
+ if self.bits not in [2, 4, 8]:
+ raise ValueError(f"Only support quantization to [2,4,8] bits but found {self.bits}")
+ if self.group_size != -1 and self.group_size <= 0:
+ raise ValueError("group_size must be greater than 0 or equal to -1")
+
+ def get_loading_attributes(self):
+ loading_attibutes_dict = {"target_backend": self.backend}
+ return loading_attibutes_dict
+
+ def to_dict(self):
+ config_dict = super().to_dict()
+ return config_dict
+
+
+class AutoRoundQuantizer(HfQuantizer):
+ """Quantizer of the AutoRound method, currently only triton and exllamav2 backend has been supported."""
+
+ requires_calibration = False
+ required_packages = ["auto_round"]
+ optimum_quantizer = None
+
+ def __init__(self, quantization_config: QuantizationConfigMixin, **kwargs):
+ self.device_map = None
+ super().__init__(quantization_config, **kwargs)
+
+ def validate_environment(self, *args, **kwargs):
+ self.device_map = kwargs.get("device_map", None)
+ if not is_auto_round_available():
+ raise ImportError("Loading a AutoRound quantized model requires auto-round library (`pip install "
+ "auto-round`)")
+ else:
+ try:
+ import auto_round
+ autoround_version = version.parse(auto_round.__version__)
+ except:
+ autoround_version = version.parse(importlib.metadata.version("auto_round"))
+ if autoround_version < version.parse("0.2.0"):
+ raise ImportError("You need a version of auto_round > 0.2.0 to use AutoRound: `pip install --upgrade "
+ "auto-round` or install from source")
+
+ def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":
+ self.target_device = infer_target_device(self.device_map)
+ if torch_dtype is None:
+ torch_dtype = torch.float16
+ elif torch_dtype != torch.float16:
+ logger.info("We suggest you to set `torch_dtype=torch.float16` for better efficiency on CUDA")
+ return torch_dtype
+
+ def post_init_model(self, model):
+ """Post-initialization that require device information, for example buffers initialization on device.
+
+ Args:
+ model (`nn.Module`):
+ The input model
+ """
+
+ class StoreAttr(object):
+ pass
+
+ model.quantize_config = StoreAttr()
+
+ post_init(model, self.used_backends)
+
+ def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs):
+ if self.pre_quantized:
+ target_device = self.target_device if hasattr(self, self.target_device) else infer_target_device(
+ self.device_map)
+ model, used_backends = convert_hf_model(model, target_device)
+ self.used_backends = used_backends
+
+ def _process_model_after_weight_loading(self, model: "PreTrainedModel", **kwargs):
+ if self.pre_quantized:
+ self.post_init_model(model)
+ else:
+ raise NotImplementedError
+
+ @property
+ def is_trainable(self, model: Optional["PreTrainedModel"] = None):
+ return True
+
+ @property
+ def is_serializable(self):
+ return True
+
+
+import transformers
+
+if version.parse(transformers.__version__) < version.parse("4.38.0"):
+ logger.error("Please upgrade transformers>=4.38.0 to support lm-head quantization")
+
+transformers.quantizers.auto.AutoHfQuantizer = AutoHfQuantizer
+transformers.modeling_utils.AutoHfQuantizer = AutoHfQuantizer
diff --git a/auto_round/backend.py b/auto_round/inference/backend.py
similarity index 57%
rename from auto_round/backend.py
rename to auto_round/inference/backend.py
index fd43af788..2f2b27f05 100644
--- a/auto_round/backend.py
+++ b/auto_round/inference/backend.py
@@ -18,6 +18,7 @@ from typing import List, Any, Optional
from transformers.utils.versions import require_version
+import auto_round_extension.cuda.gptqmodel_marlin
from auto_round.utils import get_library_version, logger
BackendInfos = {}
@@ -62,6 +63,7 @@ class BackendInfo:
sym: List[bool]
packing_format: str
bits: List[int]
+ dtype: List[str] = None
group_size: Optional[List[int]] = None
priority: int = 0 ##higher is better
convertable_format: List[str] = field(default_factory=list)
@@ -80,141 +82,167 @@ def feature_num_greater_checker(in_feature, out_feature, num):
return in_feature * out_feature > num
[email protected]_cache(None)
-def check_auto_round_exllamav2_installed():
- try:
- from autoround_exllamav2_kernels import gemm_half_q_half, make_q_matrix
- except ImportError:
- return False, ("please install from source to enable auto-round exllamav2 kernel."
- "`git clone https://github.com/intel/auto-round && cd auto-round &&"
- " pip install -vvv --no-build-isolation -e .`")
- return True, ""
-
-
feature_multiply_checker_32 = functools.partial(feature_multiply_checker, in_feature_multiplier=32)
-
+in_output_feature_multiply_checker_32 = functools.partial(feature_multiply_checker, in_feature_multiplier=32,
+ out_feature_multiplier=32)
feature_multiply_checker_marlin = functools.partial(feature_multiply_checker, in_feature_multiplier=128,
out_feature_multiplier=256)
feature_num_greater_checker_1024 = functools.partial(feature_num_greater_checker, num=1024)
+BackendInfos['auto_gptq:exllamav2'] = BackendInfo(device=["cuda"], sym=[True, False],
+ packing_format="triton_zp",
+ bits=[4],
+ priority=5,
+ dtype=["float16"],
+ group_size=[-1, 32, 64, 128, 256, 384, 512, 1024, 2048],
+ ##16 seems has accuracy issue
+ feature_checks=[feature_multiply_checker_32],
+ alias=['gptq', 'auto_gptq', 'exllamav2', "gptq:exllamav2"],
+ requirements=["auto-gptq>=0.7.1"]
+ )
-BackendInfos['auto_round:exllamav2'] = BackendInfo(device=["cuda"], sym=[True, False],
- packing_format="triton",
- bits=[4], group_size=None,
- priority=5,
- feature_checks=[feature_multiply_checker_32],
- alias=["auto_round"],
- requirements=[check_auto_round_exllamav2_installed]
- )
+BackendInfos['auto_gptq:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, False],
+ packing_format="triton_zp",
+ bits=[2, 4, 8], group_size=None,
+ dtype=["float16"],
+ priority=0, feature_checks=[feature_multiply_checker_32],
+ alias=["auto_gptq:tritonv2"],
+ requirements=["auto-gptq>=0.7.1"]
+ )
+
+BackendInfos['auto_gptq:cuda'] = BackendInfo(device=["cuda"], sym=[True, False],
+ packing_format="triton_zp",
+ bits=[2, 3, 4, 8], group_size=None,
+ priority=1, feature_checks=[feature_multiply_checker_32],
+ alias=["auto_gptq:cuda"],
+ dtype=["float16"],
+ convertable_format=["triton_zp"],
+ requirements=["auto-gptq>=0.7.1"]
+ )
BackendInfos['auto_round:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, False],
packing_format="triton",
- bits=[2, 4, 8], group_size=None,
- priority=0, feature_checks=[feature_multiply_checker_32],
- requirements=["triton<3.0,>=2.0"]
+ dtype=["float16", "bfloat16"],
+ bits=[2, 4, 8],
+ priority=1, feature_checks=[feature_multiply_checker_32],
+ alias=["auto_round", "tritonv2"],
+ requirements=["auto-round>=0.2"]
)
-BackendInfos['gptq:exllamav2'] = BackendInfo(device=["cuda"], sym=[True, False],
- packing_format="triton_zp+-1",
- bits=[4], group_size=None,
- priority=5,
- feature_checks=[feature_multiply_checker_32],
- alias=["auto_round:gptq:exllamav2", "auto_round:auto_gptq:exllamav2",
- 'gptq', 'auto_gptq', "auto_round:gptq", "auto_round:auto_gptq"],
- requirements=["auto-gptq>=0.7.1"]
- )
+BackendInfos['auto_round:tritonv2_zp'] = BackendInfo(device=["cuda"], sym=[True], ## asym has accuracy issue
+ packing_format="triton_zp",
+ dtype=["float16", "bfloat16"],
+ bits=[2, 4, 8],
+ priority=1, feature_checks=[feature_multiply_checker_32],
+ alias=["tritonv2", "tritonv2_zp"],
+ requirements=["auto-round>=0.5"]
+ )
+
+BackendInfos['gptqmodel:marlin'] = BackendInfo(device=["cuda"], sym=[True],
+ packing_format="triton",
+ bits=[4, 8],
+ group_size=[-1, 32, 64, 128],
+ dtype=["float16", "bfloat16"],
+ priority=6, feature_checks=[in_output_feature_multiply_checker_32],
+ alias=["marlin", "gptqmodel"],
+ requirements=["gptqmodel>=2.0"]
+ )
+
+BackendInfos['gptqmodel:marlin_zp'] = BackendInfo(device=["cuda"], sym=[True],
+ packing_format="triton_zp",
+ bits=[4, 8],
+ group_size=[-1, 32, 64, 128],
+ dtype=["float16", "bfloat16"],
+ priority=6, feature_checks=[in_output_feature_multiply_checker_32],
+ alias=["marlin", "gptqmodel"],
+ requirements=["gptqmodel>=2.0"]
+ )
+
+BackendInfos['gptqmodel:exllamav2'] = BackendInfo(device=["cuda"], sym=[True, False],
+ packing_format="triton",
+ bits=[4], group_size=[-1, 32, 64, 128], ##16 seems has accuracy issue
+ dtype=["float16", "bfloat16"],
+ priority=5, feature_checks=[in_output_feature_multiply_checker_32],
+ alias=["exllamav2"],
+ requirements=["gptqmodel>=2.0"]
+ )
-BackendInfos['gptq:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, False],
- packing_format="triton_zp+-1",
- bits=[2, 4, 8], group_size=None,
- priority=0, feature_checks=[feature_multiply_checker_32],
- alias=["auto_round:gptq:tritonv2", "auto_round:auto_gptq:tritonv2",
- "auto_gptq:tritonv2"],
- requirements=["auto-gptq>=0.7.1", "triton<3.0,>=2.0"]
+BackendInfos['auto_awq:gemm'] = BackendInfo(device=["cuda"], sym=[True, False], ##actrally is gemm
+ packing_format="awq",
+ bits=[4], group_size=None,
+ priority=4,
+ dtype=["float16"],
+ alias=["auto_awq:gemm", "awq",
+ "auto_awq"],
+ requirements=["autoawq"]
)
-BackendInfos['gptq:cuda'] = BackendInfo(device=["cuda"], sym=[True, False],
- packing_format="triton_zp+-1",
- bits=[2, 3, 4, 8], group_size=None,
- priority=1, feature_checks=[feature_multiply_checker_32],
- alias=["auto_round:auto_gptq:cuda", "auto_gptq:cuda", "auto_round:gptq:cuda"],
- convertable_format=["triton_zp+-1"],
- requirements=["auto-gptq>=0.7.1"]
+BackendInfos['qbits'] = BackendInfo(device=["cpu"], sym=[True, False],
+ packing_format="qbits",
+ bits=[2, 4, 8], group_size=None,
+ priority=0 if "intel" in get_cpu_manufacturer() else 5,
+ feature_checks=[],
+ alias=["itrex"],
+ dtype=["float16", "bfloat16"],
+ convertable_format=["triton"],
+ requirements=["intel-extension-for-transformers"])
+
+BackendInfos['qbits_zp'] = BackendInfo(device=["cpu"], sym=[True, False],
+ packing_format="qbits_zp",
+ bits=[2, 4, 8], group_size=None,
+ dtype=["float16", "bfloat16"],
+ priority=0 if "intel" in get_cpu_manufacturer() else 5,
+ feature_checks=[],
+ alias=["itrex"],
+ convertable_format=["triton_zp"],
+ requirements=["intel-extension-for-transformers"]
+ )
+
+# BackendInfos['auto_round:qbits_awq'] = BackendInfo(device=["cpu"], sym=[True], ## for awq, not robust
+# packing_format="awq",
+# bits=[2, 4, 8], group_size=None,
+# priority=0 if "intel" in get_cpu_manufacturer() else 5,
+# feature_checks=[],
+# requirements=["intel-extension-for-transformers"]
+# )
+
+BackendInfos['ipex_gptq'] = BackendInfo(device=["cpu", "xpu"], sym=[True, False],
+ packing_format="ipex_gptq",
+ bits=[4], group_size=None,
+ priority=5 if "intel" in get_cpu_manufacturer() else 5,
+ feature_checks=[],
+ dtype=["float16", "bfloat16"],
+ convertable_format=["triton_zp"],
+ alias=["ipex"],
+ requirements=["intel-extension-for-pytorch>=2.5"]
)
-BackendInfos['awq:gemm'] = BackendInfo(device=["cuda"], sym=[True, False], ##actrally is gemm
- packing_format="awq",
+BackendInfos['ipex_awq'] = BackendInfo(device=["cpu", "xpu"], sym=[True, False],
+ packing_format="ipex_awq",
bits=[4], group_size=None,
- priority=4,
- alias=["auto_awq:gemm", "auto_round:awq:gemm", "auto_round:auto_awq:gemm", "awq",
- "auto_awq", "auto_round:awq", "auto_round:auto_awq"],
- requirements=["autoawq"]
+ priority=1,
+ dtype=["float16", "bfloat16"],
+ feature_checks=[],
+ alias=["ipex"],
+ convertable_format=["awq"],
+ requirements=["intel-extension-for-pytorch>=2.6"]
)
-BackendInfos['auto_round:qbits'] = BackendInfo(device=["cpu"], sym=[True, False],
- packing_format="qbits",
- bits=[2, 4, 8], group_size=None,
- priority=0 if "intel" in get_cpu_manufacturer() else 5,
- feature_checks=[],
- convertable_format=["triton"],
- requirements=["intel-extension-for-transformers"])
-
-BackendInfos['auto_round:qbits_zp'] = BackendInfo(device=["cpu"], sym=[True, False],
- packing_format="qbits_zp+-1",
- bits=[2, 4, 8], group_size=None,
- priority=0 if "intel" in get_cpu_manufacturer() else 5,
- feature_checks=[],
- convertable_format=["triton_zp+-1"],
- requirements=["intel-extension-for-transformers"]
- )
-
-BackendInfos['auto_round:qbits_awq'] = BackendInfo(device=["cpu"], sym=[True],
- packing_format="awq",
- bits=[2, 4, 8], group_size=None,
- priority=0 if "intel" in get_cpu_manufacturer() else 5,
- feature_checks=[],
- requirements=["intel-extension-for-transformers"]
- )
-
-BackendInfos['auto_round:ipex_gptq'] = BackendInfo(device=["cpu", "xpu"], sym=[True, False],
- packing_format="ipex_gptq",
- bits=[4], group_size=None,
- priority=5 if "intel" in get_cpu_manufacturer() else 5,
- feature_checks=[],
- convertable_format=["triton_zp+-1"],
- requirements=["intel-extension-for-pytorch>=2.5"]
- )
-
-BackendInfos['auto_round:ipex_awq_xpu'] = BackendInfo(device=["cpu", "xpu"], sym=[True, False],
- packing_format="ipex_awq",
- bits=[4], group_size=None,
- priority=1,
- feature_checks=[],
- ##convertable_format=["triton_zp+-1", "awq"],
- convertable_format=["awq"],
- requirements=["intel-extension-for-pytorch>=2.6"]
- )
-
-# BackendInfos['auto_round:marlin'] = BackendInfo(device=["gpu"], sym=[True],
-# packing_format="marlin",
-# bits=[4], group_size=[-1, 128],
-# priority=6,
-# feature_checks=[feature_multiply_checker_marlin],
-# alias=["marlin", "auto_gptq:marlin", "auto_round:gptq:marlin",
-# "auto_round:auto_gptq:marlin"])
-
-BackendInfos['auto_round:hpu'] = BackendInfo(device=["hpu"], sym=[True, False],
- packing_format="hpu",
- bits=[4],
- priority=0,
- convertable_format=["triton"]
- )
+BackendInfos['hpu'] = BackendInfo(device=["hpu"], sym=[True, False],
+ packing_format="hpu",
+ bits=[4],
+ dtype=["bfloat16"],
+ priority=0,
+ convertable_format=["triton"]
+ )
-BackendInfos['auto_round:hpu_zp'] = BackendInfo(device=["hpu"], sym=[True, False],
- packing_format="hpu_zp+-1",
- bits=[4],
- priority=0,
- convertable_format=["triton_zp+-1"])
+BackendInfos['hpu_zp'] = BackendInfo(device=["hpu"], sym=[True, False],
+ packing_format="hpu_zp",
+ bits=[4],
+ dtype=["bfloat16"],
+ alias=["hpu"],
+ priority=0,
+ convertable_format=["triton_zp"])
def check_compatible(backend_name, device, bits, group_size, sym, packing_format, in_features, out_features,
@@ -265,16 +293,16 @@ def check_compatible(backend_name, device, bits, group_size, sym, packing_format
if sym not in backend.sym:
return False
- # Check packing format and apply feature checks
- if packing_format == backend.packing_format:
- for check in backend.feature_checks:
- if not check(in_features, out_features):
- return False
-
# Check if the format is convertible when packing formats differ
- if packing_format != backend.packing_format and packing_format not in backend.convertable_format:
+ if packing_format == backend.packing_format or packing_format in backend.convertable_format:
+ pass
+ else:
return False
+ for check in backend.feature_checks: ## convertible
+ if not check(in_features, out_features):
+ return False
+
if check_requirements and backend.requirements is not None:
for requirement in backend.requirements:
if isinstance(requirement, str):
@@ -339,16 +367,6 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):
from auto_round_extension.ipex.qlinear_ipex_awq import QuantLinear
return QuantLinear
- if "marlin" in backend:
- from transformers.utils.versions import require_version
- require_version(
- "gptqmodel",
- "marlin format requires gptqmodel to be installed, `pip install -v gptqmodel --no-build-isolation`"
- )
- from gptqmodel.nn_modules.qlinear.qlinear_marlin_inference import \
- MarlinInferenceQuantLinear # pylint: disable=E0401
- return MarlinInferenceQuantLinear
-
if "hpu" in backend:
try:
import habana_frameworks.torch.hpu # pylint: disable=E0401
@@ -361,8 +379,10 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):
else: # auto_round must be at the end
from auto_round_extension.hpu.qlinear_hpu import QuantLinear
return QuantLinear
+ if "gptqmodel" in backend:
+ return get_gptqmodel_infer_linear(backend, bits, group_size, sym)
- if "gptq" in backend:
+ if "gptq" in backend and "gptqmodel" not in backend:
return get_autogptq_infer_linear(backend, bits, group_size, sym)
if "awq" in backend:
@@ -373,13 +393,28 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):
"autoawq is required. Please install it by 'pip install autoawq' to support auto_awq format.")
return WQLinear_GEMM
- if "auto_round" in backend:
- if "exllamav2" in backend:
- import auto_round_extension.cuda.qlinear_exllamav2
- return auto_round_extension.cuda.qlinear_exllamav2.QuantLinear
- else:
- import auto_round_extension.cuda.qlinear_tritonv2
- return auto_round_extension.cuda.qlinear_tritonv2.QuantLinear
+ if backend == "auto_round:tritonv2":
+ from auto_round_extension.cuda.qlinear_tritonv2 import QuantLinear
+ return QuantLinear
+
+ if backend == "auto_round:tritonv2_zp":
+ from auto_round_extension.cuda.qlinear_tritonv2_zp import QuantLinear
+ return QuantLinear
+
+ raise ValueError(f"unsupported backend {backend}, please set it to `auto` and retry")
+
+
+def get_gptqmodel_infer_linear(backend, bits=4, group_size=128, sym=False):
+ import gptqmodel # pylint: disable=E0401
+ if "marlin" in backend:
+ return auto_round_extension.cuda.gptqmodel_marlin.get_marlin_layer()
+ # return gptqmodel.nn_modules.qlinear.marlin.MarlinQuantLinear
+ elif "exllamav2" in backend:
+ return gptqmodel.nn_modules.qlinear.exllamav2.ExllamaV2QuantLinear
+ elif "tritonv2" in backend:
+ return gptqmodel.nn_modules.qlinear.tritonv2.TritonV2QuantLinear
+ else:
+ raise ValueError(f"Unsupported {backend}")
def get_autogptq_infer_linear(backend, bits=4, group_size=128, sym=False):
@@ -469,6 +504,57 @@ def get_autogptq_infer_linear(backend, bits=4, group_size=128, sym=False):
return QuantLinear
+def find_backend(target_backend: str, orig_backend: str = None) -> str | None:
+ """
+ Finds the matching backend key based on the target backend name or its aliases.
+
+ Args:
+ target_backend (str): Name or alias of the target backend.
+ orig_backend (str, optional): Original backend name to check compatibility. Defaults to None.
+
+ Returns:
+ str or None: Matching backend key if found and compatible; otherwise, None.
+ """
+ matched_keys = [
+ key for key, info in BackendInfos.items()
+ if key == target_backend or (info.alias and target_backend in info.alias)
+ ]
+
+ if not matched_keys:
+ return None
+
+ if orig_backend is None:
+ return matched_keys[0] if len(matched_keys) >= 1 else None
+
+ orig_info = BackendInfos[orig_backend]
+
+ for key in matched_keys:
+ target_info = BackendInfos[key]
+ if (target_info.packing_format == orig_info.packing_format or
+ orig_info.packing_format in target_info.convertable_format):
+ return key
+
+ raise ValueError(
+ f"{target_backend} is not compatible with {orig_backend}. "
+ f"Please set `backend` to `auto` and retry."
+ )
+
+
+def get_all_compatible_backend(device, backend, orig_backend, bits, group_size, sym, in_features, out_features):
+ # Get packing format from the original backend
+ packing_format = BackendInfos[orig_backend].packing_format
+
+ # Find compatible backends
+ compatible_backends = [
+ key for key in BackendInfos.keys()
+ if check_compatible(key, device, bits, group_size, sym, packing_format, in_features, out_features,
+ check_requirements=False)
+ ]
+
+ # Return the first compatible backend or an empty list if none found
+ return compatible_backends
+
+
def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_features, out_features):
"""Selects the most suitable backend for the layer based on compatibility and priority.
@@ -505,30 +591,23 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_f
If no compatible backend is found for the given layer configuration.
"""
# Check if the provided backend is in BackendInfos
+ backend = find_backend(backend)
assert backend in BackendInfos.keys(), \
f"Unsupported backend {backend}, please set it to `auto` to try automatic selection"
packing_format = BackendInfos[orig_backend].packing_format
- # Check if the provided backend supports the layer configuration
- if check_compatible(backend, device, bits, group_size, sym, packing_format, in_features, out_features):
- return backend
-
# Find and store other compatible backends
supported_backends = []
for key in BackendInfos.keys():
- if key == backend:
- continue
if check_compatible(key, device, bits, group_size, sym, packing_format, in_features, out_features):
supported_backends.append(key)
# Raise an error if no compatible backends are found
if len(supported_backends) == 0:
- supported_backends_need_package = []
- for key in BackendInfos.keys():
- if check_compatible(key, device, bits, group_size, sym, packing_format, in_features, out_features,
- check_requirements=False):
- supported_backends_need_package.append(key)
+ supported_backends_need_package = get_all_compatible_backend(device, backend, orig_backend, bits, group_size,
+ sym,
+ in_features, out_features)
if len(supported_backends_need_package) > 0:
supported_backends_need_package = sorted(supported_backends_need_package,
@@ -547,7 +626,7 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_f
logger.error(str_info)
exit(-1)
- raise ValueError(f"None of the backends support this layer")
+ return None
# Sort the compatible backends by priority and return the one with the highest priority
supported_backends = sorted(supported_backends, key=lambda support_backend: BackendInfos[support_backend].priority,
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
new file mode 100644
index 000000000..ec3ed6c3f
--- /dev/null
+++ b/auto_round/inference/convert_model.py
@@ -0,0 +1,425 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from logging import getLogger
+from typing import Union
+from tqdm import tqdm
+import torch
+import torch.nn as nn
+
+from transformers.pytorch_utils import Conv1D
+
+from auto_round.utils import (get_module, set_module, is_hpu_supported, get_block_names,
+ find_matching_blocks, get_layer_names_in_block, check_to_quantized)
+
+from auto_round.inference.backend import get_layer_backend, dynamic_import_inference_linear, find_backend, BackendInfos
+
+logger = getLogger(__name__)
+
+supported_devices = ("cpu", "hpu", "xpu", "cuda")
+
+
+def get_available_devices():
+ """
+ Returns a list of available devices in the current environment.
+
+ Returns:
+ List[str]: A list of device identifiers like "cuda", "hpu", "xpu", "cpu".
+ """
+ devices = []
+
+ if torch.cuda.is_available():
+ devices.append("cuda")
+
+ if is_hpu_supported():
+ devices.append("hpu")
+
+ if hasattr(torch, "xpu") and torch.xpu.is_available():
+ devices.append("xpu")
+
+ devices.append("cpu") # Always available
+
+ return devices
+
+
+def parse_target_device_and_backend(target_backend: str):
+ """
+ Parse the target device and backend from the given target_backend string.
+
+ Args:
+ target_backend (str): A string specifying the target backend, which may include a device prefix.
+
+ Returns:
+ tuple: (target_device, backend), where:
+ - target_device (str): The identified device (e.g., "cpu", "cuda", "auto").
+ - backend (str): The backend string after removing the device prefix.
+ """
+
+ def remove_device_prefix(backend: str, prefix: str) -> str:
+ """Removes the given device prefix from the backend string, if present."""
+ return backend[len(prefix):].lstrip(":") if backend.startswith(prefix) else backend
+
+ # Handle "auto" case explicitly
+ if target_backend == "auto":
+ return "auto", ""
+ if target_backend.startswith("auto:"):
+ return "auto", target_backend[5:].lstrip(":")
+
+ # Define supported devices
+
+ # Identify the target device and adjust the backend string accordingly
+ for device in supported_devices:
+ if target_backend.startswith(device):
+ return device, remove_device_prefix(target_backend, device)
+
+ # Default case: If no known device is found, assume "auto"
+ return "auto", target_backend
+
+
+def get_layer_config(model, quantization_config):
+ """
+ get a layer-wise quantization configuration for a given model.
+
+ Args:
+ model (torch.nn.Module): The model for which quantization settings are generated.
+ quantization_config (object): An object containing quantization parameters, including:
+ - bits (int): Default bit width for quantization.
+ - group_size (int): Group size for weight quantization.
+ - data_type (str, optional): Data type for quantization (default: "int").
+ - sym (bool): Whether to use symmetric quantization.
+ - quant_block_list (list, optional): Predefined list of blocks to quantize.
+ - to_quant_block_names (list or str, optional): Blocks to quantize (if quant_block_list is None).
+ - extra_config (dict, optional): Per-layer overrides for quantization settings.
+ - modules_in_block_to_quantize (list, optional): Specific modules within a block for quantization.
+ - modules_to_not_convert (list, optional): Layers excluded from quantization (AWQ format).
+
+ Returns:
+ dict: A dictionary mapping layer names to their quantization configurations, where each layer has:
+ - "bits" (int): Bit width for quantization.
+ - "group_size" (int): Group size for quantization.
+ - "data_type" (str): Data type used for quantization.
+ - "sym" (bool): Whether symmetric quantization is applied.
+ - "clip" (bool): Whether weight clipping is enabled.
+ """
+ bits = quantization_config.bits
+ group_size = quantization_config.group_size
+ data_type = getattr(quantization_config, "data_type", "int") # Default to "int" if not specified
+ sym = quantization_config.sym
+
+ # Determine the quantization block list
+ quant_block_list = getattr(quantization_config, "quant_block_list", None)
+ if quant_block_list is None:
+ to_quant_block_names = getattr(quantization_config, "to_quant_block_names", None)
+ if isinstance(to_quant_block_names, (list, tuple)):
+ quant_block_list = to_quant_block_names
+ elif isinstance(to_quant_block_names, str):
+ # Generate quant block names based on the given layer names
+ quant_block_list = [
+ [f'{block}.{i}' for i in range(len(get_module(model, block)))]
+ for block in to_quant_block_names.split(',')
+ ]
+ else:
+ # Find matching blocks if no explicit names are provided
+ all_blocks = get_block_names(model)
+ quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)
+
+ # Get layer names that will be quantized
+ layer_names = get_layer_names_in_block(model, quant_block_list=quant_block_list)
+
+ # Load extra configuration if available
+ extra_config = getattr(quantization_config, "extra_config", {})
+
+ # Process GPTQ format: identify modules that should be quantized
+ if getattr(quantization_config, "modules_in_block_to_quantize", None):
+ modules_in_block_to_quantize = sum(quantization_config.modules_in_block_to_quantize, []) # Flatten the list
+ for layer_name in layer_names:
+ if not any(qname in layer_name for qname in modules_in_block_to_quantize):
+ extra_config[layer_name] = {"bits": 16} # Default to 16-bit for unquantized layers
+
+ # Process AWQ format: exclude specified modules from quantization
+ modules_to_not_convert = getattr(quantization_config, "modules_to_not_convert", [])
+ if modules_to_not_convert is None:
+ modules_to_not_convert = []
+ for layer_name in modules_to_not_convert:
+ extra_config[layer_name] = {"bits": 16}
+
+ # Merge and deduplicate layer names
+ layer_names = list(set(layer_names).union(extra_config.keys()))
+
+ # Construct final layer configuration
+ layer_configs = {
+ layer_name: {
+ "bits": extra_config.get(layer_name, {}).get("bits", bits),
+ "group_size": extra_config.get(layer_name, {}).get("group_size", group_size),
+ "data_type": extra_config.get(layer_name, {}).get("data_type", data_type),
+ "sym": extra_config.get(layer_name, {}).get("sym", sym),
+ "clip": extra_config.get(layer_name, {}).get("clip", False),
+ }
+ for layer_name in layer_names
+ }
+
+ return layer_configs
+
+
+def get_device(obj: Union[torch.Tensor, nn.Module]):
+ if isinstance(obj, torch.Tensor):
+ return obj.device
+ return next(obj.parameters()).device
+
+
+def _replace_by_quant_layers(module: nn.Module, layer_configs, target_backend, target_device, orig_backend):
+ """
+ Replaces linear layers in the given module with quantized layers.
+
+ Args:
+ module (nn.Module): The module containing layers to be quantized.
+ layer_configs (dict): Configuration for each layer's quantization.
+ target_backend (str): Backend for quantization (device and format).
+ target_device (str): Device for execution ('cuda', 'cpu', 'hpu').
+ orig_backend (str): Original backend of the model.
+
+ Returns:
+ dict: Flags indicating which backends were used.
+ """
+
+ used_backends = []
+ must_use_target_backend = False
+ if target_backend:
+ must_use_target_backend = True
+ layer_backend = target_backend
+ layer_backend_must = find_backend(layer_backend, orig_backend)
+ used_backends.append(layer_backend_must)
+ if layer_backend_must is None:
+ raise ValueError(
+ f"{target_backend} is not compatible, please change backend to `auto` and retry")
+ devices = BackendInfos[layer_backend_must].device
+ if target_device not in devices:
+ raise ValueError(f"{target_backend} does not support {target_device}, please change device or backend")
+
+ target_backend = target_backend or orig_backend # Default to original backend if not specified
+
+ backend_cache = {}
+
+ for layer_name, config in layer_configs.items():
+ if not check_to_quantized(config):
+ continue # Skip layers that do not require quantization
+
+ layer = get_module(module, layer_name)
+ in_features, out_features = _get_layer_features(layer)
+ if in_features is None:
+ continue # Skip unsupported layer types
+
+ if must_use_target_backend:
+ layer_backend = layer_backend_must
+ else:
+ key = f"{config['bits']}_{config['group_size']}_{config['sym']}_{in_features}_{out_features}"
+ if key in backend_cache:
+ layer_backend = backend_cache[key]
+ if layer_backend not in used_backends:
+ used_backends.append(layer_backend)
+ else:
+ # Determine backend
+ layer_backend = _get_layer_backend(target_device, target_backend, orig_backend, config,
+ in_features, out_features)
+ backend_cache[key] = layer_backend
+
+ if not layer_backend:
+ raise ValueError(f"No compatible backend found for layer {layer_name} with config {config}")
+
+ logger.debug(f"{layer_name}: {layer_backend} backend is used") ##TODO delete
+
+ # Create and replace layer
+ new_layer = _create_quant_layer(layer, layer_backend, config, in_features, out_features)
+ set_module(module, layer_name, new_layer)
+
+ return used_backends
+
+
+def _get_layer_features(layer):
+ """Extracts input and output feature dimensions for supported layers."""
+ if isinstance(layer, nn.Linear):
+ return layer.in_features, layer.out_features
+ elif isinstance(layer, Conv1D): # TODO: Verify correctness
+ return layer.weight.shape[0], layer.weight.shape[1]
+ return None, None # Unsupported layer type
+
+
+def _get_layer_backend(target_device, target_backend, orig_backend, config, in_features,
+ out_features):
+ """Determines the best backend for a given layer."""
+
+ backend = get_layer_backend(target_device, target_backend, orig_backend, config["bits"], config["group_size"],
+ config["sym"], in_features, out_features)
+
+ return backend
+
+
+def _import_exllamav2_kernels():
+ """Attempts to import ExLlamaV2 kernels for performance optimization."""
+ try:
+ from exllamav2_kernels import gemm_half_q_half, make_q_matrix # pylint: disable=E0611, E0401
+ except ImportError:
+ logger.warning_once(
+ "For better inference performance, install ExLlamaV2 kernel via: "
+ "`pip install git+https://github.com/AutoGPTQ/AutoGPTQ.git@b8b4127`"
+ )
+
+
+def _create_quant_layer(layer, layer_backend, config, in_features, out_features):
+ """Creates a quantized layer using the appropriate class."""
+ QuantLinear = dynamic_import_inference_linear(layer_backend, config["bits"], config["group_size"], config["sym"])
+ bias = layer.bias is not None
+
+ # Special handling for AWQ layers
+ from auto_round_extension.qbits.qbits_awq import QuantLinear as QBitsAWQQuantLinear
+ if "awq" in layer_backend and isinstance(QuantLinear, QBitsAWQQuantLinear):
+ return QuantLinear.from_linear(layer, config["bits"], config["group_size"], init_only=True,
+ has_zero_points=not config["sym"])
+ elif "awq" in layer_backend:
+ return QuantLinear.from_linear(layer, config["bits"], config["group_size"], init_only=True)
+ elif "gptqmodel" in layer_backend:
+ return QuantLinear(bits=config["bits"], group_size=config["group_size"], desc_act=False, sym=config["sym"],
+ in_features=in_features, out_features=out_features, bias=bias)
+ # Default quantized layer creation
+ try:
+ return QuantLinear(config["bits"], config["group_size"], in_features, out_features, bias,
+ weight_dtype=layer.weight.dtype, clip=config["clip"])
+ except: # Handle cases where `clip` is not a valid argument
+ return QuantLinear(config["bits"], config["group_size"], in_features, out_features, bias,
+ weight_dtype=layer.weight.dtype)
+
+
+def infer_target_device(device_map=None):
+ if device_map is None:
+ target_device = "cpu"
+ elif isinstance(device_map, dict):
+ devices = set(device_map.values())
+ target_device = "cpu"
+ for device in devices:
+ if device != "cpu" and device != "disk":
+ if isinstance(device, int):
+ target_device = get_available_devices()[0]
+ else:
+ target_device = str(device).split(":")[0]
+ else:
+ target_device = get_available_devices()[0]
+ assert isinstance(target_device, str)
+ return target_device
+
+
+def post_init(model, used_backends):
+ need_autogptq_init = False
+ need_gptqmodel_init = False
+ need_ipex_itrex_init = False
+ used_gptq_exllamav2 = False
+ for used_backend in used_backends:
+ if used_backend.startswith("auto_gptq"):
+ need_autogptq_init = True
+ if used_backend == "auto_gptq:exllamav2":
+ used_gptq_exllamav2 = True
+ elif used_backend.startswith("gptqmodel"):
+ need_gptqmodel_init = True
+ elif used_backend.startswith("ipex"):
+ need_ipex_itrex_init = True
+ elif used_backend.startswith("qbit"):
+ need_ipex_itrex_init = True
+ if need_autogptq_init:
+ from auto_gptq.modeling._utils import autogptq_post_init as gptq_post_init # pylint: disable=E0401
+ model = gptq_post_init(model, use_act_order=False)
+ elif need_gptqmodel_init:
+ from gptqmodel.utils.model import hf_gptqmodel_post_init as gptq_post_init # pylint: disable=E0401
+ model = gptq_post_init(model, use_act_order=False)
+ elif need_ipex_itrex_init:
+ message = "repacking to CPU/XPU format"
+ layers = [] ## ipex post_init will add one more layer
+ for n, m in model.named_modules():
+ if hasattr(m, "QUANT_TYPE") and ("qbits" in m.QUANT_TYPE or "ipex" in m.QUANT_TYPE):
+ layers.append(m)
+
+ for layer in tqdm(layers, desc=message, total=len(layers),
+ leave=True):
+ layer.post_init()
+
+ if used_gptq_exllamav2:
+ _import_exllamav2_kernels()
+
+ ## convert datatype
+ data_types=[]
+ for used_backend in used_backends:
+ backend = BackendInfos[used_backend]
+ data_types.append(backend.dtype)
+ common = set(data_types[0])
+ for l in data_types[1:]:
+ common &= set(l)
+ common = list(common)
+ if str(model.dtype).split('.')[-1] not in common:
+ if common[0]=="float16":
+ model = model.to(torch.float16)
+ logger.warning("force model to float16")
+ elif common[0]=="bfloat16":
+ model = model.to(torch.bfloat16)
+ logger.warning("force model to bfloat16")
+
+
+def convert_hf_model(model: nn.Module, target_device="cpu"):
+ """Converts the given model to an AutoRound model by replacing its layers with quantized layers.
+
+ This method extracts the quantization configuration from the model and adjusts its layers
+ according to the specified quantization parameters. It supports different backends and
+ ensures that the model's data type is compatible with the selected hardware.
+
+ Args:
+ model (nn.Module):
+ The model to be converted into an AutoRound model.
+
+ Returns:
+ nn.Module:
+ The converted AutoRound model with quantized layers.
+
+ Raises:
+ ValueError:
+ If the quantization backend is not specified in the configuration.
+ """
+
+
+ quantization_config = model.config.quantization_config
+ if not hasattr(quantization_config, "target_backend"):
+ quantization_config.target_backend = quantization_config.backend
+
+ ##target_backend could be None
+ _, target_backend = parse_target_device_and_backend(quantization_config.target_backend)
+
+ if hasattr(quantization_config, "backend"): # pragma: no cover
+ backend = quantization_config.backend
+ elif 'gptq' in quantization_config.quant_method: # pragma: no cover
+ backend = "auto_gptq"
+ elif "awq" in quantization_config.quant_method:
+ backend = "auto_awq"
+ else: # pragma: no cover
+ backend = "auto_gptq"
+ logger.warning("Quantization backend must be specified. Set it to 'auto_gptq' by default.")
+ if backend == "auto":
+ backend = "auto_gptq"
+
+ layer_configs = get_layer_config(model, quantization_config)
+ if backend.startswith("auto_round:") and ("gptq" in backend or "awq" in backend):
+ backend = backend[len("auto_round:"):]
+ backend = find_backend(backend)
+
+ if target_backend.startswith("auto_round:") and ("gptq" in backend or "awq" in backend):
+ target_backend = target_backend[len("auto_round:"):]
+
+ used_backends = _replace_by_quant_layers(model, layer_configs, target_backend, target_device, backend)
+
+ return model, used_backends
diff --git a/auto_round/mllm/eval.py b/auto_round/mllm/eval.py
index 568bbd2ba..44de86c56 100644
--- a/auto_round/mllm/eval.py
+++ b/auto_round/mllm/eval.py
@@ -103,7 +103,7 @@ def mllm_eval(
try:
from auto_round import AutoRoundConfig
except:
- from auto_round.auto_quantizer import AutoHfQuantizer
+ from auto_round.inference.auto_quantizer import AutoHfQuantizer
model = None
if data_store_dir is not None:
@@ -350,7 +350,6 @@ def lmms_eval(
use_cache=None,
apply_chat_template=False
):
- from auto_round import AutoRoundConfig
from transformers.utils.versions import require_version
require_version("lmms_eval", "lmms_eval need to be installed, `pip install lmms_eval`")
if isinstance(tasks, str):
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 62639b555..6fb6aa1b4 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -292,7 +292,7 @@ def tune(args):
"auto_round", "auto_round:auto_gptq", "auto_round:auto_awq", "auto_awq", "gguf:q4_0", "gguf:q4_1", "fake"
]
if not args.quant_nontext_module:
- supported_formats.extend(["auto_gptq", "auto_gptq:marlin"])
+ supported_formats.extend(["auto_gptq"])
formats = args.format.replace(' ', '').split(",")
for format in formats:
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 677c64e1d..9f0d58a95 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -32,11 +32,10 @@ import transformers
from auto_round.export.export_to_gguf.config import GGUF_CONFIG
supported_formats = (
- "auto_round", "auto_gptq", "auto_awq", "auto_round:auto_gptq", "auto_round:auto_awq", "auto_gptq:marlin",
+ "auto_round", "auto_gptq", "auto_awq", "auto_round:auto_gptq", "auto_round:gptqmodel", "auto_round:auto_awq",
"itrex", "itrex_xpu", "fake"
)
-
supported_formats = supported_formats + tuple(GGUF_CONFIG.keys())
supported_layer_types = (torch.nn.Linear, transformers.modeling_utils.Conv1D)
@@ -523,14 +522,13 @@ def check_to_quantized(config):
False otherwise.
"""
if isinstance(config, dict):
-
- if int(config["bits"]) > 8 and int(config["act_bits"] > 8):
- return False
- return True
+ bits = int(config.get("bits", 4))
+ act_bits = int(config.get("act_bits", 16))
else:
- if int(config.bits) > 8 and int(config.act_bits) > 8:
- return False
- return True
+ bits = int(config.bits) if hasattr(config, "bits") else 4
+ act_bits = int(config.act_bits) if hasattr(config, "act_bits") else 16
+
+ return bits <= 8 or act_bits <= 8
def detect_device_count():
@@ -1206,6 +1204,15 @@ def is_debug_mode():
return sys.gettrace() is not None or sys.flags.debug == 1
+
+def get_layer_features(layer):
+ """Extracts input and output feature dimensions for supported layers."""
+ if isinstance(layer, torch.nn.Linear):
+ return layer.in_features, layer.out_features
+ elif isinstance(layer, transformers.pytorch_utils.Conv1D): # TODO: Verify correctness
+ return layer.weight.shape[0], layer.weight.shape[1]
+ return None, None # Unsupported layer type
+
def _gguf_args_check(args):
from auto_round.utils import logger
from auto_round.export.export_to_gguf.config import GGUF_CONFIG
@@ -1256,4 +1263,5 @@ def _gguf_args_check(args):
f" reset to {', '.join(reset_list)}.")
logger.info(f"export format {format}, sym = {not args.asym}, group_size = {args.group_size}")
- return args
\ No newline at end of file
+ return args
+
diff --git a/auto_round_extension/cuda/exllamav2/config.h b/auto_round_extension/cuda/exllamav2/config.h
deleted file mode 100644
index 86baaf412..000000000
--- a/auto_round_extension/cuda/exllamav2/config.h
+++ /dev/null
@@ -1,13 +0,0 @@
-#ifndef _config_h
-#define _config_h
-
-#define MAX_Q_GEMM_ROWS 50
-
-#define QMODE_2BIT 1
-#define QMODE_3BIT 1
-#define QMODE_4BIT 1
-#define QMODE_5BIT 1
-#define QMODE_6BIT 0
-#define QMODE_8BIT 0
-
-#endif
diff --git a/auto_round_extension/cuda/exllamav2/cpp/util.h b/auto_round_extension/cuda/exllamav2/cpp/util.h
deleted file mode 100644
index 919703a89..000000000
--- a/auto_round_extension/cuda/exllamav2/cpp/util.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _util_h
-#define _util_h
-
-#define DBGS(__x) printf("%s\n", __x)
-#define DBGI(__x) printf("%s: %i\n", #__x, __x)
-#define DBGI2(__x, __y) printf("%s, %s: %i, %i\n", #__x, #__y, __x, __y)
-#define DBGI3(__x, __y, __z) printf("%s, %s, %s: %i, %i, %i\n", #__x, #__y, #__z, __x, __y, __z)
-#define DBGF(__x) printf("%s: %f\n", #__x, __x)
-#define DBGF2(__x, __y) printf("%s, %s: %f, %f\n", #__x, #__y, __x, __y)
-#define DBGF3(__x, __y, __z) printf("%s, %s, %s: %f, %f, %f\n", #__x, #__y, #__z, __x, __y, __z)
-
-#endif
diff --git a/auto_round_extension/cuda/exllamav2/cuda/compat.cuh b/auto_round_extension/cuda/exllamav2/cuda/compat.cuh
deleted file mode 100644
index 12684ff8b..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/compat.cuh
+++ /dev/null
@@ -1,56 +0,0 @@
-#ifndef _compat_cuh
-#define _compat_cuh
-
-// atomicAdd for half types, to support CC < 7.x
-
-__device__ __forceinline__ void atomicAdd_half(half* address, half val)
-{
- unsigned int * address_as_ui = (unsigned int *) ((char *)address - ((size_t)address & 2));
- unsigned int old = *address_as_ui;
- unsigned int assumed;
-
- do
- {
- assumed = old;
- __half_raw hsum;
- hsum.x = (size_t)address & 2 ? (old >> 16) : (old & 0xffff);
- half tmpres = __hadd(hsum, val);
- hsum = __half_raw(tmpres);
- old = (size_t)address & 2 ? (old & 0xffff) | (hsum.x << 16) : (old & 0xffff0000) | hsum.x;
- old = atomicCAS(address_as_ui, assumed, old);
- }
- while (assumed != old);
-}
-
-// atomicAdd for half2 types
-
-__device__ __forceinline__ void atomicAdd_half2(half2* address, half2 val)
-{
- unsigned int* address_as_ui = (unsigned int*)address;
- unsigned int old = *address_as_ui;
- unsigned int assumed;
- do
- {
- assumed = old;
- half2 old_val = *((half2*)&old);
- half2 new_val = __hadd2(old_val, val);
- old = atomicCAS(address_as_ui, assumed, *((unsigned int*)&new_val));
- }
- while (assumed != old);
-}
-
-//
-
-#if defined(__CUDA_ARCH__) || defined(USE_ROCM)
-#if __CUDA_ARCH__ < 700 || defined(USE_ROCM)
-
-__device__ __forceinline__ void atomicAdd(half* address, half val) { atomicAdd_half(address, val); }
-
-#if __CUDA_ARCH__ < 600 || defined(USE_ROCM)
-__device__ __forceinline__ void atomicAdd(half2* address, half2 val) { atomicAdd_half2(address, val); }
-#endif
-
-#endif
-#endif
-
-#endif
diff --git a/auto_round_extension/cuda/exllamav2/cuda/compat_gemm.cuh b/auto_round_extension/cuda/exllamav2/cuda/compat_gemm.cuh
deleted file mode 100644
index 19b1e4a60..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/compat_gemm.cuh
+++ /dev/null
@@ -1,38 +0,0 @@
-#ifndef _compat_gemm_cuh
-#define _compat_gemm_cuh
-
-#if defined(USE_ROCM)
-
-// For some reason this include is not present anywhere in exllama_v2 codebase, but it is required
-// for symbols as hipblasHalf.
-#include <hipblas/hipblas.h>
-
-__host__ __forceinline__ hipblasStatus_t __compat_hipblasHgemm(hipblasHandle_t handle,
- hipblasOperation_t transA,
- hipblasOperation_t transB,
- int m,
- int n,
- int k,
- const half* alpha,
- const half* AP,
- int lda,
- const half* BP,
- int ldb,
- const half* beta,
- half* CP,
- int ldc) {
- return hipblasHgemm(handle, transA, transB, m, n, k,
- reinterpret_cast<const hipblasHalf *>(alpha),
- reinterpret_cast<const hipblasHalf *>(AP), lda,
- reinterpret_cast<const hipblasHalf *>(BP), ldb,
- reinterpret_cast<const hipblasHalf *>(beta),
- reinterpret_cast<hipblasHalf *>(CP), ldc);
-}
-#define hipblasHgemm __compat_hipblasHgemm
-
-// Previous version of PyTorch were converting to rocBLAS instead of hipBLAS.
-#define rocblas_operation_none HIPBLAS_OP_N
-#define rocblas_hgemm __compat_hipblasHgemm
-#endif
-
-#endif
diff --git a/auto_round_extension/cuda/exllamav2/cuda/matrix_view.cuh b/auto_round_extension/cuda/exllamav2/cuda/matrix_view.cuh
deleted file mode 100644
index 55af84f23..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/matrix_view.cuh
+++ /dev/null
@@ -1,121 +0,0 @@
-#ifndef _matrix_view_cuh
-#define _matrix_view_cuh
-
-#include <cuda_runtime.h>
-#include <cuda_fp16.h>
-
-#include "quant/qdq_util.cuh"
-
-class MatrixView_half
-{
-public:
- const half* data;
- const int height;
- const int width;
-
- __device__ __forceinline__ MatrixView_half(const half* data, const int height, const int width)
- : data(data), height(height), width(width)
- { }
-
- __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; }
- __device__ __forceinline__ half2 item_half2(int row, int column) const { return ((half2*)data)[(row * width + column) / 2]; }
- __device__ __forceinline__ half2 item_half2half2(int row, int column) const { return __half2half2(data[row * width + column]); }
- __device__ __forceinline__ const half* item_ptr(int row, int column) const { return &data[row * width + column]; }
-
- __device__ __forceinline__ void item4(half (&items)[4], int row, int column) const
- {
- half2* ptr = (half2*) item_ptr(row, column);
- half2 i01 = ptr[0];
- half2 i23 = ptr[1];
- items[0] = __low2half(i01);
- items[1] = __high2half(i01);
- items[2] = __low2half(i23);
- items[3] = __high2half(i23);
- }
- __device__ __forceinline__ void item4_f(float (&items)[4], int row, int column) const
- {
- half2* ptr = (half2*)item_ptr(row, column);
- half2 i01 = ptr[0];
- half2 i23 = ptr[1];
- items[0] = __half2float(__low2half(i01));
- items[1] = __half2float(__high2half(i01));
- items[2] = __half2float(__low2half(i23));
- items[3] = __half2float(__high2half(i23));
- }
-
- __device__ __forceinline__ void item4_h2(half2 (&items)[4], int row, int column) const
- {
- half2* ptr = (half2*)item_ptr(row, column);
- half2 i01 = ptr[0];
- half2 i23 = ptr[1];
- items[0] = __half2half2(__low2half(i01));
- items[1] = __half2half2(__high2half(i01));
- items[2] = __half2half2(__low2half(i23));
- items[3] = __half2half2(__high2half(i23));
- }
-};
-
-class MatrixView_half_rw
-{
-public:
- half* data;
- const int height;
- const int width;
-
- __device__ __forceinline__ MatrixView_half_rw(half* data, const int height, const int width)
- : data(data), height(height), width(width)
- { }
-
- __device__ __forceinline__ half item(int row, int column) const { return data[row * width + column]; }
- __device__ __forceinline__ half2 item_half2(int row, int column) const { return ((half2*)data)[(row * width + column) / 2]; }
- __device__ __forceinline__ half* item_ptr(int row, int column) { return &data[row * width + column]; }
- __device__ __forceinline__ void set(int row, int column, half value) { data[row * width + column] = value; }
- __device__ __forceinline__ void set_half2(int row, int column, half2 value) { ((half2*)data)[(row * width + column) / 2] = value; }
-
- __device__ __forceinline__ void set4(int row, int column, half v0, half v1, half v2, half v3)
- {
- half2 v01 = __halves2half2(v0, v1);
- half2 v23 = __halves2half2(v2, v3);
- half2* ptr = (half2*) item_ptr(row, column);
- ptr[0] = v01;
- ptr[1] = v23;
- }
-};
-
-class MatrixView_q4_row
-{
-public:
- const uint32_t* data;
- const int height;
- const int width;
-
- __device__ __forceinline__ MatrixView_q4_row(const uint32_t* data, const int height, const int width)
- : data(data), height(height), width(width)
- { }
-
- __device__ __forceinline__ int item(int row, int column) const
- {
- int shift = (column & 0x07) * 4;
- return (data[row * width / 8 + column / 8] >> shift) & 0x0f;
- }
-
- __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const
- {
- int shift = (column & 0x07) * 4;
- uint32_t d = data[row * width / 8 + column / 8] >> shift;
- items[0] = d & 0x0f;
- items[1] = (d >> 4) & 0x0f;
- }
-
- __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const
- {
- int shift = (column & 0x07) * 4;
- uint32_t d = data[row * width / 8 + column / 8] >> shift;
- items[0] = d & 0x0f;
- items[1] = (d >> 4) & 0x0f;
- items[2] = (d >> 8) & 0x0f;
- items[3] = (d >> 12) & 0x0f;
- }
-};
-
-#endif
\ No newline at end of file
diff --git a/auto_round_extension/cuda/exllamav2/cuda/q_gemm.cu b/auto_round_extension/cuda/exllamav2/cuda/q_gemm.cu
deleted file mode 100644
index 351b9cd5b..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/q_gemm.cu
+++ /dev/null
@@ -1,211 +0,0 @@
-#include "q_gemm.cuh"
-#include "util.cuh"
-#include "matrix_view.cuh"
-#include "../config.h"
-
-#include "quant/qdq_2.cuh"
-#include "quant/qdq_3.cuh"
-#include "quant/qdq_4.cuh"
-#include "quant/qdq_5.cuh"
-#include "quant/qdq_6.cuh"
-#include "quant/qdq_8.cuh"
-
-#define BLOCK_KN_SIZE 128
-#define BLOCK_M_SIZE_MAX 8
-#define MAX_GROUPS_IN_BLOCK (BLOCK_KN_SIZE / 32)
-#define CLEAR_N_SIZE 256
-
-#include "q_gemm_kernel.cuh"
-#include "q_gemm_kernel_gptq.cuh"
-
-#include "compat_gemm.cuh"
-
-void gemm_half_q_half_cuda_part
-(
- const half* a,
- QMatrix* b,
- half* c,
- int size_m,
- int size_n,
- int size_k,
- int m_count,
- bool clear
-)
-{
- if (!b->is_gptq)
- {
- dim3 blockDim, gridDim;
- blockDim.x = BLOCK_KN_SIZE;
- blockDim.y = 1;
- blockDim.z = 1;
- gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4);
- gridDim.y = DIVIDE(size_m, m_count);
- gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
-
- fp_gemm_half_q_half_kernel kernel = pick_gemm_half_q_half_kernel(true, m_count);
-
- kernel<<<gridDim, blockDim>>>
- (
- a,
- b->cuda_q_weight,
- b->cuda_q_scale,
- b->cuda_q_scale_max,
- c,
- size_m,
- size_n,
- size_k,
- b->groups,
- b->groupsize,
- b->cuda_q_perm,
- b->rows_8,
- b->rows_6,
- b->rows_5,
- b->rows_4,
- b->rows_3,
- b->rows_2,
- clear
- );
- }
- else
- {
- dim3 blockDim, gridDim;
- blockDim.x = BLOCK_KN_SIZE;
- blockDim.y = 1;
- blockDim.z = 1;
- gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4);
- gridDim.y = DIVIDE(size_m, m_count);
- gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE);
-
- fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(true, m_count);
-
-// DBGX((uint64_t) b->cuda_q_perm);
-// DBGI(b->rows_4);
-// DBGI(b->height);
-
- kernel<<<gridDim, blockDim>>>
- (
- a,
- b->cuda_q_weight,
- b->cuda_gptq_qzeros,
- b->cuda_gptq_scales,
- c,
- size_m,
- size_n,
- size_k,
- b->groups,
- b->groupsize,
- b->cuda_q_perm,
- b->rows_4,
- clear
- );
- }
-}
-
-void gemm_half_q_half_cuda
-(
- cublasHandle_t cublas_handle,
- const half* a,
- QMatrix* b,
- half* c,
- int size_m,
- int size_n,
- int size_k,
- bool clear,
- half* temp_dq,
- bool force_cuda
-)
-{
- if (size_m > MAX_Q_GEMM_ROWS && !force_cuda)
- {
- //printf("cublas\n");
-
- // Reconstruct FP16 matrix, then cuBLAS
-
- if (!temp_dq) temp_dq = b->temp_dq;
- b->reconstruct(temp_dq);
-
- //cublasSetMathMode(cublas_handle, CUBLAS_TENSOR_OP_MATH);
-
- const half alpha = __float2half(1.0f);
- const half beta = clear ? __float2half(0.0f) : __float2half(1.0f);
- cublasHgemm(cublas_handle,
- CUBLAS_OP_N,
- CUBLAS_OP_N,
- size_n, size_m, size_k,
- &alpha, temp_dq, size_n,
- a, size_k,
- &beta, c, size_n);
-
- //const float alpha = 1.0f;
- //const float beta = clear ? 0.0f : 1.0f;
- //cublasSgemmEx(cublas_handle,
- // CUBLAS_OP_N,
- // CUBLAS_OP_N,
- // size_n, size_m, size_k,
- // &alpha, temp_dq, CUDA_R_16F, size_n,
- // a, CUDA_R_16F, size_k,
- // &beta, c, CUDA_R_16F, size_n);
-
- //const float alpha = 1.0f;
- //const float beta = clear ? 0.0f : 1.0f;
- //cublasGemmEx(cublas_handle,
- // CUBLAS_OP_N, CUBLAS_OP_N,
- // size_n, size_m, size_k,
- // &alpha, temp_dq, CUDA_R_16F, size_n,
- // a, CUDA_R_16F, size_k,
- // &beta, c, CUDA_R_16F, size_n,
- // CUDA_R_16F, CUBLAS_GEMM_DFALT_TENSOR_OP);
- }
- else
- {
- //printf("cuda\n");
-
- // Quantized matmul
-
- //if (clear) clear_tensor_cuda(c, size_m, size_n);
-
- int max_chunks = size_m / BLOCK_M_SIZE_MAX;
- int last_chunk = max_chunks * BLOCK_M_SIZE_MAX;
- int last_chunk_size = size_m - last_chunk;
-
- if (max_chunks)
- {
- gemm_half_q_half_cuda_part(a, b, c, last_chunk, size_n, size_k, BLOCK_M_SIZE_MAX, clear);
- }
-
- if (last_chunk_size)
- {
- gemm_half_q_half_cuda_part(a + last_chunk * size_k, b, c + last_chunk * size_n, last_chunk_size, size_n, size_k, last_chunk_size, clear);
- }
- }
-}
-
-__global__ void clear_kernel
-(
- half* __restrict__ c,
- const int size_m,
- const int size_n
-)
-{
- int m = blockIdx.y;
- int n = (blockIdx.x * CLEAR_N_SIZE + threadIdx.x) * 8;
- if (n >= size_n) return;
- int4* c_ptr = (int4*)(c + m * size_n + n);
- *c_ptr = {};
-}
-
-void clear_tensor_cuda
-(
- half* c,
- int size_m,
- int size_n
-)
-{
- return;
- dim3 blockDim, gridDim;
- blockDim.x = CLEAR_N_SIZE;
- blockDim.y = 1;
- gridDim.x = DIVIDE(size_n / 8, CLEAR_N_SIZE);
- gridDim.y = size_m;
- clear_kernel<<<gridDim, blockDim>>>(c, size_m, size_n);
-}
diff --git a/auto_round_extension/cuda/exllamav2/cuda/q_gemm.cuh b/auto_round_extension/cuda/exllamav2/cuda/q_gemm.cuh
deleted file mode 100644
index c69f1a709..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/q_gemm.cuh
+++ /dev/null
@@ -1,33 +0,0 @@
-#ifndef _q_gemm_cuh
-#define _q_gemm_cuh
-
-#include <cuda_runtime.h>
-#include <cuda_fp16.h>
-#include <cstdint>
-#include <cstdio>
-#include <ATen/cuda/CUDAContext.h>
-
-#include "q_matrix.cuh"
-
-void gemm_half_q_half_cuda
-(
- cublasHandle_t cublas_handle,
- const half* a,
- QMatrix* b,
- half* c,
- int size_m,
- int size_n,
- int size_k,
- bool clear = false,
- half* reconstruct = NULL,
- bool force_cuda = false
-);
-
-void clear_tensor_cuda
-(
- half* c,
- int size_m,
- int size_n
-);
-
-#endif
\ No newline at end of file
diff --git a/auto_round_extension/cuda/exllamav2/cuda/q_gemm_kernel.cuh b/auto_round_extension/cuda/exllamav2/cuda/q_gemm_kernel.cuh
deleted file mode 100644
index 0b899a840..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/q_gemm_kernel.cuh
+++ /dev/null
@@ -1,487 +0,0 @@
-#include "compat.cuh"
-
-#include <cuda_runtime.h>
-#include <cuda_fp16.h>
-
-__forceinline__ __device__ half2 dot22_8(half2(&dq)[4], const half* a_ptr, const half2 g_result, const half qs_h)
-{
- half2 result = {};
- const half2* a2_ptr = (const half2*)a_ptr;
- #pragma unroll
- for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
- return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
-}
-
-__forceinline__ __device__ half2 dot22_16(half2(&dq)[8], const half* a_ptr, const half2 g_result, const half qs_h)
-{
- half2 result = {};
- const half2* a2_ptr = (const half2*)a_ptr;
- #pragma unroll
- for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
- return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
-}
-
-__forceinline__ __device__ half2 dot22_32(half2(&dq)[16], const half* a_ptr, const half2 g_result, const half qs_h)
-{
- half2 result = {};
- const half2* a2_ptr = (const half2*)a_ptr;
- #pragma unroll
- for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
- return __hfma2(result, __halves2half2(qs_h, qs_h), g_result);
-}
-
-__forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr, const float g_result, const float qs_f)
-{
- half2 result = {};
- const half2* a2_ptr = (const half2*)a_ptr;
- #pragma unroll
- for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
- float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));
- return fma(result_f, qs_f, g_result);
-}
-
-__forceinline__ __device__ float dot22_16_f(half2(&dq)[8], const half* a_ptr, const float g_result, const float qs_f)
-{
- half2 result = {};
- const half2* a2_ptr = (const half2*)a_ptr;
- #pragma unroll
- for (int i = 0; i < 8; i++) result = __hfma2(dq[i], *a2_ptr++, result);
- float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));
- return fma(result_f, qs_f, g_result);
-}
-
-__forceinline__ __device__ float dot22_32_f(half2(&dq)[16], const half* a_ptr, const float g_result, const float qs_f)
-{
- half2 result = {};
- const half2* a2_ptr = (const half2*)a_ptr;
- #pragma unroll
- for (int i = 0; i < 16; i += 1) result = __hfma2(dq[i], *a2_ptr++, result);
- float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result));
- return fma(result_f, qs_f, g_result);
-}
-
-
-
-typedef void (*fp_gemm_half_q_half_kernel)
-(
- const half*,
- const uint32_t*,
- const uint32_t*,
- const half*,
- half*,
- const int,
- const int,
- const int,
- const int,
- const int,
- const uint16_t*,
- const int,
- const int,
- const int,
- const int,
- const int,
- const int,
- const bool
-);
-
-template <bool first_block, int m_count>
-__global__ void gemm_half_q_half_kernel
-(
- const half* __restrict__ a,
- const uint32_t* __restrict__ b_q_weight,
- const uint32_t* __restrict__ b_q_scale,
- const half* __restrict__ b_q_scale_max,
- half* __restrict__ c,
- const int size_m,
- const int size_n,
- const int size_k,
- const int groups,
- const int groupsize,
- const uint16_t* __restrict__ b_q_perm,
- const int rows_8,
- const int rows_6,
- const int rows_5,
- const int rows_4,
- const int rows_3,
- const int rows_2,
- const bool clear
-)
-{
- MatrixView_half a_(a, size_m, size_k);
- MatrixView_half_rw c_(c, size_m, size_n);
- MatrixView_q4_row b_q_scale_(b_q_scale, groups, size_n);
-
- int t = threadIdx.x;
-
- // Block
-
- int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
- int offset_m = blockIdx.y * m_count;
- int offset_k = blockIdx.z * BLOCK_KN_SIZE;
-
- int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
- int end_m = min(offset_m + m_count, size_m);
- int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
- int n = offset_n + t * 4;
-
- // Preload block_a
-
- __shared__ half block_a[m_count][BLOCK_KN_SIZE];
-
- if (offset_k + t < end_k)
- {
- for (int m = 0; m < m_count; ++m)
- {
- const half* a_ptr = a_.item_ptr(offset_m + m, 0);
- half* block_a_ptr = block_a[m];
- half a0 = a_ptr[b_q_perm[offset_k + t]];
- block_a_ptr[t] = a0;
- }
- }
-
- // Clear
-
- if (n >= size_n) return;
-
- if (clear && blockIdx.z == 0) // && (threadIdx.x & 1) == 0)
- {
- for (int m = 0; m < m_count; m++)
- *((uint64_t*) c_.item_ptr(offset_m + m, n)) = 0;
- }
-
- __syncthreads();
-
- // Find initial group
-
- int group = offset_k / groupsize;
-
- // Preload scales
-
- float scales[MAX_GROUPS_IN_BLOCK][4];
-
- int groups_in_block = DIVIDE((end_k - offset_k), groupsize);
- for (int g = 0; g < groups_in_block; g++)
- {
- int qscales[4];
- b_q_scale_.item4(qscales, group + g, n);
- qscales[0]++;
- qscales[1]++;
- qscales[2]++;
- qscales[3]++;
- float maxscale = __half2float(b_q_scale_max[group + g]);
- scales[g][0] = __int2float_rn(qscales[0] * qscales[0]) * maxscale;
- scales[g][1] = __int2float_rn(qscales[1] * qscales[1]) * maxscale;
- scales[g][2] = __int2float_rn(qscales[2] * qscales[2]) * maxscale;
- scales[g][3] = __int2float_rn(qscales[3] * qscales[3]) * maxscale;
- }
-
- // a, b offset
-
- int pre_rows_8 = min(rows_8, offset_k);
- int pre_rows_6 = offset_k > rows_8 ? min(rows_6, offset_k) - rows_8 : 0;
- int pre_rows_5 = offset_k > rows_6 ? min(rows_5, offset_k) - rows_6 : 0;
- int pre_rows_4 = offset_k > rows_5 ? min(rows_4, offset_k) - rows_5 : 0;
- int pre_rows_3 = offset_k > rows_4 ? min(rows_3, offset_k) - rows_4 : 0;
- int pre_rows_2 = offset_k > rows_3 ? min(rows_2, offset_k) - rows_3 : 0;
- int qk = 0;
- qk += pre_rows_8 / 32 * 8;
- qk += pre_rows_6 / 32 * 6;
- qk += pre_rows_5 / 32 * 5;
- qk += pre_rows_4 / 32 * 4;
- qk += pre_rows_3 / 32 * 3;
- qk += pre_rows_2 / 32 * 2;
-
- const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
- const half* a_ptr = &block_a[0][0];
- int a_stride = BLOCK_KN_SIZE;
-
- // Initial group
-
- int scales_idx = 0;
- float qs_f0 = scales[scales_idx][0];
- float qs_f1 = scales[scales_idx][1];
- float qs_f2 = scales[scales_idx][2];
- float qs_f3 = scales[scales_idx][3];
- int nextgroup = offset_k + groupsize;
-
- // Column result
-
- float block_c[m_count][4] = {};
-
- // Dequantize groups
-
- int k = offset_k;
-
- while (k < rows_8 && k < end_k)
- {
- if (k == nextgroup)
- {
- group++;
- scales_idx++;
- qs_f0 = scales[scales_idx][0];
- qs_f1 = scales[scales_idx][1];
- qs_f2 = scales[scales_idx][2];
- qs_f3 = scales[scales_idx][3];
- nextgroup += groupsize;
- }
-
- #pragma unroll
- for (int j = 0; j < 4; j++)
- {
- int4 load_int4[2];
- load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
- load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
-
- half2 dq[4][4];
- dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n);
- dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n);
- dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n);
- dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n);
-
- for (int m = 0; m < m_count; m++)
- {
- block_c[m][0] = dot22_8_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0);
- block_c[m][1] = dot22_8_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1);
- block_c[m][2] = dot22_8_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2);
- block_c[m][3] = dot22_8_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3);
- }
- a_ptr += 8;
- }
- k += 32;
- }
-
- while (k < rows_6 && k < end_k)
- {
- if (k == nextgroup)
- {
- group++;
- scales_idx++;
- qs_f0 = scales[scales_idx][0];
- qs_f1 = scales[scales_idx][1];
- qs_f2 = scales[scales_idx][2];
- qs_f3 = scales[scales_idx][3];
- nextgroup += groupsize;
- }
-
- #pragma unroll
- for (int j = 0; j < 2; j++)
- {
- int4 load_int4[3];
- load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
- load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
- load_int4[2] = *((int4*) b_ptr); b_ptr += size_n;
-
- half2 dq[4][8];
- dequant_6bit_16(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n);
- dequant_6bit_16(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n);
- dequant_6bit_16(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n);
- dequant_6bit_16(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n);
-
- for (int m = 0; m < m_count; m++)
- {
- block_c[m][0] = dot22_16_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0);
- block_c[m][1] = dot22_16_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1);
- block_c[m][2] = dot22_16_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2);
- block_c[m][3] = dot22_16_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3);
- }
- a_ptr += 16;
- }
- k += 32;
- }
-
- while (k < rows_5 && k < end_k)
- {
- if (k == nextgroup)
- {
- group++;
- scales_idx++;
- qs_f0 = scales[scales_idx][0];
- qs_f1 = scales[scales_idx][1];
- qs_f2 = scales[scales_idx][2];
- qs_f3 = scales[scales_idx][3];
- nextgroup += groupsize;
- }
-
- #pragma unroll
- for (int j = 0; j < 1; j++)
- {
- int4 load_int4[5];
- load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
- load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
- load_int4[2] = *((int4*) b_ptr); b_ptr += size_n;
- load_int4[3] = *((int4*) b_ptr); b_ptr += size_n;
- load_int4[4] = *((int4*) b_ptr); b_ptr += size_n;
-
- half2 dq[4][16];
- dequant_5bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, load_int4[3].x, load_int4[4].x, dq[0], size_n);
- dequant_5bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, load_int4[3].y, load_int4[4].y, dq[1], size_n);
- dequant_5bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, load_int4[3].z, load_int4[4].z, dq[2], size_n);
- dequant_5bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, load_int4[3].w, load_int4[4].w, dq[3], size_n);
-
- for (int m = 0; m < m_count; m++)
- {
- block_c[m][0] = dot22_32_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0);
- block_c[m][1] = dot22_32_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1);
- block_c[m][2] = dot22_32_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2);
- block_c[m][3] = dot22_32_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3);
- }
- a_ptr += 32;
- }
-
- k += 32;
- }
-
- while (k < rows_4 && k < end_k)
- {
- if (k == nextgroup)
- {
- group++;
- scales_idx++;
- qs_f0 = scales[scales_idx][0];
- qs_f1 = scales[scales_idx][1];
- qs_f2 = scales[scales_idx][2];
- qs_f3 = scales[scales_idx][3];
- nextgroup += groupsize;
- }
-
- #pragma unroll
- for (int j = 0; j < 4; j++)
- {
- int4 load_int4[1];
- load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
-
- half2 dq[4][4];
- dequant_4bit_8(load_int4[0].x, dq[0], size_n);
- dequant_4bit_8(load_int4[0].y, dq[1], size_n);
- dequant_4bit_8(load_int4[0].z, dq[2], size_n);
- dequant_4bit_8(load_int4[0].w, dq[3], size_n);
-
- for (int m = 0; m < m_count; m++)
- {
- block_c[m][0] = dot22_8_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0);
- block_c[m][1] = dot22_8_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1);
- block_c[m][2] = dot22_8_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2);
- block_c[m][3] = dot22_8_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3);
- }
- a_ptr += 8;
- }
- k += 32;
- }
-
- while (k < rows_3 && k < end_k)
- {
- if (k == nextgroup)
- {
- group++;
- scales_idx++;
- qs_f0 = scales[scales_idx][0];
- qs_f1 = scales[scales_idx][1];
- qs_f2 = scales[scales_idx][2];
- qs_f3 = scales[scales_idx][3];
- nextgroup += groupsize;
- }
-
- #pragma unroll
- for (int j = 0; j < 1; j++)
- {
- int4 load_int4[3];
- load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
- load_int4[1] = *((int4*) b_ptr); b_ptr += size_n;
- load_int4[2] = *((int4*) b_ptr); b_ptr += size_n;
-
- half2 dq[4][16];
- dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n);
- dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n);
- dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n);
- dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n);
-
- for (int m = 0; m < m_count; m++)
- {
- block_c[m][0] = dot22_32_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0);
- block_c[m][1] = dot22_32_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1);
- block_c[m][2] = dot22_32_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2);
- block_c[m][3] = dot22_32_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3);
- }
- a_ptr += 32;
- }
- k += 32;
- }
-
- while (k < rows_2 && k < end_k)
- {
- if (k == nextgroup)
- {
- group++;
- scales_idx++;
- qs_f0 = scales[scales_idx][0];
- qs_f1 = scales[scales_idx][1];
- qs_f2 = scales[scales_idx][2];
- qs_f3 = scales[scales_idx][3];
- nextgroup += groupsize;
- }
-
- #pragma unroll
- for (int j = 0; j < 2; j++)
- {
- int4 load_int4[1];
- load_int4[0] = *((int4*) b_ptr); b_ptr += size_n;
-
- half2 dq[4][8];
- dequant_2bit_16(load_int4[0].x, dq[0], size_n);
- dequant_2bit_16(load_int4[0].y, dq[1], size_n);
- dequant_2bit_16(load_int4[0].z, dq[2], size_n);
- dequant_2bit_16(load_int4[0].w, dq[3], size_n);
-
- for (int m = 0; m < m_count; m++)
- {
- block_c[m][0] = dot22_16_f(dq[0], a_ptr + m * a_stride, block_c[m][0], qs_f0);
- block_c[m][1] = dot22_16_f(dq[1], a_ptr + m * a_stride, block_c[m][1], qs_f1);
- block_c[m][2] = dot22_16_f(dq[2], a_ptr + m * a_stride, block_c[m][2], qs_f2);
- block_c[m][3] = dot22_16_f(dq[3], a_ptr + m * a_stride, block_c[m][3], qs_f3);
- }
-
- a_ptr += 16;
- }
- k += 32;
- }
-
- // Accumulate column sums in c
-
- for (int m = 0; m < m_count; m++)
- {
- half2* out = (half2*)c_.item_ptr(offset_m + m, n);
- half2 result01 = __halves2half2(__float2half_rn(block_c[m][0]), __float2half_rn(block_c[m][1]));
- half2 result23 = __halves2half2(__float2half_rn(block_c[m][2]), __float2half_rn(block_c[m][3]));
- atomicAdd(out , result01);
- atomicAdd(out + 1, result23);
- }
-}
-
-fp_gemm_half_q_half_kernel pick_gemm_half_q_half_kernel(bool first_block, const int m_count)
-{
- #if BLOCK_M_SIZE_MAX >= 1
- if (m_count == 1) return gemm_half_q_half_kernel<true, 1>;
- #endif
- #if BLOCK_M_SIZE_MAX >= 2
- if (m_count == 2) return gemm_half_q_half_kernel<true, 2>;
- #endif
- #if BLOCK_M_SIZE_MAX >= 3
- if (m_count == 3) return gemm_half_q_half_kernel<true, 3>;
- #endif
- #if BLOCK_M_SIZE_MAX >= 4
- if (m_count == 4) return gemm_half_q_half_kernel<true, 4>;
- #endif
- #if BLOCK_M_SIZE_MAX >= 5
- if (m_count == 5) return gemm_half_q_half_kernel<true, 5>;
- #endif
- #if BLOCK_M_SIZE_MAX >= 6
- if (m_count == 6) return gemm_half_q_half_kernel<true, 6>;
- #endif
- #if BLOCK_M_SIZE_MAX >= 7
- if (m_count == 7) return gemm_half_q_half_kernel<true, 7>;
- #endif
- #if BLOCK_M_SIZE_MAX >= 8
- if (m_count == 8) return gemm_half_q_half_kernel<true, 8>;
- #endif
- return NULL;
-}
diff --git a/auto_round_extension/cuda/exllamav2/cuda/q_gemm_kernel_gptq.cuh b/auto_round_extension/cuda/exllamav2/cuda/q_gemm_kernel_gptq.cuh
deleted file mode 100644
index 4b722ef5a..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/q_gemm_kernel_gptq.cuh
+++ /dev/null
@@ -1,223 +0,0 @@
-#include "compat.cuh"
-
-__forceinline__ __device__ half2 dot22_8(half2(&dq)[4], const half* a_ptr, const half2 g_result)
-{
- half2 result = {};
- const half2* a2_ptr = (const half2*)a_ptr;
- #pragma unroll
- for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
- return __hadd2(result, g_result);
-}
-
-__forceinline__ __device__ float dot22_8_f(half2(&dq)[4], const half* a_ptr)
-{
- half2 result = {};
- const half2* a2_ptr = (const half2*)a_ptr;
- #pragma unroll
- for (int i = 0; i < 4; i++) result = __hfma2(dq[i], *a2_ptr++, result);
- return __half2float(__low2half(result)) + __half2float(__high2half(result));
-}
-
-typedef void (*fp_gemm_half_q_half_gptq_kernel)
-(
- const half*,
- const uint32_t*,
- const uint32_t*,
- const half*,
- half*,
- const int,
- const int,
- const int,
- const int,
- const int,
- const uint16_t*,
- const int,
- const bool
-);
-
-template <bool first_block, int m_count>
-__global__ void gemm_half_q_half_gptq_kernel
-(
- const half* __restrict__ a,
- const uint32_t* __restrict__ b_q_weight,
- const uint32_t* __restrict__ b_gptq_qzeros,
- const half* __restrict__ b_gptq_scales,
- half* __restrict__ c,
- const int size_m,
- const int size_n,
- const int size_k,
- const int groups,
- const int groupsize,
- const uint16_t* __restrict__ b_q_perm,
- const int rows_4,
- const bool clear
-)
-{
- MatrixView_half a_(a, size_m, size_k);
- MatrixView_half_rw c_(c, size_m, size_n);
- MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
- MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
-
- int t = threadIdx.x;
-
- // Block
-
- int offset_n = blockIdx.x * BLOCK_KN_SIZE * 4;
- int offset_m = blockIdx.y * m_count;
- int offset_k = blockIdx.z * BLOCK_KN_SIZE;
-
- int end_n = min(offset_n + BLOCK_KN_SIZE * 4, size_n);
- int end_m = min(offset_m + m_count, size_m);
- int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
-
- int n = offset_n + t * 4;
-
- // Preload block_a
-
- __shared__ half block_a[m_count][BLOCK_KN_SIZE];
-
- if (offset_k + t < end_k)
- {
- for (int m = 0; m < m_count; ++m)
- {
- const half* a_ptr = a_.item_ptr(offset_m + m, 0);
- half* block_a_ptr = block_a[m];
-
- half a0;
- if (b_q_perm) a0 = a_ptr[b_q_perm[offset_k + t]];
- else a0 = a_ptr[offset_k + t];
- block_a_ptr[t] = a0;
- }
- }
-
- // Zero output
-
- if (n >= size_n) return;
-
- if (clear && blockIdx.z == 0) // && (threadIdx.x & 1) == 0)
- {
- for (int m = 0; m < m_count; m++)
- *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0;
- }
-
- __syncthreads();
-
- // Find initial group
-
- int group = offset_k / groupsize;
- int nextgroup = offset_k + groupsize;
-
- // a, b offset
-
- int qk = offset_k / (32 / 4);
-
- const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
- const half* a_ptr = &block_a[0][0];
- int a_stride = BLOCK_KN_SIZE;
-
- // Initial group
-
- int zeros[4];
- float scales[4];
- half2 z1z16[4][2];
- half2 y1y16[4][2];
- b_gptq_qzeros_.item4(zeros, group, n);
- b_gptq_scales_.item4_f(scales, group, n);
-
- // Avoid zeros overflow with & 0x0f.
- dequant_4bit_8_prep_zero((zeros[0]) & 0x0f, z1z16[0], y1y16[0]);
- dequant_4bit_8_prep_zero((zeros[1]) & 0x0f, z1z16[1], y1y16[1]);
- dequant_4bit_8_prep_zero((zeros[2]) & 0x0f, z1z16[2], y1y16[2]);
- dequant_4bit_8_prep_zero((zeros[3]) & 0x0f, z1z16[3], y1y16[3]);
-
-// __syncthreads();
-
- // Column result
-
- float block_c[m_count][4] = {};
-
- // Dequantize and multiply
-
- int k = offset_k;
- while (k < end_k)
- {
- if (k == nextgroup)
- {
- group++;
- nextgroup += groupsize;
- b_gptq_qzeros_.item4(zeros, group, n);
- b_gptq_scales_.item4_f(scales, group, n);
-
- // Avoid zeros overflow with & 0x0f.
- dequant_4bit_8_prep_zero((zeros[0]) & 0x0f, z1z16[0], y1y16[0]);
- dequant_4bit_8_prep_zero((zeros[1]) & 0x0f, z1z16[1], y1y16[1]);
- dequant_4bit_8_prep_zero((zeros[2]) & 0x0f, z1z16[2], y1y16[2]);
- dequant_4bit_8_prep_zero((zeros[3]) & 0x0f, z1z16[3], y1y16[3]);
- }
-
- #pragma unroll
- for (int j = 0; j < 4; j++)
- {
- const int4* b_ptr4 = (int4*) b_ptr;
- int4 load_int4 = *b_ptr4;
-
- half2 dq[4][4];
- dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false);
- dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false);
- dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false);
- dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false);
-
- #pragma unroll
- for (int m = 0; m < m_count; m++)
- {
- block_c[m][0] = fma(dot22_8_f(dq[0], a_ptr + m * a_stride), scales[0], block_c[m][0]);
- block_c[m][1] = fma(dot22_8_f(dq[1], a_ptr + m * a_stride), scales[1], block_c[m][1]);
- block_c[m][2] = fma(dot22_8_f(dq[2], a_ptr + m * a_stride), scales[2], block_c[m][2]);
- block_c[m][3] = fma(dot22_8_f(dq[3], a_ptr + m * a_stride), scales[3], block_c[m][3]);
- }
-
- b_ptr += size_n;
- a_ptr += 8;
- }
-
- k += 32;
- }
-
- for (int m = 0; m < m_count; m++)
- {
- half2 *out = (half2*) c_.item_ptr(offset_m + m, n);
- half2 result01 = __halves2half2(__float2half_rn(block_c[m][0]), __float2half_rn(block_c[m][1]));
- half2 result23 = __halves2half2(__float2half_rn(block_c[m][2]), __float2half_rn(block_c[m][3]));
- atomicAdd(out , result01);
- atomicAdd(out + 1, result23);
- }
-}
-
-fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(bool first_block, const int m_count)
-{
- #if BLOCK_M_SIZE_MAX >= 1
- if (m_count == 1) return gemm_half_q_half_gptq_kernel<true, 1>;
- #endif
- #if BLOCK_M_SIZE_MAX >= 2
- if (m_count == 2) return gemm_half_q_half_gptq_kernel<true, 2>;
- #endif
- #if BLOCK_M_SIZE_MAX >= 3
- if (m_count == 3) return gemm_half_q_half_gptq_kernel<true, 3>;
- #endif
- #if BLOCK_M_SIZE_MAX >= 4
- if (m_count == 4) return gemm_half_q_half_gptq_kernel<true, 4>;
- #endif
- #if BLOCK_M_SIZE_MAX >= 5
- if (m_count == 5) return gemm_half_q_half_gptq_kernel<true, 5>;
- #endif
- #if BLOCK_M_SIZE_MAX >= 6
- if (m_count == 6) return gemm_half_q_half_gptq_kernel<true, 6>;
- #endif
- #if BLOCK_M_SIZE_MAX >= 7
- if (m_count == 7) return gemm_half_q_half_gptq_kernel<true, 7>;
- #endif
- #if BLOCK_M_SIZE_MAX >= 8
- if (m_count == 8) return gemm_half_q_half_gptq_kernel<true, 8>;
- #endif
- return NULL;
-}
diff --git a/auto_round_extension/cuda/exllamav2/cuda/q_matrix.cu b/auto_round_extension/cuda/exllamav2/cuda/q_matrix.cu
deleted file mode 100644
index aebba7b02..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/q_matrix.cu
+++ /dev/null
@@ -1,627 +0,0 @@
-#include "q_matrix.cuh"
-#include "matrix_view.cuh"
-#include "util.cuh"
-
-#include "quant/qdq_2.cuh"
-#include "quant/qdq_3.cuh"
-#include "quant/qdq_4.cuh"
-#include "quant/qdq_5.cuh"
-#include "quant/qdq_6.cuh"
-#include "quant/qdq_8.cuh"
-
-#define BLOCK_KN_SIZE 128
-
-#define THREADS_X 32
-#define THREADS_Y 32
-
-// Shuffle quantized data on load
-
-__global__ void shuffle_kernel
-(
- uint32_t* __restrict__ b_q_weight,
- const int size_k,
- const int size_n,
- const int rows_8,
- const int rows_6,
- const int rows_5,
- const int rows_4,
- const int rows_3,
- const int rows_2
-)
-{
- int n = blockIdx.x * THREADS_X + threadIdx.x;
- if (n >= size_n) return;
- int k = 0;
- uint32_t* b_ptr = b_q_weight + n;
- while (k < rows_8) { shuffle_8bit_4 (b_ptr, size_n); b_ptr += 1 * size_n; k += 4; }
- while (k < rows_6) { shuffle_6bit_16(b_ptr, size_n); b_ptr += 3 * size_n; k += 16; }
- while (k < rows_5) { shuffle_5bit_32(b_ptr, size_n); b_ptr += 5 * size_n; k += 32; }
- while (k < rows_4) { shuffle_4bit_8 (b_ptr, size_n); b_ptr += 1 * size_n; k += 8; }
- while (k < rows_3) { shuffle_3bit_32(b_ptr, size_n); b_ptr += 3 * size_n; k += 32; }
- while (k < rows_2) { shuffle_2bit_16(b_ptr, size_n); b_ptr += 1 * size_n; k += 16; }
-}
-
-
-// QMatrix constructor
-
-QMatrix::QMatrix
-(
- const int _device,
- const int _height,
- const int _width,
- const int _groups,
-
- uint32_t* _q_weight,
- uint16_t* _q_perm,
- uint16_t* _q_invperm,
- uint32_t* _q_scale,
- half* _q_scale_max,
- uint16_t* _q_groups,
-
- uint32_t* _gptq_qzeros,
- half* _gptq_scales,
- uint32_t* _gptq_g_idx,
-
- half* _temp_dq
-) :
- device(_device),
- height(_height),
- width(_width),
- groups(_groups),
- temp_dq(_temp_dq)
-{
- cudaSetDevice(device);
-
- failed = false;
-
- cuda_q_weight = _q_weight;
- cuda_q_perm = _q_perm;
- cuda_q_invperm = _q_invperm;
- cuda_q_scale = _q_scale;
- cuda_q_scale_max = _q_scale_max;
- cuda_q_groups = _q_groups;
- cuda_gptq_qzeros = _gptq_qzeros;
- cuda_gptq_scales = _gptq_scales;
-
- is_gptq = (_gptq_qzeros != NULL);
-
- groupsize = 1;
- while (groupsize * groups < height) groupsize *= 2;
-
- // Create group map
-
- rows_8 = 0;
- rows_6 = 0;
- rows_5 = 0;
- rows_4 = 0;
- rows_3 = 0;
- rows_2 = 0;
-
- if (!is_gptq)
- {
- uint16_t* cpu_q_groups = (uint16_t*)calloc(groups * 2, sizeof(uint16_t));
- cudaMemcpy(cpu_q_groups, cuda_q_groups, groups * 2 * sizeof(uint16_t), cudaMemcpyDeviceToHost);
-
- for (int i = 0; i < groups; i++)
- {
- int bits = cpu_q_groups[i * 2];
- if (bits == 8) rows_8 += groupsize;
- if (bits == 6) rows_6 += groupsize;
- if (bits == 5) rows_5 += groupsize;
- if (bits == 4) rows_4 += groupsize;
- if (bits == 3) rows_3 += groupsize;
- if (bits == 2) rows_2 += groupsize;
- }
-
- free(cpu_q_groups);
-
- rows_6 += rows_8;
- rows_5 += rows_6;
- rows_4 += rows_5;
- rows_3 += rows_4;
- rows_2 += rows_3;
- }
- else
- {
- rows_4 = height;
- rows_3 = height;
- rows_2 = height;
-
- if (_gptq_g_idx)
- {
- if (!make_sequential(_gptq_g_idx))
- {
- failed = true;
- //printf("FAIL\n");
- return;
- }
- }
- }
-
- // Shuffle quantized data
-
- dim3 blockDim, gridDim;
- blockDim.x = THREADS_X;
- blockDim.y = 1;
- gridDim.x = DIVIDE(width, THREADS_X);
- gridDim.y = 1;
-
- shuffle_kernel<<<gridDim, blockDim>>>(cuda_q_weight, height, width, rows_8, rows_6, rows_5, rows_4, rows_3, rows_2);
-}
-
-QMatrix::~QMatrix()
-{
-}
-
-// Reconstruct b[k,n] (GPTQ)
-
-__global__ void reconstruct_gptq_kernel
-(
- const uint32_t* __restrict__ b_q_weight,
- const uint16_t* __restrict__ b_q_perm,
- const uint32_t* __restrict__ b_gptq_qzeros,
- const half* __restrict__ b_gptq_scales,
- //const uint16_t* __restrict__ b_q_groups,
- const int size_k,
- const int size_n,
- const int groupsize,
- const int groups,
- half* __restrict__ b,
- const int rows_4
-)
-{
- MatrixView_half_rw b_(b, size_k, size_n);
- MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n);
- MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n);
-
- int offset_k = BLOCK_KN_SIZE * blockIdx.y;
- int offset_n = BLOCK_KN_SIZE * blockIdx.x * 4;
-
- int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
-
- // Preload remapping table
-
- __shared__ uint16_t perm[BLOCK_KN_SIZE];
- int t = threadIdx.x;
-
- if (b_q_perm)
- {
- if (offset_k + t < size_k)
- perm[t] = b_q_perm[offset_k + t];
- }
-
- // Column
-
- int n = offset_n + t * 4;
- if (n >= size_n) return;
-
- // Find initial group
-
- int group = offset_k / groupsize;
- int nextgroup = offset_k + groupsize;
-
- // b offset
-
- int qk = offset_k / (32 / 4);
-
- const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
-
- // Initial zeros/scale
-
- int zeros[4];
- half2 scales[4];
- half2 z1z16[4][2];
- half2 y1y16[4][2];
- b_gptq_qzeros_.item4(zeros, group, n);
- b_gptq_scales_.item4_h2(scales, group, n);
-
- // Avoid zeros overflow with & 0x0f.
- dequant_4bit_8_prep_zero((zeros[0]) & 0x0f, z1z16[0], y1y16[0]);
- dequant_4bit_8_prep_zero((zeros[1]) & 0x0f, z1z16[1], y1y16[1]);
- dequant_4bit_8_prep_zero((zeros[2]) & 0x0f, z1z16[2], y1y16[2]);
- dequant_4bit_8_prep_zero((zeros[3]) & 0x0f, z1z16[3], y1y16[3]);
-
- __syncthreads();
-
- int k = offset_k;
- int lk = 0;
-
- while (k < end_k)
- {
- if (k == nextgroup)
- {
- group++;
- nextgroup += groupsize;
- b_gptq_qzeros_.item4(zeros, group, n);
- b_gptq_scales_.item4_h2(scales, group, n);
-
- // Avoid zeros overflow with & 0x0f.
- dequant_4bit_8_prep_zero((zeros[0]) & 0x0f, z1z16[0], y1y16[0]);
- dequant_4bit_8_prep_zero((zeros[1]) & 0x0f, z1z16[1], y1y16[1]);
- dequant_4bit_8_prep_zero((zeros[2]) & 0x0f, z1z16[2], y1y16[2]);
- dequant_4bit_8_prep_zero((zeros[3]) & 0x0f, z1z16[3], y1y16[3]);
- }
-
- for (int p = 0; p < 4; p++)
- {
- half2 dq[4][4];
- const int4* b_ptr4 = (int4*) b_ptr;
- int4 load_int4 = *b_ptr4;
-
- dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false);
- dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false);
- dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false);
- dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false);
-
- b_ptr += size_n;
- //half* dqh = (half*)dq;
- if (b_q_perm)
- {
- for (int j = 0; j < 4; j++)
- {
- for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
- b_.set4(perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
- b_.set4(perm[lk++], n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
- }
- }
- else
- {
- for (int j = 0; j < 4; j++)
- {
- for (int v = 0; v < 4; v++) dq[v][j] = __hmul2(scales[v], dq[v][j]);
- b_.set4(offset_k + lk++, n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j]));
- b_.set4(offset_k + lk++, n, __high2half(dq[0][j]), __high2half(dq[1][j]), __high2half(dq[2][j]), __high2half(dq[3][j]));
- }
- }
- }
- k += 32;
- }
-}
-
-
-// Reconstruct b[k,n]
-
-__global__ void reconstruct_kernel
-(
- const uint32_t* __restrict__ b_q_weight,
- const uint16_t* __restrict__ b_q_perm,
- const uint32_t* __restrict__ b_q_scale,
- const half* __restrict__ b_q_scale_max,
- //const uint16_t* __restrict__ b_q_groups,
- const int size_k,
- const int size_n,
- const int groupsize,
- const int groups,
- half* __restrict__ b,
- const int rows_8,
- const int rows_6,
- const int rows_5,
- const int rows_4,
- const int rows_3,
- const int rows_2
-)
-{
- MatrixView_half_rw b_(b, size_k, size_n);
- MatrixView_q4_row b_q_scale_(b_q_scale, groups, size_n);
-
- int offset_k = BLOCK_KN_SIZE * blockIdx.y;
- int offset_n = BLOCK_KN_SIZE * blockIdx.x;
-
- // Preload remapping table
-
- int t = threadIdx.x;
- __shared__ uint16_t perm[BLOCK_KN_SIZE];
- if (offset_k + t < size_k)
- perm[t] = b_q_perm[offset_k + t];
-
- // Column
-
- int n = offset_n + t;
- if (n >= size_n) return;
-
- // Find initial group
-
- int group = offset_k / groupsize;
-
- int pre_rows_8 = min(rows_8, offset_k);
- int pre_rows_6 = offset_k > rows_8 ? min(rows_6, offset_k) - rows_8 : 0;
- int pre_rows_5 = offset_k > rows_6 ? min(rows_5, offset_k) - rows_6 : 0;
- int pre_rows_4 = offset_k > rows_5 ? min(rows_4, offset_k) - rows_5 : 0;
- int pre_rows_3 = offset_k > rows_4 ? min(rows_3, offset_k) - rows_4 : 0;
- int pre_rows_2 = offset_k > rows_3 ? min(rows_2, offset_k) - rows_3 : 0;
- int qk = 0;
- qk += pre_rows_8 / 32 * 8;
- qk += pre_rows_6 / 32 * 6;
- qk += pre_rows_5 / 32 * 5;
- qk += pre_rows_4 / 32 * 4;
- qk += pre_rows_3 / 32 * 3;
- qk += pre_rows_2 / 32 * 2;
-
- const uint32_t* b_ptr = b_q_weight + qk * size_n + n;
-
- half qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]);
- half2 qs_h2 = __halves2half2(qs_h, qs_h);
- int nextgroup = offset_k + groupsize;
-
- int end_k = min(offset_k + BLOCK_KN_SIZE, size_k);
- int k = offset_k;
- int lk = 0;
-
- __syncthreads();
-
- while (k < rows_8 && k < end_k)
- {
- if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); }
- for (int p = 0; p < 4; p++)
- {
- half2 dq[4];
- uint32_t q_0 = *b_ptr; b_ptr += size_n;
- uint32_t q_1 = *b_ptr; b_ptr += size_n;
- dequant_8bit_8(q_0, q_1, dq, size_n);
- for (int j = 0; j < 4; j++) dq[j] = __hmul2(dq[j], qs_h2);
- half* dqh = (half*) dq;
- for (int j = 0; j < 8; j++) b_.set(perm[lk++], n, dqh[j]);
- }
- k += 32;
- }
-
- while (k < rows_6 && k < end_k)
- {
- if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); }
- for (int p = 0; p < 2; p++)
- {
- half2 dq[8];
- uint32_t q_0 = *b_ptr; b_ptr += size_n;
- uint32_t q_1 = *b_ptr; b_ptr += size_n;
- uint32_t q_2 = *b_ptr; b_ptr += size_n;
- dequant_6bit_16(q_0, q_1, q_2, dq, size_n);
- for (int j = 0; j < 8; j++) dq[j] = __hmul2(dq[j], qs_h2);
- half* dqh = (half*) dq;
- for (int j = 0; j < 16; j++) b_.set(perm[lk++], n, dqh[j]);
- }
- k += 32;
- }
-
- while (k < rows_5 && k < end_k)
- {
- if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); }
- for (int p = 0; p < 1; p++)
- {
- half2 dq[16];
- uint32_t q_0 = *b_ptr; b_ptr += size_n;
- uint32_t q_1 = *b_ptr; b_ptr += size_n;
- uint32_t q_2 = *b_ptr; b_ptr += size_n;
- uint32_t q_3 = *b_ptr; b_ptr += size_n;
- uint32_t q_4 = *b_ptr; b_ptr += size_n;
- dequant_5bit_32(q_0, q_1, q_2, q_3, q_4, dq, size_n);
- for (int j = 0; j < 16; j++) dq[j] = __hmul2(dq[j], qs_h2);
- half* dqh = (half*) dq;
- for (int j = 0; j < 32; j++) b_.set(perm[lk++], n, dqh[j]);
- }
- k += 32;
- }
-
- while (k < rows_4 && k < end_k)
- {
- if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); }
- for (int p = 0; p < 4; p++)
- {
- half2 dq[4];
- uint32_t q_0 = *b_ptr; b_ptr += size_n;
- dequant_4bit_8(q_0, dq, size_n);
- for (int j = 0; j < 4; j++) dq[j] = __hmul2(dq[j], qs_h2);
- half* dqh = (half*) dq;
- for (int j = 0; j < 8; j++) b_.set(perm[lk++], n, dqh[j]);
- }
- k += 32;
- }
-
- while (k < rows_3 && k < end_k)
- {
- if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); }
- for (int p = 0; p < 1; p++)
- {
- half2 dq[16];
- uint32_t q_0 = *b_ptr; b_ptr += size_n;
- uint32_t q_1 = *b_ptr; b_ptr += size_n;
- uint32_t q_2 = *b_ptr; b_ptr += size_n;
- dequant_3bit_32(q_0, q_1, q_2, dq, size_n);
- for (int j = 0; j < 16; j++) dq[j] = __hmul2(dq[j], qs_h2);
- half* dqh = (half*) dq;
- for (int j = 0; j < 32; j++) b_.set(perm[lk++], n, dqh[j]);
- }
- k += 32;
- }
-
- while (k < rows_2 && k < end_k)
- {
- if (k == nextgroup) { group++; qs_h = dq_scale(b_q_scale_.item(group, n), b_q_scale_max[group]); nextgroup += groupsize; qs_h2 = __halves2half2(qs_h, qs_h); }
- for (int p = 0; p < 2; p++)
- {
- half2 dq[8];
- uint32_t q_0 = *b_ptr; b_ptr += size_n;
- dequant_2bit_16(q_0, dq, size_n);
- for (int j = 0; j < 8; j++) dq[j] = __hmul2(dq[j], qs_h2);
- half* dqh = (half*) dq;
- for (int j = 0; j < 16; j++) b_.set(perm[lk++], n, dqh[j]);
- }
- k += 32;
- }
-}
-
-void QMatrix::reconstruct(half* out)
-{
- dim3 blockDim, gridDim;
- blockDim.x = BLOCK_KN_SIZE;
- blockDim.y = 1;
- gridDim.y = DIVIDE(height, BLOCK_KN_SIZE);
-
- if (!is_gptq)
- {
- gridDim.x = DIVIDE(width, BLOCK_KN_SIZE);
- reconstruct_kernel<<<gridDim, blockDim>>>
- (
- cuda_q_weight,
- cuda_q_perm,
- cuda_q_scale,
- cuda_q_scale_max,
- //cuda_q_groups,
- height,
- width,
- groupsize,
- groups,
- out,
- rows_8,
- rows_6,
- rows_5,
- rows_4,
- rows_3,
- rows_2
- );
- }
- else
- {
- gridDim.x = DIVIDE(width, BLOCK_KN_SIZE * 4);
- reconstruct_gptq_kernel<<<gridDim, blockDim>>>
- (
- cuda_q_weight,
- cuda_q_perm,
- cuda_gptq_qzeros,
- cuda_gptq_scales,
- //const uint16_t* __restrict__ b_q_groups,
- height,
- width,
- groupsize,
- groups,
- out,
- rows_4
- );
- }
-}
-
-__global__ void make_sequential_kernel
-(
- const uint32_t* __restrict__ w,
- uint32_t* __restrict__ w_new,
- const uint16_t* __restrict__ q_perm,
- const int w_height,
- const int w_width
-)
-{
- const uint64_t* w2 = (uint64_t*) w;
- uint64_t* w_new2 = (uint64_t*) w_new;
- int w2_stride = w_width >> 1;
-
- int w2_column = THREADS_X * blockIdx.x + threadIdx.x;
- if (w2_column >= w2_stride) return;
-
- int w_new2_row = blockIdx.y;
-
- int q_perm_idx = w_new2_row << 3;
-
- uint64_t dst = 0;
-
- #pragma unroll
- for (int i = 0; i < 8; i++)
- {
- int source_row = q_perm[q_perm_idx++];
-
- int w2_row = source_row >> 3;
- int w2_subrow = source_row & 0x07;
- int w2_row_shift = w2_subrow << 2;
- int wnew2_row_shift = i << 2;
-
- uint64_t src = w2[w2_row * w2_stride + w2_column];
- src >>= w2_row_shift;
- src &= 0x0000000f0000000f;
- src <<= wnew2_row_shift;
- dst |= src;
- }
-
- w_new2[w_new2_row * w2_stride + w2_column] = dst;
-}
-
-bool QMatrix::make_sequential(const uint32_t* cpu_g_idx)
-{
- uint32_t* cuda_new_qweight = NULL;
- cudaError_t err = cudaMalloc(&cuda_new_qweight, height / 8 * width * sizeof(uint32_t));
- if (err != cudaSuccess) {
- cudaError_t cuda_status = cudaGetLastError(); // Clear error
- return false;
- }
-
- uint32_t* cpu_g_idx_map = (uint32_t*) calloc(groups, sizeof(uint32_t));
- uint32_t* cpu_x_map = (uint32_t*) malloc(height * sizeof(uint32_t));
- uint32_t* cpu_x_map_inv = (uint32_t*) malloc(height * sizeof(uint32_t));
-
- // Group histogram
-
- for (int i = 0; i < height; i++) cpu_g_idx_map[cpu_g_idx[i]]++;
-
- // Group map
-
- for (int i = 0, acc = 0; i < groups; i++)
- {
- short tmp = cpu_g_idx_map[i];
- cpu_g_idx_map[i] = acc;
- acc += tmp;
- }
-
- // X map (inverse)
-
- for (int row = 0; row < height; row++)
- {
- uint32_t target_group = cpu_g_idx[row];
- uint32_t target_row = cpu_g_idx_map[target_group];
- cpu_g_idx_map[target_group]++;
- cpu_x_map_inv[row] = target_row;
- }
-
- // X map
-
- for (int row = 0; row < height; row++) cpu_x_map[cpu_x_map_inv[row]] = row;
-
- // Reduce to uint16_t
-
- uint16_t* cpu_x_map16 = (uint16_t*)cpu_x_map;
- uint16_t* cpu_x_map_inv16 = (uint16_t*)cpu_x_map_inv;
- for (int row = 0; row < height; row++) cpu_x_map16[row] = (uint16_t) cpu_x_map[row];
- for (int row = 0; row < height; row++) cpu_x_map_inv16[row] = (uint16_t) cpu_x_map_inv[row];
-
- // Move to CUDA
-
- cudaMemcpyAsync(cuda_q_perm, cpu_x_map16, height * sizeof(uint16_t), cudaMemcpyHostToDevice);
- cudaMemcpyAsync(cuda_q_invperm, cpu_x_map_inv16, height * sizeof(uint16_t), cudaMemcpyHostToDevice);
-
- // Rearrange rows in w
-
- dim3 blockDim, gridDim;
- blockDim.x = THREADS_X;
- blockDim.y = 1;
- gridDim.x = DIVIDE(width, THREADS_X);
- gridDim.y = height / 8;
-
- make_sequential_kernel<<<gridDim, blockDim>>>
- (
- cuda_q_weight,
- cuda_new_qweight,
- cuda_q_perm,
- height / 8,
- width
- );
-
- // Replace qweights
-
- cudaMemcpyAsync(cuda_q_weight, cuda_new_qweight, height / 8 * width * sizeof(uint32_t), cudaMemcpyDeviceToDevice);
-
- // Cleanup
-
- cudaDeviceSynchronize();
-
- cudaFree(cuda_new_qweight);
- free(cpu_g_idx_map);
- free(cpu_x_map);
- free(cpu_x_map_inv);
-
- return true;
-}
diff --git a/auto_round_extension/cuda/exllamav2/cuda/q_matrix.cuh b/auto_round_extension/cuda/exllamav2/cuda/q_matrix.cuh
deleted file mode 100644
index dda83a4f3..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/q_matrix.cuh
+++ /dev/null
@@ -1,73 +0,0 @@
-#ifndef _q_matrix_cuh
-#define _q_matrix_cuh
-
-#include <cuda_runtime.h>
-#include <cuda_fp16.h>
-#include <cstdint>
-#include <cstdio>
-
-#define MAX_SUPERGROUPS 16
-
-class QMatrix
-{
-public:
-
- int device;
- bool is_gptq;
-
- int height;
- int width;
- int groups;
- int groupsize;
-
- int rows_8;
- int rows_6;
- int rows_5;
- int rows_4;
- int rows_3;
- int rows_2;
-
- uint32_t* cuda_q_weight = NULL;
- uint16_t* cuda_q_perm = NULL;
- uint16_t* cuda_q_invperm = NULL;
- uint32_t* cuda_q_scale = NULL;
- half* cuda_q_scale_max = NULL;
- uint16_t* cuda_q_groups = NULL;
- uint32_t* cuda_gptq_qzeros = NULL;
- half* cuda_gptq_scales = NULL;
-
- half* temp_dq;
-
- bool failed;
-
- QMatrix
- (
- const int _device,
- const int _height,
- const int _width,
- const int _groups,
-
- uint32_t* _q_weight,
- uint16_t* _q_perm,
- uint16_t* _q_invperm,
- uint32_t* _q_scale,
- half* _q_scale_max,
- uint16_t* _q_groups,
-
- uint32_t* _gptq_qzeros,
- half* _gptq_scales,
- uint32_t* _gptq_g_idx,
-
- half* _temp_dq
- );
-
- ~QMatrix();
-
- void reconstruct(half* out);
- bool make_sequential(const uint32_t* cpu_g_idx);
-
-private:
-
-};
-
-#endif
diff --git a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_2.cuh b/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_2.cuh
deleted file mode 100644
index 3beaeefa9..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_2.cuh
+++ /dev/null
@@ -1,103 +0,0 @@
-#ifndef _qdq_2_cuh
-#define _qdq_2_cuh
-
-#include "qdq_util.cuh"
-#include "../../config.h"
-
-#if QMODE_2BIT == 1
-
-// Permutation:
-//
-// ffddbb99 77553311 eeccaa88 66442200
-
-__forceinline__ __device__ void shuffle_2bit_16
-(
- uint32_t* q,
- int stride
-)
-{
- uint32_t qa = q[0];
- uint32_t qb = 0;
-
- #pragma unroll
- for (int i = 0; i < 8; i++)
- {
- uint32_t qa0 = qa & 0x03;
- uint32_t qa1 = (qa & 0x0c) >> 2;
- qa >>= 4;
- qb |= (qa1 << (i * 2 + 16));
- qb |= (qa0 << (i * 2));
- }
- q[0] = qb;
-}
-
-__forceinline__ __device__ void dequant_2bit_16
-(
- const uint32_t q_0,
- half2 (&dq)[8],
- int stride
-)
-{
- const uint32_t c0 = 0x64006400;
- const half y4_ = __float2half_rn(1.0f / 4.0f);
- const half y16_ = __float2half_rn(1.0f / 16.0f);
- const half y64_ = __float2half_rn(1.0f / 64.0f);
- const half2 y4 = __halves2half2(y4_, y4_);
- const half2 y16 = __halves2half2(y16_, y16_);
- const half2 y64 = __halves2half2(y64_, y64_);
- const half z1_ = __float2half_rn(-1024.0f - 2.0f);
- const half z4_ = __float2half_rn(-1024.0f / 4.0f - 2.0f);
- const half z16_ = __float2half_rn(-1024.0f / 16.0f - 2.0f);
- const half z64_ = __float2half_rn(-1024.0f / 64.0f - 2.0f);
- const half2 z1 = __halves2half2(z1_, z1_);
- const half2 z4 = __halves2half2(z4_, z4_);
- const half2 z16 = __halves2half2(z16_, z16_);
- const half2 z64 = __halves2half2(z64_, z64_);
-
- uint32_t qa = q_0;
- half2_uint32 q0((qa & 0x00030003) | c0); // half2(q[ 0], q[ 1]) + 1024
- half2_uint32 q1((qa & 0x000c000c) | c0); // half2(q[ 2], q[ 3]) * 4 + 1024
- half2_uint32 q2((qa & 0x00300030) | c0); // half2(q[ 4], q[ 5]) * 16 + 1024
- half2_uint32 q3((qa & 0x00c000c0) | c0); // half2(q[ 6], q[ 7]) * 64 + 1024
- qa >>= 8;
- half2_uint32 q4((qa & 0x00030003) | c0); // half2(q[ 8], q[ 8]) + 1024
- half2_uint32 q5((qa & 0x000c000c) | c0); // half2(q[10], q[11]) * 4 + 1024
- half2_uint32 q6((qa & 0x00300030) | c0); // half2(q[12], q[13]) * 16 + 1024
- half2_uint32 q7((qa & 0x00c000c0) | c0); // half2(q[14], q[15]) * 64 + 1024
-
- dq[0] = __hadd2(q0.as_half2, z1);
- dq[1] = __hfma2(q1.as_half2, y4, z4);
- dq[2] = __hfma2(q2.as_half2, y16, z16);
- dq[3] = __hfma2(q3.as_half2, y64, z64);
- dq[4] = __hadd2(q4.as_half2, z1);
- dq[5] = __hfma2(q5.as_half2, y4, z4);
- dq[6] = __hfma2(q6.as_half2, y16, z16);
- dq[7] = __hfma2(q7.as_half2, y64, z64);
-}
-
-#else
-
-__forceinline__ __device__ void shuffle_2bit_16
-(
- uint32_t* q,
- int stride
-)
-{
-}
-
-__forceinline__ __device__ void dequant_2bit_16
-(
- const uint32_t q_0,
- half2 (&dq)[8],
- int stride
-)
-{
- half dqh[16];
- for (int i = 0; i < 16; i++) dqh[i] = dq_ns(exb(q_0, i * 2, 0x03), 2);
-
- for (int i = 0; i < 8; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
-}
-
-#endif
-
-#endif
\ No newline at end of file
diff --git a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_3.cuh b/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_3.cuh
deleted file mode 100644
index 101173763..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_3.cuh
+++ /dev/null
@@ -1,169 +0,0 @@
-#ifndef _qdq_3_cuh
-#define _qdq_3_cuh
-
-#include "qdq_util.cuh"
-#include "../../config.h"
-
-#if QMODE_3BIT == 1
-
-// Permutation:
-//
-// v9997775 55333111 u8886664 44222000 (u, v lsb)
-// vjjjhhhf ffdddbbb uiiiggge eecccaaa
-// vtttrrrp ppnnnlll usssqqqo oommmkkk
-
-__forceinline__ __device__ void shuffle_3bit_32
-(
- uint32_t* q,
- int stride
-)
-{
- uint32_t qa = q[0 * stride];
- uint32_t qb = q[1 * stride];
- uint32_t qc = q[2 * stride];
-
- // qa: aa999888 77766655 54443332 22111000
- // qb: lkkkjjji iihhhggg fffeeedd dcccbbba
- // qc: vvvuuutt tsssrrrq qqpppooo nnnmmmll
-
- uint32_t qd = qc >> 26;
- qc <<= 4;
- qc |= qb >> 28;
- qb <<= 2;
- qb |= qa >> 30;
-
- // qa: ..999888 77766655 54443332 22111000
- // qb: ..jjjiii hhhgggff feeedddc ccbbbaaa
- // qc: ..tttsss rrrqqqpp pooonnnm mmlllkkk
- // qd: vvvuuu
-
- uint32_t za = 0;
- uint32_t zb = 0;
- uint32_t zc = 0;
-
- for (int i = 0; i < 5; i++) { uint32_t t0 = qa & 0x07; uint32_t t1 = (qa & 0x38) >> 3; qa >>= 6; za |= (t0 << (i * 3)); za |= (t1 << (i * 3 + 16)); }
- for (int i = 0; i < 5; i++) { uint32_t t0 = qb & 0x07; uint32_t t1 = (qb & 0x38) >> 3; qb >>= 6; zb |= (t0 << (i * 3)); zb |= (t1 << (i * 3 + 16)); }
- for (int i = 0; i < 5; i++) { uint32_t t0 = qc & 0x07; uint32_t t1 = (qc & 0x38) >> 3; qc >>= 6; zc |= (t0 << (i * 3)); zc |= (t1 << (i * 3 + 16)); }
-
- // za: 9997775 55333111 8886664 44222000
- // zb: jjjhhhf ffdddbbb iiiggge eecccaaa
- // zc: tttrrrp ppnnnlll sssqqqo oommmkkk
- // qd: vvvuuu
-
- za |= ((qd & 0x01) >> 0) << 15;
- zb |= ((qd & 0x02) >> 1) << 15;
- zc |= ((qd & 0x04) >> 2) << 15;
- za |= ((qd & 0x08) >> 3) << 31;
- zb |= ((qd & 0x10) >> 4) << 31;
- zc |= ((qd & 0x20) >> 5) << 31;
-
- // za: v9997775 55333111 u8886664 44222000 (u, v lsb)
- // zb: vjjjhhhf ffdddbbb uiiiggge eecccaaa
- // zc: vtttrrrp ppnnnlll usssqqqo oommmkkk
-
- q[0 * stride] = za;
- q[1 * stride] = zb;
- q[2 * stride] = zc;
-}
-
-__forceinline__ __device__ void dequant_3bit_32
-(
- const uint32_t q_0,
- const uint32_t q_1,
- const uint32_t q_2,
- half2 (&dq)[16],
- int stride
-)
-{
- const uint32_t c0 = 0x64006400;
- const half y8_ = __float2half_rn(1.0f / 8.0f);
- const half y64_ = __float2half_rn(1.0f / 64.0f);
- const half2 y8 = __halves2half2(y8_, y8_);
- const half2 y64 = __halves2half2(y64_, y64_);
- const half z1_ = __float2half_rn(-1024.0f - 4.0f);
- const half z8_ = __float2half_rn(-1024.0f / 8.0f - 4.0f);
- const half z64_ = __float2half_rn(-1024.0f / 64.0f - 4.0f);
- const half2 z1 = __halves2half2(z1_, z1_);
- const half2 z8 = __halves2half2(z8_, z8_);
- const half2 z64 = __halves2half2(z64_, z64_);
-
- uint32_t qa = q_0;
- uint32_t qb = q_1;
- uint32_t qc = q_2;
-
- half2_uint32 q0((qa & 0x00070007) | c0); // half2(q[ 0], q[ 1]) + 1024
- half2_uint32 q1((qa & 0x00380038) | c0); // half2(q[ 2], q[ 3]) * 8 + 1024
- qa >>= 6;
- half2_uint32 q2((qa & 0x00070007) | c0); // half2(q[ 4], q[ 5]) + 1024
- half2_uint32 q3((qa & 0x00380038) | c0); // half2(q[ 6], q[ 7]) * 8 + 1024
- half2_uint32 q4((qa & 0x01c001c0) | c0); // half2(q[ 8], q[ 9]) * 64 + 1024
- qa >>= 9;
- qa &= 0x00010001;
- half2_uint32 q5((qb & 0x00070007) | c0); // half2(q[10], q[11]) + 1024
- half2_uint32 q6((qb & 0x00380038) | c0); // half2(q[12], q[13]) * 8 + 1024
- qb >>= 6;
- half2_uint32 q7((qb & 0x00070007) | c0); // half2(q[14], q[15]) + 1024
- half2_uint32 q8((qb & 0x00380038) | c0); // half2(q[16], q[17]) * 8 + 1024
- half2_uint32 q9((qb & 0x01c001c0) | c0); // half2(q[18], q[19]) * 64 + 1024
- qb >>= 8;
- qb &= 0x00020002;
- half2_uint32 q10((qc & 0x00070007) | c0); // half2(q[20], q[21]) + 1024
- half2_uint32 q11((qc & 0x00380038) | c0); // half2(q[22], q[23]) * 8 + 1024
- qc >>= 6;
- half2_uint32 q12((qc & 0x00070007) | c0); // half2(q[24], q[25]) + 1024
- half2_uint32 q13((qc & 0x00380038) | c0); // half2(q[26], q[27]) * 8 + 1024
- half2_uint32 q14((qc & 0x01c001c0) | c0); // half2(q[28], q[29]) * 64 + 1024
- qc >>= 7;
- qc &= 0x00040004;
- half2_uint32 q15((qa | qb | qc) | c0);
-
- dq[ 0] = __hadd2( q0.as_half2, z1);
- dq[ 1] = __hfma2( q1.as_half2, y8, z8);
- dq[ 2] = __hadd2( q2.as_half2, z1);
- dq[ 3] = __hfma2( q3.as_half2, y8, z8);
- dq[ 4] = __hfma2( q4.as_half2, y64, z64);
- dq[ 5] = __hadd2( q5.as_half2, z1);
- dq[ 6] = __hfma2( q6.as_half2, y8, z8);
- dq[ 7] = __hadd2( q7.as_half2, z1);
- dq[ 8] = __hfma2( q8.as_half2, y8, z8);
- dq[ 9] = __hfma2( q9.as_half2, y64, z64);
- dq[10] = __hadd2(q10.as_half2, z1);
- dq[11] = __hfma2(q11.as_half2, y8, z8);
- dq[12] = __hadd2(q12.as_half2, z1);
- dq[13] = __hfma2(q13.as_half2, y8, z8);
- dq[14] = __hfma2(q14.as_half2, y64, z64);
- dq[15] = __hadd2(q15.as_half2, z1);
-}
-
-#else
-
-__forceinline__ __device__ void shuffle_3bit_32
-(
- uint32_t* q,
- int stride
-)
-{
-}
-
-__forceinline__ __device__ void dequant_3bit_32
-(
- const uint32_t q_0,
- const uint32_t q_1,
- const uint32_t q_2,
- half2 (&dq)[16],
- int stride
-)
-{
- half dqh[32];
- for (int i = 0; i < 10; i++) dqh[ i] = dq_ns(exb( q_0, i * 3 , 0x07), 4);
- dqh[10 ] = dq_ns(exb(q_1, q_0, 30, 0x07), 4);
- for (int i = 0; i < 10; i++) dqh[11 + i] = dq_ns(exb( q_1, i * 3 + 1, 0x07), 4);
- dqh[21 ] = dq_ns(exb(q_2, q_1, 31, 0x07), 4);
- for (int i = 0; i < 10; i++) dqh[22 + i] = dq_ns(exb( q_2, i * 3 + 2, 0x07), 4);
-
- for (int i = 0; i < 16; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
-}
-
-#endif
-
-#endif
diff --git a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_4.cuh b/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_4.cuh
deleted file mode 100644
index 5fb070d06..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_4.cuh
+++ /dev/null
@@ -1,227 +0,0 @@
-#ifndef _qdq_4_cuh
-#define _qdq_4_cuh
-
-#include "qdq_util.cuh"
-#include "../../config.h"
-
-#if QMODE_4BIT == 1
-
-// Permutation:
-//
-// 77775555 33331111 66664444 22220000
-
-__forceinline__ __device__ void shuffle_4bit_8
-(
- uint32_t* q,
- int stride
-)
-{
- uint32_t qa = q[0];
- uint32_t qb = 0;
-
- #pragma unroll
- for (int i = 0; i < 4; i++)
- {
- uint32_t qa0 = qa & 0x0f;
- uint32_t qa1 = (qa & 0xf0) >> 4;
- qa >>= 8;
- qb |= (qa1 << (i * 4 + 16));
- qb |= (qa0 << (i * 4));
- }
- q[0] = qb;
-}
-
-__forceinline__ __device__ void dequant_4bit_8
-(
- const uint32_t q_0,
- half2 (&dq)[4],
- int stride
-)
-{
- const uint32_t c0 = 0x64006400;
- const half y16_ = __float2half_rn(1.0f / 16.0f);
- const half2 y16 = __halves2half2(y16_, y16_);
- const half z1_ = __float2half_rn(-1024.0f - 8.0f);
- const half z16_ = __float2half_rn(-1024.0f / 16.0f - 8.0f);
- const half2 z1 = __halves2half2(z1_, z1_);
- const half2 z16 = __halves2half2(z16_, z16_);
-
- uint32_t qa = q_0;
- half2_uint32 q0((qa & 0x000f000f) | c0); // half2(q[ 0], q[ 1]) + 1024
- half2_uint32 q1((qa & 0x00f000f0) | c0); // half2(q[ 2], q[ 3]) * 16 + 1024
- qa >>= 8;
- half2_uint32 q2((qa & 0x000f000f) | c0); // half2(q[ 4], q[ 5]) + 1024
- half2_uint32 q3((qa & 0x00f000f0) | c0); // half2(q[ 6], q[ 7]) * 16 + 1024
-
- dq[0] = __hadd2(q0.as_half2, z1);
- dq[1] = __hfma2(q1.as_half2, y16, z16);
- dq[2] = __hadd2(q2.as_half2, z1);
- dq[3] = __hfma2(q3.as_half2, y16, z16);
-}
-
-__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale
-(
- const uint32_t zero,
- const half scale,
- half2 (&z1z16)[2],
- half2 (&y1y16)[2]
-)
-{
- half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);
- half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));
-
- half2 scale2 = __half2half2(scale);
-
- z1z16[0] = __hmul2(scale2, __half2half2(z1.as_half));
- z1z16[1] = __hmul2(scale2, __half2half2(z16));
-
- const half y1 = __float2half_rn(1.0f);
- const half y16 = __float2half_rn(1.0f / 16.0f);
-
- y1y16[0] = __hmul2(scale2, __half2half2(y1));
- y1y16[1] = __hmul2(scale2, __half2half2(y16));
-}
-
-__forceinline__ __device__ void dequant_4bit_8_prep_zero
-(
- const uint32_t zero,
- half2(&z1z16)[2],
- half2(&y1y16)[2]
-)
-{
- half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero);
- half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero));
-
- z1z16[0] = __half2half2(z1.as_half);
- z1z16[1] = __half2half2(z16);
-
- const half y1 = __float2half_rn(1.0f);
- const half y16 = __float2half_rn(1.0f / 16.0f);
-
- y1y16[0] = __half2half2(y1);
- y1y16[1] = __half2half2(y16);
-}
-
-
-__forceinline__ __device__ void dequant_4bit_8_gptq
-(
- const uint32_t q_0,
- half2 (&dq)[4],
- half2 (&z1z16)[2],
- half2 (&y1y16)[2],
- int stride,
- bool scaled
-)
-{
- const uint32_t c0 = 0x64006400;
-
- uint32_t qa = q_0;
- half2_uint32 q0((qa & 0x000f000f) | c0); // half2( q[0] + 1024, q[1] + 1024 )
- half2_uint32 q1((qa & 0x00f000f0) | c0); // half2( q[2] * 16 + 1024, q[3] * 16 + 1024 )
- qa >>= 8;
- half2_uint32 q2((qa & 0x000f000f) | c0); // half2( q[4] + 1024, q[5] + 1024 )
- half2_uint32 q3((qa & 0x00f000f0) | c0); // half2( q[6] * 16 + 1024, q[7] * 16 + 1024 )
-
- if (scaled)
- {
- dq[0] = __hfma2(q0.as_half2, y1y16[0], z1z16[0]); // half2( q[0] * s - z * s, q[1] * s - z * s)
- dq[1] = __hfma2(q1.as_half2, y1y16[1], z1z16[1]); // half2( q[2] * s - z * s, q[3] * s - z * s)
- dq[2] = __hfma2(q2.as_half2, y1y16[0], z1z16[0]);
- dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]);
- }
- else
- {
- dq[0] = __hadd2(q0.as_half2, z1z16[0]); // half2( q[0] - z, q[1] - z )
- dq[1] = __hfma2(q1.as_half2, y1y16[1], z1z16[1]); // half2( q[2] - z, q[3] - z )
- dq[2] = __hadd2(q2.as_half2, z1z16[0]); // half2( q[4] - z, q[5] - z )
- dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]); // half2( q[6] - z, q[7] - z )
- }
-}
-
-#else
-
-__forceinline__ __device__ void shuffle_4bit_8
-(
- uint32_t* q,
- int stride
-)
-{
-}
-
-__forceinline__ __device__ void dequant_4bit_8
-(
- const uint32_t q_0,
- half2 (&dq)[4],
- int stride
-)
-{
- half dqh[8];
- for (int i = 0; i < 8; i++) dqh[i] = dq_ns(exb(q_0, i * 4, 0x0f), 8);
-
- for (int i = 0; i < 4; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
-}
-
-__forceinline__ __device__ void dequant_4bit_8_prep_zero_scale
-(
- const uint32_t zero,
- const half scale,
- half2 (&z1)[2],
- half2 (&y1)[2]
-)
-{
- half z = __int2half_rn(-((int)zero));
- z = __hmul(z, scale);
- z1[0] = __half2half2(z);
- y1[0] = __half2half2(scale);
-}
-
-__forceinline__ __device__ void dequant_4bit_8_prep_zero
-(
- const uint32_t zero,
- half2(&z1)[2],
- half2(&y1)[2]
-)
-{
- half z = __int2half_rn(-((int)zero));
- z1[0] = __half2half2(z);
-}
-
-__forceinline__ __device__ void dequant_4bit_8_gptq
-(
- const uint32_t q_0,
- half2 (&dq)[4],
- half2 (&z1)[2],
- half2 (&y1)[2],
- int stride,
- bool scaled
-)
-{
- half2 dqh2[8];
-
- uint32_t qa = q_0;
- for (int i = 0; i < 4; i++)
- {
- half d0 = __int2half_rn(qa & 0x0f); qa >>= 4;
- half d1 = __int2half_rn(qa & 0x0f); qa >>= 4;
- dqh2[i] = __halves2half2(d0, d1);
- }
-
- if (scaled)
- {
- dq[0] = __hfma2(dqh2[0], y1[0], z1[0]);
- dq[1] = __hfma2(dqh2[1], y1[0], z1[0]);
- dq[2] = __hfma2(dqh2[2], y1[0], z1[0]);
- dq[3] = __hfma2(dqh2[3], y1[0], z1[0]);
- }
- else
- {
- dq[0] = __hadd2(dqh2[0], z1[0]);
- dq[1] = __hadd2(dqh2[1], z1[0]);
- dq[2] = __hadd2(dqh2[2], z1[0]);
- dq[3] = __hadd2(dqh2[3], z1[0]);
- }
-}
-
-#endif
-
-#endif
\ No newline at end of file
diff --git a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_5.cuh b/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_5.cuh
deleted file mode 100644
index 454e4b93b..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_5.cuh
+++ /dev/null
@@ -1,207 +0,0 @@
-#ifndef _qdq_5_cuh
-#define _qdq_5_cuh
-
-#include "qdq_util.cuh"
-#include "../../config.h"
-
-#if QMODE_5BIT == 1
-
-// Permutation:
-//
-// v5555533 33311111 u4444422 22200000 (u, v lsb)
-// vbbbbb99 99977777 uaaaaa88 88866666
-// vhhhhhff fffddddd ugggggee eeeccccc
-// vnnnnnll llljjjjj ummmmmkk kkkiiiii
-// vtttttrr rrrppppp usssssqq qqqooooo
-
-__forceinline__ __device__ void shuffle_5bit_32
-(
- uint32_t* q,
- int stride
-)
-{
- uint32_t qa = q[0 * stride];
- uint32_t qb = q[1 * stride];
- uint32_t qc = q[2 * stride];
- uint32_t qd = q[3 * stride];
- uint32_t qe = q[4 * stride];
-
- // qa: 66555554 44443333 32222211 11100000
- // qb: ccccbbbb baaaaa99 99988888 77777666
- // qc: jiiiiihh hhhggggg fffffeee eedddddc
- // qd: pppooooo nnnnnmmm mmlllllk kkkkjjjj
- // qe: vvvvvuuu uuttttts ssssrrrr rqqqqqpp
-
- uint32_t qf = qe >> 22;
- qe <<= 8;
- qe |= qd >> 24;
- qd <<= 6;
- qd |= qc >> 26;
- qc <<= 4;
- qc |= qb >> 28;
- qb <<= 2;
- qb |= qa >> 30;
-
- // qa: 555554 44443333 32222211 11100000
- // qb: bbbbba aaaa9999 98888877 77766666
- // qc: hhhhhg ggggffff feeeeedd dddccccc
- // qd: nnnnnm mmmmllll lkkkkkjj jjjiiiii
- // qe: ttttts ssssrrrr rqqqqqpp pppooooo
- // qf: vv vvvuuuuu
-
- uint32_t za = 0;
- uint32_t zb = 0;
- uint32_t zc = 0;
- uint32_t zd = 0;
- uint32_t ze = 0;
-
- for (int i = 0; i < 3; i++) { uint32_t t0 = qa & 0x1f; uint32_t t1 = (qa & 0x3e0) >> 5; qa >>= 10; za |= (t0 << (i * 5)); za |= (t1 << (i * 5 + 16)); }
- for (int i = 0; i < 3; i++) { uint32_t t0 = qb & 0x1f; uint32_t t1 = (qb & 0x3e0) >> 5; qb >>= 10; zb |= (t0 << (i * 5)); zb |= (t1 << (i * 5 + 16)); }
- for (int i = 0; i < 3; i++) { uint32_t t0 = qc & 0x1f; uint32_t t1 = (qc & 0x3e0) >> 5; qc >>= 10; zc |= (t0 << (i * 5)); zc |= (t1 << (i * 5 + 16)); }
- for (int i = 0; i < 3; i++) { uint32_t t0 = qd & 0x1f; uint32_t t1 = (qd & 0x3e0) >> 5; qd >>= 10; zd |= (t0 << (i * 5)); zd |= (t1 << (i * 5 + 16)); }
- for (int i = 0; i < 3; i++) { uint32_t t0 = qe & 0x1f; uint32_t t1 = (qe & 0x3e0) >> 5; qe >>= 10; ze |= (t0 << (i * 5)); ze |= (t1 << (i * 5 + 16)); }
-
- // za: 5555533 33311111 4444422 22200000
- // zb: bbbbb99 99977777 aaaaa88 88866666
- // zc: hhhhhff fffddddd gggggee eeeccccc
- // zd: nnnnnll llljjjjj mmmmmkk kkkiiiii
- // ze: tttttrr rrrppppp sssssqq qqqooooo
- // qf: vv vvvuuuuu
-
- za |= ((qf & 0x001) >> 0) << 15;
- zb |= ((qf & 0x002) >> 1) << 15;
- zc |= ((qf & 0x004) >> 2) << 15;
- zd |= ((qf & 0x008) >> 3) << 15;
- ze |= ((qf & 0x010) >> 4) << 15;
- za |= ((qf & 0x020) >> 5) << 31;
- zb |= ((qf & 0x040) >> 6) << 31;
- zc |= ((qf & 0x080) >> 7) << 31;
- zd |= ((qf & 0x100) >> 8) << 31;
- ze |= ((qf & 0x200) >> 9) << 31;
-
- // za: v5555533 33311111 u4444422 22200000 (u, v lsb)
- // zb: vbbbbb99 99977777 uaaaaa88 88866666
- // zc: vhhhhhff fffddddd ugggggee eeeccccc
- // zd: vnnnnnll llljjjjj ummmmmkk kkkiiiii
- // ze: vtttttrr rrrppppp usssssqq qqqooooo
-
- q[0 * stride] = za;
- q[1 * stride] = zb;
- q[2 * stride] = zc;
- q[3 * stride] = zd;
- q[4 * stride] = ze;
-}
-
-__forceinline__ __device__ void dequant_5bit_32
-(
- const uint32_t q_0,
- const uint32_t q_1,
- const uint32_t q_2,
- const uint32_t q_3,
- const uint32_t q_4,
- half2 (&dq)[16],
- int stride
-)
-{
- const uint32_t c0 = 0x64006400;
- const half y32_ = __float2half_rn(1.0f / 32.0f);
- const half2 y32 = __halves2half2(y32_, y32_);
- const half z1_ = __float2half_rn(-1024.0f - 16.0f);
- const half z32_ = __float2half_rn(-1024.0f / 32.0f - 16.0f);
- const half2 z1 = __halves2half2(z1_, z1_);
- const half2 z32 = __halves2half2(z32_, z32_);
-
- uint32_t qa = q_0;
- uint32_t qb = q_1;
- uint32_t qc = q_2;
- uint32_t qd = q_3;
- uint32_t qe = q_4;
-
- half2_uint32 q0 ((qa & 0x001f001f) | c0); // half2(q[ 0], q[ 1]) + 1024
- half2_uint32 q1 ((qa & 0x03e003e0) | c0); // half2(q[ 2], q[ 3]) * 32 + 1024
- qa >>= 10;
- half2_uint32 q2 ((qa & 0x001f001f) | c0); // half2(q[ 4], q[ 5]) + 1024
- qa >>= 5;
- qa &= 0x00010001;
- half2_uint32 q3 ((qb & 0x001f001f) | c0); // half2(q[ 6], q[ 7]) + 1024
- half2_uint32 q4 ((qb & 0x03e003e0) | c0); // half2(q[ 8], q[ 9]) * 32 + 1024
- qb >>= 10;
- half2_uint32 q5 ((qb & 0x001f001f) | c0); // half2(q[10], q[11]) + 1024
- qb >>= 4;
- qb &= 0x00020002;
- half2_uint32 q6 ((qc & 0x001f001f) | c0); // half2(q[12], q[13]) + 1024
- half2_uint32 q7 ((qc & 0x03e003e0) | c0); // half2(q[14], q[15]) * 32 + 1024
- qc >>= 10;
- half2_uint32 q8 ((qc & 0x001f001f) | c0); // half2(q[16], q[17]) + 1024
- qc >>= 3;
- qc &= 0x00040004;
- half2_uint32 q9 ((qd & 0x001f001f) | c0); // half2(q[18], q[19]) + 1024
- half2_uint32 q10((qd & 0x03e003e0) | c0); // half2(q[20], q[21]) * 32 + 1024
- qd >>= 10;
- half2_uint32 q11((qd & 0x001f001f) | c0); // half2(q[22], q[23]) + 1024
- qd >>= 2;
- qd &= 0x00080008;
- half2_uint32 q12((qe & 0x001f001f) | c0); // half2(q[24], q[25]) + 1024
- half2_uint32 q13((qe & 0x03e003e0) | c0); // half2(q[26], q[27]) * 32 + 1024
- qe >>= 10;
- half2_uint32 q14((qe & 0x001f001f) | c0); // half2(q[28], q[29]) + 1024
- qe >>= 1;
- qe &= 0x00100010;
- half2_uint32 q15((qa | qb | qc | qd | qe) | c0);
-
- dq[ 0] = __hadd2( q0.as_half2, z1);
- dq[ 1] = __hfma2( q1.as_half2, y32, z32);
- dq[ 2] = __hadd2( q2.as_half2, z1);
- dq[ 3] = __hadd2( q3.as_half2, z1);
- dq[ 4] = __hfma2( q4.as_half2, y32, z32);
- dq[ 5] = __hadd2( q5.as_half2, z1);
- dq[ 6] = __hadd2( q6.as_half2, z1);
- dq[ 7] = __hfma2( q7.as_half2, y32, z32);
- dq[ 8] = __hadd2( q8.as_half2, z1);
- dq[ 9] = __hadd2( q9.as_half2, z1);
- dq[10] = __hfma2(q10.as_half2, y32, z32);
- dq[11] = __hadd2(q11.as_half2, z1);
- dq[12] = __hadd2(q12.as_half2, z1);
- dq[13] = __hfma2(q13.as_half2, y32, z32);
- dq[14] = __hadd2(q14.as_half2, z1);
- dq[15] = __hadd2(q15.as_half2, z1);
-}
-
-#else
-
-__forceinline__ __device__ void shuffle_5bit_32
-(
- uint32_t* q,
- int stride
-)
-{
-}
-
-__forceinline__ __device__ void dequant_5bit_32
-(
- const uint32_t q_0,
- const uint32_t q_1,
- const uint32_t q_2,
- const uint32_t q_3,
- const uint32_t q_4,
- half2 (&dq)[16],
- int stride
-)
-{
- half dqh[32];
- for (int i = 0; i < 6; i++) dqh[ i] = dq_ns(exb( q_0, i * 5 , 0x1f), 16);
- dqh[ 6 ] = dq_ns(exb(q_1, q_0, 30, 0x1f), 16);
- for (int i = 0; i < 5; i++) dqh[ 7 + i] = dq_ns(exb( q_1, i * 5 + 3, 0x1f), 16);
- dqh[12 ] = dq_ns(exb(q_2, q_1, 28, 0x1f), 16);
- for (int i = 0; i < 6; i++) dqh[13 + i] = dq_ns(exb( q_2, i * 5 + 1, 0x1f), 16);
- dqh[19 ] = dq_ns(exb(q_3, q_2, 31, 0x1f), 16);
- for (int i = 0; i < 5; i++) dqh[20 + i] = dq_ns(exb( q_3, i * 5 + 4, 0x1f), 16);
- dqh[25 ] = dq_ns(exb(q_4, q_3, 29, 0x1f), 16);
- for (int i = 0; i < 6; i++) dqh[26 + i] = dq_ns(exb( q_4, i * 5 + 2, 0x1f), 16);
-
- for (int i = 0; i < 16; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
-}
-
-#endif
-
-#endif
\ No newline at end of file
diff --git a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_6.cuh b/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_6.cuh
deleted file mode 100644
index c2eb8cfbf..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_6.cuh
+++ /dev/null
@@ -1,44 +0,0 @@
-#ifndef _qdq_6_cuh
-#define _qdq_6_cuh
-
-#include "qdq_util.cuh"
-#include "../../config.h"
-
-#if QMODE_6BIT == 1
-
- // Not implemented
-
-#else
-
-__forceinline__ __device__ void shuffle_6bit_16
-(
- uint32_t* q,
- int stride
-)
-{
-}
-
-__forceinline__ __device__ void dequant_6bit_16
-(
- const uint32_t q_0,
- const uint32_t q_1,
- const uint32_t q_2,
- half2 (&dq)[8],
- int stride
-)
-{
- half dqh[16];
- for (int i = 0; i < 5; i++) dqh[ i] = dq_ns(exb( q_0, i * 6 , 0x3f), 32);
- dqh[ 5 ] = dq_ns(exb(q_1, q_0, 30, 0x3f), 32);
- for (int i = 0; i < 4; i++) dqh[ 6 + i] = dq_ns(exb( q_1, i * 6 + 4, 0x3f), 32);
- dqh[10 ] = dq_ns(exb(q_2, q_1, 28, 0x3f), 32);
- for (int i = 0; i < 5; i++) dqh[11 + i] = dq_ns(exb( q_2, i * 6 + 2, 0x3f), 32);
-
- for (int i = 0; i < 8; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
-}
-
-#endif
-
-#endif
-
-
diff --git a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_8.cuh b/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_8.cuh
deleted file mode 100644
index e2409efac..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_8.cuh
+++ /dev/null
@@ -1,38 +0,0 @@
-#ifndef _qdq_8_cuh
-#define _qdq_8_cuh
-
-#include "qdq_util.cuh"
-#include "../../config.h"
-
-#if QMODE_8BIT == 1
-
- // Not implemented
-
-#else
-
-__forceinline__ __device__ void shuffle_8bit_4
-(
- uint32_t* q,
- int stride
-)
-{
-}
-
-__forceinline__ __device__ void dequant_8bit_8
-(
- const uint32_t q_0,
- const uint32_t q_1,
- half2 (&dq)[4],
- int stride
-)
-{
- half dqh[8];
- for (int i = 0; i < 4; i++) dqh[i ] = dq_ns(exb(q_0, i * 8, 0xff), 128);
- for (int i = 0; i < 4; i++) dqh[i + 4] = dq_ns(exb(q_1, i * 8, 0xff), 128);
-
- for (int i = 0; i < 4; i++) dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]);
-}
-
-#endif
-
-#endif
\ No newline at end of file
diff --git a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_util.cuh b/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_util.cuh
deleted file mode 100644
index 71657191b..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/quant/qdq_util.cuh
+++ /dev/null
@@ -1,51 +0,0 @@
-#ifndef _qdq_util_cuh
-#define _qdq_util_cuh
-
-union half2_uint32
-{
- uint32_t as_uint32;
- half2 as_half2;
- __device__ half2_uint32(uint32_t val) : as_uint32(val) {}
- __device__ half2_uint32(half2 val) : as_half2(val) {}
-};
-
-union half_uint16
-{
- uint16_t as_uint16;
- half as_half;
- __device__ half_uint16(uint16_t val) : as_uint16(val) {}
- __device__ half_uint16(half val) : as_half(val) {}
-};
-
-// Max_scale premultiplied by 1/256
-
-__forceinline__ __device__ half dq_scale(const int qs, const half max_scale)
-{
- int qs_i = qs + 1;
- half qs_h = __int2half_rn(qs_i * qs_i);
- qs_h = __hmul(qs_h, max_scale);
- return qs_h;
-}
-
-__forceinline__ __device__ half dq(const int q, const int qzero, const half scale)
-{
- return __hmul(__int2half_rn(q - qzero), scale);
-}
-
-__forceinline__ __device__ half dq_ns(const int q, const int qzero)
-{
- //return __hsub(__int2half_rn(q), __int2half_rn(qzero));
- return __int2half_rn(q - qzero);
-}
-
-__forceinline__ __device__ int exb(const uint32_t q, const int shift, const int mask)
-{
- return (int)((q >> shift) & mask);
-}
-
-__forceinline__ __device__ int exb(const uint32_t q1, const uint32_t q0, const int shift, const int mask)
-{
- return (int)(__funnelshift_rc(q0, q1, shift) & mask);
-}
-
-#endif
diff --git a/auto_round_extension/cuda/exllamav2/cuda/util.cuh b/auto_round_extension/cuda/exllamav2/cuda/util.cuh
deleted file mode 100644
index 36be0e242..000000000
--- a/auto_round_extension/cuda/exllamav2/cuda/util.cuh
+++ /dev/null
@@ -1,42 +0,0 @@
-
-#define DIVIDE(x, size) (((x) + (size) - 1) / (size))
-
-#define DBGS(__x) printf("%s\n", __x)
-#define DBGI(__x) printf("%s: %i\n", #__x, __x)
-#define DBGI2(__x, __y) printf("%s, %s: %i, %i\n", #__x, #__y, __x, __y)
-#define DBGI3(__x, __y, __z) printf("%s, %s, %s: %i, %i, %i\n", #__x, #__y, #__z, __x, __y, __z)
-#define DBGX(__x) printf("%s: %x\n", #__x, __x)
-#define DBGX2(__x, __y) printf("%s, %s: %x, %x\n", #__x, #__y, __x, __y)
-#define DBGX3(__x, __y, __z) printf("%s, %s, %s: %x, %x, %x\n", #__x, #__y, #__z, __x, __y, __z)
-#define DBGF(__x) printf("%s: %f\n", #__x, __x)
-#define DBGF2(__x, __y) printf("%s, %s: %f, %f\n", #__x, #__y, __x, __y)
-#define DBGF3(__x, __y, __z) printf("%s, %s, %s: %f, %f, %f\n", #__x, #__y, #__z, __x, __y, __z)
-#define DBGH(__x) printf("%s: %f\n", #__x, __half2float(__x))
-#define DBGH2(__x, __y) printf("%s, %s: %f, %f\n", #__x, #__y, __half2float(__x), __half2float(__y))
-#define DBGH3(__x, __y, __z) printf("%s, %s, %s: %f, %f, %f\n", #__x, #__y, #__z, __half2float(__x), __half2float(__y), __half2float(__z))
-
-#define DBGIH(__x, __y) printf("%s, %s: %i, %f\n", #__x, #__y, __x, __half2float(__y))
-#define DBGIH2(__x, __y, __z) printf("%s, %s, %s: %i, %f, %f\n", #__x, #__y, #__z, __x, __half2float(__y), __half2float(__z))
-
-__forceinline__ __device__ half dq_scale_(const int qs, const half max_scale)
-{
- half qs_h = __hmul(__int2half_rn(qs + 1), __float2half_rn(1.0f / 16.0f));
- qs_h = __hmul(qs_h, qs_h);
- qs_h = __hmul(qs_h, max_scale);
- return qs_h;
-}
-
-__forceinline__ __device__ float clamp(float x, float a, float b)
-{
- return fmaxf(a, fminf(b, x));
-}
-
-#define cuda_check(res) { gpu_assert((res), __FILE__, __LINE__); }
-inline void gpu_assert(cudaError_t code, const char *file, int line, bool abort=true)
-{
- if (code != cudaSuccess)
- {
- fprintf(stderr,"CUDA error: %s %s %d\n", cudaGetErrorString(code), file, line);
- if (abort) exit(code);
- }
-}
diff --git a/auto_round_extension/cuda/exllamav2/ext.cpp b/auto_round_extension/cuda/exllamav2/ext.cpp
deleted file mode 100644
index 5e52e6abd..000000000
--- a/auto_round_extension/cuda/exllamav2/ext.cpp
+++ /dev/null
@@ -1,134 +0,0 @@
-#include <torch/extension.h>
-#include <c10/cuda/CUDAGuard.h>
-#include <ATen/cuda/CUDAContext.h>
-#include <cuda_runtime.h>
-#include <cuda_fp16.h>
-#include <cstdint>
-#include <cstdio>
-
-#include "config.h"
-
-#include "cuda/q_matrix.cuh"
-#include "cuda/q_gemm.cuh"
-
-#include "cpp/util.h"
-
-// Some decluttering macros
-
-#define TORCH_CHECK_DTYPE(__x, __dtype) TORCH_CHECK((__x).dtype() == torch::__dtype, #__x " is incorrect datatype, must be " #__dtype)
-#define TORCH_CHECK_DTYPE_OPT(__x, __dtype) TORCH_CHECK((__x).device().is_meta() || (__x).dtype() == torch::__dtype, #__x " is incorrect datatype, must be " #__dtype)
-#define TORCH_CHECK_SHAPES(__x, __dim_x, __y, __dim_y, __scale_y) TORCH_CHECK((__x).size(__dim_x) == (__y).size(__dim_y) * __scale_y, #__x " and " #__y " have incompatible shapes")
-#define TORCH_CHECK_SHAPES_OPT(__x, __dim_x, __y, __dim_y, __scale_y) TORCH_CHECK((__x).device().is_meta() || (__x).size(__dim_x) == (__y).size(__dim_y) * __scale_y, #__x " and " #__y " have incompatible shapes")
-
-
-// Quant matrix
-
-uintptr_t make_q_matrix
-(
- torch::Tensor q_weight,
- torch::Tensor q_perm,
- torch::Tensor q_invperm,
- torch::Tensor q_scale,
- torch::Tensor q_scale_max,
- torch::Tensor q_groups,
- torch::Tensor gptq_qzeros,
- torch::Tensor gptq_scales,
- torch::Tensor gptq_g_idx,
- torch::Tensor temp_dq
-)
-{
- TORCH_CHECK_DTYPE(q_weight, kInt);
- TORCH_CHECK_DTYPE_OPT(q_perm, kShort);
- TORCH_CHECK_DTYPE_OPT(q_invperm, kShort);
- TORCH_CHECK_DTYPE_OPT(q_scale, kInt);
- TORCH_CHECK_DTYPE_OPT(q_scale_max, kHalf);
- TORCH_CHECK_DTYPE_OPT(q_groups, kShort);
- TORCH_CHECK_DTYPE_OPT(gptq_qzeros, kInt);
- TORCH_CHECK_DTYPE_OPT(gptq_scales, kHalf);
- TORCH_CHECK_DTYPE_OPT(gptq_g_idx, kInt);
-
- TORCH_CHECK_SHAPES(q_perm, 0, q_invperm, 0, 1);
-
- int device = q_weight.device().index();
- int width = q_weight.size(1);
- int groups;
- int height;
-
- if (!q_scale.device().is_meta())
- {
- TORCH_CHECK_SHAPES(q_weight, 1, q_scale, 1, 8);
- TORCH_CHECK_SHAPES(q_scale_max, 0, q_scale, 0, 1);
- groups = q_scale.size(0);
- height = q_invperm.size(0);
- }
- else
- {
- TORCH_CHECK_SHAPES(q_weight, 1, gptq_qzeros, 1, 8);
- TORCH_CHECK_SHAPES(q_weight, 1, gptq_scales, 1, 1);
- groups = gptq_qzeros.size(0);
- height = q_weight.size(0) * 8;
- }
-
- TORCH_CHECK(temp_dq.size(0) >= width * height, "Insufficient size of temp_dq buffer")
-
- QMatrix* m = new QMatrix
- (
- device,
- height,
- width,
- groups,
- (uint32_t*) q_weight.data_ptr(),
- q_perm.device().is_meta() ? NULL : (uint16_t*) q_perm.data_ptr(),
- q_invperm.device().is_meta() ? NULL : (uint16_t*) q_invperm.data_ptr(),
- q_scale.device().is_meta() ? NULL : (uint32_t*) q_scale.data_ptr(),
- q_scale_max.device().is_meta() ? NULL : (half*) q_scale_max.data_ptr(),
- q_groups.device().is_meta() ? NULL : (uint16_t*) q_groups.data_ptr(),
- gptq_qzeros.device().is_meta() ? NULL : (uint32_t*) gptq_qzeros.data_ptr(),
- gptq_scales.device().is_meta() ? NULL : (half*) gptq_scales.data_ptr(),
- gptq_g_idx.device().is_meta() ? NULL : (uint32_t*) gptq_g_idx.data_ptr(),
- (half*) temp_dq.data_ptr()
- );
-
- return reinterpret_cast<uintptr_t> (m);
-}
-
-void gemm_half_q_half
-(
- torch::Tensor a,
- uintptr_t b,
- torch::Tensor c,
- bool force_cuda
-)
-{
- QMatrix* qm = reinterpret_cast<QMatrix*> (b);
-
- TORCH_CHECK_DTYPE(a, kHalf);
- TORCH_CHECK_DTYPE(c, kHalf);
- TORCH_CHECK_SHAPES(a, 0, c, 0, 1);
- TORCH_CHECK(qm->height == a.size(1), "a and b have incompatible shapes")
- TORCH_CHECK(qm->width == c.size(1), "b and c have incompatible shapes")
-
- const at::cuda::OptionalCUDAGuard device_guard(device_of(a));
-
- gemm_half_q_half_cuda
- (
- at::cuda::getCurrentCUDABlasHandle(),
- (const half*) a.data_ptr(),
- qm,
- (half*) c.data_ptr(),
- c.size(0), // m
- c.size(1), // n
- a.size(1), // k
- true,
- NULL,
- force_cuda
- );
-}
-
-// Bindings
-
-PYBIND11_MODULE(TORCH_EXTENSION_NAME, m)
-{
- m.def("make_q_matrix", &make_q_matrix, "make_q_matrix");
- m.def("gemm_half_q_half", &gemm_half_q_half, "gemm_half_q_half");
-}
diff --git a/auto_round_extension/cuda/gptqmodel_marlin.py b/auto_round_extension/cuda/gptqmodel_marlin.py
new file mode 100644
index 000000000..242037d9b
--- /dev/null
+++ b/auto_round_extension/cuda/gptqmodel_marlin.py
@@ -0,0 +1,486 @@
+# Copyright 2024-2025 ModelCloud.ai
+# Copyright 2024-2025 [email protected]
+# Contact: [email protected], x.com/qubitium
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Adapted from vllm
+# at https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/gptq_marlin.py
+
+from typing import Any, Dict, List, Optional, Tuple
+
+import numpy as np
+import torch
+
+
+def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demand
+ from gptqmodel.models._const import DEVICE, PLATFORM # pylint: disable=E0401
+ from gptqmodel.nn_modules.qlinear import BaseQuantLinear # pylint: disable=E0401
+ from gptqmodel.utils.backend import BACKEND # pylint: disable=E0401
+
+ marlin_import_exception = None
+ try:
+ import gptqmodel_marlin_kernels # pylint: disable=E0401
+ except ImportError as e:
+ marlin_import_exception = e
+
+ from auto_round.utils import logger
+
+ GPTQ_MARLIN_TILE = 16
+ GPTQ_MARLIN_MIN_THREAD_N = 64
+ GPTQ_MARLIN_MIN_THREAD_K = 128
+ GPTQ_MARLIN_MAX_PARALLEL = 16
+
+ def set_weight_attrs(
+ weight: torch.Tensor,
+ weight_attrs: Optional[Dict[str, Any]],
+ ):
+ """Set attributes on a weight tensor.
+
+ This method is used to set attributes on a weight tensor. This method
+ will not overwrite existing attributes.
+
+ Args:
+ weight: The weight tensor.
+ weight_attrs: A dictionary of attributes to set on the weight tensor.
+ """
+ if weight_attrs is None:
+ return
+ for key, value in weight_attrs.items():
+ assert not hasattr(
+ weight, key), (f"Overwriting existing tensor attribute: {key}")
+ setattr(weight, key, value)
+
+ def marlin_is_k_full(act_order: bool, is_row_parallel: bool) -> bool:
+ return (not act_order) or (act_order and not is_row_parallel)
+
+ def marlin_repeat_scales_on_all_ranks(act_order: bool, group_size: int,
+ is_row_parallel: bool) -> bool:
+ # Need to repeat scales on every rank if act_ordering or
+ # channelwise and RowParallelLinear
+ is_channelwise = group_size == -1
+ return act_order or (is_channelwise and is_row_parallel)
+
+ def marlin_make_workspace(output_size_per_partition: int,
+ device: torch.device) -> torch.Tensor:
+ max_workspace_size = (output_size_per_partition //
+ GPTQ_MARLIN_MIN_THREAD_N) * GPTQ_MARLIN_MAX_PARALLEL
+
+ return torch.zeros(max_workspace_size,
+ dtype=torch.int,
+ device=device,
+ requires_grad=False)
+
+ def marlin_sort_g_idx(
+ g_idx: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+ g_idx_sort_indices = torch.argsort(g_idx).to(torch.int)
+ return g_idx[g_idx_sort_indices], g_idx_sort_indices
+
+ def marlin_make_empty_g_idx(device: torch.device) -> torch.Tensor:
+ return torch.nn.Parameter(torch.empty(0, dtype=torch.int, device=device),
+ requires_grad=False)
+
+ # Newly generated tensors need to replace existing tensors that are
+ # already registered as parameters by vLLM (and won't be freed)
+ def replace_tensor(layer: torch.nn.Module, name: str,
+ new_t: torch.Tensor) -> None:
+ # It is important to use resize_() here since it ensures
+ # the same buffer is reused
+ getattr(layer, name).resize_(new_t.shape)
+ getattr(layer, name).copy_(new_t)
+ del new_t
+
+ def marlin_permute_scales(s: torch.Tensor, size_k: int, size_n: int,
+ group_size: int) -> torch.Tensor:
+
+ scale_perm, scale_perm_single = get_scale_perms()
+ if group_size < size_k and group_size != -1:
+ s = s.reshape((-1, len(scale_perm)))[:, scale_perm]
+ else:
+ s = s.reshape((-1, len(scale_perm_single)))[:, scale_perm_single]
+ s = s.reshape((-1, size_n)).contiguous()
+
+ return s
+
+ def get_scale_perms():
+ scale_perm: List[int] = []
+ for i in range(8):
+ scale_perm.extend([i + 8 * j for j in range(8)])
+ scale_perm_single: List[int] = []
+ for i in range(4):
+ scale_perm_single.extend(
+ [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
+ return scale_perm, scale_perm_single
+
+ def apply_gptq_marlin_linear(
+ input: torch.Tensor,
+ weight: torch.Tensor,
+ weight_scale: torch.Tensor,
+ weight_zp: torch.Tensor,
+ g_idx: torch.Tensor,
+ g_idx_sort_indices: torch.Tensor,
+ workspace: torch.Tensor,
+ num_bits: int,
+ output_size_per_partition: int,
+ input_size_per_partition: int,
+ is_k_full: bool,
+ bias: torch.Tensor,
+ fp32: bool,
+ ) -> torch.Tensor:
+
+ reshaped_x = input.reshape(-1, input.shape[-1])
+ out_shape = input.shape[:-1] + (output_size_per_partition,)
+
+ output = gptqmodel_marlin_kernels.gptq_marlin_gemm(
+ reshaped_x,
+ weight,
+ weight_scale,
+ weight_zp,
+ g_idx,
+ g_idx_sort_indices,
+ workspace,
+ num_bits,
+ reshaped_x.shape[0],
+ output_size_per_partition,
+ input_size_per_partition,
+ is_k_full,
+ False,
+ fp32, # <- True: enable fp32 reduce for higher accuracy, False: fp16
+ )
+
+ if bias is not None:
+ output.add_(bias) # In-place add
+
+ return output.reshape(out_shape)
+
+ class MarlinQuantLinear(BaseQuantLinear):
+ SUPPORTS_BITS = [4, 8]
+ SUPPORTS_GROUP_SIZE = [-1, 32, 64, 128]
+ SUPPORTS_DESC_ACT = [True, False]
+ SUPPORTS_SYM = [True]
+ SUPPORTS_SHARDS = True
+ SUPPORTS_TRAINING = False
+ SUPPORTS_AUTO_PADDING = False
+ SUPPORTS_IN_FEATURES_DIVISIBLE_BY = [1]
+ SUPPORTS_OUT_FEATURES_DIVISIBLE_BY = [64]
+
+ SUPPORTS_DEVICES = [DEVICE.CUDA]
+ SUPPORTS_PLATFORM = [PLATFORM.LINUX]
+ SUPPORTS_PACK_DTYPES = [torch.int32]
+
+ SUPPORTS_DTYPES = [torch.float16, torch.bfloat16]
+
+ # for transformers/optimum tests compat
+ QUANT_TYPE = "marlin"
+
+ def __init__(
+ self, bits: int,
+ group_size: int,
+ desc_act: bool,
+ sym: bool,
+ in_features: int,
+ out_features: int,
+ bias: bool = False,
+ pack_dtype: torch.dtype = torch.int32,
+ **kwargs):
+ if marlin_import_exception is not None:
+ raise ValueError(
+ f"Trying to use the marlin backend, but could not "
+ f"import the C++/CUDA dependencies with the following error: {marlin_import_exception}"
+ )
+
+ # self.original_in_features = in_features
+ # self.original_out_features = out_features
+
+ if desc_act and group_size == -1:
+ # In this case, act_order == True is the same as act_order == False
+ # (since we have only one group per output channel)
+ desc_act = False
+
+ super().__init__(
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ desc_act=desc_act,
+ in_features=in_features,
+ out_features=out_features,
+ bias=bias,
+ pack_dtype=pack_dtype,
+ backend=kwargs.pop("backend", BACKEND.MARLIN),
+ adapter=None,
+ register_buffers=False,
+ **kwargs)
+
+ # toggle fp32 mode depending on MARLIN or MARLIN_FP16 backend
+ self.fp32 = True if self.backend in [BACKEND.MARLIN, BACKEND.AUTO] else False
+
+ if not self.fp32:
+ logger.warning_once(
+ "Kernel: Marlin FP16 mode is activated with reduced accuracy. "
+ "Use default Marlin model for improved inference quality.")
+
+ # Determine sharding
+ if marlin_repeat_scales_on_all_ranks(desc_act,
+ self.group_size,
+ is_row_parallel=False):
+ # By setting scale_dim == None, weight_loader will
+ # repeat the scales on each GPU in TP>1 case.
+ scales_and_zp_input_dim = None
+ scales_and_zp_size = self.in_features // self.group_size
+ else:
+ # By setting scale_dim == 0, weight_loader will
+ # shard the scales in TP>1 case.
+ scales_and_zp_input_dim = 0
+ scales_and_zp_size = self.in_features // self.group_size
+
+ # Quantized weights
+ self.register_buffer(
+ "qweight",
+ torch.empty(
+ self.in_features // self.pack_factor,
+ self.out_features,
+ dtype=torch.int32,
+ ),
+ )
+
+ # Activation order
+ # self.register_buffer(
+ # "g_idx_temp",
+ # torch.empty(
+ # 0,
+ # dtype=torch.int32,
+ # ),
+ # )
+
+ # Ignore warning from fused linear layers such as QKVParallelLinear.
+ # set_weight_attrs(
+ # self.g_idx_temp,
+ # {
+ # "input_dim": 0,
+ # "ignore_warning": True
+ # },
+ # )
+
+ set_weight_attrs(
+ self.qweight,
+ {
+ "input_dim": 0,
+ "output_dim": 1,
+ "packed_dim": 0,
+ "pack_factor": self.pack_factor,
+ },
+ )
+
+ #
+ # # Ignore warning from fused linear layers such as QKVParallelLinear.
+ # set_weight_attrs(
+ # self.g_idx,
+ # {
+ # "input_dim": 0,
+ # "ignore_warning": True
+ # },
+ # )
+
+ # Scales
+ self.register_buffer(
+ "scales",
+ torch.empty(
+ scales_and_zp_size,
+ self.out_features,
+ dtype=torch.float16,
+ ),
+ )
+
+ set_weight_attrs(
+ self.scales,
+ {
+ "input_dim": scales_and_zp_input_dim,
+ "output_dim": 1,
+ },
+ )
+
+ # Quantized zero-points
+ self.register_buffer(
+ "qzeros",
+ torch.empty(
+ scales_and_zp_size,
+ self.out_features // self.pack_factor,
+ dtype=torch.int32,
+ ),
+ )
+
+ set_weight_attrs(
+ self.qzeros,
+ {
+ "input_dim": scales_and_zp_input_dim,
+ "output_dim": 1,
+ "packed_dim": 1,
+ "pack_factor": self.pack_factor,
+ },
+ )
+
+ self.is_k_full = marlin_is_k_full(self.desc_act, is_row_parallel=False)
+
+ if bias:
+ self.register_buffer("bias", torch.zeros((self.out_features), dtype=torch.float16))
+ else:
+ self.bias = None
+
+ self.is_lm_head = False
+ if kwargs.get("name") is not None and kwargs.get("lm_head_name") is not None:
+ self.is_lm_head = kwargs["name"] == kwargs["lm_head_name"]
+
+ # auto-optimize on post init
+ # self.optimize()
+
+ # def optimize(self, backend: str = "inductor", mode: str = None, fullgraph: bool = False):
+ # if self.optimized:
+ # return
+ #
+ # # compile dequantize
+ # self.forward = torch_compile(self.forward, backend=backend, mode=mode, fullgraph=fullgraph)
+ #
+ # super().optimize()
+
+ @classmethod
+ # internal method and should not be overridden
+ def verify_supports_params(cls):
+ return
+
+ def post_init(self):
+ device = self.qweight.device
+ # Allocate marlin workspace
+ self.workspace = marlin_make_workspace(
+ self.out_features, device)
+
+ # Handle sorting for activation reordering if needed.
+
+ # self.g_idx = marlin_make_empty_g_idx(device)
+ g_idx_sort_indices = marlin_make_empty_g_idx(device)
+
+ # No zero-point
+ # self.zp = marlin_make_empty_g_idx(device)
+
+ # Repack weights from autogptq format to marlin format.
+ marlin_qweight = gptqmodel_marlin_kernels.gptq_marlin_repack(
+ self.qweight,
+ g_idx_sort_indices,
+ self.in_features,
+ self.out_features,
+ self.bits,
+ self.pack_dtype_bits)
+ replace_tensor(self, "qweight", marlin_qweight)
+
+ # Permute scales from autogptq format to marlin format.
+ marlin_scales = marlin_permute_scales(
+ self.scales,
+ size_k=self.in_features,
+ size_n=self.out_features,
+ group_size=self.group_size)
+ replace_tensor(self, "scales", marlin_scales)
+
+ super().post_init()
+
+ def list_buffers(self) -> List:
+ buf = super().list_buffers()
+ if hasattr(self, "workspace") and self.workspace is not None:
+ buf.append(self.workspace)
+ if hasattr(self, "zp") and self.zp is not None:
+ buf.append(self.zp)
+ return buf
+
+ def forward(self, x: torch.Tensor):
+ # TODO FIXME: parent should never call us if there is no data to process
+ # check: https://github.com/ModelCloud/GPTQModel/issues/1361
+ if x.shape[0] == 0:
+ return torch.empty((0, self.out_features), dtype=x.dtype, device=x.device)
+
+ # make sure scales is synced with x/input
+ if x.dtype != self.scales.dtype:
+ self.scales = self.scales.to(dtype=x.dtype)
+ g_idx = marlin_make_empty_g_idx(self.qweight.device)
+ g_idx_sort_indices = marlin_make_empty_g_idx(self.qweight.device)
+ zp = marlin_make_empty_g_idx(self.qweight.device)
+ out = apply_gptq_marlin_linear(
+ input=x.contiguous() if self.is_lm_head else x,
+ weight=self.qweight,
+ weight_scale=self.scales,
+ weight_zp=zp,
+ g_idx=g_idx,
+ g_idx_sort_indices=g_idx_sort_indices,
+ workspace=self.workspace,
+ num_bits=self.bits,
+ output_size_per_partition=self.out_features,
+ input_size_per_partition=self.in_features,
+ is_k_full=self.is_k_full,
+ bias=self.bias,
+ fp32=self.fp32,
+ )
+
+ if self.adapter:
+ out = self.adapter.apply(x=x, out=out)
+
+ return out
+
+ # Precompute permutations for Marlin weight and scale shuffling
+ def _get_perms():
+ perm = []
+ for i in range(32):
+ perm1 = []
+ col = i // 4
+ for block in [0, 1]:
+ for row in [
+ 2 * (i % 4),
+ 2 * (i % 4) + 1,
+ 2 * (i % 4 + 4),
+ 2 * (i % 4 + 4) + 1,
+ ]:
+ perm1.append(16 * row + col + 8 * block)
+ for j in range(4):
+ perm.extend([p + 256 * j for p in perm1])
+
+ perm = np.array(perm)
+ interleave = np.array([0, 2, 4, 6, 1, 3, 5, 7])
+ perm = perm.reshape((-1, 8))[:, interleave].ravel()
+ perm = torch.from_numpy(perm)
+ scale_perm = []
+ for i in range(8):
+ scale_perm.extend([i + 8 * j for j in range(8)])
+ scale_perm_single = []
+ for i in range(4):
+ scale_perm_single.extend([2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
+ return perm, scale_perm, scale_perm_single
+
+ def unpack_qzeros(qzeros):
+ unpacked_zeros = torch.zeros(
+ (qzeros.shape[0], qzeros.shape[1] * 8),
+ dtype=torch.int8,
+ device=qzeros.device,
+ requires_grad=False,
+ )
+
+ for col in range(unpacked_zeros.shape[1]):
+ i = col % 8
+ unpacked_zeros[:, col] = (qzeros[:, col // 8] >> (4 * i)) & 0xF
+
+ return unpacked_zeros
+
+ def dequantize_qzeros(layer):
+ qzeros = layer.qzeros
+ unpacked_qzeros = unpack_qzeros(qzeros)
+ group_size = layer.group_size
+ unpacked_qzeros = unpacked_qzeros.repeat_interleave(group_size, dim=0)
+
+ return unpacked_qzeros
+
+ return MarlinQuantLinear
diff --git a/auto_round_extension/cuda/post_init.py b/auto_round_extension/cuda/post_init.py
deleted file mode 100644
index 6bad13639..000000000
--- a/auto_round_extension/cuda/post_init.py
+++ /dev/null
@@ -1,157 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# MIT License
-#
-# Copyright (c) 2023 潘其威(William)
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-import torch
-
-EXLLAMA_DEFAULT_MAX_INPUT_LENGTH = 2048
-
-
-def autoround_post_init(model):
- """
- The max_input_length argument is specific to the exllama backend, that requires to initialize a buffer temp_state.
- """
- device_to_buffers_size = {}
-
- model_uses_exllama = False
- for name, submodule in model.named_modules():
- if hasattr(submodule, "QUANT_TYPE") and submodule.QUANT_TYPE == "exllama":
- model_uses_exllama = True
- device = submodule.qweight.device
- if device not in device_to_buffers_size:
- device_to_buffers_size[device] = {
- "max_dq_buffer_size": 1,
- "max_inner_outer_dim": 1,
- }
-
- submodule._use_act_order = False
-
- # Disable this heuristic for detecting act_order, but it could be used instead of the config.
- """
- if submodule.g_idx is None:
- submodule.act_order = False
- elif submodule.g_idx is not None and ((submodule.g_idx == 0).all() or
- torch.equal(submodule.g_idx.cpu(),
- torch.tensor([i // submodule.group_size for i in range(submodule.g_idx.shape[0])], dtype=torch.int32))):
- submodule.g_idx = None
- submodule.act_order = False
- else:
- submodule.act_order = True
- """
-
- device_to_buffers_size[device]["max_dq_buffer_size"] = max(
- device_to_buffers_size[device]["max_dq_buffer_size"],
- submodule.qweight.numel() * 8,
- )
-
- if model_uses_exllama:
- # To be honest this is quite ugly, not proud of this.
- try:
- from exllama_kernels import prepare_buffers, set_tuning_params
- except ImportError as e:
- raise ImportError(
- f"Could not import exllama backend dependencies prepare_buffers, set_tuning_params with the following "
- f"error: {e}"
- )
-
- device_to_buffers = {}
-
- max_input_len = 1
-
- for device, buffers_size in device_to_buffers_size.items():
- # The temp_state buffer is required to reorder X in the act-order case.
- # The temp_dq buffer is required to dequantize weights when using cuBLAS, typically for the prefill.
- device_to_buffers[device] = {
- "temp_state": torch.zeros(
- (max_input_len, buffers_size["max_inner_outer_dim"]),
- dtype=torch.float16,
- device=device,
- ),
- "temp_dq": torch.zeros(
- (1, buffers_size["max_dq_buffer_size"]),
- dtype=torch.float16,
- device=device,
- ),
- "max_dq_buffer_size": buffers_size["max_dq_buffer_size"],
- "max_inner_outer_dim": buffers_size["max_inner_outer_dim"],
- }
-
- # Buffers need to be persistent to avoid any bug.
- model.device_to_buffers = device_to_buffers
-
- for device, buffers in model.device_to_buffers.items():
- prepare_buffers(device, buffers["temp_state"], buffers["temp_dq"])
-
- # Using the default from exllama repo here.
- matmul_recons_thd = 8
- matmul_fused_remap = False
- matmul_no_half2 = False
- set_tuning_params(matmul_recons_thd, matmul_fused_remap, matmul_no_half2)
-
- # The buffers need to have been initialized first before calling make_q4.
- for name, submodule in model.named_modules():
- if hasattr(submodule, "QUANT_TYPE") and submodule.QUANT_TYPE == "exllama":
- submodule.post_init()
-
- ## exllamav2
- fixed_bytes = {}
- model_uses_exllamav2 = False
-
- for _, submodule in model.named_modules():
- if hasattr(submodule, "QUANT_TYPE"):
- if submodule.QUANT_TYPE == "exllamav2":
- model_uses_exllamav2 = True
- device = submodule.qweight.device
- scratch_fixed = submodule.scratch_space_fixed()
- fixed_bytes[device] = max(scratch_fixed, fixed_bytes.get(device, 0))
- elif submodule.QUANT_TYPE == "hpu":
- submodule.post_init()
-
- if model_uses_exllamav2:
- from auto_round_extension.cuda.qlinear_exllamav2 import ExLlamaV2DeviceTensors
-
- device_tensors = {}
- for device, scratch_bytes in fixed_bytes.items():
- device_tensors[device] = ExLlamaV2DeviceTensors(device.index, scratch_bytes)
-
- # have persistent buffers, otherwise we will get OOM
- model.device_tensors = device_tensors
-
- for _, submodule in model.named_modules():
- if hasattr(submodule, "QUANT_TYPE") and submodule.QUANT_TYPE == "exllamav2":
- device = submodule.qweight.device
- submodule.post_init(temp_dq=model.device_tensors[device])
- torch.cuda.empty_cache()
-
- return model
diff --git a/auto_round_extension/cuda/qlinear_exllamav2.py b/auto_round_extension/cuda/qlinear_exllamav2.py
deleted file mode 100644
index 3bc415971..000000000
--- a/auto_round_extension/cuda/qlinear_exllamav2.py
+++ /dev/null
@@ -1,277 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Adapted from turboderp exllama: https://github.com/turboderp/exllamav2
-# MIT License
-#
-# Copyright (c) 2023 潘其威(William)
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-import math
-from logging import getLogger
-
-import torch
-import torch.nn as nn
-
-logger = getLogger(__name__)
-
-try:
- from autoround_exllamav2_kernels import gemm_half_q_half, make_q_matrix
-except ImportError as e:
- exllama_v2_import_exception = e
-
-
- def error_raiser_exllama(*args, **kwargs):
- raise ValueError(
- f"Trying to use the exllama v2 backend, but could not import the C++/CUDA dependencies with the following "
- f"error: {exllama_v2_import_exception}"
- )
-
-
- make_q_matrix = error_raiser_exllama
- gemm_half_q_half = error_raiser_exllama
-
-# Dummy tensor to pass instead of g_idx since there is no way to pass "None" to a C++ extension
-none_tensor = torch.empty((1, 1), device="meta")
-
-
-def _torch_device(idx):
- if idx == -1:
- return "cpu"
- return f"cuda:{idx}"
-
-
-def ext_gemm_half_q_half(x, q_handle, q4_width, force_cuda):
- """Matrix multiplication, returns x @ q4"""
- output_shape = x.shape[:-1] + (q4_width,)
- x = x.view(-1, x.shape[-1])
- output = torch.empty((x.shape[0], q4_width), dtype=torch.half, device=x.device)
- gemm_half_q_half(x, q_handle, output, force_cuda)
- return output.view(output_shape)
-
-
-def ext_make_q_matrix(w: dict, temp_dq, key: str = None):
- """
- Create Q matrix
- """
- # EXL2
- # won't work as the moment because the tensors are not the same.
- if "q_weight" in w:
- w["q_scale_max"] /= 256
- w["q_perm"] = w["q_perm"].short()
- w["q_invperm"] = w["q_invperm"].short()
- return make_q_matrix(
- w["q_weight"],
- w["q_perm"],
- w["q_invperm"],
- w["q_scale"],
- w["q_scale_max"],
- w["q_groups"],
- none_tensor,
- none_tensor,
- none_tensor,
- temp_dq,
- )
- # GPTQ
- elif "qweight" in w:
- if w["scales"].dtype == torch.float:
- w["scales"] = w["scales"].half()
-
- # GPTQ with g_idx (act_order)
- if "g_idx" in w and not (w["g_idx"] == 0).all().item():
- w["q_perm"] = torch.empty(
- (w["qweight"].shape[0] * 8,),
- dtype=torch.short,
- device=w["qweight"].device,
- )
- w["q_invperm"] = torch.empty_like(w["q_perm"])
- # make_q4 segfaults if g_idx is not on cpu in the act-order case. In the non act-order case, None needs
- # to be passed for g_idx.
- return make_q_matrix(
- w["qweight"],
- w["q_perm"],
- w["q_invperm"],
- none_tensor,
- none_tensor,
- none_tensor,
- w["qzeros"],
- w["scales"],
- w["g_idx"].cpu(),
- temp_dq,
- )
- # GPTQ without g_idx
- else:
- return make_q_matrix(
- w["qweight"],
- none_tensor,
- none_tensor,
- none_tensor,
- none_tensor,
- none_tensor,
- w["qzeros"],
- w["scales"],
- none_tensor,
- temp_dq,
- )
-
-
-class QuantLinear(nn.Module):
- QUANT_TYPE = "exllamav2"
-
- """Linear layer implementation with per-group 4-bit quantization of the weights"""
-
- def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):
- super().__init__()
- if bits != 4:
- raise ValueError(
- f"Exllamav2 kernel supports only bits=4, requested bits={bits}. Something is wrong in the model "
- f"initialization."
- )
- if trainable:
- raise NotImplementedError("Exllamav2 kernel does not support training.")
-
- self.q_handle = None
- self.q_tensors = None
-
- self.padding = -outfeatures % 32
- self.outfeatures = outfeatures + self.padding
- outfeatures = self.outfeatures
-
- self.infeatures = infeatures
- self.bits = bits
- self.group_size = group_size if group_size != -1 else infeatures
- self.trainable = trainable
- self.maxq = 2 ** self.bits - 1
-
- assert infeatures % 32 == 0
- assert infeatures % self.group_size == 0
- assert outfeatures % 32 == 0
-
- # I need to register the tensors, otherwise, we won't be able to load them easily using transformers ...
- self.register_buffer(
- "qweight",
- torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),
- )
- self.register_buffer(
- "qzeros",
- torch.zeros(
- (
- math.ceil(infeatures / self.group_size),
- outfeatures // 32 * self.bits,
- ),
- dtype=torch.int32,
- ),
- )
- self.register_buffer(
- "scales",
- torch.zeros(
- (math.ceil(infeatures / self.group_size), outfeatures),
- dtype=torch.float16,
- ),
- )
- self.register_buffer(
- "g_idx",
- torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),
- )
-
- if bias:
- self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))
- else:
- self.bias = None
- self.clip = kwargs.get("clip", False)
-
- def post_init(self, temp_dq):
- assert self.qweight.device.type == "cuda"
- assert self.qweight.device.index is not None
- self.q_tensors = {
- "qweight": self.qweight,
- "qzeros": self.qzeros,
- "scales": self.scales,
- "g_idx": self.g_idx,
- }
- temp_dq = temp_dq.get_scratch_slice(self.temp_dq_size())
- self.q_handle = ext_make_q_matrix(self.q_tensors, temp_dq)
-
- def forward(self, x, force_cuda=False):
- orig_dtype = x.dtype
- if x.dtype != torch.float16:
- logger.warning_once(
- f"The exllama v2 kernel for GPTQ requires a float16 input activation, while {x.dtype} was passed. "
- f"Casting to float16.\nMake sure you loaded your model with torch_dtype=torch.float16, that the model "
- f"definition does not inadvertently cast to float32, or disable AMP Autocast that may produce float32 "
- f"intermediate activations in the model."
- )
-
- x = x.half()
-
- output = ext_gemm_half_q_half(x, self.q_handle, self.outfeatures, force_cuda)
- if orig_dtype != torch.float16:
- output = output.to(orig_dtype)
- if self.bias is not None:
- output.add_(self.bias)
- if self.clip:
- output = torch.clip(output, -65504, 65504)
- return output
-
- def temp_dq_size(self):
- return self.infeatures * self.outfeatures * 2 + 128
-
- def temp_fwd_size(self, max_input_len, max_batch_size):
- return self.outfeatures * max_input_len * max_batch_size * 4 + 128
-
- def scratch_space_fixed(self, max_input_len=2048, max_batch_size=8):
- return self.temp_dq_size() + self.temp_fwd_size(max_input_len, max_batch_size)
-
-
-class ExLlamaV2DeviceTensors:
- device_idx: int
- scratch_bytes: int
- scratch_idx: int
- scratch: torch.tensor = None
-
- def __init__(self, device_idx, scratch_bytes):
- self.device_idx = device_idx
- self.scratch_bytes = scratch_bytes
-
- def prepare(self):
- self.scratch = torch.empty(
- (self.scratch_bytes // 2,),
- dtype=torch.half,
- device=_torch_device(self.device_idx),
- )
-
- def get_scratch_slice(self, size_bytes):
- if self.scratch is None:
- self.prepare()
-
- size_bytes = ((size_bytes + 127) // 128) * 128
- size_half = size_bytes // 2
- scratch_slice = self.scratch.narrow(0, 0, size_half)
- return scratch_slice
diff --git a/auto_round_extension/cuda/qlinear_triton.py b/auto_round_extension/cuda/qlinear_triton.py
deleted file mode 100644
index 006911f2f..000000000
--- a/auto_round_extension/cuda/qlinear_triton.py
+++ /dev/null
@@ -1,326 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# MIT License
-#
-# Copyright (c) 2023 潘其威(William)
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-import math
-from logging import getLogger
-
-import numpy as np
-import torch
-import torch.nn as nn
-import transformers
-
-from auto_round_extension.cuda.triton_utils.mixin import TritonModuleMixin
-
-
-logger = getLogger(__name__)
-
-try:
- from auto_round_extension.cuda.triton_utils.kernels import (
- QuantLinearFunction,
- QuantLinearInferenceOnlyFunction,
- quant_matmul_248,
- quant_matmul_inference_only_248,
- transpose_quant_matmul_248,
- )
-except ImportError as e:
- triton_import_exception = e
-
- def error_raiser_triton(*args, **kwargs):
- raise ValueError(
- f'Trying to use the triton backend, but could not import triton '
- f'dependencies with the following error: {triton_import_exception}'
- )
-
- class FakeTriton:
- def __getattr__(self, name):
- raise ImportError(
- f"Trying to use the triton backend, but could not import triton "
- f"dependencies with the following error: {triton_import_exception}"
- )
-
- quant_matmul_248 = error_raiser_triton
- transpose_quant_matmul_248 = error_raiser_triton
- quant_matmul_inference_only_248 = error_raiser_triton
- QuantLinearFunction = FakeTriton
- QuantLinearInferenceOnlyFunction = FakeTriton
-
-
-class QuantLinear(nn.Module, TritonModuleMixin):
- QUANT_TYPE = "triton"
-
- def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):
- super().__init__()
- if bits not in [2, 4, 8]:
- raise NotImplementedError("Only 2,4,8 bits are supported.")
- if infeatures % 32 != 0 or outfeatures % 32 != 0:
- raise NotImplementedError("in_feature and out_feature must be divisible by 32.")
- self.infeatures = infeatures
- self.outfeatures = outfeatures
- self.bits = bits
- self.group_size = group_size if group_size != -1 else infeatures
- self.maxq = 2**self.bits - 1
-
- self.register_buffer(
- "qweight",
- torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),
- )
- self.register_buffer(
- "qzeros",
- torch.zeros(
- (
- math.ceil(infeatures / self.group_size),
- outfeatures // 32 * self.bits,
- ),
- dtype=torch.int32,
- ),
- )
- self.register_buffer(
- "scales",
- torch.zeros(
- (math.ceil(infeatures / self.group_size), outfeatures),
- dtype=torch.float16,
- ),
- )
- self.register_buffer(
- "g_idx",
- torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),
- )
- if bias:
- self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))
- else:
- self.bias = None
-
- self.trainable = trainable
-
- def post_init(self):
- pass
-
- def pack_cpu(self, linear, scales, zeros, g_idx=None):
- W = linear.weight.data.clone()
- if isinstance(linear, nn.Conv2d):
- W = W.flatten(1)
- if isinstance(linear, transformers.pytorch_utils.Conv1D):
- W = W.t()
-
- self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
-
- scales = scales.t().contiguous()
- if isinstance(zeros,torch.Tensor):
- zeros = zeros.t().contiguous()
- scale_zeros = zeros * scales
- else:
- scale_zeros = scales*zeros
- self.scales = scales.clone().half()
- if linear.bias is not None:
- self.bias = linear.bias.clone().half()
-
- intweight = []
- for idx in range(self.infeatures):
- intweight.append(
- torch.round((W[:, idx] + scale_zeros[self.g_idx[idx]]) / self.scales[self.g_idx[idx]]).to(torch.int)[
- :, None
- ]
- )
- intweight = torch.cat(intweight, dim=1)
- intweight = intweight.t().contiguous()
- intweight = intweight.numpy().astype(np.uint32)
-
- i = 0
- row = 0
- qweight = np.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32)
- while row < qweight.shape[0]:
- for j in range(i, i + (32 // self.bits)):
- qweight[row] |= intweight[j] << (self.bits * (j - i))
- i += 32 // self.bits
- row += 1
-
- qweight = qweight.astype(np.int32)
- self.qweight = torch.from_numpy(qweight)
-
- if isinstance(zeros, torch.Tensor):
- zeros = zeros.numpy().astype(np.uint32)
- qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)
- i = 0
- col = 0
- while col < qzeros.shape[1]:
- for j in range(i, i + (32 // self.bits)):
- qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
- i += 32 // self.bits
- col += 1
-
- qzeros = qzeros.astype(np.int32)
- self.qzeros = torch.from_numpy(qzeros)
- else:
- shape = scales.shape
- value = 0
- for j in range(0, (32 // self.bits)):
- value |= zeros << (self.bits * j)
- qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value
- qzeros = qzeros.astype(np.int32)
- self.qzeros = torch.from_numpy(qzeros)
-
-
-
- def pack(self, linear, scales, zeros, g_idx):
- if torch.cuda.is_available():
- return self.pack_cuda(linear, scales, zeros, g_idx)
- else:
- return self.pack_cpu(linear, scales, zeros, g_idx)
-
- def pack_cuda(self, linear, scales, zeros, g_idx=None):
- W = linear.weight.data.clone()
- if isinstance(linear, nn.Conv2d):
- W = W.flatten(1)
- if isinstance(linear, transformers.pytorch_utils.Conv1D):
- W = W.t()
- scales_t = scales.t().contiguous()
- self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
- if linear.bias is not None:
- self.bias = linear.bias.clone().half()
- self.scales = scales_t.clone().half()
-
- repeat_scales = scales.to("cuda:0").repeat_interleave(self.group_size, 1)
- if isinstance(zeros, torch.Tensor):
- repeat_zeros = zeros.to("cuda:0").repeat_interleave(self.group_size, 1)
- else:
- repeat_zeros = zeros
-
- intweight = torch.round(W.to("cuda:0") / repeat_scales + repeat_zeros).to(torch.int).t().contiguous().to("cpu")
- intweight = intweight.numpy().astype(np.uint32)
- del repeat_scales
-
- i = 0
- row = 0
- qweight = np.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32)
- # pack_array_with_numba_b4_c32(intweight, qweight)
- while row < qweight.shape[0]:
- for j in range(i, i + (32 // self.bits)):
- qweight[row] |= intweight[j] << (self.bits * (j - i))
- i += 32 // self.bits
- row += 1
-
- qweight = qweight.astype(np.int32)
- self.qweight = torch.from_numpy(qweight)
-
- if isinstance(zeros, torch.Tensor):
- zeros = zeros.t().contiguous()
- zeros = zeros.numpy().astype(np.uint32)
- qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)
- i = 0
- col = 0
- while col < qzeros.shape[1]:
- for j in range(i, i + (32 // self.bits)):
- qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
- i += 32 // self.bits
- col += 1
-
- qzeros = qzeros.astype(np.int32)
- self.qzeros = torch.from_numpy(qzeros)
- else:
- shape = scales_t.shape
- value = 0
- for j in range(0, (32 // self.bits)):
- value |= zeros << (self.bits * j)
- qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value
- qzeros = qzeros.astype(np.int32)
- self.qzeros = torch.from_numpy(qzeros)
-
- def forward(self, x):
- out_shape = x.shape[:-1] + (self.outfeatures,)
- quant_linear_fn = QuantLinearFunction if self.trainable else QuantLinearInferenceOnlyFunction
- out = quant_linear_fn.apply(
- x.reshape(-1, x.shape[-1]),
- self.qweight,
- self.scales,
- self.qzeros,
- self.g_idx,
- self.bits,
- self.maxq,
- )
- out = out.half().reshape(out_shape)
- out = out + self.bias if self.bias is not None else out
- return out
-
- @classmethod
- def warmup(cls, model, transpose=False, seqlen=2048):
- """
- Pre-tunes the quantized kernel
- """
- from tqdm import tqdm
-
- kn_values = {}
-
- for _, m in model.named_modules():
- if not isinstance(m, cls):
- continue
-
- k = m.infeatures
- n = m.outfeatures
-
- if (k, n) not in kn_values:
- kn_values[(k, n)] = (
- m.qweight,
- m.scales,
- m.qzeros,
- m.g_idx,
- m.bits,
- m.maxq,
- )
-
- logger.info(f"Found {len(kn_values)} unique KN Linear values.")
- logger.info("Warming up autotune cache ...")
- with torch.no_grad():
- for m in tqdm(range(0, math.ceil(math.log2(seqlen)) + 1)):
- m = 2**m
- for (k, n), (
- qweight,
- scales,
- qzeros,
- g_idx,
- bits,
- maxq,
- ) in kn_values.items():
- if transpose:
- a = torch.randn(m, k, dtype=torch.float16, device=model.device)
- quant_matmul_248(a, qweight, scales, qzeros, g_idx, bits, maxq)
- a = torch.randn(m, n, dtype=torch.float16, device=model.device)
- transpose_quant_matmul_248(a, qweight, scales, qzeros, g_idx, bits, maxq)
- else:
- a = torch.randn(m, k, dtype=torch.float16, device=model.device)
- quant_matmul_inference_only_248(a, qweight, scales, qzeros, g_idx, bits, maxq)
- del kn_values
-
-
-__all__ = ["QuantLinear"]
diff --git a/auto_round_extension/cuda/qlinear_tritonv2.py b/auto_round_extension/cuda/qlinear_tritonv2.py
index 9bc86bb40..8cb4b05b6 100644
--- a/auto_round_extension/cuda/qlinear_tritonv2.py
+++ b/auto_round_extension/cuda/qlinear_tritonv2.py
@@ -81,12 +81,6 @@ class QuantLinear(nn.Module, TritonModuleMixin):
dtype=torch.float16,
),
)
- self.register_buffer(
- "g_idx",
- torch.tensor(
- [i // self.group_size for i in range(infeatures)], dtype=torch.int32
- ),
- )
if bias:
self.register_buffer(
"bias", torch.zeros((outfeatures), dtype=torch.float16)
@@ -100,67 +94,64 @@ class QuantLinear(nn.Module, TritonModuleMixin):
pass
def pack(self, linear, scales, zeros, g_idx=None):
- W = linear.weight.data.clone()
+ scales_t = scales.t().contiguous()
+ if linear.bias is not None:
+ self.bias = linear.bias.clone().half()
+ self.scales = scales_t.clone().half()
+ device = "cpu"
+ if torch.cuda.is_available():
+ device = "cuda:0"
+ elif torch.xpu.is_available():
+ device = "xpu:0"
+
+ W = linear.weight.data.to(device).clone()
if isinstance(linear, nn.Conv2d):
W = W.flatten(1)
if isinstance(linear, transformers.pytorch_utils.Conv1D):
W = W.t()
- self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
-
- scales = scales.t().contiguous()
- zeros = zeros.t().contiguous()
- scale_zeros = zeros * scales
- self.scales = scales.clone().half()
- if linear.bias is not None:
- self.bias = linear.bias.clone().half()
-
- intweight = []
- for idx in range(self.infeatures):
- intweight.append(
- torch.round(
- (W[:, idx] + scale_zeros[self.g_idx[idx]])
- / self.scales[self.g_idx[idx]]
- ).to(torch.int)[:, None]
- )
- intweight = torch.cat(intweight, dim=1)
- intweight = intweight.t().contiguous()
- intweight = intweight.numpy().astype(np.uint32)
-
- i = 0
- row = 0
- qweight = np.zeros(
- (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32
- )
- while row < qweight.shape[0]:
- if self.bits in [2, 4, 8]:
- for j in range(i, i + (32 // self.bits)):
- qweight[row] |= intweight[j] << (self.bits * (j - i))
- i += 32 // self.bits
- row += 1
- else:
- raise NotImplementedError("Only 2,4,8 bits are supported.")
-
- qweight = qweight.astype(np.int32)
- self.qweight = torch.from_numpy(qweight)
-
- zeros = zeros.numpy().astype(np.uint32)
- qzeros = np.zeros(
- (zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32
- )
- i = 0
- col = 0
- while col < qzeros.shape[1]:
- if self.bits in [2, 4, 8]:
+ repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)
+ if isinstance(zeros, torch.Tensor):
+ repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)
+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros[:, :W.shape[1]]).to(
+ torch.int32)
+ else:
+ repeat_zeros = zeros
+ intweight = torch.round(W.to(device) / repeat_scales[:, :W.shape[1]] + repeat_zeros).to(
+ torch.int32)
+
+ del repeat_scales
+ intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)
+ order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits
+ intweight = intweight.to(torch.int32)
+ intweight = intweight << order_map
+ intweight = torch.sum(intweight, dim=-1)
+
+ intweight = intweight.t().contiguous().to(torch.int32)
+ self.qweight = intweight.to("cpu")
+
+ if isinstance(zeros, torch.Tensor):
+ zeros = zeros.t().contiguous()
+ zeros = zeros.numpy().astype(np.uint32)
+ qzeros = np.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32)
+ i = 0
+ col = 0
+ while col < qzeros.shape[1]:
for j in range(i, i + (32 // self.bits)):
qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
i += 32 // self.bits
col += 1
- else:
- raise NotImplementedError("Only 2,4,8 bits are supported.")
- qzeros = qzeros.astype(np.int32)
- self.qzeros = torch.from_numpy(qzeros)
+ qzeros = qzeros.astype(np.int32)
+ self.qzeros = torch.from_numpy(qzeros)
+ else:
+ shape = scales_t.shape
+ value = 0
+ for j in range(0, (32 // self.bits)):
+ value |= zeros << (self.bits * j)
+ qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value
+ qzeros = qzeros.astype(np.int32)
+ self.qzeros = torch.from_numpy(qzeros)
def forward(self, x):
out_shape = x.shape[:-1] + (self.outfeatures,)
@@ -177,7 +168,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):
)
out = out.half().reshape(out_shape)
out = out + self.bias if self.bias is not None else out
- return out
+ return out.to(x.dtype)
@classmethod
def warmup(cls, model, transpose=False, seqlen=2048):
diff --git a/auto_round_extension/cuda/qlinear_tritonv2_zp.py b/auto_round_extension/cuda/qlinear_tritonv2_zp.py
new file mode 100644
index 000000000..0436394ee
--- /dev/null
+++ b/auto_round_extension/cuda/qlinear_tritonv2_zp.py
@@ -0,0 +1,227 @@
+import math
+from logging import getLogger
+
+import numpy as np
+import torch
+import torch.nn as nn
+import transformers
+
+
+from auto_round_extension.cuda.triton_utils_zp.mixin import TritonModuleMixin
+
+
+logger = getLogger(__name__)
+
+try:
+ from auto_round_extension.cuda.triton_utils_zp.dequant import QuantLinearFunction, quant_matmul_248
+except ImportError as e:
+ triton_import_exception = e
+
+ def error_raiser_triton(*args, **kwargs):
+ raise ValueError(
+ f"Trying to use the triton backend, but could not import triton dependencies with the following error: {triton_import_exception}"
+ )
+
+ class FakeTriton:
+ def __getattr__(self, name):
+ raise ImportError(
+ f"Trying to use the triton backend, but could not import triton dependencies with the following error: {triton_import_exception}"
+ )
+
+ quant_matmul_248 = error_raiser_triton
+ QuantLinearFunction = FakeTriton
+ QuantLinearInferenceOnlyFunction = FakeTriton
+
+
+class QuantLinear(nn.Module, TritonModuleMixin):
+ """
+ Triton v2 quantized linear layer.
+
+ Calls dequant kernel (see triton_utils/dequant) to dequantize the weights then uses
+ torch.matmul to compute the output whereas original `triton` quantized linear layer fused
+ dequant and matmul into single kernel.add()
+ """
+
+ QUANT_TYPE = "tritonv2"
+
+ def __init__(
+ self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs
+ ):
+ super().__init__()
+ if bits not in [2, 4, 8]:
+ raise NotImplementedError("Only 2,4,8 bits are supported.")
+ if infeatures % 32 != 0 or outfeatures % 32 != 0:
+ raise NotImplementedError(
+ "in_feature and out_feature must be divisible by 32."
+ )
+ self.infeatures = infeatures
+ self.outfeatures = outfeatures
+ self.bits = bits
+ self.group_size = group_size if group_size != -1 else infeatures
+ self.maxq = 2**self.bits - 1
+
+ self.register_buffer(
+ "qweight",
+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),
+ )
+ self.register_buffer(
+ "qzeros",
+ torch.zeros(
+ (
+ math.ceil(infeatures / self.group_size),
+ outfeatures // 32 * self.bits,
+ ),
+ dtype=torch.int32,
+ ),
+ )
+ self.register_buffer(
+ "scales",
+ torch.zeros(
+ (math.ceil(infeatures / self.group_size), outfeatures),
+ dtype=torch.float16,
+ ),
+ )
+ self.register_buffer(
+ "g_idx",
+ torch.tensor(
+ [i // self.group_size for i in range(infeatures)], dtype=torch.int32
+ ),
+ )
+ if bias:
+ self.register_buffer(
+ "bias", torch.zeros((outfeatures), dtype=torch.float16)
+ )
+ else:
+ self.bias = None
+
+ self.trainable = trainable
+
+ def post_init(self):
+ pass
+
+ def pack(self, linear, scales, zeros, g_idx=None):
+ W = linear.weight.data.clone()
+ if isinstance(linear, nn.Conv2d):
+ W = W.flatten(1)
+ if isinstance(linear, transformers.pytorch_utils.Conv1D):
+ W = W.t()
+
+ self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
+
+ scales = scales.t().contiguous()
+ zeros = zeros.t().contiguous()
+ scale_zeros = zeros * scales
+ self.scales = scales.clone().half()
+ if linear.bias is not None:
+ self.bias = linear.bias.clone().half()
+
+ intweight = []
+ for idx in range(self.infeatures):
+ intweight.append(
+ torch.round(
+ (W[:, idx] + scale_zeros[self.g_idx[idx]])
+ / self.scales[self.g_idx[idx]]
+ ).to(torch.int)[:, None]
+ )
+ intweight = torch.cat(intweight, dim=1)
+ intweight = intweight.t().contiguous()
+ intweight = intweight.numpy().astype(np.uint32)
+
+ i = 0
+ row = 0
+ qweight = np.zeros(
+ (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32
+ )
+ while row < qweight.shape[0]:
+ if self.bits in [2, 4, 8]:
+ for j in range(i, i + (32 // self.bits)):
+ qweight[row] |= intweight[j] << (self.bits * (j - i))
+ i += 32 // self.bits
+ row += 1
+ else:
+ raise NotImplementedError("Only 2,4,8 bits are supported.")
+
+ qweight = qweight.astype(np.int32)
+ self.qweight = torch.from_numpy(qweight)
+
+ zeros -= 1
+ zeros = zeros.numpy().astype(np.uint32)
+ qzeros = np.zeros(
+ (zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32
+ )
+ i = 0
+ col = 0
+ while col < qzeros.shape[1]:
+ if self.bits in [2, 4, 8]:
+ for j in range(i, i + (32 // self.bits)):
+ qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
+ i += 32 // self.bits
+ col += 1
+ else:
+ raise NotImplementedError("Only 2,4,8 bits are supported.")
+
+ qzeros = qzeros.astype(np.int32)
+ self.qzeros = torch.from_numpy(qzeros)
+
+ def forward(self, x):
+ out_shape = x.shape[:-1] + (self.outfeatures,)
+ quant_linear_fn = QuantLinearFunction
+
+ out = quant_linear_fn.apply(
+ x.reshape(-1, x.shape[-1]),
+ self.qweight,
+ self.scales,
+ self.qzeros,
+ self.g_idx,
+ self.bits,
+ self.maxq,
+ )
+ out = out.half().reshape(out_shape)
+ out = out + self.bias if self.bias is not None else out
+ return out.to(x.dtype)
+
+ @classmethod
+ def warmup(cls, model, transpose=False, seqlen=2048):
+ """
+ Pre-tunes the quantized kernel
+ """
+ from tqdm import tqdm
+
+ kn_values = {}
+
+ for _, m in model.named_modules():
+ if not isinstance(m, cls):
+ continue
+
+ k = m.infeatures
+ n = m.outfeatures
+
+ if (k, n) not in kn_values:
+ kn_values[(k, n)] = (
+ m.qweight,
+ m.scales,
+ m.qzeros,
+ m.g_idx,
+ m.bits,
+ m.maxq,
+ )
+
+ logger.info(f"Found {len(kn_values)} unique KN Linear values.")
+ logger.info("Warming up autotune cache ...")
+ with torch.no_grad():
+ for m in tqdm(range(0, math.ceil(math.log2(seqlen)) + 1)):
+ m = 2**m
+ for (k, n), (
+ qweight,
+ scales,
+ qzeros,
+ g_idx,
+ bits,
+ maxq,
+ ) in kn_values.items():
+ a = torch.randn(m, k, dtype=torch.float16, device=model.device)
+ quant_matmul_248(a, qweight, scales, qzeros, g_idx, bits, maxq)
+ del kn_values
+
+
+__all__ = ["QuantLinear"]
\ No newline at end of file
diff --git a/auto_round_extension/cuda/triton_utils/dequant.py b/auto_round_extension/cuda/triton_utils/dequant.py
index b7c6316de..d971059d3 100644
--- a/auto_round_extension/cuda/triton_utils/dequant.py
+++ b/auto_round_extension/cuda/triton_utils/dequant.py
@@ -54,17 +54,17 @@ DEFAULT_DEQUANT_CONFIGS = make_dequant_configs([128, 256, 512, 1024], [4, 8])
@triton.autotune(DEFAULT_DEQUANT_CONFIGS, key=["numels"])
@triton.jit
def dequant_kernel_248(
- g_idx_ptr,
- scales_ptr,
- qweight_ptr,
- qzeros_ptr,
- out_ptr,
- numels,
- maxq: tl.constexpr,
- bits: tl.constexpr,
- outfeatures: tl.constexpr,
- num_groups: tl.constexpr,
- X_BLOCK: tl.constexpr,
+ g_idx_ptr,
+ scales_ptr,
+ qweight_ptr,
+ qzeros_ptr,
+ out_ptr,
+ numels,
+ maxq: tl.constexpr,
+ bits: tl.constexpr,
+ outfeatures: tl.constexpr,
+ num_groups: tl.constexpr,
+ X_BLOCK: tl.constexpr,
):
# Block indexing
xoffset = tl.program_id(0) * X_BLOCK
@@ -110,7 +110,7 @@ def dequant_kernel_248(
zeros = qzeros >> wf_zeros
zeros = zeros & maxq
- # Dequantize
+ # # ##Dequantize
# zeros = zeros + 1
weights = weights - zeros
weights = weights.to(tl.float32)
@@ -119,7 +119,7 @@ def dequant_kernel_248(
tl.store(out_ptr + (x_index), weights, mask=xmask)
-def dequant248(qweight, scales, qzeros, g_idx, bits, maxq=None):
+def dequant248(qweight, scales, qzeros, g_idx, bits, maxq=None, input_dtype=torch.float16):
"""
Launcher for triton dequant kernel. Only valid for bits = 2, 4, 8
"""
@@ -128,9 +128,9 @@ def dequant248(qweight, scales, qzeros, g_idx, bits, maxq=None):
outfeatures = scales.shape[1]
infeatures = g_idx.shape[0]
- out = torch.empty((infeatures, outfeatures), device="cuda", dtype=torch.float16)
+ out = torch.empty((infeatures, outfeatures), device="cuda", dtype=input_dtype)
numels = out.numel()
- maxq = 2**bits - 1 if maxq is None else maxq
+ maxq = 2 ** bits - 1 if maxq is None else maxq
grid = lambda meta: (triton.cdiv(numels, meta["X_BLOCK"]),) # noqa: E731
dequant_kernel_248[grid](
@@ -149,12 +149,15 @@ def dequant248(qweight, scales, qzeros, g_idx, bits, maxq=None):
def quant_matmul_248(
- input, qweight, scales, qzeros, g_idx, bits, maxq=None, transpose=False
+ input, qweight, scales, qzeros, g_idx, bits, maxq=None, transpose=False
):
- W = dequant248(qweight, scales, qzeros, g_idx, bits, maxq=maxq)
+ input_dtype = input.dtype
+ W = dequant248(qweight, scales, qzeros, g_idx, bits, maxq=maxq, input_dtype=input_dtype)
+ orig_device = input.device
if transpose:
- return input @ W.t()
- return input @ W
+ return (input.to(W.device) @ W.t()).to(orig_device)
+
+ return (input.to(W.device) @ W).to(orig_device)
class QuantLinearFunction(torch.autograd.Function):
diff --git a/auto_round_extension/cuda/triton_utils_zp/__init__.py b/auto_round_extension/cuda/triton_utils_zp/__init__.py
new file mode 100644
index 000000000..2045808ac
--- /dev/null
+++ b/auto_round_extension/cuda/triton_utils_zp/__init__.py
@@ -0,0 +1,14 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
diff --git a/auto_round_extension/cuda/triton_utils_zp/custom_autotune.py b/auto_round_extension/cuda/triton_utils_zp/custom_autotune.py
new file mode 100644
index 000000000..b511579cc
--- /dev/null
+++ b/auto_round_extension/cuda/triton_utils_zp/custom_autotune.py
@@ -0,0 +1,254 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# MIT License
+#
+# Copyright (c) 2023 潘其威(William)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import builtins
+import math
+import time
+from typing import Dict
+
+import triton
+
+
+# code based https://github.com/fpgaminer/GPTQ-triton
+"""
+Mostly the same as the autotuner in Triton, but with a few changes like using 40 runs instead of 100.
+"""
+
+
+class CustomizedTritonAutoTuner(triton.KernelInterface):
+ def __init__(
+ self,
+ fn,
+ arg_names,
+ configs,
+ key,
+ reset_to_zero,
+ prune_configs_by: Dict = None,
+ nearest_power_of_two: bool = False,
+ ):
+ if not configs:
+ self.configs = [triton.Config({}, num_warps=4, num_stages=2)]
+ else:
+ self.configs = configs
+ self.key_idx = [arg_names.index(k) for k in key]
+ self.nearest_power_of_two = nearest_power_of_two
+ self.cache = {}
+ # hook to reset all required tensor to zeros before relaunching a kernel
+ self.hook = lambda args: 0
+ if reset_to_zero is not None:
+ self.reset_idx = [arg_names.index(k) for k in reset_to_zero]
+
+ def _hook(args):
+ for i in self.reset_idx:
+ args[i].zero_()
+
+ self.hook = _hook
+ self.arg_names = arg_names
+ # prune configs
+ if prune_configs_by:
+ perf_model, top_k = (
+ prune_configs_by["perf_model"],
+ prune_configs_by["top_k"],
+ )
+ if "early_config_prune" in prune_configs_by:
+ early_config_prune = prune_configs_by["early_config_prune"]
+ else:
+ perf_model, top_k, early_config_prune = None, None, None
+ self.perf_model, self.configs_top_k = perf_model, top_k
+ self.early_config_prune = early_config_prune
+ self.fn = fn
+
+ def _bench(self, *args, config, **meta):
+ # check for conflicts, i.e. meta-parameters both provided
+ # as kwargs and by the autotuner
+ conflicts = meta.keys() & config.kwargs.keys()
+ if conflicts:
+ raise ValueError(
+ f"Conflicting meta-parameters: {', '.join(conflicts)}."
+ " Make sure that you don't re-define auto-tuned symbols."
+ )
+ # augment meta-parameters with tunable ones
+ current = dict(meta, **config.kwargs)
+
+ def kernel_call():
+ if config.pre_hook:
+ config.pre_hook(self.nargs)
+ self.hook(args)
+ self.fn.run(
+ *args,
+ num_warps=config.num_warps,
+ num_stages=config.num_stages,
+ **current,
+ )
+
+ try:
+ # In testings using only 40 reps seems to be close enough and it appears to be what PyTorch uses
+ # PyTorch also sets fast_flush to True, but I didn't see any speedup so I'll leave the default
+ return triton.testing.do_bench(kernel_call, quantiles=(0.5, 0.2, 0.8), rep=40)
+ except triton.OutOfResources:
+ return (float("inf"), float("inf"), float("inf"))
+
+ def run(self, *args, **kwargs):
+ self.nargs = dict(zip(self.arg_names, args))
+ if len(self.configs) > 1:
+ key = tuple(args[i] for i in self.key_idx)
+
+ # This reduces the amount of autotuning by rounding the keys to the nearest power of two
+ # In my testing this gives decent results, and greatly reduces the amount of tuning required
+ if self.nearest_power_of_two:
+ key = tuple([2 ** int(math.log2(x) + 0.5) for x in key])
+
+ if key not in self.cache:
+ # prune configs
+ pruned_configs = self.prune_configs(kwargs)
+ bench_start = time.time()
+ timings = {config: self._bench(*args, config=config, **kwargs) for config in pruned_configs}
+ bench_end = time.time()
+ self.bench_time = bench_end - bench_start
+ self.cache[key] = builtins.min(timings, key=timings.get)
+ self.hook(args)
+ self.configs_timings = timings
+ config = self.cache[key]
+ else:
+ config = self.configs[0]
+ self.best_config = config
+ if config.pre_hook is not None:
+ config.pre_hook(self.nargs)
+ return self.fn.run(
+ *args,
+ num_warps=config.num_warps,
+ num_stages=config.num_stages,
+ **kwargs,
+ **config.kwargs,
+ )
+
+ def prune_configs(self, kwargs):
+ pruned_configs = self.configs
+ if self.early_config_prune:
+ pruned_configs = self.early_config_prune(self.configs, self.nargs)
+ if self.perf_model:
+ top_k = self.configs_top_k
+ if isinstance(top_k, float) and top_k <= 1.0:
+ top_k = int(len(self.configs) * top_k)
+ if len(pruned_configs) > top_k:
+ est_timing = {
+ config: self.perf_model(
+ **self.nargs,
+ **kwargs,
+ **config.kwargs,
+ num_stages=config.num_stages,
+ num_warps=config.num_warps,
+ )
+ for config in pruned_configs
+ }
+ pruned_configs = sorted(est_timing.keys(), key=lambda x: est_timing[x])[:top_k]
+ return pruned_configs
+
+ def warmup(self, *args, **kwargs):
+ self.nargs = dict(zip(self.arg_names, args))
+ for config in self.prune_configs(kwargs):
+ self.fn.warmup(
+ *args,
+ num_warps=config.num_warps,
+ num_stages=config.num_stages,
+ **kwargs,
+ **config.kwargs,
+ )
+ self.nargs = None
+
+
+def autotune(configs, key, prune_configs_by=None, reset_to_zero=None, nearest_power_of_two=False):
+ def decorator(fn):
+ return CustomizedTritonAutoTuner(
+ fn,
+ fn.arg_names,
+ configs,
+ key,
+ reset_to_zero,
+ prune_configs_by,
+ nearest_power_of_two,
+ )
+
+ return decorator
+
+
+def matmul248_kernel_config_pruner(configs, nargs):
+ """
+ The main purpose of this function is to shrink BLOCK_SIZE_* when the corresponding dimension is smaller.
+ """
+ m = max(2 ** int(math.ceil(math.log2(nargs["M"]))), 16)
+ n = max(2 ** int(math.ceil(math.log2(nargs["N"]))), 16)
+ k = max(2 ** int(math.ceil(math.log2(nargs["K"]))), 16)
+
+ used = set()
+ for config in configs:
+ block_size_m = min(m, config.kwargs["BLOCK_SIZE_M"])
+ block_size_n = min(n, config.kwargs["BLOCK_SIZE_N"])
+ block_size_k = min(k, config.kwargs["BLOCK_SIZE_K"])
+ group_size_m = config.kwargs["GROUP_SIZE_M"]
+
+ if (
+ block_size_m,
+ block_size_n,
+ block_size_k,
+ group_size_m,
+ config.num_stages,
+ config.num_warps,
+ ) in used:
+ continue
+
+ used.add(
+ (
+ block_size_m,
+ block_size_n,
+ block_size_k,
+ group_size_m,
+ config.num_stages,
+ config.num_warps,
+ )
+ )
+ yield triton.Config(
+ {
+ "BLOCK_SIZE_M": block_size_m,
+ "BLOCK_SIZE_N": block_size_n,
+ "BLOCK_SIZE_K": block_size_k,
+ "GROUP_SIZE_M": group_size_m,
+ },
+ num_stages=config.num_stages,
+ num_warps=config.num_warps,
+ )
+
+
+__all__ = ["autotune"]
diff --git a/auto_round_extension/cuda/triton_utils_zp/dequant.py b/auto_round_extension/cuda/triton_utils_zp/dequant.py
new file mode 100644
index 000000000..4ef9936ce
--- /dev/null
+++ b/auto_round_extension/cuda/triton_utils_zp/dequant.py
@@ -0,0 +1,183 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# MIT License
+#
+# Copyright (c) 2023 潘其威(William)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import itertools
+
+import torch
+import triton
+import triton.language as tl
+from torch.cuda.amp import custom_bwd, custom_fwd
+
+
+def make_dequant_configs(block_sizes, num_warps):
+ configs = []
+ for bs, ws in itertools.product(block_sizes, num_warps):
+ configs.append(triton.Config({"X_BLOCK": bs}, num_warps=ws))
+ return configs
+
+
+DEFAULT_DEQUANT_CONFIGS = make_dequant_configs([128, 256, 512, 1024], [4, 8])
+
+
[email protected](DEFAULT_DEQUANT_CONFIGS, key=["numels"])
+def dequant_kernel_248(
+ g_idx_ptr,
+ scales_ptr,
+ qweight_ptr,
+ qzeros_ptr,
+ out_ptr,
+ numels,
+ maxq: tl.constexpr,
+ bits: tl.constexpr,
+ outfeatures: tl.constexpr,
+ num_groups: tl.constexpr,
+ X_BLOCK: tl.constexpr,
+):
+ # Block indexing
+ xoffset = tl.program_id(0) * X_BLOCK
+ x_index = xoffset + tl.arange(0, X_BLOCK)
+ xmask = x_index < numels
+ row_idx = x_index // outfeatures
+ col_idx = x_index % outfeatures
+
+ elements_per_feature: tl.constexpr = 32 // bits
+
+ # Load parameters
+ g_idx = tl.load(g_idx_ptr + (row_idx), None, eviction_policy="evict_last")
+ qweights = tl.load(
+ qweight_ptr + (col_idx + (outfeatures * (row_idx // elements_per_feature))),
+ None,
+ )
+
+ wf_weights = (row_idx % elements_per_feature) * bits
+
+ wf_zeros = (col_idx % elements_per_feature) * bits
+
+ tmp1 = g_idx + num_groups
+ tmp2 = g_idx < 0
+ tl.device_assert(g_idx >= 0, "index out of bounds: 0 <= tmp0 < 0")
+ groups = tl.where(tmp2, tmp1, g_idx) # tmp3 are g_idx
+
+ scales = tl.load(scales_ptr + (col_idx + (outfeatures * groups)), None).to(
+ tl.float32
+ )
+
+ # Unpack weights
+ weights = qweights >> wf_weights # bit shift qweight
+
+ weights = weights & maxq
+
+ # Unpack zeros
+ qzero_ncols: tl.constexpr = outfeatures // elements_per_feature
+ qzeros = tl.load(
+ qzeros_ptr + ((qzero_ncols * groups) + (col_idx // elements_per_feature)),
+ None,
+ eviction_policy="evict_last",
+ )
+ zeros = qzeros >> wf_zeros
+ zeros = zeros & maxq
+
+ ##Dequantize
+ zeros = zeros + 1
+ weights = weights - zeros
+ weights = weights.to(tl.float32)
+ weights = scales * weights
+
+ tl.store(out_ptr + (x_index), weights, mask=xmask)
+
+
+def dequant248(qweight, scales, qzeros, g_idx, bits, maxq=None, input_dtype=torch.float16):
+ """
+ Launcher for triton dequant kernel. Only valid for bits = 2, 4, 8
+ """
+
+ num_groups = scales.shape[0]
+ outfeatures = scales.shape[1]
+ infeatures = g_idx.shape[0]
+
+ out = torch.empty((infeatures, outfeatures), device="cuda", dtype=input_dtype)
+ numels = out.numel()
+ maxq = 2 ** bits - 1 if maxq is None else maxq
+ grid = lambda meta: (triton.cdiv(numels, meta["X_BLOCK"]),) # noqa: E731
+
+ dequant_kernel_248[grid](
+ g_idx,
+ scales,
+ qweight,
+ qzeros,
+ out,
+ numels,
+ maxq=maxq,
+ bits=bits,
+ outfeatures=outfeatures,
+ num_groups=num_groups,
+ )
+ return out
+
+
+def quant_matmul_248(
+ input, qweight, scales, qzeros, g_idx, bits, maxq=None, transpose=False
+):
+ input_dtype = input.dtype
+ W = dequant248(qweight, scales, qzeros, g_idx, bits, maxq=maxq, input_dtype=input_dtype)
+ orig_device = input.device
+ if transpose:
+ return (input.to(W.device) @ W.t()).to(orig_device)
+
+ return (input.to(W.device) @ W).to(orig_device)
+
+
+class QuantLinearFunction(torch.autograd.Function):
+ @staticmethod
+ @custom_fwd
+ def forward(ctx, input, qweight, scales, qzeros, g_idx, bits, maxq):
+ output = quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxq)
+ ctx.save_for_backward(qweight, scales, qzeros, g_idx)
+ ctx.bits, ctx.maxq = bits, maxq
+ return output
+
+ @staticmethod
+ @custom_bwd
+ def backward(ctx, grad_output):
+ qweight, scales, qzeros, g_idx = ctx.saved_tensors
+ bits, maxq = ctx.bits, ctx.maxq
+ grad_input = None
+
+ if ctx.needs_input_grad[0]:
+ grad_input = quant_matmul_248(
+ grad_output, qweight, scales, qzeros, g_idx, bits, maxq, transpose=True
+ )
+ return grad_input, None, None, None, None, None, None
diff --git a/auto_round_extension/cuda/triton_utils_zp/kernels.py b/auto_round_extension/cuda/triton_utils_zp/kernels.py
new file mode 100644
index 000000000..44981c7a3
--- /dev/null
+++ b/auto_round_extension/cuda/triton_utils_zp/kernels.py
@@ -0,0 +1,500 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# MIT License
+#
+# Copyright (c) 2023 潘其威(William)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from logging import getLogger
+
+import torch
+import triton
+import triton.language as tl
+from torch.cuda.amp import custom_bwd, custom_fwd
+
+from . import custom_autotune
+
+
+logger = getLogger(__name__)
+
+
+# code based https://github.com/fpgaminer/GPTQ-triton
+
+
+@custom_autotune.autotune(
+ configs=[
+ triton.Config(
+ {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 256,
+ "BLOCK_SIZE_K": 32,
+ "GROUP_SIZE_M": 8,
+ },
+ num_stages=4,
+ num_warps=4,
+ ),
+ triton.Config(
+ {
+ "BLOCK_SIZE_M": 128,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 32,
+ "GROUP_SIZE_M": 8,
+ },
+ num_stages=4,
+ num_warps=4,
+ ),
+ triton.Config(
+ {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 32,
+ "GROUP_SIZE_M": 8,
+ },
+ num_stages=4,
+ num_warps=4,
+ ),
+ triton.Config(
+ {
+ "BLOCK_SIZE_M": 128,
+ "BLOCK_SIZE_N": 32,
+ "BLOCK_SIZE_K": 32,
+ "GROUP_SIZE_M": 8,
+ },
+ num_stages=4,
+ num_warps=4,
+ ),
+ triton.Config(
+ {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 64,
+ "BLOCK_SIZE_K": 32,
+ "GROUP_SIZE_M": 8,
+ },
+ num_stages=4,
+ num_warps=4,
+ ),
+ triton.Config(
+ {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 128,
+ "BLOCK_SIZE_K": 32,
+ "GROUP_SIZE_M": 8,
+ },
+ num_stages=2,
+ num_warps=8,
+ ),
+ ],
+ key=["M", "N", "K"],
+ nearest_power_of_two=True,
+ prune_configs_by={
+ "early_config_prune": custom_autotune.matmul248_kernel_config_pruner,
+ "perf_model": None,
+ "top_k": None,
+ },
+)
+def quant_matmul_248_kernel(
+ a_ptr,
+ b_ptr,
+ c_ptr,
+ scales_ptr,
+ zeros_ptr,
+ g_ptr,
+ M,
+ N,
+ K,
+ bits,
+ maxq,
+ stride_am,
+ stride_ak,
+ stride_bk,
+ stride_bn,
+ stride_cm,
+ stride_cn,
+ stride_scales,
+ stride_zeros,
+ BLOCK_SIZE_M: tl.constexpr,
+ BLOCK_SIZE_N: tl.constexpr,
+ BLOCK_SIZE_K: tl.constexpr,
+ GROUP_SIZE_M: tl.constexpr,
+):
+ """
+ Compute the matrix multiplication C = A x B.
+ A is of shape (M, K) float16
+ B is of shape (K//8, N) int32
+ C is of shape (M, N) float16
+ scales is of shape (G, N) float16
+ zeros is of shape (G, N) float16
+ g_ptr is of shape (K) int32
+ """
+ infearure_per_bits = 32 // bits
+
+ pid = tl.program_id(axis=0)
+ num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
+ num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
+ num_pid_k = tl.cdiv(K, BLOCK_SIZE_K)
+ num_pid_in_group = GROUP_SIZE_M * num_pid_n
+ group_id = pid // num_pid_in_group
+ first_pid_m = group_id * GROUP_SIZE_M
+ group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
+ pid_m = first_pid_m + (pid % group_size_m)
+ pid_n = (pid % num_pid_in_group) // group_size_m
+
+ offs_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
+ offs_bn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
+ offs_k = tl.arange(0, BLOCK_SIZE_K)
+ a_ptrs = a_ptr + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) # (BLOCK_SIZE_M, BLOCK_SIZE_K)
+ a_mask = offs_am[:, None] < M
+ # b_ptrs is set up such that it repeats elements along the K axis 8 times
+ b_ptrs = b_ptr + (
+ (offs_k[:, None] // infearure_per_bits) * stride_bk + offs_bn[None, :] * stride_bn
+ ) # (BLOCK_SIZE_K, BLOCK_SIZE_N)
+ g_ptrs = g_ptr + offs_k
+ # shifter is used to extract the N bits of each element in the 32-bit word from B
+ scales_ptrs = scales_ptr + offs_bn[None, :]
+ zeros_ptrs = zeros_ptr + (offs_bn[None, :] // infearure_per_bits)
+
+ shifter = (offs_k % infearure_per_bits) * bits
+ zeros_shifter = (offs_bn % infearure_per_bits) * bits
+ accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
+
+ for k in range(0, num_pid_k):
+ g_idx = tl.load(g_ptrs)
+
+ # Fetch scales and zeros; these are per-outfeature and thus reused in the inner loop
+ scales = tl.load(scales_ptrs + g_idx[:, None] * stride_scales) # (BLOCK_SIZE_K, BLOCK_SIZE_N,)
+ zeros = tl.load(zeros_ptrs + g_idx[:, None] * stride_zeros) # (BLOCK_SIZE_K, BLOCK_SIZE_N,)
+
+ zeros = (zeros >> zeros_shifter[None, :]) & maxq
+ # zeros = zeros + 1
+
+ a = tl.load(a_ptrs, mask=a_mask, other=0.0) # (BLOCK_SIZE_M, BLOCK_SIZE_K)
+ b = tl.load(b_ptrs) # (BLOCK_SIZE_K, BLOCK_SIZE_N), but repeated
+
+ # Now we need to unpack b (which is N-bit values) into 32-bit values
+ b = (b >> shifter[:, None]) & maxq # Extract the N-bit values
+ b = (b - zeros) * scales # Scale and shift
+
+ accumulator += tl.dot(a, b)
+ a_ptrs += BLOCK_SIZE_K
+ b_ptrs += (BLOCK_SIZE_K // infearure_per_bits) * stride_bk
+ g_ptrs += BLOCK_SIZE_K
+
+ c_ptrs = c_ptr + stride_cm * offs_am[:, None] + stride_cn * offs_bn[None, :]
+ c_mask = (offs_am[:, None] < M) & (offs_bn[None, :] < N)
+ tl.store(c_ptrs, accumulator, mask=c_mask)
+
+
+@custom_autotune.autotune(
+ configs=[
+ triton.Config(
+ {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 32,
+ "BLOCK_SIZE_K": 256,
+ "GROUP_SIZE_M": 8,
+ },
+ num_stages=4,
+ num_warps=4,
+ ),
+ triton.Config(
+ {
+ "BLOCK_SIZE_M": 128,
+ "BLOCK_SIZE_N": 32,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ },
+ num_stages=4,
+ num_warps=4,
+ ),
+ triton.Config(
+ {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 32,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ },
+ num_stages=4,
+ num_warps=4,
+ ),
+ triton.Config(
+ {
+ "BLOCK_SIZE_M": 128,
+ "BLOCK_SIZE_N": 32,
+ "BLOCK_SIZE_K": 32,
+ "GROUP_SIZE_M": 8,
+ },
+ num_stages=4,
+ num_warps=4,
+ ),
+ triton.Config(
+ {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 32,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 8,
+ },
+ num_stages=4,
+ num_warps=4,
+ ),
+ triton.Config(
+ {
+ "BLOCK_SIZE_M": 64,
+ "BLOCK_SIZE_N": 32,
+ "BLOCK_SIZE_K": 128,
+ "GROUP_SIZE_M": 8,
+ },
+ num_stages=2,
+ num_warps=8,
+ ),
+ ],
+ key=["M", "N", "K"],
+ nearest_power_of_two=True,
+)
+def transpose_quant_matmul_248_kernel(
+ a_ptr,
+ b_ptr,
+ c_ptr,
+ scales_ptr,
+ zeros_ptr,
+ g_ptr,
+ M,
+ N,
+ K,
+ bits,
+ maxq,
+ stride_am,
+ stride_ak,
+ stride_bk,
+ stride_bn,
+ stride_cm,
+ stride_cn,
+ stride_scales,
+ stride_zeros,
+ BLOCK_SIZE_M: tl.constexpr,
+ BLOCK_SIZE_N: tl.constexpr,
+ BLOCK_SIZE_K: tl.constexpr,
+ GROUP_SIZE_M: tl.constexpr,
+):
+ """
+ Compute the matrix multiplication C = A x B.
+ A is of shape (M, N) float16
+ B is of shape (K//8, N) int32
+ C is of shape (M, K) float16
+ scales is of shape (G, N) float16
+ zeros is of shape (G, N) float16
+ g_ptr is of shape (K) int32
+ """
+ infearure_per_bits = 32 // bits
+
+ pid = tl.program_id(axis=0)
+ num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
+ num_pid_k = tl.cdiv(K, BLOCK_SIZE_K)
+ num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
+ num_pid_in_group = GROUP_SIZE_M * num_pid_k
+ group_id = pid // num_pid_in_group
+ first_pid_m = group_id * GROUP_SIZE_M
+ group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
+ pid_m = first_pid_m + (pid % group_size_m)
+ pid_k = (pid % num_pid_in_group) // group_size_m
+
+ offs_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
+ offs_bk = pid_k * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K)
+ offs_n = tl.arange(0, BLOCK_SIZE_N)
+ a_ptrs = a_ptr + (offs_am[:, None] * stride_am + offs_n[None, :] * stride_ak) # (BLOCK_SIZE_M, BLOCK_SIZE_N)
+ a_mask = offs_am[:, None] < M
+ # b_ptrs is set up such that it repeats elements along the K axis 8 times
+ b_ptrs = b_ptr + (
+ (offs_bk[:, None] // infearure_per_bits) * stride_bk + offs_n[None, :] * stride_bn
+ ) # (BLOCK_SIZE_K, BLOCK_SIZE_N)
+ g_ptrs = g_ptr + offs_bk
+ g_idx = tl.load(g_ptrs)
+
+ # shifter is used to extract the N bits of each element in the 32-bit word from B
+ scales_ptrs = scales_ptr + offs_n[None, :] + g_idx[:, None] * stride_scales
+ zeros_ptrs = zeros_ptr + (offs_n[None, :] // infearure_per_bits) + g_idx[:, None] * stride_zeros
+
+ shifter = (offs_bk % infearure_per_bits) * bits
+ zeros_shifter = (offs_n % infearure_per_bits) * bits
+ accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_K), dtype=tl.float32)
+
+ for k in range(0, num_pid_n):
+ # Fetch scales and zeros; these are per-outfeature and thus reused in the inner loop
+ scales = tl.load(scales_ptrs) # (BLOCK_SIZE_K, BLOCK_SIZE_N,)
+ zeros = tl.load(zeros_ptrs) # (BLOCK_SIZE_K, BLOCK_SIZE_N,)
+
+ zeros = (zeros >> zeros_shifter[None, :]) & maxq
+ # zeros = zeros + 1
+
+ a = tl.load(a_ptrs, mask=a_mask, other=0.0) # (BLOCK_SIZE_M, BLOCK_SIZE_N)
+ b = tl.load(b_ptrs) # (BLOCK_SIZE_K, BLOCK_SIZE_N), but repeated
+
+ # Now we need to unpack b (which is N-bit values) into 32-bit values
+ b = (b >> shifter[:, None]) & maxq # Extract the N-bit values
+ b = (b - zeros) * scales # Scale and shift
+ b = tl.trans(b)
+
+ accumulator += tl.dot(a, b)
+ a_ptrs += BLOCK_SIZE_N
+ b_ptrs += BLOCK_SIZE_N
+ scales_ptrs += BLOCK_SIZE_N
+ zeros_ptrs += BLOCK_SIZE_N // infearure_per_bits
+
+ c_ptrs = c_ptr + stride_cm * offs_am[:, None] + stride_cn * offs_bk[None, :]
+ c_mask = (offs_am[:, None] < M) & (offs_bk[None, :] < K)
+ tl.store(c_ptrs, accumulator, mask=c_mask)
+
+
+def silu(x):
+ return x * tl.sigmoid(x)
+
+
+def quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxq):
+ with torch.cuda.device(input.device):
+ output = torch.empty((input.shape[0], qweight.shape[1]), device=input.device, dtype=input.dtype)
+ grid = lambda META: ( # noqa: E731
+ triton.cdiv(input.shape[0], META["BLOCK_SIZE_M"]) * triton.cdiv(qweight.shape[1], META["BLOCK_SIZE_N"]),
+ )
+ quant_matmul_248_kernel[grid](
+ input,
+ qweight,
+ output,
+ scales.to(input.dtype),
+ qzeros,
+ g_idx,
+ input.shape[0],
+ qweight.shape[1],
+ input.shape[1],
+ bits,
+ maxq,
+ input.stride(0),
+ input.stride(1),
+ qweight.stride(0),
+ qweight.stride(1),
+ output.stride(0),
+ output.stride(1),
+ scales.stride(0),
+ qzeros.stride(0),
+ )
+ return output
+
+
+def transpose_quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxq):
+ with torch.cuda.device(input.device):
+ output_dim = (qweight.shape[0] * 32) // bits
+ output = torch.empty((input.shape[0], output_dim), device=input.device, dtype=input.dtype)
+ grid = lambda META: ( # noqa: E731
+ triton.cdiv(input.shape[0], META["BLOCK_SIZE_M"]) * triton.cdiv(output_dim, META["BLOCK_SIZE_K"]),
+ )
+ transpose_quant_matmul_248_kernel[grid](
+ input,
+ qweight,
+ output,
+ scales.to(input.dtype),
+ qzeros,
+ g_idx,
+ input.shape[0],
+ qweight.shape[1],
+ output_dim,
+ bits,
+ maxq,
+ input.stride(0),
+ input.stride(1),
+ qweight.stride(0),
+ qweight.stride(1),
+ output.stride(0),
+ output.stride(1),
+ scales.stride(0),
+ qzeros.stride(0),
+ )
+ return output
+
+
+class QuantLinearFunction(torch.autograd.Function):
+ @staticmethod
+ @custom_fwd
+ def forward(ctx, input, qweight, scales, qzeros, g_idx, bits, maxq):
+ output = quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxq)
+ ctx.save_for_backward(qweight, scales, qzeros, g_idx)
+ ctx.bits, ctx.maxq = bits, maxq
+ return output
+
+ @staticmethod
+ @custom_bwd
+ def backward(ctx, grad_output):
+ qweight, scales, qzeros, g_idx = ctx.saved_tensors
+ bits, maxq = ctx.bits, ctx.maxq
+ grad_input = None
+
+ if ctx.needs_input_grad[0]:
+ grad_input = transpose_quant_matmul_248(grad_output, qweight, scales, qzeros, g_idx, bits, maxq)
+ return grad_input, None, None, None, None, None, None
+
+
+def quant_matmul_inference_only_248(input, qweight, scales, qzeros, g_idx, bits, maxq):
+ with torch.cuda.device(input.device):
+ output = torch.empty((input.shape[0], qweight.shape[1]), device=input.device, dtype=torch.float16)
+ grid = lambda META: ( # noqa: E731
+ triton.cdiv(input.shape[0], META["BLOCK_SIZE_M"]) * triton.cdiv(qweight.shape[1], META["BLOCK_SIZE_N"]),
+ )
+ quant_matmul_248_kernel[grid](
+ input,
+ qweight,
+ output,
+ scales,
+ qzeros,
+ g_idx,
+ input.shape[0],
+ qweight.shape[1],
+ input.shape[1],
+ bits,
+ maxq,
+ input.stride(0),
+ input.stride(1),
+ qweight.stride(0),
+ qweight.stride(1),
+ output.stride(0),
+ output.stride(1),
+ scales.stride(0),
+ qzeros.stride(0),
+ )
+ return output
+
+
+class QuantLinearInferenceOnlyFunction(torch.autograd.Function):
+ @staticmethod
+ @custom_fwd(cast_inputs=torch.float16)
+ def forward(ctx, input, qweight, scales, qzeros, g_idx, bits, maxq):
+ output = quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxq)
+ return output
diff --git a/auto_round_extension/cuda/triton_utils_zp/mixin.py b/auto_round_extension/cuda/triton_utils_zp/mixin.py
new file mode 100644
index 000000000..557d3b480
--- /dev/null
+++ b/auto_round_extension/cuda/triton_utils_zp/mixin.py
@@ -0,0 +1,39 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# MIT License
+#
+# Copyright (c) 2023 潘其威(William)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+class TritonModuleMixin:
+ @classmethod
+ def warmup(cls, model, transpose=False, seqlen=2048):
+ pass
diff --git a/auto_round_extension/ipex/qlinear_ipex_awq.py b/auto_round_extension/ipex/qlinear_ipex_awq.py
index d224234e7..b03d2104f 100644
--- a/auto_round_extension/ipex/qlinear_ipex_awq.py
+++ b/auto_round_extension/ipex/qlinear_ipex_awq.py
@@ -2,7 +2,7 @@ import torch
import torch.nn as nn
class QuantLinear(nn.Module):
-
+ QUANT_TYPE = "ipex_awq"
def __init__(self, w_bit, group_size, in_features, out_features, bias, dev):
super().__init__()
assert w_bit == 4, "Only 4 bit are supported for now."
diff --git a/auto_round_extension/ipex/qlinear_ipex_gptq.py b/auto_round_extension/ipex/qlinear_ipex_gptq.py
index c486638ae..042f0226f 100644
--- a/auto_round_extension/ipex/qlinear_ipex_gptq.py
+++ b/auto_round_extension/ipex/qlinear_ipex_gptq.py
@@ -78,10 +78,10 @@ class QuantLinear(nn.Module):
dtype=weight_dtype,
),
)
- self.register_buffer(
- "g_idx",
- torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),
- )
+ # self.register_buffer(
+ # "g_idx",
+ # torch.tensor([i // self.group_size for i in range(infeatures)], dtype=torch.int32),
+ # )
if bias:
self.register_buffer("bias", torch.zeros((outfeatures), dtype=weight_dtype))
else:
@@ -172,7 +172,7 @@ class QuantLinear(nn.Module):
if isinstance(linear, transformers.pytorch_utils.Conv1D):
W = W.t()
- self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
+ # self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
scales = scales.t().contiguous()
zeros = zeros.t().contiguous()
diff --git a/auto_round_extension/qbits/qbits_awq.py b/auto_round_extension/qbits/qbits_awq.py
index 82110e934..0b3adefb0 100644
--- a/auto_round_extension/qbits/qbits_awq.py
+++ b/auto_round_extension/qbits/qbits_awq.py
@@ -68,6 +68,7 @@ def convert_dtype_torch2str(dtype):
class QuantLinear(nn.Module):
+ QUANT_TYPE = "qbits_awq"
def __init__(self, w_bit, group_size, in_features, out_features, bias, zero_point, dev):
super().__init__()
diff --git a/auto_round_extension/qbits/qlinear_qbits_gptq.py b/auto_round_extension/qbits/qlinear_qbits_gptq.py
index 658adead7..0afca0288 100644
--- a/auto_round_extension/qbits/qlinear_qbits_gptq.py
+++ b/auto_round_extension/qbits/qlinear_qbits_gptq.py
@@ -29,7 +29,7 @@ BITS_DTYPE_MAPPING = {
class QuantLinear(nn.Module):
- QUANT_TYPE = "qbits"
+ QUANT_TYPE = "qbits_gptq"
def __init__(
self,
diff --git a/test/test_basic_usage.py b/test/test_basic_usage.py
index 83de4bbe1..e468d173c 100644
--- a/test/test_basic_usage.py
+++ b/test/test_basic_usage.py
@@ -20,56 +20,56 @@ class TestAutoRoundCmd(unittest.TestCase):
python_path = sys.executable
##test llm script
- res = os.system(
- f"cd .. && {python_path} -m auto_round -h")
- if res > 0 or res == -1:
- assert False, "cmd line test fail, please have a check"
-
+ # res = os.system(
+ # f"cd .. && {python_path} -m auto_round -h")
+ # if res > 0 or res == -1:
+ # assert False, "cmd line test fail, please have a check"
+ #
res = os.system(
f"cd .. && {python_path} -m auto_round --model 'facebook/opt-125m' --seqlen 32 --iter 2 --nsamples 1 --format auto_gptq,auto_round --output_dir ./saved --tasks piqa")
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
- res = os.system(
- f"cd .. && {python_path} -m auto_round --model 'facebook/opt-125m' --seqlen 8 --iter 1 --nsamples 1 --eval_task_by_task --tasks openbookqa --bs 32"
- )
- if res > 0 or res == -1:
- assert False, "cmd line test fail, please have a check"
-
- res = os.system(
- f"cd .. && {python_path} -c 'from auto_round.__main__ import run_light; run_light()' --seqlen 8 --iter 2 --nsamples 8 --output_dir ./saved --tasks lambada_openai")
- if res > 0 or res == -1:
- assert False, "cmd line test fail, please have a check"
-
- # test mllm script
- # test auto_round_mllm help
- res = os.system(
- f"cd .. && {python_path} -m auto_round --mllm -h")
- if res > 0 or res == -1:
- assert False, "cmd line test fail, please have a check"
-
- # test auto_round_mllm --eval help
- res = os.system(
- f"cd .. && {python_path} -m auto_round --mllm --eval -h")
- if res > 0 or res == -1:
- assert False, "cmd line test fail, please have a check"
-
- # test auto_round_mllm --lmms help
- res = os.system(
- f"cd .. && {python_path} -m auto_round --mllm --lmms -h")
- if res > 0 or res == -1:
- assert False, "cmd line test fail, please have a check"
-
- res = os.system(
- f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 32 --format auto_round --output_dir ./saved")
- if res > 0 or res == -1:
- assert False, "cmd line test fail, please have a check"
-
- res = os.system(
- f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 256 --format auto_round"
- " --quant_nontext_module --output_dir ./saved ")
- if res > 0 or res == -1:
- assert False, "cmd line test fail, please have a check"
+ # res = os.system(
+ # f"cd .. && {python_path} -m auto_round --model 'facebook/opt-125m' --seqlen 8 --iter 1 --nsamples 1 --eval_task_by_task --tasks openbookqa --bs 32"
+ # )
+ # if res > 0 or res == -1:
+ # assert False, "cmd line test fail, please have a check"
+ #
+ # res = os.system(
+ # f"cd .. && {python_path} -c 'from auto_round.__main__ import run_light; run_light()' --seqlen 8 --iter 2 --nsamples 8 --output_dir ./saved --tasks lambada_openai")
+ # if res > 0 or res == -1:
+ # assert False, "cmd line test fail, please have a check"
+ #
+ # # test mllm script
+ # # test auto_round_mllm help
+ # res = os.system(
+ # f"cd .. && {python_path} -m auto_round --mllm -h")
+ # if res > 0 or res == -1:
+ # assert False, "cmd line test fail, please have a check"
+ #
+ # # test auto_round_mllm --eval help
+ # res = os.system(
+ # f"cd .. && {python_path} -m auto_round --mllm --eval -h")
+ # if res > 0 or res == -1:
+ # assert False, "cmd line test fail, please have a check"
+ #
+ # # test auto_round_mllm --lmms help
+ # res = os.system(
+ # f"cd .. && {python_path} -m auto_round --mllm --lmms -h")
+ # if res > 0 or res == -1:
+ # assert False, "cmd line test fail, please have a check"
+ #
+ # res = os.system(
+ # f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 32 --format auto_round --output_dir ./saved")
+ # if res > 0 or res == -1:
+ # assert False, "cmd line test fail, please have a check"
+ #
+ # res = os.system(
+ # f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 256 --format auto_round"
+ # " --quant_nontext_module --output_dir ./saved ")
+ # if res > 0 or res == -1:
+ # assert False, "cmd line test fail, please have a check"
if __name__ == "__main__":
diff --git a/test/test_block_names.py b/test/test_block_names.py
index 5cdbd9c93..0a106de6c 100644
--- a/test/test_block_names.py
+++ b/test/test_block_names.py
@@ -1,4 +1,3 @@
-import copy
import shutil
import sys
import unittest
@@ -172,8 +171,7 @@ class TestQuantizationBlocks(unittest.TestCase):
return
quantized_model_path = "./saved"
autoround.save_quantized(quantized_model_path, inplace=False, safe_serialization=False, format="auto_round")
-
- from auto_round.auto_quantizer import AutoHfQuantizer
+
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
diff --git a/test/test_export.py b/test/test_export.py
index 0394886bf..3bde40491 100644
--- a/test/test_export.py
+++ b/test/test_export.py
@@ -1,11 +1,9 @@
-import copy
import shutil
import sys
import unittest
sys.path.insert(0, "..")
import torch
-import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
@@ -112,7 +110,6 @@ class TestAutoRound(unittest.TestCase):
except:
return
- from auto_round.auto_quantizer import AutoHfQuantizer
device = "auto" ##cpu, hpu, cuda
from auto_round import AutoRoundConfig
quantization_config = AutoRoundConfig(
diff --git a/test/test_generation.py b/test/test_generation.py
index 8772d41b4..4f1dbb140 100644
--- a/test/test_generation.py
+++ b/test/test_generation.py
@@ -27,142 +27,107 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):
self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
self.llm_dataloader = LLMDataLoader()
+ self.save_folder = "./saved"
@classmethod
def tearDownClass(self):
- shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
- @unittest.skipIf(torch.cuda.is_available() is False, "Skipping because no cuda")
- def test_llm_generation_sym_gpu_gptq(self):
+
+ def test_4bits_asym(self):
bits = 4
- group_size = 32
+ group_size = 128
+ sym = False
autoround = AutoRound(
self.model,
self.tokenizer,
bits=bits,
group_size=group_size,
- sym=True,
+ sym=sym,
iters=1,
seqlen=2,
dataset=self.llm_dataloader,
)
- autoround.quantize()
- quantized_model_path = "./saved"
+ quantized_model_path = self.save_folder
+
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round", inplace=False)
- autoround.save_quantized(output_dir=quantized_model_path, format="auto_round:gptq",inplace=False)
- device = "auto" ##cpu, hpu, cuda
from auto_round import AutoRoundConfig
quantization_config = AutoRoundConfig(
- backend=device
+ backend="auto"
)
-
model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
- device_map=device, quantization_config=quantization_config)
+ device_map="cpu", quantization_config=quantization_config)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- text = "There is a girl who likes adventure,"
+ text = "My name is "
inputs = tokenizer(text, return_tensors="pt").to(model.device)
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
print(res)
assert ("!!!" not in res)
- @unittest.skipIf(torch.cuda.is_available() is False, "Skipping because no cuda")
- def test_llm_generation_asym_gpu_awq(self):
+
+ def test_4bits_sym(self):
bits = 4
- group_size = 32
+ group_size = 128
+ sym = True
autoround = AutoRound(
self.model,
self.tokenizer,
bits=bits,
group_size=group_size,
- sym=True,
+ sym=sym,
iters=1,
seqlen=2,
dataset=self.llm_dataloader,
)
- autoround.quantize()
- quantized_model_path = "./saved"
+ quantized_model_path = self.save_folder
+
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round", inplace=False)
- autoround.save_quantized(output_dir=quantized_model_path, format="auto_round:awq",inplace=False)
- device = "auto" ##cpu, hpu, cuda
from auto_round import AutoRoundConfig
quantization_config = AutoRoundConfig(
- backend=device
+ backend="ipex"
)
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
+ device_map="cpu", quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "My name is "
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
+ print(res)
+ assert ("!!!" not in res)
model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
- device_map=device, quantization_config=quantization_config)
+ device_map="cpu", quantization_config=quantization_config,
+ torch_dtype=torch.float16)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
print(res)
assert ("!!!" not in res)
-
- def test_llm_generation_asym_qbits(self):
- try:
- import intel_extension_for_transformers
- except:
- return
- bits = 4
- group_size = 32
- autoround = AutoRound(
- self.model,
- self.tokenizer,
- bits=bits,
- group_size=group_size,
- sym=True,
- iters=1,
- seqlen=2,
- dataset=self.llm_dataloader,
- )
- autoround.quantize()
- quantized_model_path = "./saved"
- autoround.save_quantized(output_dir=quantized_model_path, format="auto_round",inplace=False)
- device = "cpu" ##cpu, hpu, cuda
+
from auto_round import AutoRoundConfig
quantization_config = AutoRoundConfig(
- backend="cpu:auto_round:qbits_zp"
+ backend="itrex"
)
-
model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
device_map="cpu", quantization_config=quantization_config)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- text = "There is a girl who likes adventure,"
+ text = "My name is "
inputs = tokenizer(text, return_tensors="pt").to(model.device)
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
print(res)
assert ("!!!" not in res)
-
- def test_force_to_autoround_format(self):
- bits = 4
- group_size = 128
- autoround = AutoRound(
- self.model,
- self.tokenizer,
- bits=bits,
- group_size=group_size,
- sym=True,
- iters=1,
- seqlen=2,
- dataset=self.llm_dataloader,
- )
- autoround.quantize()
- quantized_model_path = "./saved"
- autoround.save_quantized(output_dir=quantized_model_path, format="auto_awq", inplace=False)
- device = "auto" ##cpu, hpu, cuda, auto
- from auto_round import AutoRoundConfig
- quantization_config = AutoRoundConfig(
- # backend="auto_round:ipex_awq",
- backend="auto"
- )
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device,
- quantization_config=quantization_config)
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
+ device_map="cpu", quantization_config=quantization_config,
+ torch_dtype=torch.float16)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
print(res)
assert ("!!!" not in res)
-
diff --git a/test/test_hpu.py b/test/test_hpu.py
index e7b355b28..f7c87f0d4 100644
--- a/test/test_hpu.py
+++ b/test/test_hpu.py
@@ -1,11 +1,9 @@
-import copy
import shutil
import sys
import unittest
sys.path.insert(0, "..")
import torch
-import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
@@ -63,9 +61,6 @@ class TestAutoRound(unittest.TestCase):
quantized_model_path = "./saved"
autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_gptq")
- from auto_round.auto_quantizer import AutoHfQuantizer
- import habana_frameworks.torch.core as htcore
- import habana_frameworks.torch.hpu as hthpu
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", \
trust_remote_code=True).to('hpu').to(torch.float32)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
@@ -94,9 +89,6 @@ class TestAutoRound(unittest.TestCase):
autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")
- from auto_round.auto_quantizer import AutoHfQuantizer
- import habana_frameworks.torch.core as htcore
- import habana_frameworks.torch.hpu as hthpu
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, \
device_map="auto").to('hpu').to(torch.float32)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
diff --git a/test_cuda/test_auto_round_format.py b/test_cuda/test_auto_round_format.py
new file mode 100644
index 000000000..3c90204b2
--- /dev/null
+++ b/test_cuda/test_auto_round_format.py
@@ -0,0 +1,418 @@
+import copy
+import shutil
+import sys
+import unittest
+
+from auto_round.eval.evaluation import simple_evaluate_user_model
+
+sys.path.insert(0, "..")
+import torch
+import transformers
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+from auto_round import AutoRoundConfig
+
+
+class LLMDataLoader:
+ def __init__(self):
+ self.batch_size = 1
+
+ def __iter__(self):
+ for i in range(2):
+ yield torch.ones([1, 10], dtype=torch.long)
+
+
+class TestAutoRound(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.model_name = "/models/opt-125m"
+ ##self.model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"
+
+ self.llm_dataloader = LLMDataLoader()
+ self.save_folder = "./saved"
+
+ def model_infer(self, model, tokenizer):
+ prompts = [
+ "Hello,my name is",
+ # "The president of the United States is",
+ # "The capital of France is",
+ # "The future of AI is",
+ ]
+
+ ##texts = []
+ # for prompt in prompts:
+ # messages = [
+ # {"role": "user", "content": prompt}
+ # ]
+ # text = tokenizer.apply_chat_template(
+ # messages,
+ # tokenize=False,
+ # add_generation_prompt=True
+ # )
+ # texts.append(text)
+
+ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)
+
+ outputs = model.generate(
+ input_ids=inputs["input_ids"].to(model.device),
+ attention_mask=inputs["attention_mask"].to(model.device),
+ do_sample=False, ## change this to follow official usage
+ max_new_tokens=5
+ )
+ generated_ids = [
+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)
+ ]
+
+ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+
+ for i, prompt in enumerate(prompts):
+ print(f"Prompt: {prompt}")
+ print(f"Generated: {decoded_outputs[i]}")
+ print("-" * 50)
+
+ @classmethod
+ def tearDownClass(self):
+ return
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_device_map(self):
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+ model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"
+
+ device_map = {}
+ for i in range(0, 32):
+ key = f"model.layers.{str(i)}"
+ device_map[key] = "cuda"
+ device_map["model.layers.1"] = "cpu"
+ device_map["model.layers.2"] = "cpu"
+ device_map["model.layers.3"] = "cpu"
+ device_map["model.layers.4"] = "cpu"
+ device_map["model.layers.5"] = "cpu"
+ device_map["model.layers.6"] = "cpu"
+ device_map["model.layers.27"] = "cpu"
+ device_map["model.layers.28"] = "cpu"
+ device_map["model.layers.30"] = "cpu"
+ device_map["model.layers.31"] = "cpu"
+ device_map["lm_head"] = "cuda"
+ device_map["model.norm"] = "cuda"
+ device_map["model.rotary_emb"] = "cuda"
+ device_map["model.embed_tokens"] = "cuda"
+
+ device_map1 = {}
+ for i in range(0, 32):
+ key = f"model.layers.{str(i)}"
+ device_map1[key] = "cuda"
+ device_map1["model.layers.1"] = "cuda:1"
+ device_map1["model.layers.2"] = "cuda:1"
+ device_map1["model.layers.3"] = "cuda:1"
+ device_map1["model.layers.4"] = "cuda:1"
+ device_map1["model.layers.5"] = "cuda:1"
+ device_map1["model.layers.6"] = "cuda:1"
+ device_map1["model.layers.27"] = "cuda:1"
+ device_map1["model.layers.28"] = "cuda:1"
+ device_map1["model.layers.30"] = "cuda:1"
+ device_map1["model.layers.31"] = "cuda:1"
+ device_map1["lm_head"] = "cuda:1"
+ device_map1["model.norm"] = "cuda"
+ device_map1["model.rotary_emb"] = "cuda"
+ device_map1["model.embed_tokens"] = "cuda"
+
+ for tmp_device_map in [device_map1, device_map, None, 0, "balanced", "balanced_low_0", "sequential", "cpu",
+ "cuda:0", "cuda", "auto",
+ ]:
+ model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ torch_dtype="auto",
+ device_map=tmp_device_map
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+ prompts = [
+ "Hello,my name is",
+ # "The president of the United States is",
+ # "The capital of France is",
+ # "The future of AI is",
+ ]
+
+ texts = []
+ for prompt in prompts:
+ messages = [
+ {"role": "user", "content": prompt}
+ ]
+ text = tokenizer.apply_chat_template(
+ messages,
+ tokenize=False,
+ add_generation_prompt=True
+ )
+ texts.append(text)
+
+ inputs = tokenizer(texts, return_tensors="pt", padding=False, truncation=True)
+
+ outputs = model.generate(
+ input_ids=inputs["input_ids"].to(model.device),
+ attention_mask=inputs["attention_mask"].to(model.device),
+ do_sample=False, ## change this to follow official usage
+ max_new_tokens=5
+ )
+ generated_ids = [
+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)
+ ]
+
+ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+
+ for i, prompt in enumerate(prompts):
+ print(f"Prompt: {prompt}")
+ print(f"Generated: {decoded_outputs[i]}")
+ print("-" * 50)
+ model = None
+ del model
+ torch.cuda.empty_cache()
+
+ def test_device_map_for_triton(self):
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+ model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"
+
+ device_map = {}
+ for i in range(0, 32):
+ key = f"model.layers.{str(i)}"
+ device_map[key] = "cuda"
+ device_map["model.layers.1"] = "cpu"
+ device_map["model.layers.2"] = "cpu"
+ device_map["model.layers.3"] = "cpu"
+ device_map["model.layers.4"] = "cpu"
+ device_map["model.layers.5"] = "cpu"
+ device_map["model.layers.6"] = "cpu"
+ device_map["model.layers.27"] = "cpu"
+ device_map["model.layers.28"] = "cpu"
+ device_map["model.layers.30"] = "cpu"
+ device_map["model.layers.31"] = "cpu"
+ device_map["lm_head"] = "cuda"
+ device_map["model.norm"] = "cuda"
+ device_map["model.rotary_emb"] = "cuda"
+ device_map["model.embed_tokens"] = "cuda"
+
+ device_map1 = {}
+ for i in range(0, 32):
+ key = f"model.layers.{str(i)}"
+ device_map1[key] = "cuda"
+ device_map1["model.layers.1"] = "cuda:1"
+ device_map1["model.layers.2"] = "cuda:1"
+ device_map1["model.layers.3"] = "cuda:1"
+ device_map1["model.layers.4"] = "cuda:1"
+ device_map1["model.layers.5"] = "cuda:1"
+ device_map1["model.layers.6"] = "cuda:1"
+ device_map1["model.layers.27"] = "cuda:1"
+ device_map1["model.layers.28"] = "cuda:1"
+ device_map1["model.layers.30"] = "cuda:1"
+ device_map1["model.layers.31"] = "cuda:1"
+ device_map1["lm_head"] = "cuda:1"
+ device_map1["model.norm"] = "cuda"
+ device_map1["model.rotary_emb"] = "cuda"
+ device_map1["model.embed_tokens"] = "cuda"
+ from auto_round import AutoRoundConfig
+ quantization_config = AutoRoundConfig(backend="tritonv2")
+
+ for tmp_device_map in [device_map1, 0, "balanced", "balanced_low_0", "sequential",
+ "cuda:0", "cuda", "auto",
+ ]:
+ model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ torch_dtype="auto",
+ device_map=tmp_device_map,
+ quantization_config = quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+ prompts = [
+ "Hello,my name is",
+ # "The president of the United States is",
+ # "The capital of France is",
+ # "The future of AI is",
+ ]
+
+ texts = []
+ for prompt in prompts:
+ messages = [
+ {"role": "user", "content": prompt}
+ ]
+ text = tokenizer.apply_chat_template(
+ messages,
+ tokenize=False,
+ add_generation_prompt=True
+ )
+ texts.append(text)
+
+ inputs = tokenizer(texts, return_tensors="pt", padding=False, truncation=True)
+
+ outputs = model.generate(
+ input_ids=inputs["input_ids"].to(model.device),
+ attention_mask=inputs["attention_mask"].to(model.device),
+ do_sample=False, ## change this to follow official usage
+ max_new_tokens=5
+ )
+ generated_ids = [
+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)
+ ]
+
+ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+
+ for i, prompt in enumerate(prompts):
+ print(f"Prompt: {prompt}")
+ print(f"Generated: {decoded_outputs[i]}")
+ print("-" * 50)
+ model = None
+ del model
+ torch.cuda.empty_cache()
+
+ def test_awq_backend(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 4, 128, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ iters=1,
+ nsamples=1,
+ sym=sym,
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:auto_awq")
+
+ quantization_config = AutoRoundConfig(backend="auto")
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.18)
+ torch.cuda.empty_cache()
+
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.bfloat16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+
+ def test_tritonv2_bf16(self):
+ model_name = "/data5/wenhuach/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"
+ quantization_config = AutoRoundConfig(backend="tritonv2")
+ model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ torch_dtype=torch.bfloat16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ self.model_infer(model, tokenizer)
+
+ torch.cuda.empty_cache()
+
+
+
+ def test_autoround_gptq_sym_format(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 4, 128, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = "./saved"
+
+ autoround.quantize_and_save(output_dir=quantized_model_path)
+
+ from auto_round import AutoRoundConfig
+ quantization_config = AutoRoundConfig(backend="ipex_gptq")
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", trust_remote_code=True,
+ quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
+ print(res)
+ assert ("!!!" not in res)
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
+ print(res)
+ assert ("!!!" not in res)
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", trust_remote_code=True,
+ quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
+ print(res)
+ assert ("!!!" not in res)
+
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_autoround_awq_sym_format(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 4, 128, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = "./saved"
+
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:auto_awq")
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
+ print(res)
+ assert ("!!!" not in res)
+
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", trust_remote_code=True,
+ torch_dtype=torch.bfloat16)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
+ print(res)
+ assert ("!!!" not in res)
+
+ shutil.rmtree("./saved", ignore_errors=True)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test_cuda/test_exllamav2_backend.py b/test_cuda/test_exllamav2_backend.py
new file mode 100644
index 000000000..b07ba65f6
--- /dev/null
+++ b/test_cuda/test_exllamav2_backend.py
@@ -0,0 +1,181 @@
+import shutil
+import sys
+import unittest
+import pytest
+from auto_round.eval.evaluation import simple_evaluate_user_model
+
+sys.path.insert(0, "..")
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+from auto_round import AutoRoundConfig
+
+
+class LLMDataLoader:
+ def __init__(self):
+ self.batch_size = 1
+
+ def __iter__(self):
+ for i in range(2):
+ yield torch.ones([1, 10], dtype=torch.long)
+
+
+class TestAutoRoundMarlinBackend(unittest.TestCase):
+
+ @classmethod
+ def setUpClass(self):
+ self.model_name = "/models/opt-125m"
+ self.save_folder = "./saved"
+ self.llm_dataloader = LLMDataLoader()
+
+ def model_infer(self, model, tokenizer):
+ prompts = [
+ "Hello,my name is",
+ # "The president of the United States is",
+ # "The capital of France is",
+ # "The future of AI is",
+ ]
+
+ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)
+
+ outputs = model.generate(
+ input_ids=inputs["input_ids"].to(model.device),
+ attention_mask=inputs["attention_mask"].to(model.device),
+ do_sample=False, ## change this to follow official usage
+ max_new_tokens=5
+ )
+ generated_ids = [
+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)
+ ]
+
+ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+
+ for i, prompt in enumerate(prompts):
+ print(f"Prompt: {prompt}")
+ print(f"Generated: {decoded_outputs[i]}")
+ print("-" * 50)
+ return decoded_outputs[0]
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_gptqmodel_exllmav2_4bits_asym(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 4, 128, False
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=1,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+
+ quantization_config = AutoRoundConfig(backend="gptqmodel:exllamav2")
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.35)
+ torch.cuda.empty_cache()
+
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.bfloat16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.35)
+ torch.cuda.empty_cache()
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_gptq_exllamav2_4bits_sym(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 4, 128, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=1,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round") ##will convert to gptq model
+
+ quantization_config = AutoRoundConfig(backend="gptq:exllamav2") ## or exllamav2
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)
+ torch.cuda.empty_cache()
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+
+ def test_gptq_exllamav2_4bits_sym_group_size(self):
+ for group_size in [32, 512, 1024]:
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 4, group_size, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ iters=1,
+ nsamples=1,
+ group_size=group_size,
+ sym=sym,
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path,
+ format="auto_round") ##will convert to gptq model
+
+ quantization_config = AutoRoundConfig(backend="gptq:exllamav2") ## or exllamav2
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.15)
+ torch.cuda.empty_cache()
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test_cuda/test_marlin_backend.py b/test_cuda/test_marlin_backend.py
new file mode 100644
index 000000000..3a231044f
--- /dev/null
+++ b/test_cuda/test_marlin_backend.py
@@ -0,0 +1,226 @@
+import shutil
+import sys
+import unittest
+import pytest
+from auto_round.eval.evaluation import simple_evaluate_user_model
+
+sys.path.insert(0, "..")
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+from auto_round import AutoRoundConfig
+
+
+class LLMDataLoader:
+ def __init__(self):
+ self.batch_size = 1
+
+ def __iter__(self):
+ for i in range(2):
+ yield torch.ones([1, 10], dtype=torch.long)
+
+
+class TestAutoRoundMarlinBackend(unittest.TestCase):
+
+ def test_marlin_group_size(self):
+ for group_size in [-1, 64]:
+ print(f"{group_size}!!!!!!!!!!!!!!!!!")
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 4, group_size, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=1,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")
+
+ quantization_config = AutoRoundConfig(backend="marlin")
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.14)
+
+ for group_size in [32, 128]:
+ print(f"{group_size}!!!!!!!!!!!!!!!!!")
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 4, group_size, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=1,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+
+ quantization_config = AutoRoundConfig(backend="marlin")
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.14)
+
+ @classmethod
+ def setUpClass(self):
+ self.model_name = "/models/opt-125m"
+ self.save_folder = "./saved"
+ self.llm_dataloader = LLMDataLoader()
+
+ def model_infer(self, model, tokenizer):
+ prompts = [
+ "Hello,my name is",
+ # "The president of the United States is",
+ # "The capital of France is",
+ # "The future of AI is",
+ ]
+
+ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)
+
+ outputs = model.generate(
+ input_ids=inputs["input_ids"].to(model.device),
+ attention_mask=inputs["attention_mask"].to(model.device),
+ do_sample=False, ## change this to follow official usage
+ max_new_tokens=5
+ )
+ generated_ids = [
+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)
+ ]
+
+ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+
+ for i, prompt in enumerate(prompts):
+ print(f"Prompt: {prompt}")
+ print(f"Generated: {decoded_outputs[i]}")
+ print("-" * 50)
+ return decoded_outputs[0]
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_marlin_4bits_sym_with_zp_m_1(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 4, 128, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=1,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_gptq")
+
+ quantization_config = AutoRoundConfig(backend="marlin")
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)
+ torch.cuda.empty_cache()
+
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.bfloat16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)
+ torch.cuda.empty_cache()
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ # def test_marlin_4bits_sym(self):
+ # model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ # tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ # bits, group_size, sym = 4, 128, True
+ # autoround = AutoRound(
+ # model,
+ # tokenizer,
+ # bits=bits,
+ # group_size=group_size,
+ # sym=sym,
+ # iters=1,
+ # seqlen=2,
+ # dataset=self.llm_dataloader,
+ # )
+ # quantized_model_path = self.save_folder
+ # autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+ #
+ # quantization_config = AutoRoundConfig(backend="marlin")
+ # model = AutoModelForCausalLM.from_pretrained(
+ # self.save_folder,
+ # torch_dtype=torch.float16,
+ # device_map="auto",
+ # quantization_config=quantization_config
+ # )
+ #
+ # tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ # self.model_infer(model, tokenizer)
+ # result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ # print(result['results']['lambada_openai']['acc,none'])
+ # self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)
+ # torch.cuda.empty_cache()
+ #
+ # model = AutoModelForCausalLM.from_pretrained(
+ # self.save_folder,
+ # torch_dtype=torch.bfloat16,
+ # device_map="auto",
+ # quantization_config=quantization_config
+ # )
+ #
+ # tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ # self.model_infer(model, tokenizer)
+ # result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ # print(result['results']['lambada_openai']['acc,none'])
+ # self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)
+ # torch.cuda.empty_cache()
+ # shutil.rmtree("./saved", ignore_errors=True)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test_cuda/test_triton_backend.py b/test_cuda/test_triton_backend.py
new file mode 100644
index 000000000..324d3ceaf
--- /dev/null
+++ b/test_cuda/test_triton_backend.py
@@ -0,0 +1,296 @@
+
+import shutil
+import sys
+import unittest
+
+from auto_round.eval.evaluation import simple_evaluate_user_model
+
+sys.path.insert(0, "..")
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+from auto_round import AutoRoundConfig
+
+
+class LLMDataLoader:
+ def __init__(self):
+ self.batch_size = 1
+
+ def __iter__(self):
+ for i in range(2):
+ yield torch.ones([1, 10], dtype=torch.long)
+
+
+class TestAutoRoundTritonBackend(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.model_name = "/models/opt-125m"
+ self.save_folder = "./saved"
+ self.llm_dataloader = LLMDataLoader()
+
+ def model_infer(self, model, tokenizer):
+ prompts = [
+ "Hello,my name is",
+ # "The president of the United States is",
+ # "The capital of France is",
+ # "The future of AI is",
+ ]
+
+ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)
+
+ outputs = model.generate(
+ input_ids=inputs["input_ids"].to(model.device),
+ attention_mask=inputs["attention_mask"].to(model.device),
+ do_sample=False, ## change this to follow official usage
+ max_new_tokens=5
+ )
+ generated_ids = [
+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)
+ ]
+
+ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+
+ for i, prompt in enumerate(prompts):
+ print(f"Prompt: {prompt}")
+ print(f"Generated: {decoded_outputs[i]}")
+ print("-" * 50)
+ return decoded_outputs[0]
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree("./saved", ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_tritonv2_4bits_asym(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 4, 128, False
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=1,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path)
+
+ quantization_config = AutoRoundConfig(backend="tritonv2")
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.34)
+ torch.cuda.empty_cache()
+
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.bfloat16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.34)
+ torch.cuda.empty_cache()
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_tritonv2_2bits_asym(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 2, 32, False
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path)
+
+ quantization_config = AutoRoundConfig(backend="tritonv2")
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.20)
+ torch.cuda.empty_cache()
+
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.bfloat16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.20)
+ torch.cuda.empty_cache()
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_tritonv2_4bits_sym(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 4, 128, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=1,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path)
+
+ quantization_config = AutoRoundConfig(backend="tritonv2")
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ # print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.26)
+ torch.cuda.empty_cache()
+
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.bfloat16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ # print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.26)
+ torch.cuda.empty_cache()
+
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_tritonv2_8bits_sym(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 4, 256, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ nsamples=1,
+ iters=1
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path)
+
+ quantization_config = AutoRoundConfig(backend="tritonv2")
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)
+ torch.cuda.empty_cache()
+
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.bfloat16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ # print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.27)
+ torch.cuda.empty_cache()
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_tritonv2_2bits_sym(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = 2, 64, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path)
+
+ quantization_config = AutoRoundConfig(backend="tritonv2")
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.18)
+ torch.cuda.empty_cache()
+
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.bfloat16,
+ device_map="auto",
+ quantization_config=quantization_config
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ self.model_infer(model, tokenizer)
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="lambada_openai")
+ # print(result['results']['lambada_openai']['acc,none'])
+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.18)
+ torch.cuda.empty_cache()
+ shutil.rmtree("./saved", ignore_errors=True)
+
+
+if __name__ == "__main__":
+ unittest.main()