PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine inference step 2 (#498)
Wenhua Cheng37341f5ee2ac
diff --git a/README.md b/README.mdindex 3868178fa..99e9a4ba3 100644--- a/README.md+++ b/README.md@@ -73,7 +73,7 @@ pip install auto-round-lib## Model Quantization-### Basic Usage (Gaudi2/CPU/GPU)+### Basic Usage (Gaudi2/CPU/XPU/GPU)A user guide detailing the full list of supported arguments is provided by calling ```auto-round -h``` on the terminal.Set the format you want in `format` and@@ -268,7 +268,7 @@ autoround.save_quantized(output_dir, format='auto_round', inplace=True)### Export Formats**AutoRound Format**: This format is well-suited for CPU, HPU devices, 2 bits, as well as mixed-precision-inference. **[2,4] bits are supported**. However, it has not yet gained widespread community adoption.+inference. **[2,3,4,8] bits are supported**. However, it has not yet gained widespread community adoption.**AutoGPTQ Format**: This format is well-suited for symmetric quantization on CUDA devices and is widely adopted by thecommunity, **[2,3,4,8] bits are supported**. However, **the@@ -324,14 +324,9 @@ in [Gaudi Guide](https://docs.habana.ai/en/latest/).from transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundConfig-backend = "auto" ##cpu, hpu, cuda-quantization_config = AutoRoundConfig(- backend=backend-)quantized_model_path = "./tmp_autoround"model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map=backend.split(':')[0],- quantization_config=quantization_config)+ device_map="auto")tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)diff --git a/auto_round/__init__.py b/auto_round/__init__.pyindex 8a2d74b4d..f492c5cf9 100644--- a/auto_round/__init__.py+++ b/auto_round/__init__.py@@ -15,5 +15,14 @@ from .autoround import AutoRound, AutoRoundAdam, AutoRoundOPTfrom .mllm import AutoRoundMLLMfrom auto_round.utils import LazyImport-from auto_round.inference.auto_quantizer import AutoHfQuantizer,AutoRoundConfig+def __getattr__(name):+ if name == 'AutoHfQuantizer':+ from auto_round.inference.auto_quantizer import AutoHfQuantizer+ return AutoHfQuantizer+ if name == 'AutoRoundConfig':+ from auto_round.inference.auto_quantizer import AutoRoundConfig+ return AutoRoundConfig++ raise AttributeError(f"auto-round has no attribute '{name}'")+from .version import __version__diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 47f97dcf3..a80392302 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -475,8 +475,8 @@ class AutoRound(object):for index in range(len(formats)):format = formats[index]if "auto_round" in format:- if self.sym and ("gptq" not in format and "awq" not in format):- format = format.replace('auto_round', 'auto_round:gptq')+ if (self.sym and ("gptq" not in format and "awq" not in format)) or self.bits==3:+ format = format.replace('auto_round', 'auto_round:auto_gptq')formats[index] = format# Remove duplicates from formats list@@ -496,6 +496,13 @@ class AutoRound(object):# Save the quantized model in the specified formatsfolders = []for format in formats:+ if "gptq" in format and not self.sym:+ logger.warning(+ "The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"+ " particularly for 2-bit quantization and smaller models."+ " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "+ "the AutoRound format(2/4/8 bits)."+ )save_format_ = format.replace(":", "-").replace("_", "-")save_folder = os.path.join(output_dir, save_format_) if len(formats) > 1 else output_dirself.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)@@ -1598,8 +1605,8 @@ class AutoRound(object):logger.warning("The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"" particularly for 2-bit quantization and smaller models."- " We recommend exporting to either the AutoAWQ format (4 bits) or "- "the AutoRound format (2 bits) to enhance performance."+ " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "+ "the AutoRound format(2/4/8 bits).")if "awq" in format and not self.bits == 4:raise ValueError("The AWQ format only supports W4 quantization ")diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 9c68a96d6..5c86d1fd7 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -23,7 +23,7 @@ import transformersimport auto_round.export.export_to_autoround.qlinear_triton_actimport auto_round_extension.cuda.qlinear_tritonv2-from auto_round.utils import get_layer_names_in_block, get_module, logger, set_module, supported_layer_types+from auto_round.utils import get_module, logger, set_module, supported_layer_typesimport threadpoolctl as tctlimport inspectfrom tqdm import tqdm@@ -75,15 +75,15 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_from auto_round_extension.cuda.qlinear_tritonv2 import QuantLinearreturn QuantLinear- elif "auto_round" in backend and "gptq" in backend:- from auto_round.export.export_to_autoround.qlinear_triton import QuantLinear ##no g_idx- return QuantLinear+ elif "auto_round" in backend and "gptq" in backend and bits in (2, 4, 8):+ from auto_round.export.export_to_autoround.qlinear_triton import QuantLinear ##no g_idx+ return QuantLinearelif "awq" in backend:from ..export_to_awq.utils import WQLinear_GEMMreturn WQLinear_GEMMelif "gptqmodel" in backend:return auto_round_extension.cuda.qlinear_tritonv2.QuantLinear- elif "gptq" in backend and not "gptqmodel" in backend: ## have g_idx+ elif "gptq" in backend and not "gptqmodel" in backend: ## have g_idxreturn get_autogptq_packing_qlinear(backend, bits, group_size, sym)else:assert False, f"only support auto_gptq, auto_awq and auto_round backend"@@ -190,7 +190,9 @@ def pack_layer(layer_name, model, backend):new_layer.device = deviceset_module(model, layer_name, new_layer)qlayer = new_layer- if sym:+ import auto_round.export.export_to_autoround.qlinear_triton+ if sym and isinstance(QuantLinear, (auto_round.export.export_to_autoround.qlinear_triton.QuantLinear,+ auto_round_extension.cuda.qlinear_tritonv2.QuantLinear)):zp = int(zp.flatten()[0])qlayer.to("cpu")@@ -248,7 +250,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex##if using sym, we change to gptq sym kernel to avoid compiling from auto_round sourceif (kwargs.get("sym") is None or kwargs.get("sym") == True) and ("gptq" not in backend and "awq" not in backend):- backend = backend.replace('auto_round', 'auto_round:gptq')+ backend = backend.replace('auto_round', 'auto_round:auto_gptq')model = kwargs["model"]safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]@@ -260,6 +262,9 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exquantization_config["quant_method"] = "intel/auto-round"quantization_config["backend"] = backend+ if quantization_config["bits"]==3:+ backend = "auto_round:auto_gptq"+tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)extra_config = {}diff --git a/auto_round/inference/__init__.py b/auto_round/inference/__init__.pyindex 2b414eb72..789a8cdc7 100644--- a/auto_round/inference/__init__.py+++ b/auto_round/inference/__init__.py@@ -11,4 +11,5 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+from auto_round.inference.convert_model import convert_hf_model, infer_target_device, post_initdiff --git a/auto_round/inference/auto_quantizer.py b/auto_round/inference/auto_quantizer.pyindex 90fd551bd..8dc62efb3 100644--- a/auto_round/inference/auto_quantizer.py+++ b/auto_round/inference/auto_quantizer.py@@ -155,26 +155,11 @@ class AutoHfQuantizer:loading_attr_dict = quantization_config_from_args.get_loading_attributes() \if quantization_config_from_args is not None else Noneif isinstance(quantization_config, dict):- if "auto-round" in quantization_config["quant_method"]:+ if "auto-round" in quantization_config[+ "quant_method"] or quantization_config_from_args.__class__.__name__ == "AutoRoundConfig":quantization_config = AutoRoundConfig.from_dict(quantization_config)else:- if isinstance(quantization_config_from_args, (AutoRoundConfig)):- logger.info(f"Loading quantized model in auto_round format.")- tmp_backend = quantization_config["quant_method"]- if "auto-round" not in tmp_backend and "gptq" not in tmp_backend and "awq" not in tmp_backend:- logger.error("could not convert to auto_round format, currently only supports `gptq`,`awq` or "- "`auto-round` format")- exit(-1)- target_backend = quantization_config["backend"] if "backend" in quantization_config else "auto"- if loading_attr_dict is not None and "backend" in loading_attr_dict:- target_backend = loading_attr_dict["backend"]- loading_attr_dict.pop("backend")- if "auto_round" not in target_backend:- target_backend = f"auto_round:{tmp_backend}" #- quantization_config = AutoRoundConfig.from_dict(quantization_config)- setattr(quantization_config, "backend", target_backend)- else:- quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101+ quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101if isinstance(quantization_config,(GPTQConfig, AwqConfig, AutoRoundConfig)) and quantization_config_from_args is not None:@@ -265,8 +250,8 @@ class AutoRoundConfig(QuantizationConfigMixin):def post_init(self):r"""Safety checker that arguments are correct."""- if self.bits not in [2, 4, 8]:- raise ValueError(f"Only support quantization to [2,4,8] bits but found {self.bits}")+ if self.bits not in [2, 3, 4, 8]:+ raise ValueError(f"Only support quantization to [2,3,4,8] bits but found {self.bits}")if self.group_size != -1 and self.group_size <= 0:raise ValueError("group_size must be greater than 0 or equal to -1")@@ -278,6 +263,26 @@ class AutoRoundConfig(QuantizationConfigMixin):config_dict = super().to_dict()return config_dict+ @classmethod+ def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs):+ quant_method = config_dict["quant_method"]+ if "auto-round" not in quant_method and "gptq" not in quant_method and "awq" not in quant_method:+ raise NotImplementedError(+ "Failed to convert to auto_round format. Only `gptqv1`, `awq`, and `auto-round` formats are supported."+ )++ if "gptq" in quant_method and "meta" in config_dict:+ raise NotImplementedError(+ "Failed to convert gptq format to auto_round format. Only supports `gptqv1`")++ if "awq" in quant_method and config_dict.get("version", "gemm") != "gemm":+ raise NotImplementedError(+ "Failed to convert awq format to auto_round format. Only supports awq format with gemm version")++ if "auto-round" not in quant_method:+ config_dict["backend"] = f"auto_round:{quant_method}"+ return super().from_dict(config_dict, return_unused_kwargs=return_unused_kwargs, **kwargs)+class AutoRoundQuantizer(HfQuantizer):"""Quantizer of the AutoRound method, currently only triton and exllamav2 backend has been supported."""@@ -306,7 +311,6 @@ class AutoRoundQuantizer(HfQuantizer):"auto-round` or install from source")def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":- self.target_device = infer_target_device(self.device_map)if torch_dtype is None:torch_dtype = torch.float16elif torch_dtype != torch.float16:@@ -330,8 +334,7 @@ class AutoRoundQuantizer(HfQuantizer):def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs):if self.pre_quantized:- target_device = self.target_device if hasattr(self, self.target_device) else infer_target_device(- self.device_map)+ target_device = infer_target_device(self.device_map)model, used_backends = convert_hf_model(model, target_device)self.used_backends = used_backendsdiff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex 2f2b27f05..4a4020426 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -126,7 +126,7 @@ BackendInfos['auto_round:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, Fabits=[2, 4, 8],priority=1, feature_checks=[feature_multiply_checker_32],alias=["auto_round", "tritonv2"],- requirements=["auto-round>=0.2"]+ requirements=["auto-round>=0.5.0"])BackendInfos['auto_round:tritonv2_zp'] = BackendInfo(device=["cuda"], sym=[True], ## asym has accuracy issue@@ -135,7 +135,7 @@ BackendInfos['auto_round:tritonv2_zp'] = BackendInfo(device=["cuda"], sym=[True]bits=[2, 4, 8],priority=1, feature_checks=[feature_multiply_checker_32],alias=["tritonv2", "tritonv2_zp"],- requirements=["auto-round>=0.5"]+ requirements=["auto-round>=0.5.0"])BackendInfos['gptqmodel:marlin'] = BackendInfo(device=["cuda"], sym=[True],@@ -145,7 +145,7 @@ BackendInfos['gptqmodel:marlin'] = BackendInfo(device=["cuda"], sym=[True],dtype=["float16", "bfloat16"],priority=6, feature_checks=[in_output_feature_multiply_checker_32],alias=["marlin", "gptqmodel"],- requirements=["gptqmodel>=2.0"]+ requirements=["gptqmodel>=2.0"],)BackendInfos['gptqmodel:marlin_zp'] = BackendInfo(device=["cuda"], sym=[True],@@ -504,7 +504,7 @@ def get_autogptq_infer_linear(backend, bits=4, group_size=128, sym=False):return QuantLinear-def find_backend(target_backend: str, orig_backend: str = None) -> str | None:+def find_backend(target_backend: str, orig_backend: str = None):"""Finds the matching backend key based on the target backend name or its aliases.@@ -620,7 +620,10 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_ftry:require_version(requirement)except ImportError:- logger.error(f"pip install '{requirement}' ")+ if "gptqmodel" in requirement:+ logger.error(f"pip install -v '{requirement}' --no-build-isolation")+ else:+ logger.error(f"pip install '{requirement}' ")else:str_info = requirement()[1]logger.error(str_info)@@ -633,3 +636,63 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_freverse=True)return supported_backends[0]+++def get_highest_priority_backend(bits, sym, group_size, device, packing_format):+ supported_backends = []+ for key in BackendInfos.keys():+ backend = BackendInfos[key]+ # Check if device is supported by the backend+ if device not in backend.device:+ continue++ # Check if bit-width is supported+ if bits not in backend.bits:+ continue++ # Check if group_size is valid (if required by backend)+ if backend.group_size is not None and group_size not in backend.group_size:+ continue++ # Check if symmetric/asymmetric quantization is supported+ if sym not in backend.sym:+ continue++ # Check if the format is convertible when packing formats differ+ if packing_format == backend.packing_format or packing_format in backend.convertable_format:+ pass+ else:+ continue+ supported_backends.append(key)++ if len(supported_backends) > 0:++ supported_backends = sorted(supported_backends,+ key=lambda support_backend: BackendInfos[support_backend].priority,+ reverse=True)+ return supported_backends[0]+ else:+ return None+++def process_requirement(requirements: list):+ gptqmodel_requirements = None+ other_requirements = []+ for requirement in requirements:+ if "gptqmodel" in requirement:+ gptqmodel_requirements = requirement+ else:+ other_requirements.append(requirement)++ infos = []++ if gptqmodel_requirements is not None:+ infos.append(f"pip install -v '{gptqmodel_requirements}' --no-build-isolation")+ infos.append(f"pip install 'numpy<2.0'")++ other_info = f"pip install"+ if len(other_requirements) > 0:+ for requirement in other_requirements:+ other_info += f" {requirement}"+ infos.append(other_info)+ return infosdiff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex ec3ed6c3f..5024d2f27 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -23,7 +23,8 @@ from transformers.pytorch_utils import Conv1Dfrom auto_round.utils import (get_module, set_module, is_hpu_supported, get_block_names,find_matching_blocks, get_layer_names_in_block, check_to_quantized)-from auto_round.inference.backend import get_layer_backend, dynamic_import_inference_linear, find_backend, BackendInfos+from auto_round.inference.backend import (get_layer_backend, dynamic_import_inference_linear,+ find_backend, BackendInfos, get_highest_priority_backend, process_requirement)logger = getLogger(__name__)@@ -355,7 +356,7 @@ def post_init(model, used_backends):_import_exllamav2_kernels()## convert datatype- data_types=[]+ data_types = []for used_backend in used_backends:backend = BackendInfos[used_backend]data_types.append(backend.dtype)@@ -364,10 +365,10 @@ def post_init(model, used_backends):common &= set(l)common = list(common)if str(model.dtype).split('.')[-1] not in common:- if common[0]=="float16":+ if common[0] == "float16":model = model.to(torch.float16)logger.warning("force model to float16")- elif common[0]=="bfloat16":+ elif common[0] == "bfloat16":model = model.to(torch.bfloat16)logger.warning("force model to bfloat16")@@ -392,8 +393,15 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):If the quantization backend is not specified in the configuration."""-quantization_config = model.config.quantization_config++ if hasattr(quantization_config, "desc_act") and quantization_config.desc_act == True:+ ##check static_group+ if (hasattr(quantization_config, "static_groups") and not quantization_config.static_groups) or (+ not hasattr(quantization_config, "static_groups")):+ raise NotImplementedError(+ "This GPTQ model may contain a non-dummy g_idx, which is not yet supported by AutoRound")+if not hasattr(quantization_config, "target_backend"):quantization_config.target_backend = quantization_config.backend@@ -422,4 +430,17 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):used_backends = _replace_by_quant_layers(model, layer_configs, target_backend, target_device, backend)+ if target_backend == "auto" or target_backend == "":+ best_backend = get_highest_priority_backend(quantization_config.bits, quantization_config.sym,+ quantization_config.group_size, target_device,+ BackendInfos[backend].packing_format)+ if best_backend is not None and best_backend not in used_backends:+ info = f"better backend is found, please install all the following requirements to enable it,"+ extra_info = info+ requirements = BackendInfos[best_backend].requirements+ requirements_info = process_requirement(requirements)+ for req in requirements_info:+ extra_info += (f"`{req}`") + " "+ logger.warning(extra_info)+return model, used_backendsdiff --git a/auto_round_extension/cuda/qlinear_tritonv2.py b/auto_round_extension/cuda/qlinear_tritonv2.pyindex 8cb4b05b6..655957e86 100644--- a/auto_round_extension/cuda/qlinear_tritonv2.py+++ b/auto_round_extension/cuda/qlinear_tritonv2.py@@ -156,6 +156,10 @@ class QuantLinear(nn.Module, TritonModuleMixin):def forward(self, x):out_shape = x.shape[:-1] + (self.outfeatures,)quant_linear_fn = QuantLinearFunction+ if not hasattr(self,"g_idx"):+ self.g_idx = torch.tensor(+ [i // self.group_size for i in range(self.infeatures)], dtype=torch.int32+ ).to(self.qweight.device)out = quant_linear_fn.apply(x.reshape(-1, x.shape[-1]),diff --git a/auto_round_extension/cuda/qlinear_tritonv2_zp.py b/auto_round_extension/cuda/qlinear_tritonv2_zp.pyindex 0436394ee..2dadfc0ae 100644--- a/auto_round_extension/cuda/qlinear_tritonv2_zp.py+++ b/auto_round_extension/cuda/qlinear_tritonv2_zp.py@@ -81,12 +81,12 @@ class QuantLinear(nn.Module, TritonModuleMixin):dtype=torch.float16,),)- self.register_buffer(- "g_idx",- torch.tensor(- [i // self.group_size for i in range(infeatures)], dtype=torch.int32- ),- )+ # self.register_buffer(+ # "g_idx",+ # torch.tensor(+ # [i // self.group_size for i in range(infeatures)], dtype=torch.int32+ # ),+ # )if bias:self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16)@@ -98,74 +98,78 @@ class QuantLinear(nn.Module, TritonModuleMixin):def post_init(self):pass-- def pack(self, linear, scales, zeros, g_idx=None):- W = linear.weight.data.clone()- if isinstance(linear, nn.Conv2d):- W = W.flatten(1)- if isinstance(linear, transformers.pytorch_utils.Conv1D):- W = W.t()-- self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx-- scales = scales.t().contiguous()- zeros = zeros.t().contiguous()- scale_zeros = zeros * scales- self.scales = scales.clone().half()- if linear.bias is not None:- self.bias = linear.bias.clone().half()-- intweight = []- for idx in range(self.infeatures):- intweight.append(- torch.round(- (W[:, idx] + scale_zeros[self.g_idx[idx]])- / self.scales[self.g_idx[idx]]- ).to(torch.int)[:, None]- )- intweight = torch.cat(intweight, dim=1)- intweight = intweight.t().contiguous()- intweight = intweight.numpy().astype(np.uint32)-- i = 0- row = 0- qweight = np.zeros(- (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32- )- while row < qweight.shape[0]:- if self.bits in [2, 4, 8]:- for j in range(i, i + (32 // self.bits)):- qweight[row] |= intweight[j] << (self.bits * (j - i))- i += 32 // self.bits- row += 1- else:- raise NotImplementedError("Only 2,4,8 bits are supported.")-- qweight = qweight.astype(np.int32)- self.qweight = torch.from_numpy(qweight)-- zeros -= 1- zeros = zeros.numpy().astype(np.uint32)- qzeros = np.zeros(- (zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32- )- i = 0- col = 0- while col < qzeros.shape[1]:- if self.bits in [2, 4, 8]:- for j in range(i, i + (32 // self.bits)):- qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))- i += 32 // self.bits- col += 1- else:- raise NotImplementedError("Only 2,4,8 bits are supported.")-- qzeros = qzeros.astype(np.int32)- self.qzeros = torch.from_numpy(qzeros)+ #+ # def pack(self, linear, scales, zeros, g_idx=None):+ # W = linear.weight.data.clone()+ # if isinstance(linear, nn.Conv2d):+ # W = W.flatten(1)+ # if isinstance(linear, transformers.pytorch_utils.Conv1D):+ # W = W.t()+ #+ # self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx+ #+ # scales = scales.t().contiguous()+ # zeros = zeros.t().contiguous()+ # scale_zeros = zeros * scales+ # self.scales = scales.clone().half()+ # if linear.bias is not None:+ # self.bias = linear.bias.clone().half()+ #+ # intweight = []+ # for idx in range(self.infeatures):+ # intweight.append(+ # torch.round(+ # (W[:, idx] + scale_zeros[self.g_idx[idx]])+ # / self.scales[self.g_idx[idx]]+ # ).to(torch.int)[:, None]+ # )+ # intweight = torch.cat(intweight, dim=1)+ # intweight = intweight.t().contiguous()+ # intweight = intweight.numpy().astype(np.uint32)+ #+ # i = 0+ # row = 0+ # qweight = np.zeros(+ # (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32+ # )+ # while row < qweight.shape[0]:+ # if self.bits in [2, 4, 8]:+ # for j in range(i, i + (32 // self.bits)):+ # qweight[row] |= intweight[j] << (self.bits * (j - i))+ # i += 32 // self.bits+ # row += 1+ # else:+ # raise NotImplementedError("Only 2,4,8 bits are supported.")+ #+ # qweight = qweight.astype(np.int32)+ # self.qweight = torch.from_numpy(qweight)+ #+ # zeros -= 1+ # zeros = zeros.numpy().astype(np.uint32)+ # qzeros = np.zeros(+ # (zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32+ # )+ # i = 0+ # col = 0+ # while col < qzeros.shape[1]:+ # if self.bits in [2, 4, 8]:+ # for j in range(i, i + (32 // self.bits)):+ # qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))+ # i += 32 // self.bits+ # col += 1+ # else:+ # raise NotImplementedError("Only 2,4,8 bits are supported.")+ #+ # qzeros = qzeros.astype(np.int32)+ # self.qzeros = torch.from_numpy(qzeros)def forward(self, x):out_shape = x.shape[:-1] + (self.outfeatures,)quant_linear_fn = QuantLinearFunction+ if not hasattr(self, "g_idx"):+ self.g_idx = torch.tensor(+ [i // self.group_size for i in range(self.infeatures)], dtype=torch.int32+ ).to(self.qweight.device)out = quant_linear_fn.apply(x.reshape(-1, x.shape[-1]),diff --git a/test/test_generation.py b/test/test_generation.pyindex 4f1dbb140..a4afc278f 100644--- a/test/test_generation.py+++ b/test/test_generation.py@@ -23,9 +23,9 @@ class LLMDataLoader:class TestAutoRoundFormatGeneration(unittest.TestCase):@classmethoddef setUpClass(self):- model_name = "facebook/opt-125m"- self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)- self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ self.model_name = "facebook/opt-125m"+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)self.llm_dataloader = LLMDataLoader()self.save_folder = "./saved"@@ -66,6 +66,7 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):print(res)assert ("!!!" not in res)+ shutil.rmtree(self.save_folder, ignore_errors=True)def test_4bits_sym(self):bits = 4@@ -131,3 +132,63 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])print(res)assert ("!!!" not in res)+ shutil.rmtree(self.save_folder, ignore_errors=True)++ def test_autoround_asym(self):+ for bits in [2, 4, 8]:+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = bits, 128, False+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = "./saved"++ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",+ trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ print(res)+ assert ("!!!" not in res)+ shutil.rmtree(self.save_folder, ignore_errors=True)++ def test_autoround_sym(self):+ for bits in [2, 4, 8]:+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = bits, 128, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = "./saved"++ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",+ trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ print(res)+ assert ("!!!" not in res)+ shutil.rmtree(self.save_folder, ignore_errors=True)+diff --git a/test/test_load_awq_gptq.py b/test/test_load_awq_gptq.pynew file mode 100644index 000000000..71e7e9a01--- /dev/null+++ b/test/test_load_awq_gptq.py@@ -0,0 +1,93 @@+import shutil+import sys+import unittest++sys.path.insert(0, "..")++from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRoundConfig+++class TestAutoRound(unittest.TestCase):+ def model_infer(self, model, tokenizer):+ prompts = [+ "Hello,my name is",+ # "The president of the United States is",+ # "The capital of France is",+ # "The future of AI is",+ ]++ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)++ outputs = model.generate(+ input_ids=inputs["input_ids"].to(model.device),+ attention_mask=inputs["attention_mask"].to(model.device),+ do_sample=False, ## change this to follow official usage+ max_new_tokens=5+ )+ generated_ids = [+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)+ ]++ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)++ for i, prompt in enumerate(prompts):+ print(f"Prompt: {prompt}")+ print(f"Generated: {decoded_outputs[i]}")+ print("-" * 50)++ @classmethod+ def tearDownClass(self):+ shutil.rmtree("runs", ignore_errors=True)++ def test_load_gptq_model_4bits(self):+ model_name = "ybelkada/opt-125m-gptq-4bit"+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,+ device_map="cpu",+ quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ self.model_infer(model, tokenizer)++ def test_load_gptq_model_8bits(self):+ model_name = "acloudfan/opt-125m-gptq-8bit"+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,+ device_map="cpu",+ quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ self.model_infer(model, tokenizer)+++ def test_load_gptq_model_2bits(self):+ model_name = "LucasSantiago257/gemma-2b-2bits-gptq"+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,+ device_map="cpu",+ quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ self.model_infer(model, tokenizer)+++++ def test_load_gptq_no_dummy_gidx_model(self):+ model_name = "ModelCloud/Llama-3.2-1B-Instruct-gptqmodel-4bit-vortex-v1"+ quantization_config = AutoRoundConfig()+ with self.assertRaises(NotImplementedError) as cm:+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,+ device_map="cpu",+ quantization_config=quantization_config)++ def test_load_awq(self):+ model_name = "casperhansen/opt-125m-awq"+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,+ device_map="cpu",+ quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ self.model_infer(model, tokenizer)+++diff --git a/test_cuda/test_auto_round_format.py b/test_cuda/test_auto_round_format.pyindex 3c90204b2..e2f0929d9 100644--- a/test_cuda/test_auto_round_format.py+++ b/test_cuda/test_auto_round_format.py@@ -73,7 +73,6 @@ class TestAutoRound(unittest.TestCase):@classmethoddef tearDownClass(self):- returnshutil.rmtree(self.save_folder, ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)@@ -212,7 +211,7 @@ class TestAutoRound(unittest.TestCase):device_map1["model.norm"] = "cuda"device_map1["model.rotary_emb"] = "cuda"device_map1["model.embed_tokens"] = "cuda"- from auto_round import AutoRoundConfig+ from auto_round import AutoRoundConfigquantization_config = AutoRoundConfig(backend="tritonv2")for tmp_device_map in [device_map1, 0, "balanced", "balanced_low_0", "sequential",@@ -222,7 +221,7 @@ class TestAutoRound(unittest.TestCase):model_name,torch_dtype="auto",device_map=tmp_device_map,- quantization_config = quantization_config+ quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(model_name)@@ -324,8 +323,6 @@ class TestAutoRound(unittest.TestCase):torch.cuda.empty_cache()--def test_autoround_gptq_sym_format(self):model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)@@ -401,7 +398,6 @@ class TestAutoRound(unittest.TestCase):print(res)assert ("!!!" not in res)-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", trust_remote_code=True,torch_dtype=torch.bfloat16)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)@@ -413,6 +409,71 @@ class TestAutoRound(unittest.TestCase):shutil.rmtree("./saved", ignore_errors=True)+ def test_autoround_asym(self):+ for bits in [2, 3, 4, 8]:+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = bits, 128, False+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = "./saved"++ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",+ trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ print(res)+ assert ("!!!" not in res)++ def test_autoround_sym(self):+ for bits in [2, 3, 4, 8]:+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)+ bits, group_size, sym = bits, 128, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = "./saved"++ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",+ trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])+ print(res)+ assert ("!!!" not in res)++ def test_load_gptq_model_3bits(self):+ model_name = "LucasSantiago257/gemma-2b-2bits-gptq"+ quantization_config = AutoRoundConfig()+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,+ device_map="auto",+ quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ self.model_infer(model, tokenizer)+if __name__ == "__main__":unittest.main()