PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add tritonv2, improve packing and pbar (#239)
wenhuach21a5337c884536
diff --git a/README.md b/README.mdindex 083fbe581..8a3f3a93e 100644--- a/README.md+++ b/README.md@@ -137,33 +137,41 @@ install from the source.**AutoGPTQ Format**: This format is well-suited for symmetric quantization on CUDA devices and is widely adopted by thecommunity. It also benefits from the Marlin kernel, which can boost inference performance notably. However, the-asymmetric kernel has issues that can cause considerable accuracy drops, particularly at 2-bit quantization.+asymmetric kernel has issues that can cause considerable accuracy drops, particularly at 2-bit quantization and small models.Additionally, symmetric quantization tends to perform poorly at 2-bit precision.**AutoAWQ format**: This format is well-suited for asymmetric 4-bit quantization on CUDA devices and is widely adoptedwithin the community. Asymmetric quantization typically improves accuracy but may reduce inference speed. It features-specialized layer fusion tailored for Llama models. However, it supports only 4-bit asymmetric quantization and is not-compatible with some models, such as Phi.+specialized layer fusion tailored for Llama models. However, it supports only 4-bit asymmetric quantization. Currently, please manually install autoawq via `pip install autoawq` before exporting.## Model inferencePlease run the quantization code first### AutoGPTQ/AutoAWQ format+```python+from transformers import AutoModelForCausalLM, AutoTokenizer-Refer to their repositories to inference the model.-+quantized_model_path = "./tmp_autoround"+model = AutoModelForCausalLM.from_pretrained(quantized_model_path,+ device_map="auto")+tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+text = "There is a girl who likes adventure,"+inputs = tokenizer(text, return_tensors="pt").to(model.device)+print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+```### AutoRound format-**cuda**: git clone https://github.com/intel/auto-round.git && cd auto-round && pip install -vvv --no-build-isolation--e .--**cpu**: no extra operations+**CPU**: no extra operations-**hpu**: docker image with Gaudi Software Stack is recommended. More details can be found+**HPU**: docker image with Gaudi Software Stack is recommended. More details can be foundin [Gaudi Guide](https://docs.habana.ai/en/latest/).-#### Gaudi2/ CPU/ GPU on 0.3.0++**CUDA**: git clone https://github.com/intel/auto-round.git && cd auto-round && pip install -vvv --no-build-isolation+-e .+++#### CPU/HPU/CUDA on 0.3.0+```pythonfrom transformers import AutoModelForCausalLM, AutoTokenizer@@ -182,7 +190,7 @@ inputs = tokenizer(text, return_tensors="pt").to(model.device)print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))```-#### Gaudi2/ CPU/ GPU on 0.3.0+#### CPU/HPU/CUDA on 0.3.0```pythonfrom transformers import AutoModelForCausalLM, AutoTokenizerdiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 9a232dc66..00cc0e308 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -21,7 +21,7 @@ import torchimport transformersfrom transformers import set_seedfrom torch import autocast-+from tqdm import tqdmfrom .calib_dataset import get_dataloaderfrom .quantizer import WrapperMultiblock, wrapper_block, unwrapper_block, WrapperLinear, unwrapper_layer, \WrapperTransformerConv1d@@ -972,7 +972,7 @@ class AutoRound(object):f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "f"layers in the block, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}")- logger.info(dump_info)+ logger.debug(dump_info)if len(unquantized_layer_names) != 0:logger.info(f"{unquantized_layer_names} have not been quantized")with torch.no_grad():@@ -1041,15 +1041,15 @@ class AutoRound(object):elif isinstance(input_others[key], list):for i in range(len(input_others[key])):input_others[key][i].to(tmp_dtype)-- for i in range(0, len(block_names), nblocks):+ pbar = tqdm(range(0, len(block_names), nblocks))+ for i in pbar:if nblocks == 1:n = block_names[i]- logger.info(f"quantizing {i + 1}/{len(block_names)}, {n}")+ pbar.set_description(f"Quantizing {n}")m = get_module(model, n)else:names = block_names[i: i + nblocks]- logger.info(names)+ pbar.set_description(f"Quantizing [{i + 1}-{i + nblocks}]/{len(block_names)}")modules = [get_module(model, n) for n in names]m = WrapperMultiblock(modules)@@ -1545,4 +1545,3 @@ class AutoAdamRound(AutoOPTRound):optimizer,**kwargs,)-diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 3e18bf9b7..feb71bae1 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -48,7 +48,8 @@ import transformersfrom auto_round.export.register import register_formatimport threadpoolctl as tctlimport inspect-+from tqdm import tqdm+from concurrent.futures import ThreadPoolExecutordef get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):"""@@ -99,6 +100,58 @@ def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):return QuantLinear+def pack_layer(name, model, layer_config, backend, pbar):+ with tctl.threadpool_limits(limits=1):+ pbar.set_description(f"packing {name}")+ if name == "lm_head": ##dese not support lm-head+ pbar.update(1)+ return+ config = layer_config[name]+ if config["bits"] > 8:+ pbar.update(1)+ return+ bits = config["bits"]+ group_size = config["group_size"]+ sym = config["sym"]++ layer = get_module(model, name)+ device = layer.weight.device++ QuantLinear = get_autogptq_packing_qlinear(backend, bits, group_size, sym)++ if isinstance(layer, nn.Linear):+ in_features = layer.in_features+ out_features = layer.out_features+ elif isinstance(layer, nn.Conv2d):+ in_features = layer.in_channels+ out_features = layer.out_channels+ elif isinstance(layer, transformers.pytorch_utils.Conv1D):+ in_features = layer.weight.shape[0]+ out_features = layer.weight.shape[1]++ ##bias = layer.bias is not None and torch.any(layer.bias)+ bias = True ## if using the above, llama3 lambada RTN will be NAN , TODO why?+ new_layer = QuantLinear( ##pylint: disable=E1123+ bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype+ )++ new_layer.device = device+ set_module(model, name, new_layer)+ qlayer = new_layer+ scale = layer_config[name]["scale"]+ zero = layer_config[name]["zp"]+ # so far can only pack layer on CPU+ qlayer.to("cpu")+ ##force to float32 to be compatible with torch 2.0+ layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)+ sig = inspect.signature(qlayer.pack)+ param_count = len(sig.parameters)+ if param_count == 2:+ qlayer.pack(layer, scale)+ else:+ qlayer.pack(layer, scale, zero, None)+ qlayer.to(device)+ pbar.update(1)@register_format("auto_gptq")def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllamav2",**kwargs):@@ -141,57 +194,14 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllmodel = copy.deepcopy(model.to("cpu"))layer_config = kwargs["layer_config"]-- with tctl.threadpool_limits(limits=1):- for name in layer_config.keys():- if name == "lm_head": ##dese not support lm-head- continue- config = kwargs["layer_config"][name]- if config["bits"] > 8:- continue- logger.info(f"packing {name}")-- bits = config["bits"]- group_size = config["group_size"]- sym = config["sym"]-- layer = get_module(model, name)- device = layer.weight.device-- QuantLinear = get_autogptq_packing_qlinear(backend, bits, group_size, sym)-- if isinstance(layer, nn.Linear):- in_features = layer.in_features- out_features = layer.out_features- elif isinstance(layer, nn.Conv2d):- in_features = layer.in_channels- out_features = layer.out_channels- elif isinstance(layer, transformers.pytorch_utils.Conv1D):- in_features = layer.weight.shape[0]- out_features = layer.weight.shape[1]-- ##bias = layer.bias is not None and torch.any(layer.bias)- bias = True ## if using the above, llama3 lambada RTN will be NAN , TODO why?- new_layer = QuantLinear( ##pylint: disable=E1123- bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype- )-- new_layer.device = device- set_module(model, name, new_layer)- qlayer = new_layer- scale = layer_config[name]["scale"]- zero = layer_config[name]["zp"]- # so far can only pack layer on CPU- qlayer.to("cpu")- ##force to float32 to be compatible with torch 2.0- layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)- sig = inspect.signature(qlayer.pack)- param_count = len(sig.parameters)- if param_count == 2:- qlayer.pack(layer, scale)- else:- qlayer.pack(layer, scale, zero, None)- qlayer.to(device)+ names = list(layer_config.keys())+ with ThreadPoolExecutor(max_workers=2) as executor:+ with tqdm(total=len(names), leave=True) as pbar:+ def wrapper(name):+ pack_layer(name, model, layer_config, backend, pbar)++ for _ in executor.map(wrapper, names):+ passif output_dir is None:return modelquantization_config = kwargs["serialization_dict"]diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 7a0061d17..f47a8b9b3 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -25,6 +25,8 @@ from auto_round.export.register import register_formatfrom auto_round.utils import get_layer_names_in_block, get_module, logger, set_moduleimport threadpoolctl as tctlimport inspect+from tqdm import tqdm+from concurrent.futures import ThreadPoolExecutordef check_neq_config(config, data_type, bits, group_size, sym):@@ -41,11 +43,11 @@ def check_neq_config(config, data_type, bits, group_size, sym):Returns:list: A list of strings indicating which configuration parameters do not match."""- expected_config = {"data_type": data_type,- "bits": bits,- "group_size": group_size,- "sym": sym- }+ expected_config = {"data_type": data_type,+ "bits": bits,+ "group_size": group_size,+ "sym": sym+ }return [key for key, expected_value in expected_config.items() if config.get(key) != expected_value]@@ -85,7 +87,6 @@ def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):disable_exllamav2 = Truedisable_exllamav1 = True-from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401QuantLinear = dynamically_import_QuantLinear(use_triton=use_triton,@@ -121,7 +122,7 @@ def dynamic_import_quantLinear_for_packing(backend, bits, group_size, sym):if not ("triton" in backend or "exllamav2" in backend):logger.warning_once(f"autoround format does not support {backend}, try to pack with autogptq")return get_autogptq_packing_qlinear(backend, bits, group_size, sym)- from auto_round_extension.cuda.qliner_triton import QuantLinear+ from auto_round_extension.cuda.qlinear_triton import QuantLinearreturn QuantLinearelif "awq" in backend:try:@@ -136,6 +137,75 @@ def dynamic_import_quantLinear_for_packing(backend, bits, group_size, sym):assert False, f"only support gptq and autoround backend"+def pack_layer(name, model, layer_config, backend, pbar):+ with tctl.threadpool_limits(limits=1):+ pbar.set_description(f"packing {name}")+ config = layer_config[name]+ if config["bits"] > 8:+ pbar.update(1)+ return++ bits = config["bits"]+ group_size = config["group_size"]+ sym = config["sym"]++ layer = get_module(model, name)+ device = layer.weight.device++ QuantLinear = dynamic_import_quantLinear_for_packing(backend, bits, group_size, sym)++ if isinstance(layer, nn.Linear):+ in_features = layer.in_features+ out_features = layer.out_features+ elif isinstance(layer, nn.Conv2d):+ in_features = layer.in_channels+ out_features = layer.out_channels+ elif isinstance(layer, transformers.pytorch_utils.Conv1D):+ in_features = layer.weight.shape[0]+ out_features = layer.weight.shape[1]+ bias = layer.bias is not None and torch.any(layer.bias)++ if "awq" not in backend:+ new_layer = QuantLinear( ##pylint: disable=E1123+ bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype+ )+ new_layer.device = device+ set_module(model, name, new_layer)+ qlayer = new_layer+ scale = layer_config[name]["scale"]+ zero = layer_config[name]["zp"]+ # so far can only pack layer on CPU+ qlayer.to("cpu")+ ##force to float32 to be compatible with torch 2.0+ layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)+ sig = inspect.signature(qlayer.pack)+ param_count = len(sig.parameters)+ if param_count == 2:+ qlayer.pack(layer, scale)+ else:+ qlayer.pack(layer, scale, zero, None)+ qlayer.to(device)+ else:+ from awq.utils.utils import clear_memory # pylint: disable=E0401+ scale, zp = layer_config[name]["scale"].to(torch.float32), layer_config[name]["zp"].to(torch.float32)+ scale = scale.t().contiguous()+ zp = zp.t().contiguous()+ if bits != 4:+ logger.error("AutoAWQ format only supports 4-bits quantization.")+ qlayer = QuantLinear.from_linear(+ linear=layer,+ w_bit=bits,+ group_size=group_size,+ init_only=False,+ scales=scale,+ zeros=zp,+ )+ qlayer.to(device)+ set_module(model, name, qlayer)+ clear_memory()+ pbar.update(1)++@register_format("auto_round")def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exllamav2", **kwargs):"""@@ -169,7 +239,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exmodel = kwargs["model"]quant_block_list = kwargs["quant_block_list"]- safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]+ safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]if not inplace:model = copy.deepcopy(model.to("cpu"))layer_names_in_block = get_layer_names_in_block(model, quant_block_list=quant_block_list)@@ -201,84 +271,28 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exextra_config[layer_name][key] = layer_config[layer_name][key]if len(extra_config) > 0:quantization_config["extra_config"] = extra_config- with tctl.threadpool_limits(limits=1):- modules_to_not_convert = []- for name in layer_config.keys():- config = kwargs["layer_config"][name]- if config["bits"] > 8:- if "awq" in backend:- modules_to_not_convert.append(name)- continue- logger.info(f"packing {name}")-- bits = config["bits"]- group_size = config["group_size"]- sym = config["sym"]-- layer = get_module(model, name)- device = layer.weight.device-- QuantLinear = dynamic_import_quantLinear_for_packing(backend, bits, group_size, sym)-- if isinstance(layer, nn.Linear):- in_features = layer.in_features- out_features = layer.out_features- elif isinstance(layer, nn.Conv2d):- in_features = layer.in_channels- out_features = layer.out_channels- elif isinstance(layer, transformers.pytorch_utils.Conv1D):- in_features = layer.weight.shape[0]- out_features = layer.weight.shape[1]- bias = layer.bias is not None and torch.any(layer.bias)-- if "awq" not in backend:- new_layer = QuantLinear( ##pylint: disable=E1123- bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype- )- new_layer.device = device- set_module(model, name, new_layer)- qlayer = new_layer- scale = layer_config[name]["scale"]- zero = layer_config[name]["zp"]- # so far can only pack layer on CPU- qlayer.to("cpu")- ##force to float32 to be compatible with torch 2.0- layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)- sig = inspect.signature(qlayer.pack)- param_count = len(sig.parameters)- if param_count == 2:- qlayer.pack(layer, scale)- else:- qlayer.pack(layer, scale, zero, None)- qlayer.to(device)- else:- from awq.utils.utils import clear_memory # pylint: disable=E0401- scale, zp = layer_config[name]["scale"].to(torch.float32), layer_config[name]["zp"].to(torch.float32)- scale = scale.t().contiguous()- zp = zp.t().contiguous()- if bits != 4:- logger.error("AutoAWQ format only supports 4-bits quantization.")- qlayer = QuantLinear.from_linear(- linear=layer,- w_bit=bits,- group_size=group_size,- init_only=False,- scales=scale,- zeros=zp,- )- qlayer.to(device)- set_module(model, name, qlayer)- clear_memory()+ names = list(layer_config.keys())+ with ThreadPoolExecutor(max_workers=2) as executor:+ with tqdm(total=len(names), leave=True) as pbar:+ def wrapper(name):+ pack_layer(name, model, layer_config, backend, pbar)+ for _ in executor.map(wrapper, names):+ passif hasattr(model, "config"):model.config.quantization_config = quantization_configtokenizer = kwargs["tokenizer"]if tokenizer is not None:tokenizer.save_pretrained(output_dir)+ modules_to_not_convert = []if "awq" not in backend:save(model, output_dir, safe_serialization=safe_serialization)else:+ for name in layer_config.keys():+ config = kwargs["layer_config"][name]+ if config["bits"] > 8:+ modules_to_not_convert.append(name)save_awq(model, output_dir, modules_to_not_convert=modules_to_not_convert)@@ -310,13 +324,12 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serijson.dump(model.config.quantization_config, f, indent=2)-def save_awq(- model: nn.Module,- save_dir: str,- max_shard_size: str = "5GB",- safe_serialization: bool = True,- modules_to_not_convert: list = [],+ model: nn.Module,+ save_dir: str,+ max_shard_size: str = "5GB",+ safe_serialization: bool = True,+ modules_to_not_convert: list = [],):"""Save model state dict and configs.@@ -347,5 +360,3 @@ def save_awq(if hasattr(model, "config") and hasattr(model.config, "quantization_config"):with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:json.dump(quantization_config, f, indent=2)--diff --git a/auto_round/utils.py b/auto_round/utils.pyindex bd54cdf45..f123d3b87 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -342,43 +342,6 @@ def collect_best_params(block):params[n][key]=copy.deepcopy(m.params[key].data)return params--# def collect_round_v(block):-# """Collects the round values for wrapped linear modules in the given block.-#-# Args:-# block: The input block.-#-# Returns:-# vs: A dictionary of round values for the wrapped linear modules.-# """-# vs = {}-# for n, m in block.named_modules():-# if hasattr(m, "orig_layer"):-# v = m.value.data-# vs[n] = copy.deepcopy(v)-# return vs-#-#-# def collect_minmax_scale(block):-# """Collects the min-max scaling values for wrapped linear modules in the given block.-#-# Args:-# block: The input block.-#-# Returns:-# min_scales: A dictionary of minimum scaling values.-# max_scales: A dictionary of maximum scaling values.-# """-# min_scales = {}-# max_scales = {}-# for n, m in block.named_modules():-# if hasattr(m, "orig_layer"):-# min_scales[n] = copy.deepcopy(torch.clamp(m.min_scale.data, 0, 1.0))-# max_scales[n] = copy.deepcopy(torch.clamp(m.max_scale.data, 0, 1.0))-# return min_scales, max_scales--@torch.no_grad()def sampling_inputs(input_ids, input_others, indices, seqlen,share_attention_mask_flag=False, not_share_position_ids_flag=False, input_dim=0):@@ -834,11 +797,11 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):support auto_awq format.")return WQLinear_GEMMif bits == 4 and exllama2_available and "exllamav2" in backend:- from auto_round_extension.cuda.qliner_exllamav2 import QuantLinear+ from auto_round_extension.cuda.qlinear_exllamav2 import QuantLinearelif bits == 4 and "exllamav2" in backend:logger.warning_once("Please install auto-round from source to enable exllamav2 kernels, switch to triton ""kernels for now")- from auto_round_extension.cuda.qliner_triton import QuantLinear+ from auto_round_extension.cuda.qlinear_tritonv2 import QuantLinearelse:- from auto_round_extension.cuda.qliner_triton import QuantLinear+ from auto_round_extension.cuda.qlinear_tritonv2 import QuantLinearreturn QuantLineardiff --git a/auto_round_extension/cuda/post_init.py b/auto_round_extension/cuda/post_init.pyindex 4b74596a5..6bad13639 100644--- a/auto_round_extension/cuda/post_init.py+++ b/auto_round_extension/cuda/post_init.py@@ -139,7 +139,7 @@ def autoround_post_init(model):submodule.post_init()if model_uses_exllamav2:- from auto_round_extension.cuda.qliner_exllamav2 import ExLlamaV2DeviceTensors+ from auto_round_extension.cuda.qlinear_exllamav2 import ExLlamaV2DeviceTensorsdevice_tensors = {}for device, scratch_bytes in fixed_bytes.items():diff --git a/auto_round_extension/cuda/qliner_exllamav2.py b/auto_round_extension/cuda/qlinear_exllamav2.pysimilarity index 100%rename from auto_round_extension/cuda/qliner_exllamav2.pyrename to auto_round_extension/cuda/qlinear_exllamav2.pydiff --git a/auto_round_extension/cuda/qliner_triton.py b/auto_round_extension/cuda/qlinear_triton.pysimilarity index 100%rename from auto_round_extension/cuda/qliner_triton.pyrename to auto_round_extension/cuda/qlinear_triton.pydiff --git a/auto_round_extension/cuda/qlinear_tritonv2.py b/auto_round_extension/cuda/qlinear_tritonv2.pynew file mode 100644index 000000000..9bc86bb40--- /dev/null+++ b/auto_round_extension/cuda/qlinear_tritonv2.py@@ -0,0 +1,226 @@+import math+from logging import getLogger++import numpy as np+import torch+import torch.nn as nn+import transformers+++from auto_round_extension.cuda.triton_utils.mixin import TritonModuleMixin+++logger = getLogger(__name__)++try:+ from auto_round_extension.cuda.triton_utils.dequant import QuantLinearFunction, quant_matmul_248+except ImportError as e:+ triton_import_exception = e++ def error_raiser_triton(*args, **kwargs):+ raise ValueError(+ f"Trying to use the triton backend, but could not import triton dependencies with the following error: {triton_import_exception}"+ )++ class FakeTriton:+ def __getattr__(self, name):+ raise ImportError(+ f"Trying to use the triton backend, but could not import triton dependencies with the following error: {triton_import_exception}"+ )++ quant_matmul_248 = error_raiser_triton+ QuantLinearFunction = FakeTriton+ QuantLinearInferenceOnlyFunction = FakeTriton+++class QuantLinear(nn.Module, TritonModuleMixin):+ """+ Triton v2 quantized linear layer.++ Calls dequant kernel (see triton_utils/dequant) to dequantize the weights then uses+ torch.matmul to compute the output whereas original `triton` quantized linear layer fused+ dequant and matmul into single kernel.add()+ """++ QUANT_TYPE = "tritonv2"++ def __init__(+ self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs+ ):+ super().__init__()+ if bits not in [2, 4, 8]:+ raise NotImplementedError("Only 2,4,8 bits are supported.")+ if infeatures % 32 != 0 or outfeatures % 32 != 0:+ raise NotImplementedError(+ "in_feature and out_feature must be divisible by 32."+ )+ self.infeatures = infeatures+ self.outfeatures = outfeatures+ self.bits = bits+ self.group_size = group_size if group_size != -1 else infeatures+ self.maxq = 2**self.bits - 1++ self.register_buffer(+ "qweight",+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),+ )+ self.register_buffer(+ "qzeros",+ torch.zeros(+ (+ math.ceil(infeatures / self.group_size),+ outfeatures // 32 * self.bits,+ ),+ dtype=torch.int32,+ ),+ )+ self.register_buffer(+ "scales",+ torch.zeros(+ (math.ceil(infeatures / self.group_size), outfeatures),+ dtype=torch.float16,+ ),+ )+ self.register_buffer(+ "g_idx",+ torch.tensor(+ [i // self.group_size for i in range(infeatures)], dtype=torch.int32+ ),+ )+ if bias:+ self.register_buffer(+ "bias", torch.zeros((outfeatures), dtype=torch.float16)+ )+ else:+ self.bias = None++ self.trainable = trainable++ def post_init(self):+ pass++ def pack(self, linear, scales, zeros, g_idx=None):+ W = linear.weight.data.clone()+ if isinstance(linear, nn.Conv2d):+ W = W.flatten(1)+ if isinstance(linear, transformers.pytorch_utils.Conv1D):+ W = W.t()++ self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx++ scales = scales.t().contiguous()+ zeros = zeros.t().contiguous()+ scale_zeros = zeros * scales+ self.scales = scales.clone().half()+ if linear.bias is not None:+ self.bias = linear.bias.clone().half()++ intweight = []+ for idx in range(self.infeatures):+ intweight.append(+ torch.round(+ (W[:, idx] + scale_zeros[self.g_idx[idx]])+ / self.scales[self.g_idx[idx]]+ ).to(torch.int)[:, None]+ )+ intweight = torch.cat(intweight, dim=1)+ intweight = intweight.t().contiguous()+ intweight = intweight.numpy().astype(np.uint32)++ i = 0+ row = 0+ qweight = np.zeros(+ (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32+ )+ while row < qweight.shape[0]:+ if self.bits in [2, 4, 8]:+ for j in range(i, i + (32 // self.bits)):+ qweight[row] |= intweight[j] << (self.bits * (j - i))+ i += 32 // self.bits+ row += 1+ else:+ raise NotImplementedError("Only 2,4,8 bits are supported.")++ qweight = qweight.astype(np.int32)+ self.qweight = torch.from_numpy(qweight)++ zeros = zeros.numpy().astype(np.uint32)+ qzeros = np.zeros(+ (zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32+ )+ i = 0+ col = 0+ while col < qzeros.shape[1]:+ if self.bits in [2, 4, 8]:+ for j in range(i, i + (32 // self.bits)):+ qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))+ i += 32 // self.bits+ col += 1+ else:+ raise NotImplementedError("Only 2,4,8 bits are supported.")++ qzeros = qzeros.astype(np.int32)+ self.qzeros = torch.from_numpy(qzeros)++ def forward(self, x):+ out_shape = x.shape[:-1] + (self.outfeatures,)+ quant_linear_fn = QuantLinearFunction++ out = quant_linear_fn.apply(+ x.reshape(-1, x.shape[-1]),+ self.qweight,+ self.scales,+ self.qzeros,+ self.g_idx,+ self.bits,+ self.maxq,+ )+ out = out.half().reshape(out_shape)+ out = out + self.bias if self.bias is not None else out+ return out++ @classmethod+ def warmup(cls, model, transpose=False, seqlen=2048):+ """+ Pre-tunes the quantized kernel+ """+ from tqdm import tqdm++ kn_values = {}++ for _, m in model.named_modules():+ if not isinstance(m, cls):+ continue++ k = m.infeatures+ n = m.outfeatures++ if (k, n) not in kn_values:+ kn_values[(k, n)] = (+ m.qweight,+ m.scales,+ m.qzeros,+ m.g_idx,+ m.bits,+ m.maxq,+ )++ logger.info(f"Found {len(kn_values)} unique KN Linear values.")+ logger.info("Warming up autotune cache ...")+ with torch.no_grad():+ for m in tqdm(range(0, math.ceil(math.log2(seqlen)) + 1)):+ m = 2**m+ for (k, n), (+ qweight,+ scales,+ qzeros,+ g_idx,+ bits,+ maxq,+ ) in kn_values.items():+ a = torch.randn(m, k, dtype=torch.float16, device=model.device)+ quant_matmul_248(a, qweight, scales, qzeros, g_idx, bits, maxq)+ del kn_values+++__all__ = ["QuantLinear"]\ No newline at end of filediff --git a/requirements.txt b/requirements.txtindex 35bba235b..1edf3fd76 100644--- a/requirements.txt+++ b/requirements.txt@@ -7,4 +7,5 @@ transformerstritonnumpy < 2.0threadpoolctl-intel-extension-for-transformers\ No newline at end of file+intel-extension-for-transformers+tqdm\ No newline at end of file