Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add tritonv2, improve packing and pbar (#239)

wenhuach21a5337c884536
diff --git a/README.md b/README.md
index 083fbe581..8a3f3a93e 100644
--- a/README.md
+++ b/README.md
@@ -137,33 +137,41 @@ install from the source.
**AutoGPTQ Format**: This format is well-suited for symmetric quantization on CUDA devices and is widely adopted by the
community. It also benefits from the Marlin kernel, which can boost inference performance notably. However, the
-asymmetric kernel has issues that can cause considerable accuracy drops, particularly at 2-bit quantization.
+asymmetric kernel has issues that can cause considerable accuracy drops, particularly at 2-bit quantization and small models.
Additionally, symmetric quantization tends to perform poorly at 2-bit precision.
**AutoAWQ format**: This format is well-suited for asymmetric 4-bit quantization on CUDA devices and is widely adopted
within the community. Asymmetric quantization typically improves accuracy but may reduce inference speed. It features
-specialized layer fusion tailored for Llama models. However, it supports only 4-bit asymmetric quantization and is not
-compatible with some models, such as Phi.
+specialized layer fusion tailored for Llama models. However, it supports only 4-bit asymmetric quantization. Currently, please manually install autoawq via `pip install autoawq` before exporting.
## Model inference
Please run the quantization code first
### AutoGPTQ/AutoAWQ format
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
-Refer to their repositories to inference the model.
-
+quantized_model_path = "./tmp_autoround"
+model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
+ device_map="auto")
+tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+text = "There is a girl who likes adventure,"
+inputs = tokenizer(text, return_tensors="pt").to(model.device)
+print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+```
### AutoRound format
-**cuda**: git clone https://github.com/intel/auto-round.git && cd auto-round && pip install -vvv --no-build-isolation
--e .
-
-**cpu**: no extra operations
+**CPU**: no extra operations
-**hpu**: docker image with Gaudi Software Stack is recommended. More details can be found
+**HPU**: docker image with Gaudi Software Stack is recommended. More details can be found
in [Gaudi Guide](https://docs.habana.ai/en/latest/).
-#### Gaudi2/ CPU/ GPU on 0.3.0+
+**CUDA**: git clone https://github.com/intel/auto-round.git && cd auto-round && pip install -vvv --no-build-isolation
+-e .
+
+
+#### CPU/HPU/CUDA on 0.3.0+
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -182,7 +190,7 @@ inputs = tokenizer(text, return_tensors="pt").to(model.device)
print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
```
-#### Gaudi2/ CPU/ GPU on 0.3.0
+#### CPU/HPU/CUDA on 0.3.0
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 9a232dc66..00cc0e308 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -21,7 +21,7 @@ import torch
import transformers
from transformers import set_seed
from torch import autocast
-
+from tqdm import tqdm
from .calib_dataset import get_dataloader
from .quantizer import WrapperMultiblock, wrapper_block, unwrapper_block, WrapperLinear, unwrapper_layer, \
WrapperTransformerConv1d
@@ -972,7 +972,7 @@ class AutoRound(object):
f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "
f"layers in the block, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
)
- logger.info(dump_info)
+ logger.debug(dump_info)
if len(unquantized_layer_names) != 0:
logger.info(f"{unquantized_layer_names} have not been quantized")
with torch.no_grad():
@@ -1041,15 +1041,15 @@ class AutoRound(object):
elif isinstance(input_others[key], list):
for i in range(len(input_others[key])):
input_others[key][i].to(tmp_dtype)
-
- for i in range(0, len(block_names), nblocks):
+ pbar = tqdm(range(0, len(block_names), nblocks))
+ for i in pbar:
if nblocks == 1:
n = block_names[i]
- logger.info(f"quantizing {i + 1}/{len(block_names)}, {n}")
+ pbar.set_description(f"Quantizing {n}")
m = get_module(model, n)
else:
names = block_names[i: i + nblocks]
- logger.info(names)
+ pbar.set_description(f"Quantizing [{i + 1}-{i + nblocks}]/{len(block_names)}")
modules = [get_module(model, n) for n in names]
m = WrapperMultiblock(modules)
@@ -1545,4 +1545,3 @@ class AutoAdamRound(AutoOPTRound):
optimizer,
**kwargs,
)
-
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index 3e18bf9b7..feb71bae1 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -48,7 +48,8 @@ import transformers
from auto_round.export.register import register_format
import threadpoolctl as tctl
import inspect
-
+from tqdm import tqdm
+from concurrent.futures import ThreadPoolExecutor
def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):
"""
@@ -99,6 +100,58 @@ def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):
return QuantLinear
+def pack_layer(name, model, layer_config, backend, pbar):
+ with tctl.threadpool_limits(limits=1):
+ pbar.set_description(f"packing {name}")
+ if name == "lm_head": ##dese not support lm-head
+ pbar.update(1)
+ return
+ config = layer_config[name]
+ if config["bits"] > 8:
+ pbar.update(1)
+ return
+ bits = config["bits"]
+ group_size = config["group_size"]
+ sym = config["sym"]
+
+ layer = get_module(model, name)
+ device = layer.weight.device
+
+ QuantLinear = get_autogptq_packing_qlinear(backend, bits, group_size, sym)
+
+ if isinstance(layer, nn.Linear):
+ in_features = layer.in_features
+ out_features = layer.out_features
+ elif isinstance(layer, nn.Conv2d):
+ in_features = layer.in_channels
+ out_features = layer.out_channels
+ elif isinstance(layer, transformers.pytorch_utils.Conv1D):
+ in_features = layer.weight.shape[0]
+ out_features = layer.weight.shape[1]
+
+ ##bias = layer.bias is not None and torch.any(layer.bias)
+ bias = True ## if using the above, llama3 lambada RTN will be NAN , TODO why?
+ new_layer = QuantLinear( ##pylint: disable=E1123
+ bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype
+ )
+
+ new_layer.device = device
+ set_module(model, name, new_layer)
+ qlayer = new_layer
+ scale = layer_config[name]["scale"]
+ zero = layer_config[name]["zp"]
+ # so far can only pack layer on CPU
+ qlayer.to("cpu")
+ ##force to float32 to be compatible with torch 2.0
+ layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)
+ sig = inspect.signature(qlayer.pack)
+ param_count = len(sig.parameters)
+ if param_count == 2:
+ qlayer.pack(layer, scale)
+ else:
+ qlayer.pack(layer, scale, zero, None)
+ qlayer.to(device)
+ pbar.update(1)
@register_format("auto_gptq")
def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllamav2",
**kwargs):
@@ -141,57 +194,14 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
model = copy.deepcopy(model.to("cpu"))
layer_config = kwargs["layer_config"]
-
- with tctl.threadpool_limits(limits=1):
- for name in layer_config.keys():
- if name == "lm_head": ##dese not support lm-head
- continue
- config = kwargs["layer_config"][name]
- if config["bits"] > 8:
- continue
- logger.info(f"packing {name}")
-
- bits = config["bits"]
- group_size = config["group_size"]
- sym = config["sym"]
-
- layer = get_module(model, name)
- device = layer.weight.device
-
- QuantLinear = get_autogptq_packing_qlinear(backend, bits, group_size, sym)
-
- if isinstance(layer, nn.Linear):
- in_features = layer.in_features
- out_features = layer.out_features
- elif isinstance(layer, nn.Conv2d):
- in_features = layer.in_channels
- out_features = layer.out_channels
- elif isinstance(layer, transformers.pytorch_utils.Conv1D):
- in_features = layer.weight.shape[0]
- out_features = layer.weight.shape[1]
-
- ##bias = layer.bias is not None and torch.any(layer.bias)
- bias = True ## if using the above, llama3 lambada RTN will be NAN , TODO why?
- new_layer = QuantLinear( ##pylint: disable=E1123
- bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype
- )
-
- new_layer.device = device
- set_module(model, name, new_layer)
- qlayer = new_layer
- scale = layer_config[name]["scale"]
- zero = layer_config[name]["zp"]
- # so far can only pack layer on CPU
- qlayer.to("cpu")
- ##force to float32 to be compatible with torch 2.0
- layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)
- sig = inspect.signature(qlayer.pack)
- param_count = len(sig.parameters)
- if param_count == 2:
- qlayer.pack(layer, scale)
- else:
- qlayer.pack(layer, scale, zero, None)
- qlayer.to(device)
+ names = list(layer_config.keys())
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ with tqdm(total=len(names), leave=True) as pbar:
+ def wrapper(name):
+ pack_layer(name, model, layer_config, backend, pbar)
+
+ for _ in executor.map(wrapper, names):
+ pass
if output_dir is None:
return model
quantization_config = kwargs["serialization_dict"]
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 7a0061d17..f47a8b9b3 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -25,6 +25,8 @@ from auto_round.export.register import register_format
from auto_round.utils import get_layer_names_in_block, get_module, logger, set_module
import threadpoolctl as tctl
import inspect
+from tqdm import tqdm
+from concurrent.futures import ThreadPoolExecutor
def check_neq_config(config, data_type, bits, group_size, sym):
@@ -41,11 +43,11 @@ def check_neq_config(config, data_type, bits, group_size, sym):
Returns:
list: A list of strings indicating which configuration parameters do not match.
"""
- expected_config = {"data_type": data_type,
- "bits": bits,
- "group_size": group_size,
- "sym": sym
- }
+ expected_config = {"data_type": data_type,
+ "bits": bits,
+ "group_size": group_size,
+ "sym": sym
+ }
return [key for key, expected_value in expected_config.items() if config.get(key) != expected_value]
@@ -85,7 +87,6 @@ def get_autogptq_packing_qlinear(backend, bits=4, group_size=128, sym=False):
disable_exllamav2 = True
disable_exllamav1 = True
-
from auto_gptq.utils.import_utils import dynamically_import_QuantLinear # pylint: disable=E0401
QuantLinear = dynamically_import_QuantLinear(
use_triton=use_triton,
@@ -121,7 +122,7 @@ def dynamic_import_quantLinear_for_packing(backend, bits, group_size, sym):
if not ("triton" in backend or "exllamav2" in backend):
logger.warning_once(f"autoround format does not support {backend}, try to pack with autogptq")
return get_autogptq_packing_qlinear(backend, bits, group_size, sym)
- from auto_round_extension.cuda.qliner_triton import QuantLinear
+ from auto_round_extension.cuda.qlinear_triton import QuantLinear
return QuantLinear
elif "awq" in backend:
try:
@@ -136,6 +137,75 @@ def dynamic_import_quantLinear_for_packing(backend, bits, group_size, sym):
assert False, f"only support gptq and autoround backend"
+def pack_layer(name, model, layer_config, backend, pbar):
+ with tctl.threadpool_limits(limits=1):
+ pbar.set_description(f"packing {name}")
+ config = layer_config[name]
+ if config["bits"] > 8:
+ pbar.update(1)
+ return
+
+ bits = config["bits"]
+ group_size = config["group_size"]
+ sym = config["sym"]
+
+ layer = get_module(model, name)
+ device = layer.weight.device
+
+ QuantLinear = dynamic_import_quantLinear_for_packing(backend, bits, group_size, sym)
+
+ if isinstance(layer, nn.Linear):
+ in_features = layer.in_features
+ out_features = layer.out_features
+ elif isinstance(layer, nn.Conv2d):
+ in_features = layer.in_channels
+ out_features = layer.out_channels
+ elif isinstance(layer, transformers.pytorch_utils.Conv1D):
+ in_features = layer.weight.shape[0]
+ out_features = layer.weight.shape[1]
+ bias = layer.bias is not None and torch.any(layer.bias)
+
+ if "awq" not in backend:
+ new_layer = QuantLinear( ##pylint: disable=E1123
+ bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype
+ )
+ new_layer.device = device
+ set_module(model, name, new_layer)
+ qlayer = new_layer
+ scale = layer_config[name]["scale"]
+ zero = layer_config[name]["zp"]
+ # so far can only pack layer on CPU
+ qlayer.to("cpu")
+ ##force to float32 to be compatible with torch 2.0
+ layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)
+ sig = inspect.signature(qlayer.pack)
+ param_count = len(sig.parameters)
+ if param_count == 2:
+ qlayer.pack(layer, scale)
+ else:
+ qlayer.pack(layer, scale, zero, None)
+ qlayer.to(device)
+ else:
+ from awq.utils.utils import clear_memory # pylint: disable=E0401
+ scale, zp = layer_config[name]["scale"].to(torch.float32), layer_config[name]["zp"].to(torch.float32)
+ scale = scale.t().contiguous()
+ zp = zp.t().contiguous()
+ if bits != 4:
+ logger.error("AutoAWQ format only supports 4-bits quantization.")
+ qlayer = QuantLinear.from_linear(
+ linear=layer,
+ w_bit=bits,
+ group_size=group_size,
+ init_only=False,
+ scales=scale,
+ zeros=zp,
+ )
+ qlayer.to(device)
+ set_module(model, name, qlayer)
+ clear_memory()
+ pbar.update(1)
+
+
@register_format("auto_round")
def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exllamav2", **kwargs):
"""
@@ -169,7 +239,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
model = kwargs["model"]
quant_block_list = kwargs["quant_block_list"]
- safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]
+ safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]
if not inplace:
model = copy.deepcopy(model.to("cpu"))
layer_names_in_block = get_layer_names_in_block(model, quant_block_list=quant_block_list)
@@ -201,84 +271,28 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
extra_config[layer_name][key] = layer_config[layer_name][key]
if len(extra_config) > 0:
quantization_config["extra_config"] = extra_config
- with tctl.threadpool_limits(limits=1):
- modules_to_not_convert = []
- for name in layer_config.keys():
- config = kwargs["layer_config"][name]
- if config["bits"] > 8:
- if "awq" in backend:
- modules_to_not_convert.append(name)
- continue
- logger.info(f"packing {name}")
-
- bits = config["bits"]
- group_size = config["group_size"]
- sym = config["sym"]
-
- layer = get_module(model, name)
- device = layer.weight.device
-
- QuantLinear = dynamic_import_quantLinear_for_packing(backend, bits, group_size, sym)
-
- if isinstance(layer, nn.Linear):
- in_features = layer.in_features
- out_features = layer.out_features
- elif isinstance(layer, nn.Conv2d):
- in_features = layer.in_channels
- out_features = layer.out_channels
- elif isinstance(layer, transformers.pytorch_utils.Conv1D):
- in_features = layer.weight.shape[0]
- out_features = layer.weight.shape[1]
- bias = layer.bias is not None and torch.any(layer.bias)
-
- if "awq" not in backend:
- new_layer = QuantLinear( ##pylint: disable=E1123
- bits, group_size, in_features, out_features, bias, weight_dtype=layer.weight.dtype
- )
- new_layer.device = device
- set_module(model, name, new_layer)
- qlayer = new_layer
- scale = layer_config[name]["scale"]
- zero = layer_config[name]["zp"]
- # so far can only pack layer on CPU
- qlayer.to("cpu")
- ##force to float32 to be compatible with torch 2.0
- layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)
- sig = inspect.signature(qlayer.pack)
- param_count = len(sig.parameters)
- if param_count == 2:
- qlayer.pack(layer, scale)
- else:
- qlayer.pack(layer, scale, zero, None)
- qlayer.to(device)
- else:
- from awq.utils.utils import clear_memory # pylint: disable=E0401
- scale, zp = layer_config[name]["scale"].to(torch.float32), layer_config[name]["zp"].to(torch.float32)
- scale = scale.t().contiguous()
- zp = zp.t().contiguous()
- if bits != 4:
- logger.error("AutoAWQ format only supports 4-bits quantization.")
- qlayer = QuantLinear.from_linear(
- linear=layer,
- w_bit=bits,
- group_size=group_size,
- init_only=False,
- scales=scale,
- zeros=zp,
- )
- qlayer.to(device)
- set_module(model, name, qlayer)
- clear_memory()
+ names = list(layer_config.keys())
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ with tqdm(total=len(names), leave=True) as pbar:
+ def wrapper(name):
+ pack_layer(name, model, layer_config, backend, pbar)
+ for _ in executor.map(wrapper, names):
+ pass
if hasattr(model, "config"):
model.config.quantization_config = quantization_config
tokenizer = kwargs["tokenizer"]
if tokenizer is not None:
tokenizer.save_pretrained(output_dir)
+ modules_to_not_convert = []
if "awq" not in backend:
save(model, output_dir, safe_serialization=safe_serialization)
else:
+ for name in layer_config.keys():
+ config = kwargs["layer_config"][name]
+ if config["bits"] > 8:
+ modules_to_not_convert.append(name)
save_awq(model, output_dir, modules_to_not_convert=modules_to_not_convert)
@@ -310,13 +324,12 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri
json.dump(model.config.quantization_config, f, indent=2)
-
def save_awq(
- model: nn.Module,
- save_dir: str,
- max_shard_size: str = "5GB",
- safe_serialization: bool = True,
- modules_to_not_convert: list = [],
+ model: nn.Module,
+ save_dir: str,
+ max_shard_size: str = "5GB",
+ safe_serialization: bool = True,
+ modules_to_not_convert: list = [],
):
"""Save model state dict and configs.
@@ -347,5 +360,3 @@ def save_awq(
if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
json.dump(quantization_config, f, indent=2)
-
-
diff --git a/auto_round/utils.py b/auto_round/utils.py
index bd54cdf45..f123d3b87 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -342,43 +342,6 @@ def collect_best_params(block):
params[n][key]=copy.deepcopy(m.params[key].data)
return params
-
-# def collect_round_v(block):
-# """Collects the round values for wrapped linear modules in the given block.
-#
-# Args:
-# block: The input block.
-#
-# Returns:
-# vs: A dictionary of round values for the wrapped linear modules.
-# """
-# vs = {}
-# for n, m in block.named_modules():
-# if hasattr(m, "orig_layer"):
-# v = m.value.data
-# vs[n] = copy.deepcopy(v)
-# return vs
-#
-#
-# def collect_minmax_scale(block):
-# """Collects the min-max scaling values for wrapped linear modules in the given block.
-#
-# Args:
-# block: The input block.
-#
-# Returns:
-# min_scales: A dictionary of minimum scaling values.
-# max_scales: A dictionary of maximum scaling values.
-# """
-# min_scales = {}
-# max_scales = {}
-# for n, m in block.named_modules():
-# if hasattr(m, "orig_layer"):
-# min_scales[n] = copy.deepcopy(torch.clamp(m.min_scale.data, 0, 1.0))
-# max_scales[n] = copy.deepcopy(torch.clamp(m.max_scale.data, 0, 1.0))
-# return min_scales, max_scales
-
-
@torch.no_grad()
def sampling_inputs(input_ids, input_others, indices, seqlen,
share_attention_mask_flag=False, not_share_position_ids_flag=False, input_dim=0):
@@ -834,11 +797,11 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):
support auto_awq format.")
return WQLinear_GEMM
if bits == 4 and exllama2_available and "exllamav2" in backend:
- from auto_round_extension.cuda.qliner_exllamav2 import QuantLinear
+ from auto_round_extension.cuda.qlinear_exllamav2 import QuantLinear
elif bits == 4 and "exllamav2" in backend:
logger.warning_once("Please install auto-round from source to enable exllamav2 kernels, switch to triton "
"kernels for now")
- from auto_round_extension.cuda.qliner_triton import QuantLinear
+ from auto_round_extension.cuda.qlinear_tritonv2 import QuantLinear
else:
- from auto_round_extension.cuda.qliner_triton import QuantLinear
+ from auto_round_extension.cuda.qlinear_tritonv2 import QuantLinear
return QuantLinear
diff --git a/auto_round_extension/cuda/post_init.py b/auto_round_extension/cuda/post_init.py
index 4b74596a5..6bad13639 100644
--- a/auto_round_extension/cuda/post_init.py
+++ b/auto_round_extension/cuda/post_init.py
@@ -139,7 +139,7 @@ def autoround_post_init(model):
submodule.post_init()
if model_uses_exllamav2:
- from auto_round_extension.cuda.qliner_exllamav2 import ExLlamaV2DeviceTensors
+ from auto_round_extension.cuda.qlinear_exllamav2 import ExLlamaV2DeviceTensors
device_tensors = {}
for device, scratch_bytes in fixed_bytes.items():
diff --git a/auto_round_extension/cuda/qliner_exllamav2.py b/auto_round_extension/cuda/qlinear_exllamav2.py
similarity index 100%
rename from auto_round_extension/cuda/qliner_exllamav2.py
rename to auto_round_extension/cuda/qlinear_exllamav2.py
diff --git a/auto_round_extension/cuda/qliner_triton.py b/auto_round_extension/cuda/qlinear_triton.py
similarity index 100%
rename from auto_round_extension/cuda/qliner_triton.py
rename to auto_round_extension/cuda/qlinear_triton.py
diff --git a/auto_round_extension/cuda/qlinear_tritonv2.py b/auto_round_extension/cuda/qlinear_tritonv2.py
new file mode 100644
index 000000000..9bc86bb40
--- /dev/null
+++ b/auto_round_extension/cuda/qlinear_tritonv2.py
@@ -0,0 +1,226 @@
+import math
+from logging import getLogger
+
+import numpy as np
+import torch
+import torch.nn as nn
+import transformers
+
+
+from auto_round_extension.cuda.triton_utils.mixin import TritonModuleMixin
+
+
+logger = getLogger(__name__)
+
+try:
+ from auto_round_extension.cuda.triton_utils.dequant import QuantLinearFunction, quant_matmul_248
+except ImportError as e:
+ triton_import_exception = e
+
+ def error_raiser_triton(*args, **kwargs):
+ raise ValueError(
+ f"Trying to use the triton backend, but could not import triton dependencies with the following error: {triton_import_exception}"
+ )
+
+ class FakeTriton:
+ def __getattr__(self, name):
+ raise ImportError(
+ f"Trying to use the triton backend, but could not import triton dependencies with the following error: {triton_import_exception}"
+ )
+
+ quant_matmul_248 = error_raiser_triton
+ QuantLinearFunction = FakeTriton
+ QuantLinearInferenceOnlyFunction = FakeTriton
+
+
+class QuantLinear(nn.Module, TritonModuleMixin):
+ """
+ Triton v2 quantized linear layer.
+
+ Calls dequant kernel (see triton_utils/dequant) to dequantize the weights then uses
+ torch.matmul to compute the output whereas original `triton` quantized linear layer fused
+ dequant and matmul into single kernel.add()
+ """
+
+ QUANT_TYPE = "tritonv2"
+
+ def __init__(
+ self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs
+ ):
+ super().__init__()
+ if bits not in [2, 4, 8]:
+ raise NotImplementedError("Only 2,4,8 bits are supported.")
+ if infeatures % 32 != 0 or outfeatures % 32 != 0:
+ raise NotImplementedError(
+ "in_feature and out_feature must be divisible by 32."
+ )
+ self.infeatures = infeatures
+ self.outfeatures = outfeatures
+ self.bits = bits
+ self.group_size = group_size if group_size != -1 else infeatures
+ self.maxq = 2**self.bits - 1
+
+ self.register_buffer(
+ "qweight",
+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),
+ )
+ self.register_buffer(
+ "qzeros",
+ torch.zeros(
+ (
+ math.ceil(infeatures / self.group_size),
+ outfeatures // 32 * self.bits,
+ ),
+ dtype=torch.int32,
+ ),
+ )
+ self.register_buffer(
+ "scales",
+ torch.zeros(
+ (math.ceil(infeatures / self.group_size), outfeatures),
+ dtype=torch.float16,
+ ),
+ )
+ self.register_buffer(
+ "g_idx",
+ torch.tensor(
+ [i // self.group_size for i in range(infeatures)], dtype=torch.int32
+ ),
+ )
+ if bias:
+ self.register_buffer(
+ "bias", torch.zeros((outfeatures), dtype=torch.float16)
+ )
+ else:
+ self.bias = None
+
+ self.trainable = trainable
+
+ def post_init(self):
+ pass
+
+ def pack(self, linear, scales, zeros, g_idx=None):
+ W = linear.weight.data.clone()
+ if isinstance(linear, nn.Conv2d):
+ W = W.flatten(1)
+ if isinstance(linear, transformers.pytorch_utils.Conv1D):
+ W = W.t()
+
+ self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
+
+ scales = scales.t().contiguous()
+ zeros = zeros.t().contiguous()
+ scale_zeros = zeros * scales
+ self.scales = scales.clone().half()
+ if linear.bias is not None:
+ self.bias = linear.bias.clone().half()
+
+ intweight = []
+ for idx in range(self.infeatures):
+ intweight.append(
+ torch.round(
+ (W[:, idx] + scale_zeros[self.g_idx[idx]])
+ / self.scales[self.g_idx[idx]]
+ ).to(torch.int)[:, None]
+ )
+ intweight = torch.cat(intweight, dim=1)
+ intweight = intweight.t().contiguous()
+ intweight = intweight.numpy().astype(np.uint32)
+
+ i = 0
+ row = 0
+ qweight = np.zeros(
+ (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32
+ )
+ while row < qweight.shape[0]:
+ if self.bits in [2, 4, 8]:
+ for j in range(i, i + (32 // self.bits)):
+ qweight[row] |= intweight[j] << (self.bits * (j - i))
+ i += 32 // self.bits
+ row += 1
+ else:
+ raise NotImplementedError("Only 2,4,8 bits are supported.")
+
+ qweight = qweight.astype(np.int32)
+ self.qweight = torch.from_numpy(qweight)
+
+ zeros = zeros.numpy().astype(np.uint32)
+ qzeros = np.zeros(
+ (zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32
+ )
+ i = 0
+ col = 0
+ while col < qzeros.shape[1]:
+ if self.bits in [2, 4, 8]:
+ for j in range(i, i + (32 // self.bits)):
+ qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
+ i += 32 // self.bits
+ col += 1
+ else:
+ raise NotImplementedError("Only 2,4,8 bits are supported.")
+
+ qzeros = qzeros.astype(np.int32)
+ self.qzeros = torch.from_numpy(qzeros)
+
+ def forward(self, x):
+ out_shape = x.shape[:-1] + (self.outfeatures,)
+ quant_linear_fn = QuantLinearFunction
+
+ out = quant_linear_fn.apply(
+ x.reshape(-1, x.shape[-1]),
+ self.qweight,
+ self.scales,
+ self.qzeros,
+ self.g_idx,
+ self.bits,
+ self.maxq,
+ )
+ out = out.half().reshape(out_shape)
+ out = out + self.bias if self.bias is not None else out
+ return out
+
+ @classmethod
+ def warmup(cls, model, transpose=False, seqlen=2048):
+ """
+ Pre-tunes the quantized kernel
+ """
+ from tqdm import tqdm
+
+ kn_values = {}
+
+ for _, m in model.named_modules():
+ if not isinstance(m, cls):
+ continue
+
+ k = m.infeatures
+ n = m.outfeatures
+
+ if (k, n) not in kn_values:
+ kn_values[(k, n)] = (
+ m.qweight,
+ m.scales,
+ m.qzeros,
+ m.g_idx,
+ m.bits,
+ m.maxq,
+ )
+
+ logger.info(f"Found {len(kn_values)} unique KN Linear values.")
+ logger.info("Warming up autotune cache ...")
+ with torch.no_grad():
+ for m in tqdm(range(0, math.ceil(math.log2(seqlen)) + 1)):
+ m = 2**m
+ for (k, n), (
+ qweight,
+ scales,
+ qzeros,
+ g_idx,
+ bits,
+ maxq,
+ ) in kn_values.items():
+ a = torch.randn(m, k, dtype=torch.float16, device=model.device)
+ quant_matmul_248(a, qweight, scales, qzeros, g_idx, bits, maxq)
+ del kn_values
+
+
+__all__ = ["QuantLinear"]
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 35bba235b..1edf3fd76 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,4 +7,5 @@ transformers
triton
numpy < 2.0
threadpoolctl
-intel-extension-for-transformers
\ No newline at end of file
+intel-extension-for-transformers
+tqdm
\ No newline at end of file