Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refine inference step 2 (#498)

Wenhua Cheng37341f5ee2ac
diff --git a/README.md b/README.md
index 3868178fa..99e9a4ba3 100644
--- a/README.md
+++ b/README.md
@@ -73,7 +73,7 @@ pip install auto-round-lib
## Model Quantization
-### Basic Usage (Gaudi2/CPU/GPU)
+### Basic Usage (Gaudi2/CPU/XPU/GPU)
A user guide detailing the full list of supported arguments is provided by calling ```auto-round -h``` on the terminal.
Set the format you want in `format` and
@@ -268,7 +268,7 @@ autoround.save_quantized(output_dir, format='auto_round', inplace=True)
### Export Formats
**AutoRound Format**: This format is well-suited for CPU, HPU devices, 2 bits, as well as mixed-precision
-inference. **[2,4] bits are supported**. However, it has not yet gained widespread community adoption.
+inference. **[2,3,4,8] bits are supported**. However, it has not yet gained widespread community adoption.
**AutoGPTQ Format**: This format is well-suited for symmetric quantization on CUDA devices and is widely adopted by the
community, **[2,3,4,8] bits are supported**. However, **the
@@ -324,14 +324,9 @@ in [Gaudi Guide](https://docs.habana.ai/en/latest/).
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRoundConfig
-backend = "auto" ##cpu, hpu, cuda
-quantization_config = AutoRoundConfig(
- backend=backend
-)
quantized_model_path = "./tmp_autoround"
model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
- device_map=backend.split(':')[0],
- quantization_config=quantization_config)
+ device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
diff --git a/auto_round/__init__.py b/auto_round/__init__.py
index 8a2d74b4d..f492c5cf9 100644
--- a/auto_round/__init__.py
+++ b/auto_round/__init__.py
@@ -15,5 +15,14 @@ from .autoround import AutoRound, AutoRoundAdam, AutoRoundOPT
from .mllm import AutoRoundMLLM
from auto_round.utils import LazyImport
-from auto_round.inference.auto_quantizer import AutoHfQuantizer,AutoRoundConfig
+def __getattr__(name):
+ if name == 'AutoHfQuantizer':
+ from auto_round.inference.auto_quantizer import AutoHfQuantizer
+ return AutoHfQuantizer
+ if name == 'AutoRoundConfig':
+ from auto_round.inference.auto_quantizer import AutoRoundConfig
+ return AutoRoundConfig
+
+ raise AttributeError(f"auto-round has no attribute '{name}'")
+
from .version import __version__
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 47f97dcf3..a80392302 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -475,8 +475,8 @@ class AutoRound(object):
for index in range(len(formats)):
format = formats[index]
if "auto_round" in format:
- if self.sym and ("gptq" not in format and "awq" not in format):
- format = format.replace('auto_round', 'auto_round:gptq')
+ if (self.sym and ("gptq" not in format and "awq" not in format)) or self.bits==3:
+ format = format.replace('auto_round', 'auto_round:auto_gptq')
formats[index] = format
# Remove duplicates from formats list
@@ -496,6 +496,13 @@ class AutoRound(object):
# Save the quantized model in the specified formats
folders = []
for format in formats:
+ if "gptq" in format and not self.sym:
+ logger.warning(
+ "The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"
+ " particularly for 2-bit quantization and smaller models."
+ " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "
+ "the AutoRound format(2/4/8 bits)."
+ )
save_format_ = format.replace(":", "-").replace("_", "-")
save_folder = os.path.join(output_dir, save_format_) if len(formats) > 1 else output_dir
self.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)
@@ -1598,8 +1605,8 @@ class AutoRound(object):
logger.warning(
"The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"
" particularly for 2-bit quantization and smaller models."
- " We recommend exporting to either the AutoAWQ format (4 bits) or "
- "the AutoRound format (2 bits) to enhance performance."
+ " We recommend exporting to either the AutoAWQ format ( only 4 bits) or "
+ "the AutoRound format(2/4/8 bits)."
)
if "awq" in format and not self.bits == 4:
raise ValueError("The AWQ format only supports W4 quantization ")
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 9c68a96d6..5c86d1fd7 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -23,7 +23,7 @@ import transformers
import auto_round.export.export_to_autoround.qlinear_triton_act
import auto_round_extension.cuda.qlinear_tritonv2
-from auto_round.utils import get_layer_names_in_block, get_module, logger, set_module, supported_layer_types
+from auto_round.utils import get_module, logger, set_module, supported_layer_types
import threadpoolctl as tctl
import inspect
from tqdm import tqdm
@@ -75,15 +75,15 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_
from auto_round_extension.cuda.qlinear_tritonv2 import QuantLinear
return QuantLinear
- elif "auto_round" in backend and "gptq" in backend:
- from auto_round.export.export_to_autoround.qlinear_triton import QuantLinear ##no g_idx
- return QuantLinear
+ elif "auto_round" in backend and "gptq" in backend and bits in (2, 4, 8):
+ from auto_round.export.export_to_autoround.qlinear_triton import QuantLinear ##no g_idx
+ return QuantLinear
elif "awq" in backend:
from ..export_to_awq.utils import WQLinear_GEMM
return WQLinear_GEMM
elif "gptqmodel" in backend:
return auto_round_extension.cuda.qlinear_tritonv2.QuantLinear
- elif "gptq" in backend and not "gptqmodel" in backend: ## have g_idx
+ elif "gptq" in backend and not "gptqmodel" in backend: ## have g_idx
return get_autogptq_packing_qlinear(backend, bits, group_size, sym)
else:
assert False, f"only support auto_gptq, auto_awq and auto_round backend"
@@ -190,7 +190,9 @@ def pack_layer(layer_name, model, backend):
new_layer.device = device
set_module(model, layer_name, new_layer)
qlayer = new_layer
- if sym:
+ import auto_round.export.export_to_autoround.qlinear_triton
+ if sym and isinstance(QuantLinear, (auto_round.export.export_to_autoround.qlinear_triton.QuantLinear,
+ auto_round_extension.cuda.qlinear_tritonv2.QuantLinear)):
zp = int(zp.flatten()[0])
qlayer.to("cpu")
@@ -248,7 +250,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
##if using sym, we change to gptq sym kernel to avoid compiling from auto_round source
if (kwargs.get("sym") is None or kwargs.get("sym") == True) and ("gptq" not in backend and "awq" not in backend):
- backend = backend.replace('auto_round', 'auto_round:gptq')
+ backend = backend.replace('auto_round', 'auto_round:auto_gptq')
model = kwargs["model"]
safe_serialization = True if 'safe_serialization' not in kwargs.keys() else kwargs["safe_serialization"]
@@ -260,6 +262,9 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
quantization_config["quant_method"] = "intel/auto-round"
quantization_config["backend"] = backend
+ if quantization_config["bits"]==3:
+ backend = "auto_round:auto_gptq"
+
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
extra_config = {}
diff --git a/auto_round/inference/__init__.py b/auto_round/inference/__init__.py
index 2b414eb72..789a8cdc7 100644
--- a/auto_round/inference/__init__.py
+++ b/auto_round/inference/__init__.py
@@ -11,4 +11,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from auto_round.inference.convert_model import convert_hf_model, infer_target_device, post_init
diff --git a/auto_round/inference/auto_quantizer.py b/auto_round/inference/auto_quantizer.py
index 90fd551bd..8dc62efb3 100644
--- a/auto_round/inference/auto_quantizer.py
+++ b/auto_round/inference/auto_quantizer.py
@@ -155,26 +155,11 @@ class AutoHfQuantizer:
loading_attr_dict = quantization_config_from_args.get_loading_attributes() \
if quantization_config_from_args is not None else None
if isinstance(quantization_config, dict):
- if "auto-round" in quantization_config["quant_method"]:
+ if "auto-round" in quantization_config[
+ "quant_method"] or quantization_config_from_args.__class__.__name__ == "AutoRoundConfig":
quantization_config = AutoRoundConfig.from_dict(quantization_config)
else:
- if isinstance(quantization_config_from_args, (AutoRoundConfig)):
- logger.info(f"Loading quantized model in auto_round format.")
- tmp_backend = quantization_config["quant_method"]
- if "auto-round" not in tmp_backend and "gptq" not in tmp_backend and "awq" not in tmp_backend:
- logger.error("could not convert to auto_round format, currently only supports `gptq`,`awq` or "
- "`auto-round` format")
- exit(-1)
- target_backend = quantization_config["backend"] if "backend" in quantization_config else "auto"
- if loading_attr_dict is not None and "backend" in loading_attr_dict:
- target_backend = loading_attr_dict["backend"]
- loading_attr_dict.pop("backend")
- if "auto_round" not in target_backend:
- target_backend = f"auto_round:{tmp_backend}" #
- quantization_config = AutoRoundConfig.from_dict(quantization_config)
- setattr(quantization_config, "backend", target_backend)
- else:
- quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101
+ quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101
if isinstance(quantization_config,
(GPTQConfig, AwqConfig, AutoRoundConfig)) and quantization_config_from_args is not None:
@@ -265,8 +250,8 @@ class AutoRoundConfig(QuantizationConfigMixin):
def post_init(self):
r"""Safety checker that arguments are correct."""
- if self.bits not in [2, 4, 8]:
- raise ValueError(f"Only support quantization to [2,4,8] bits but found {self.bits}")
+ if self.bits not in [2, 3, 4, 8]:
+ raise ValueError(f"Only support quantization to [2,3,4,8] bits but found {self.bits}")
if self.group_size != -1 and self.group_size <= 0:
raise ValueError("group_size must be greater than 0 or equal to -1")
@@ -278,6 +263,26 @@ class AutoRoundConfig(QuantizationConfigMixin):
config_dict = super().to_dict()
return config_dict
+ @classmethod
+ def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs):
+ quant_method = config_dict["quant_method"]
+ if "auto-round" not in quant_method and "gptq" not in quant_method and "awq" not in quant_method:
+ raise NotImplementedError(
+ "Failed to convert to auto_round format. Only `gptqv1`, `awq`, and `auto-round` formats are supported."
+ )
+
+ if "gptq" in quant_method and "meta" in config_dict:
+ raise NotImplementedError(
+ "Failed to convert gptq format to auto_round format. Only supports `gptqv1`")
+
+ if "awq" in quant_method and config_dict.get("version", "gemm") != "gemm":
+ raise NotImplementedError(
+ "Failed to convert awq format to auto_round format. Only supports awq format with gemm version")
+
+ if "auto-round" not in quant_method:
+ config_dict["backend"] = f"auto_round:{quant_method}"
+ return super().from_dict(config_dict, return_unused_kwargs=return_unused_kwargs, **kwargs)
+
class AutoRoundQuantizer(HfQuantizer):
"""Quantizer of the AutoRound method, currently only triton and exllamav2 backend has been supported."""
@@ -306,7 +311,6 @@ class AutoRoundQuantizer(HfQuantizer):
"auto-round` or install from source")
def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":
- self.target_device = infer_target_device(self.device_map)
if torch_dtype is None:
torch_dtype = torch.float16
elif torch_dtype != torch.float16:
@@ -330,8 +334,7 @@ class AutoRoundQuantizer(HfQuantizer):
def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs):
if self.pre_quantized:
- target_device = self.target_device if hasattr(self, self.target_device) else infer_target_device(
- self.device_map)
+ target_device = infer_target_device(self.device_map)
model, used_backends = convert_hf_model(model, target_device)
self.used_backends = used_backends
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index 2f2b27f05..4a4020426 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -126,7 +126,7 @@ BackendInfos['auto_round:tritonv2'] = BackendInfo(device=["cuda"], sym=[True, Fa
bits=[2, 4, 8],
priority=1, feature_checks=[feature_multiply_checker_32],
alias=["auto_round", "tritonv2"],
- requirements=["auto-round>=0.2"]
+ requirements=["auto-round>=0.5.0"]
)
BackendInfos['auto_round:tritonv2_zp'] = BackendInfo(device=["cuda"], sym=[True], ## asym has accuracy issue
@@ -135,7 +135,7 @@ BackendInfos['auto_round:tritonv2_zp'] = BackendInfo(device=["cuda"], sym=[True]
bits=[2, 4, 8],
priority=1, feature_checks=[feature_multiply_checker_32],
alias=["tritonv2", "tritonv2_zp"],
- requirements=["auto-round>=0.5"]
+ requirements=["auto-round>=0.5.0"]
)
BackendInfos['gptqmodel:marlin'] = BackendInfo(device=["cuda"], sym=[True],
@@ -145,7 +145,7 @@ BackendInfos['gptqmodel:marlin'] = BackendInfo(device=["cuda"], sym=[True],
dtype=["float16", "bfloat16"],
priority=6, feature_checks=[in_output_feature_multiply_checker_32],
alias=["marlin", "gptqmodel"],
- requirements=["gptqmodel>=2.0"]
+ requirements=["gptqmodel>=2.0"],
)
BackendInfos['gptqmodel:marlin_zp'] = BackendInfo(device=["cuda"], sym=[True],
@@ -504,7 +504,7 @@ def get_autogptq_infer_linear(backend, bits=4, group_size=128, sym=False):
return QuantLinear
-def find_backend(target_backend: str, orig_backend: str = None) -> str | None:
+def find_backend(target_backend: str, orig_backend: str = None):
"""
Finds the matching backend key based on the target backend name or its aliases.
@@ -620,7 +620,10 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_f
try:
require_version(requirement)
except ImportError:
- logger.error(f"pip install '{requirement}' ")
+ if "gptqmodel" in requirement:
+ logger.error(f"pip install -v '{requirement}' --no-build-isolation")
+ else:
+ logger.error(f"pip install '{requirement}' ")
else:
str_info = requirement()[1]
logger.error(str_info)
@@ -633,3 +636,63 @@ def get_layer_backend(device, backend, orig_backend, bits, group_size, sym, in_f
reverse=True)
return supported_backends[0]
+
+
+def get_highest_priority_backend(bits, sym, group_size, device, packing_format):
+ supported_backends = []
+ for key in BackendInfos.keys():
+ backend = BackendInfos[key]
+ # Check if device is supported by the backend
+ if device not in backend.device:
+ continue
+
+ # Check if bit-width is supported
+ if bits not in backend.bits:
+ continue
+
+ # Check if group_size is valid (if required by backend)
+ if backend.group_size is not None and group_size not in backend.group_size:
+ continue
+
+ # Check if symmetric/asymmetric quantization is supported
+ if sym not in backend.sym:
+ continue
+
+ # Check if the format is convertible when packing formats differ
+ if packing_format == backend.packing_format or packing_format in backend.convertable_format:
+ pass
+ else:
+ continue
+ supported_backends.append(key)
+
+ if len(supported_backends) > 0:
+
+ supported_backends = sorted(supported_backends,
+ key=lambda support_backend: BackendInfos[support_backend].priority,
+ reverse=True)
+ return supported_backends[0]
+ else:
+ return None
+
+
+def process_requirement(requirements: list):
+ gptqmodel_requirements = None
+ other_requirements = []
+ for requirement in requirements:
+ if "gptqmodel" in requirement:
+ gptqmodel_requirements = requirement
+ else:
+ other_requirements.append(requirement)
+
+ infos = []
+
+ if gptqmodel_requirements is not None:
+ infos.append(f"pip install -v '{gptqmodel_requirements}' --no-build-isolation")
+ infos.append(f"pip install 'numpy<2.0'")
+
+ other_info = f"pip install"
+ if len(other_requirements) > 0:
+ for requirement in other_requirements:
+ other_info += f" {requirement}"
+ infos.append(other_info)
+ return infos
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index ec3ed6c3f..5024d2f27 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -23,7 +23,8 @@ from transformers.pytorch_utils import Conv1D
from auto_round.utils import (get_module, set_module, is_hpu_supported, get_block_names,
find_matching_blocks, get_layer_names_in_block, check_to_quantized)
-from auto_round.inference.backend import get_layer_backend, dynamic_import_inference_linear, find_backend, BackendInfos
+from auto_round.inference.backend import (get_layer_backend, dynamic_import_inference_linear,
+ find_backend, BackendInfos, get_highest_priority_backend, process_requirement)
logger = getLogger(__name__)
@@ -355,7 +356,7 @@ def post_init(model, used_backends):
_import_exllamav2_kernels()
## convert datatype
- data_types=[]
+ data_types = []
for used_backend in used_backends:
backend = BackendInfos[used_backend]
data_types.append(backend.dtype)
@@ -364,10 +365,10 @@ def post_init(model, used_backends):
common &= set(l)
common = list(common)
if str(model.dtype).split('.')[-1] not in common:
- if common[0]=="float16":
+ if common[0] == "float16":
model = model.to(torch.float16)
logger.warning("force model to float16")
- elif common[0]=="bfloat16":
+ elif common[0] == "bfloat16":
model = model.to(torch.bfloat16)
logger.warning("force model to bfloat16")
@@ -392,8 +393,15 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):
If the quantization backend is not specified in the configuration.
"""
-
quantization_config = model.config.quantization_config
+
+ if hasattr(quantization_config, "desc_act") and quantization_config.desc_act == True:
+ ##check static_group
+ if (hasattr(quantization_config, "static_groups") and not quantization_config.static_groups) or (
+ not hasattr(quantization_config, "static_groups")):
+ raise NotImplementedError(
+ "This GPTQ model may contain a non-dummy g_idx, which is not yet supported by AutoRound")
+
if not hasattr(quantization_config, "target_backend"):
quantization_config.target_backend = quantization_config.backend
@@ -422,4 +430,17 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):
used_backends = _replace_by_quant_layers(model, layer_configs, target_backend, target_device, backend)
+ if target_backend == "auto" or target_backend == "":
+ best_backend = get_highest_priority_backend(quantization_config.bits, quantization_config.sym,
+ quantization_config.group_size, target_device,
+ BackendInfos[backend].packing_format)
+ if best_backend is not None and best_backend not in used_backends:
+ info = f"better backend is found, please install all the following requirements to enable it,"
+ extra_info = info
+ requirements = BackendInfos[best_backend].requirements
+ requirements_info = process_requirement(requirements)
+ for req in requirements_info:
+ extra_info += (f"`{req}`") + " "
+ logger.warning(extra_info)
+
return model, used_backends
diff --git a/auto_round_extension/cuda/qlinear_tritonv2.py b/auto_round_extension/cuda/qlinear_tritonv2.py
index 8cb4b05b6..655957e86 100644
--- a/auto_round_extension/cuda/qlinear_tritonv2.py
+++ b/auto_round_extension/cuda/qlinear_tritonv2.py
@@ -156,6 +156,10 @@ class QuantLinear(nn.Module, TritonModuleMixin):
def forward(self, x):
out_shape = x.shape[:-1] + (self.outfeatures,)
quant_linear_fn = QuantLinearFunction
+ if not hasattr(self,"g_idx"):
+ self.g_idx = torch.tensor(
+ [i // self.group_size for i in range(self.infeatures)], dtype=torch.int32
+ ).to(self.qweight.device)
out = quant_linear_fn.apply(
x.reshape(-1, x.shape[-1]),
diff --git a/auto_round_extension/cuda/qlinear_tritonv2_zp.py b/auto_round_extension/cuda/qlinear_tritonv2_zp.py
index 0436394ee..2dadfc0ae 100644
--- a/auto_round_extension/cuda/qlinear_tritonv2_zp.py
+++ b/auto_round_extension/cuda/qlinear_tritonv2_zp.py
@@ -81,12 +81,12 @@ class QuantLinear(nn.Module, TritonModuleMixin):
dtype=torch.float16,
),
)
- self.register_buffer(
- "g_idx",
- torch.tensor(
- [i // self.group_size for i in range(infeatures)], dtype=torch.int32
- ),
- )
+ # self.register_buffer(
+ # "g_idx",
+ # torch.tensor(
+ # [i // self.group_size for i in range(infeatures)], dtype=torch.int32
+ # ),
+ # )
if bias:
self.register_buffer(
"bias", torch.zeros((outfeatures), dtype=torch.float16)
@@ -98,74 +98,78 @@ class QuantLinear(nn.Module, TritonModuleMixin):
def post_init(self):
pass
-
- def pack(self, linear, scales, zeros, g_idx=None):
- W = linear.weight.data.clone()
- if isinstance(linear, nn.Conv2d):
- W = W.flatten(1)
- if isinstance(linear, transformers.pytorch_utils.Conv1D):
- W = W.t()
-
- self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
-
- scales = scales.t().contiguous()
- zeros = zeros.t().contiguous()
- scale_zeros = zeros * scales
- self.scales = scales.clone().half()
- if linear.bias is not None:
- self.bias = linear.bias.clone().half()
-
- intweight = []
- for idx in range(self.infeatures):
- intweight.append(
- torch.round(
- (W[:, idx] + scale_zeros[self.g_idx[idx]])
- / self.scales[self.g_idx[idx]]
- ).to(torch.int)[:, None]
- )
- intweight = torch.cat(intweight, dim=1)
- intweight = intweight.t().contiguous()
- intweight = intweight.numpy().astype(np.uint32)
-
- i = 0
- row = 0
- qweight = np.zeros(
- (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32
- )
- while row < qweight.shape[0]:
- if self.bits in [2, 4, 8]:
- for j in range(i, i + (32 // self.bits)):
- qweight[row] |= intweight[j] << (self.bits * (j - i))
- i += 32 // self.bits
- row += 1
- else:
- raise NotImplementedError("Only 2,4,8 bits are supported.")
-
- qweight = qweight.astype(np.int32)
- self.qweight = torch.from_numpy(qweight)
-
- zeros -= 1
- zeros = zeros.numpy().astype(np.uint32)
- qzeros = np.zeros(
- (zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32
- )
- i = 0
- col = 0
- while col < qzeros.shape[1]:
- if self.bits in [2, 4, 8]:
- for j in range(i, i + (32 // self.bits)):
- qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
- i += 32 // self.bits
- col += 1
- else:
- raise NotImplementedError("Only 2,4,8 bits are supported.")
-
- qzeros = qzeros.astype(np.int32)
- self.qzeros = torch.from_numpy(qzeros)
+ #
+ # def pack(self, linear, scales, zeros, g_idx=None):
+ # W = linear.weight.data.clone()
+ # if isinstance(linear, nn.Conv2d):
+ # W = W.flatten(1)
+ # if isinstance(linear, transformers.pytorch_utils.Conv1D):
+ # W = W.t()
+ #
+ # self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
+ #
+ # scales = scales.t().contiguous()
+ # zeros = zeros.t().contiguous()
+ # scale_zeros = zeros * scales
+ # self.scales = scales.clone().half()
+ # if linear.bias is not None:
+ # self.bias = linear.bias.clone().half()
+ #
+ # intweight = []
+ # for idx in range(self.infeatures):
+ # intweight.append(
+ # torch.round(
+ # (W[:, idx] + scale_zeros[self.g_idx[idx]])
+ # / self.scales[self.g_idx[idx]]
+ # ).to(torch.int)[:, None]
+ # )
+ # intweight = torch.cat(intweight, dim=1)
+ # intweight = intweight.t().contiguous()
+ # intweight = intweight.numpy().astype(np.uint32)
+ #
+ # i = 0
+ # row = 0
+ # qweight = np.zeros(
+ # (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32
+ # )
+ # while row < qweight.shape[0]:
+ # if self.bits in [2, 4, 8]:
+ # for j in range(i, i + (32 // self.bits)):
+ # qweight[row] |= intweight[j] << (self.bits * (j - i))
+ # i += 32 // self.bits
+ # row += 1
+ # else:
+ # raise NotImplementedError("Only 2,4,8 bits are supported.")
+ #
+ # qweight = qweight.astype(np.int32)
+ # self.qweight = torch.from_numpy(qweight)
+ #
+ # zeros -= 1
+ # zeros = zeros.numpy().astype(np.uint32)
+ # qzeros = np.zeros(
+ # (zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32
+ # )
+ # i = 0
+ # col = 0
+ # while col < qzeros.shape[1]:
+ # if self.bits in [2, 4, 8]:
+ # for j in range(i, i + (32 // self.bits)):
+ # qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
+ # i += 32 // self.bits
+ # col += 1
+ # else:
+ # raise NotImplementedError("Only 2,4,8 bits are supported.")
+ #
+ # qzeros = qzeros.astype(np.int32)
+ # self.qzeros = torch.from_numpy(qzeros)
def forward(self, x):
out_shape = x.shape[:-1] + (self.outfeatures,)
quant_linear_fn = QuantLinearFunction
+ if not hasattr(self, "g_idx"):
+ self.g_idx = torch.tensor(
+ [i // self.group_size for i in range(self.infeatures)], dtype=torch.int32
+ ).to(self.qweight.device)
out = quant_linear_fn.apply(
x.reshape(-1, x.shape[-1]),
diff --git a/test/test_generation.py b/test/test_generation.py
index 4f1dbb140..a4afc278f 100644
--- a/test/test_generation.py
+++ b/test/test_generation.py
@@ -23,9 +23,9 @@ class LLMDataLoader:
class TestAutoRoundFormatGeneration(unittest.TestCase):
@classmethod
def setUpClass(self):
- model_name = "facebook/opt-125m"
- self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
- self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ self.model_name = "facebook/opt-125m"
+ self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
self.llm_dataloader = LLMDataLoader()
self.save_folder = "./saved"
@@ -66,6 +66,7 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):
print(res)
assert ("!!!" not in res)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
def test_4bits_sym(self):
bits = 4
@@ -131,3 +132,63 @@ class TestAutoRoundFormatGeneration(unittest.TestCase):
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
print(res)
assert ("!!!" not in res)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+
+ def test_autoround_asym(self):
+ for bits in [2, 4, 8]:
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = bits, 128, False
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = "./saved"
+
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",
+ trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
+ print(res)
+ assert ("!!!" not in res)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+
+ def test_autoround_sym(self):
+ for bits in [2, 4, 8]:
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = bits, 128, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = "./saved"
+
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",
+ trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
+ print(res)
+ assert ("!!!" not in res)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+
diff --git a/test/test_load_awq_gptq.py b/test/test_load_awq_gptq.py
new file mode 100644
index 000000000..71e7e9a01
--- /dev/null
+++ b/test/test_load_awq_gptq.py
@@ -0,0 +1,93 @@
+import shutil
+import sys
+import unittest
+
+sys.path.insert(0, "..")
+
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRoundConfig
+
+
+class TestAutoRound(unittest.TestCase):
+ def model_infer(self, model, tokenizer):
+ prompts = [
+ "Hello,my name is",
+ # "The president of the United States is",
+ # "The capital of France is",
+ # "The future of AI is",
+ ]
+
+ inputs = tokenizer(prompts, return_tensors="pt", padding=False, truncation=True)
+
+ outputs = model.generate(
+ input_ids=inputs["input_ids"].to(model.device),
+ attention_mask=inputs["attention_mask"].to(model.device),
+ do_sample=False, ## change this to follow official usage
+ max_new_tokens=5
+ )
+ generated_ids = [
+ output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)
+ ]
+
+ decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
+
+ for i, prompt in enumerate(prompts):
+ print(f"Prompt: {prompt}")
+ print(f"Generated: {decoded_outputs[i]}")
+ print("-" * 50)
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_load_gptq_model_4bits(self):
+ model_name = "ybelkada/opt-125m-gptq-4bit"
+ quantization_config = AutoRoundConfig()
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,
+ device_map="cpu",
+ quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ self.model_infer(model, tokenizer)
+
+ def test_load_gptq_model_8bits(self):
+ model_name = "acloudfan/opt-125m-gptq-8bit"
+ quantization_config = AutoRoundConfig()
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,
+ device_map="cpu",
+ quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ self.model_infer(model, tokenizer)
+
+
+ def test_load_gptq_model_2bits(self):
+ model_name = "LucasSantiago257/gemma-2b-2bits-gptq"
+ quantization_config = AutoRoundConfig()
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,
+ device_map="cpu",
+ quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ self.model_infer(model, tokenizer)
+
+
+
+
+ def test_load_gptq_no_dummy_gidx_model(self):
+ model_name = "ModelCloud/Llama-3.2-1B-Instruct-gptqmodel-4bit-vortex-v1"
+ quantization_config = AutoRoundConfig()
+ with self.assertRaises(NotImplementedError) as cm:
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,
+ device_map="cpu",
+ quantization_config=quantization_config)
+
+ def test_load_awq(self):
+ model_name = "casperhansen/opt-125m-awq"
+ quantization_config = AutoRoundConfig()
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,
+ device_map="cpu",
+ quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ self.model_infer(model, tokenizer)
+
+
+
diff --git a/test_cuda/test_auto_round_format.py b/test_cuda/test_auto_round_format.py
index 3c90204b2..e2f0929d9 100644
--- a/test_cuda/test_auto_round_format.py
+++ b/test_cuda/test_auto_round_format.py
@@ -73,7 +73,6 @@ class TestAutoRound(unittest.TestCase):
@classmethod
def tearDownClass(self):
- return
shutil.rmtree(self.save_folder, ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
@@ -212,7 +211,7 @@ class TestAutoRound(unittest.TestCase):
device_map1["model.norm"] = "cuda"
device_map1["model.rotary_emb"] = "cuda"
device_map1["model.embed_tokens"] = "cuda"
- from auto_round import AutoRoundConfig
+ from auto_round import AutoRoundConfig
quantization_config = AutoRoundConfig(backend="tritonv2")
for tmp_device_map in [device_map1, 0, "balanced", "balanced_low_0", "sequential",
@@ -222,7 +221,7 @@ class TestAutoRound(unittest.TestCase):
model_name,
torch_dtype="auto",
device_map=tmp_device_map,
- quantization_config = quantization_config
+ quantization_config=quantization_config
)
tokenizer = AutoTokenizer.from_pretrained(model_name)
@@ -324,8 +323,6 @@ class TestAutoRound(unittest.TestCase):
torch.cuda.empty_cache()
-
-
def test_autoround_gptq_sym_format(self):
model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
@@ -401,7 +398,6 @@ class TestAutoRound(unittest.TestCase):
print(res)
assert ("!!!" not in res)
-
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", trust_remote_code=True,
torch_dtype=torch.bfloat16)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
@@ -413,6 +409,71 @@ class TestAutoRound(unittest.TestCase):
shutil.rmtree("./saved", ignore_errors=True)
+ def test_autoround_asym(self):
+ for bits in [2, 3, 4, 8]:
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = bits, 128, False
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = "./saved"
+
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",
+ trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
+ print(res)
+ assert ("!!!" not in res)
+
+ def test_autoround_sym(self):
+ for bits in [2, 3, 4, 8]:
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+ bits, group_size, sym = bits, 128, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = "./saved"
+
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto",
+ trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
+ print(res)
+ assert ("!!!" not in res)
+
+ def test_load_gptq_model_3bits(self):
+ model_name = "LucasSantiago257/gemma-2b-2bits-gptq"
+ quantization_config = AutoRoundConfig()
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True,
+ device_map="auto",
+ quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ self.model_infer(model, tokenizer)
+
if __name__ == "__main__":
unittest.main()