Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refine code and fix activation quantization eval regression (#762)

Wenhua Cheng305923bc808a
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index fed33df34..1df45bf74 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -19,7 +19,7 @@ import re
import sys
import time
import traceback
-from typing import Any, Union
+from typing import Any, Callable, Union
import accelerate
import torch
@@ -307,7 +307,7 @@ class AutoRound(object):
self.not_use_best_mse = not_use_best_mse
self.dynamic_max_gap = dynamic_max_gap
self.lr_scheduler = lr_scheduler
- self.optimizer = self.get_optimizer(None)
+ self.optimizer = self._get_optimizer(None)
self.super_bits = super_bits
self.super_group_size = super_group_size
self.disable_opt_rtn = disable_opt_rtn
@@ -345,7 +345,7 @@ class AutoRound(object):
self.device_map = device_map
self._set_device_map_in_blocks(self.device_map)
self.scale_dtype = convert_dtype_str2torch(scale_dtype)
- self.set_amp_dtype()
+ self._set_amp_dtype()
self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device
if self.act_bits <= 8 and self.amp_dtype == torch.float16:
logger.warning("Force to use bf16 to for quantization tuning when enabling activation quantization")
@@ -537,10 +537,11 @@ class AutoRound(object):
if self.nsamples < self.gradient_accumulate_steps * self.batch_size:
if self.batch_size > self.nsamples:
- logger.warning(
- f"Reset `batch_size` to {self.nsamples} as `nsamples`({self.nsamples})"
- f" is smaller than batch_size({self.batch_size})"
- )
+ if self.iters > 0: # GGUF should log this warning, but we don't know the format here
+ logger.warning(
+ f"Reset `batch_size` to {self.nsamples} as `nsamples`({self.nsamples})"
+ f" is smaller than batch_size({self.batch_size})"
+ )
self.batch_size = self.nsamples
if self.gradient_accumulate_steps > self.nsamples // self.batch_size:
self.gradient_accumulate_steps = self.nsamples // self.batch_size
@@ -751,7 +752,9 @@ class AutoRound(object):
return format
- def quantize_and_save(self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace=True, **kwargs):
+ def quantize_and_save(
+ self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace: bool = True, **kwargs
+ ) -> tuple[torch.nn.Module, dict[str, Any]]:
"""Quantizes the model and saves it in the specified format(s).
This function checks the validity of the requested format(s), quantizes
@@ -805,14 +808,14 @@ class AutoRound(object):
" We recommend exporting to either the AutoAWQ format ( only 4 bits) or "
"the AutoRound format(2/3/4/8 bits)."
)
- save_folder = self.get_save_folder_name(format)
+ save_folder = self._get_save_folder_name(format)
self.save_quantized(save_folder, format=format, inplace=inplace, **kwargs)
folders.append(save_folder)
return model, folders
- def get_save_folder_name(self, format_str: str) -> str:
+ def _get_save_folder_name(self, format_str: str) -> str:
"""Generates the save folder name based on the provided format string.
If there are multiple formats to handle, the function creates a subfolder
@@ -835,7 +838,7 @@ class AutoRound(object):
return self.orig_output_dir
@torch.inference_mode()
- def quantize_embedding_layer(self):
+ def _quantize_embedding_layer(self):
"""Quantizes embedding layers in the model according to the configuration.
This method iterates through all modules in the model, identifies embedding
@@ -879,7 +882,7 @@ class AutoRound(object):
except RuntimeError as e:
cuda_error_msg = traceback.format_exc()
try:
- logger.info("out of VRAM, falling back to CPU.")
+ logger.info("out of VRAM, falling back to CPU")
weight, scale, zp = quant_func(
module.weight.to("cpu"),
**{
@@ -912,7 +915,7 @@ class AutoRound(object):
return is_quantized
- def quant_rtn_with_imatrix(self, all_to_quantized_module_names: list[str]) -> None:
+ def _quant_rtn_with_imatrix(self, all_to_quantized_module_names: list[str]) -> None:
"""Performs RTN quantization using input activation statistics (imatrix).
This method accumulates per-channel second-moment activation statistics (imatrix)
@@ -926,7 +929,7 @@ class AutoRound(object):
Returns:
None
"""
- logger.info("start to compute imatrix for GGUF quantization.")
+ logger.info("start to compute imatrix for GGUF quantization")
# Load dataset
from auto_round.calib_dataset import get_dataloader
@@ -1015,7 +1018,7 @@ class AutoRound(object):
cnt = 1
for name in pbar:
pbar.set_description(f"Quantizing {name}")
- self.quantize_layer_via_rtn(name)
+ self._quantize_layer_via_rtn(name)
if cnt % clear_mem_freq == 0:
clear_memory()
cnt = 1
@@ -1030,7 +1033,7 @@ class AutoRound(object):
logger.warning("Out of VRAM, falling back to blockwise quantization. Accuracy may degrade.")
model = model.to("cpu")
clear_memory()
- self.quantize_via_rtn_blockwise(all_to_quantized_module_names)
+ self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
except RuntimeError as e:
cuda_error_msg = traceback.format_exc()
try:
@@ -1048,7 +1051,7 @@ class AutoRound(object):
orig_device = self.device
self.device = "cpu"
- self.quantize_via_rtn_blockwise(all_to_quantized_module_names)
+ self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
self.device = orig_device
except Exception as e:
logger.error(cuda_error_msg)
@@ -1058,11 +1061,7 @@ class AutoRound(object):
for hook in hooks:
hook.remove()
- # Move back to CPU and free memory
- model.to("cpu")
- clear_memory()
-
- def check_need_to_quantize_lm_head_embedding(self) -> bool:
+ def _check_need_to_quantize_lm_head_embedding(self) -> bool:
"""Checks if LM head and embedding layers need quantization for GGUF format.
This function inspects the current model's formats and determines whether
@@ -1142,7 +1141,7 @@ class AutoRound(object):
setattr(get_module(self.model, layer_name), "act_bits", act_bits)
setattr(get_module(self.model, layer_name), "scale_dtype", scale_dtype)
- def quantize_layer_via_rtn(self, name: str) -> None:
+ def _quantize_layer_via_rtn(self, name: str) -> None:
"""Quantizes a layer using RTN (Round-To-Nearest) if available.
This function attempts to quantize a layer by switching its data type to a
@@ -1218,7 +1217,7 @@ class AutoRound(object):
if has_gguf:
from auto_round.export.export_to_gguf.export import pack_gguf_layer
- output_dir = self.get_save_folder_name(self.formats[0])
+ output_dir = self._get_save_folder_name(self.formats[0])
model_type = ModelType.MMPROJ if self.vlm else ModelType.TEXT
pack_gguf_layer(
name,
@@ -1240,7 +1239,7 @@ class AutoRound(object):
set_module(self.model, name, m)
@torch.inference_mode()
- def quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:
+ def _quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:
"""Quantize all modules in the model using RTN (Round-To-Nearest) strategy.
If the target format includes GGUF with `k`, and optimized RTN is enabled,
@@ -1271,17 +1270,17 @@ class AutoRound(object):
has_gguf_k = any("gguf" in fmt and "k" in fmt for fmt in getattr(self, "formats", []))
- self.quantize_embedding_layer()
+ self._quantize_embedding_layer()
self.model.to("cpu")
if has_gguf_k and not self.disable_opt_rtn:
- self.quant_rtn_with_imatrix(all_to_quantized_module_names)
+ self._quant_rtn_with_imatrix(all_to_quantized_module_names)
elif self.act_bits <= 8 and check_need_act_calibration(
self.act_dynamic, self.act_data_type
): ##TODO, mixed datatype has bug
- hook_handles = self.register_act_max_hook(self.model)
+ hook_handles = self._register_act_max_hook(self.model)
try:
- self.quantize_via_rtn_blockwise(all_to_quantized_module_names)
+ self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
except RuntimeError as e:
logger.warning("Fallback to CPU. Consider using more GPUs via `--device 0,1,2,3`.")
self.model = self.model.to("cpu")
@@ -1292,7 +1291,7 @@ class AutoRound(object):
accelerate.hooks.remove_hook_from_submodules(self.model)
orig_device = self.device
self.device = "cpu"
- self.quantize_via_rtn_blockwise(all_to_quantized_module_names)
+ self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
self.device = orig_device
for handle in hook_handles:
handle.remove()
@@ -1305,7 +1304,7 @@ class AutoRound(object):
cnt = 1
for name in pbar:
pbar.set_description(f"Quantizing {name}")
- self.quantize_layer_via_rtn(name)
+ self._quantize_layer_via_rtn(name)
if cnt % clear_mem_freq == 0:
clear_memory()
cnt = 1
@@ -1316,7 +1315,7 @@ class AutoRound(object):
self.quantized = True
return self.model, self.layer_config
- def quantize_via_rtn_blockwise(self, all_to_quantized_module_names: list[str]) -> None:
+ def _quantize_via_rtn_blockwise(self, all_to_quantized_module_names: list[str]) -> None:
"""Quantize model layers block by block using cached inputs and imatrix.
Args:
@@ -1330,7 +1329,7 @@ class AutoRound(object):
all_first_block_names = [block[0] for block in all_blocks]
if self.act_bits < 16 and not self.act_dynamic:
- layer_names = self.get_quantized_layer_names_outside_blocks()
+ layer_names = self._get_quantized_layer_names_outside_blocks()
if len(layer_names) > 0:
logger.warning(
"quantize layers outside blocks for static activation quantizaiton"
@@ -1393,7 +1392,7 @@ class AutoRound(object):
add_hook_to_module(m, hook, True)
else:
block = block.to(self.device)
- input_ids = self.get_block_outputs(
+ input_ids = self._get_block_outputs(
block,
input_ids,
input_others,
@@ -1409,14 +1408,14 @@ class AutoRound(object):
) or is_static_afp8(self):
from auto_round.utils import set_amax_for_all_moe_layers
- # enable moe experts act_max automatic generation for linears
+ # enable moe experts act_max automatic generation for Linear
set_amax_for_all_moe_layers(block, attr_name="act_max")
# Normalize imatrix and quantize layers
for _, m in block.named_modules():
if hasattr(m, "imatrix"):
m.imatrix /= m.imatrix_cnt
if hasattr(m, "tmp_name") and m.tmp_name in all_to_quantized_module_names:
- self.quantize_layer_via_rtn(m.tmp_name)
+ self._quantize_layer_via_rtn(m.tmp_name)
all_to_quantized_module_names.remove(m.tmp_name)
mv_module_from_gpu(block, self.low_cpu_mem_usage)
@@ -1430,13 +1429,13 @@ class AutoRound(object):
clear_mem_freq = 1
# Process remaining layers not in blocks
for name in all_to_quantized_module_names:
- self.quantize_layer_via_rtn(name)
+ self._quantize_layer_via_rtn(name)
if cnt % clear_mem_freq == 0:
clear_memory()
cnt = 1
cnt += 1
- def quantize(self):
+ def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:
"""Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.
Returns:
The quantized model and layer configurations.
@@ -1444,7 +1443,7 @@ class AutoRound(object):
for n, m in self.model.named_modules():
m.tmp_name = n
self._check_compatibility()
- self.has_qlayer_outside_block = self.set_layerwise_config(self.layer_config)
+ self.has_qlayer_outside_block = self._set_layerwise_config(self.layer_config)
if not hasattr(self, "formats"):
logger.warning("this API is deprecated, please use `quantize_and_save` instead")
else:
@@ -1478,7 +1477,7 @@ class AutoRound(object):
):
self.is_packing_immediate = True
if self.iters == 0:
- return self.quantize_rtn()
+ return self._quantize_rtn()
if bool(self.quant_block_list):
all_blocks = self.quant_block_list
@@ -1492,7 +1491,7 @@ class AutoRound(object):
if self.amp:
self.model = self.model.to(self.amp_dtype)
- layer_names = self.get_quantized_layer_names_outside_blocks()
+ layer_names = self._get_quantized_layer_names_outside_blocks()
self.start_time = time.time()
all_first_block_names = [block[0] for block in all_blocks]
if len(layer_names) > 0:
@@ -1502,7 +1501,7 @@ class AutoRound(object):
else:
logger.info("start to cache block inputs")
all_inputs = self.try_cache_inter_data_gpucpu(all_first_block_names, self.nsamples, layer_names=layer_names)
- is_quantized_embedding = self.quantize_embedding_layer()
+ is_quantized_embedding = self._quantize_embedding_layer()
all_q_inputs = None
if is_quantized_embedding:
all_inputs = copy.deepcopy(self.inputs)
@@ -1547,7 +1546,7 @@ class AutoRound(object):
self.batch_size = total_samples
logger.warning(f"force the train batch size to {total_samples}")
- self.quantize_blocks(
+ self._quantize_blocks(
self.model,
inputs,
block_names,
@@ -1562,7 +1561,7 @@ class AutoRound(object):
f"but got {len(self.formats)} formats."
)
- self.quant_layers(layer_names, all_inputs) ##TODO pack layer immediately
+ self._quantize_layers(layer_names, all_inputs) ##TODO pack layer immediately
if hasattr(self.model, "is_fp8"):
for n, m in self.model.named_modules():
@@ -1595,7 +1594,7 @@ class AutoRound(object):
self.quantized = True
return self.model, self.layer_config
- def quant_layers(self, layer_names, layer_inputs):
+ def _quantize_layers(self, layer_names: list, layer_inputs: dict) -> None:
"""Quantizes specified layers based on inputs and configuration.
Args:
@@ -1660,9 +1659,9 @@ class AutoRound(object):
self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
clear_memory()
if self.enable_torch_compile:
- quant_layer = compile_func(self.quant_layer, self.device)
+ quant_layer = compile_func(self._quantize_layer, self.device)
else:
- quant_layer = self.quant_layer
+ quant_layer = self._quantize_layer
for layer_name in layer_names:
layer_input = layer_inputs[layer_name]
layer_input = to_device(layer_input, self.cache_device)
@@ -1672,7 +1671,7 @@ class AutoRound(object):
del layer_input
clear_memory(q_layer_input)
- def set_layerwise_config(self, layer_config):
+ def _set_layerwise_config(self, layer_config: dict) -> bool:
"""
Sets the layer-wise configuration based on the provided `layer_config`.
By default, only quantize layers in blocks.
@@ -1786,7 +1785,7 @@ class AutoRound(object):
# Apply the configuration to the corresponding layer in the model
for key in keys:
setattr(m, key, layer_config[n][key])
- need_to_quantize_lm_head = self.check_need_to_quantize_lm_head_embedding()
+ need_to_quantize_lm_head = self._check_need_to_quantize_lm_head_embedding()
if need_to_quantize_lm_head:
has_qlayer_outside_block = True
@@ -1794,7 +1793,16 @@ class AutoRound(object):
return has_qlayer_outside_block
@torch.no_grad()
- def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_device, save_output=True):
+ def _get_block_outputs(
+ self,
+ block: torch.nn.Module,
+ input_ids: torch.Tensor,
+ input_others: torch.Tensor,
+ bs: int,
+ device: Union[str, torch.device],
+ cache_device: Union[str, torch.device],
+ save_output: bool = True,
+ ):
"""Compute the output of a given block of the model for a given input.
Args:
@@ -1815,7 +1823,7 @@ class AutoRound(object):
for i in range(0, nsamples, bs):
end_index = min(nsamples, i + bs)
indices = torch.arange(i, end_index).to(torch.long)
- tmp_input_ids, tmp_input_others = AutoRound.sampling_inputs(
+ tmp_input_ids, tmp_input_others = AutoRound._sampling_inputs(
input_ids, input_others, indices, self.seqlen, self.batch_dim, share_cache_keys=self.shared_cache_keys
)
tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(
@@ -2046,7 +2054,7 @@ class AutoRound(object):
return res
@torch.no_grad()
- def get_block_forward_func(self, name):
+ def _get_block_forward_func(self, name: str) -> Callable:
"""Gets the forward function.
Args:
@@ -2192,13 +2200,15 @@ class AutoRound(object):
for n, m in self.model.named_modules():
if n in self.to_cached_layers and not isinstance(m, tuple(self.supported_types)): ##block
m.orig_forward = m.forward
- m.forward = partial(self.get_block_forward_func(n), m)
+ m.forward = partial(self._get_block_forward_func(n), m)
elif n in self.to_cached_layers: ##linear layer or conv1d layer
hook_func = self._get_cache_data_hook_for_layer(n)
hook_handle = m.register_forward_hook(hook_func)
self.hook_handles.append(hook_handle)
- def quant_layer(self, layer_name, inputs, q_inputs=None, device=torch.device("cpu")):
+ def _quantize_layer(
+ self, layer_name: str, inputs: torch.Tensor, q_inputs: torch.Tensor = None, device: str = "cpu"
+ ):
"""Quantize a specific layer of the model using the provided inputs.
Args:
@@ -2255,7 +2265,7 @@ class AutoRound(object):
last_best_iter = 0
best_loss = torch.finfo(torch.float).max
mse_loss = torch.nn.MSELoss().to(device)
- scaler = self.get_scaler() # pylint: disable=assignment-from-none
+ scaler = self._get_scaler() # pylint: disable=assignment-from-none
init_loss = None
# best_v, best_min_scale, best_max_scale = torch.tensor(0), torch.tensor(1.0), torch.tensor(1.0)
gradient_accumulate_steps = self.batch_size ##Force to low gpu
@@ -2306,7 +2316,7 @@ class AutoRound(object):
)
total_loss += loss.item() / num_elm
- self.scale_loss_and_backward(scaler, loss)
+ self._scale_loss_and_backward(scaler, loss)
if i == 0:
init_loss = total_loss
@@ -2321,7 +2331,7 @@ class AutoRound(object):
if not self.not_use_best_mse:
if 0 < self.dynamic_max_gap <= i - last_best_iter:
break
- self.step(scaler, optimizer, lr_schedule)
+ self._step(scaler, optimizer, lr_schedule)
last_loss = total_loss
best_iter = self.iters
@@ -2334,7 +2344,7 @@ class AutoRound(object):
dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
logger.info(dump_info)
- def register_act_max_hook(self, model):
+ def _register_act_max_hook(self, model):
def get_act_max_hook(module, input, output):
if isinstance(input, (tuple, list)):
input = input[0]
@@ -2380,7 +2390,14 @@ class AutoRound(object):
continue
return hook_handles
- def quantize_block(self, block, input_ids, input_others, q_input=None, device=torch.device("cpu")):
+ def _quantize_block(
+ self,
+ block: torch.nn.Module,
+ input_ids: list[torch.Tensor],
+ input_others: dict,
+ q_input: Union[None, torch.Tensor] = None,
+ device: Union[str, torch.device] = "cpu",
+ ):
"""Quantize the weights of a given block of the model.
Args:
@@ -2411,21 +2428,21 @@ class AutoRound(object):
add_hook_to_module(m, hook, True)
if q_input is None:
- hook_handles = self.register_act_max_hook(block)
+ hook_handles = self._register_act_max_hook(block)
- output = self.get_block_outputs(
+ output = self._get_block_outputs(
block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device, self.cache_device
)
for handle in hook_handles:
handle.remove()
else:
- output = self.get_block_outputs(
+ output = self._get_block_outputs(
block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device, self.cache_device
)
- hook_handles = self.register_act_max_hook(block)
+ hook_handles = self._register_act_max_hook(block)
if hook_handles:
- self.get_block_outputs(
+ self._get_block_outputs(
block,
q_input,
input_others,
@@ -2502,7 +2519,7 @@ class AutoRound(object):
if self.gradient_accumulate_steps != 1:
mse_reduction = "sum"
mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)
- scaler = self.get_scaler() # pylint: disable=assignment-from-none
+ scaler = self._get_scaler() # pylint: disable=assignment-from-none
init_loss = None
best_params = {}
total_loss = 0
@@ -2517,7 +2534,7 @@ class AutoRound(object):
for tmp_step in range(self.gradient_accumulate_steps):
indices = whole_indices[tmp_step * self.batch_size : (tmp_step + 1) * self.batch_size]
- current_input_ids, current_input_others = AutoRound.sampling_inputs(
+ current_input_ids, current_input_others = AutoRound._sampling_inputs(
input_ids,
input_others,
indices,
@@ -2542,7 +2559,7 @@ class AutoRound(object):
)
total_loss += loss.item() / num_elm
- self.scale_loss_and_backward(scaler, loss)
+ self._scale_loss_and_backward(scaler, loss)
if i == 0:
init_loss = total_loss
@@ -2560,7 +2577,7 @@ class AutoRound(object):
if not self.not_use_best_mse:
if 0 < self.dynamic_max_gap <= i - last_best_iter:
break
- self.step(scaler, optimizer, lr_schedule)
+ self._step(scaler, optimizer, lr_schedule)
last_loss = total_loss
best_iter = self.iters
@@ -2584,7 +2601,7 @@ class AutoRound(object):
if self.low_cpu_mem_usage:
block = block.to(device)
clear_memory()
- q_outputs = self.get_block_outputs(
+ q_outputs = self._get_block_outputs(
block,
input_ids,
input_others,
@@ -2606,8 +2623,15 @@ class AutoRound(object):
clear_memory(input_ids)
return None, output
- def quantize_blocks(
- self, model: torch.nn.Module, inputs, block_names, q_input=None, nblocks=1, device="cpu", pbar=None
+ def _quantize_blocks(
+ self,
+ model: torch.nn.Module,
+ inputs: dict,
+ block_names: list,
+ q_input: torch.Tensor = None,
+ nblocks: int = 1,
+ device: str = "cpu",
+ pbar: tqdm = None,
):
"""Quantize and dequantize the weights of the specified blocks in the model.
@@ -2663,13 +2687,13 @@ class AutoRound(object):
else:
logger.info("using algorithm extension for quantization.")
except (ImportError, ModuleNotFoundError):
- quantize_block = self.quantize_block
+ quantize_block = self._quantize_block
if self.enable_torch_compile:
quantize_block = compile_func(quantize_block, device)
else:
quantize_block = quantize_block
else:
- quantize_block = self.quantize_block
+ quantize_block = self._quantize_block
if self.enable_torch_compile:
quantize_block = compile_func(quantize_block, device)
@@ -2703,27 +2727,28 @@ class AutoRound(object):
from auto_round.export import PACKING_LAYER_WITH_FORMAT
for _, tmp_m in m.named_modules():
- if hasattr(tmp_m, "bits") and check_to_quantized(tmp_m):
- target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]
- has_gguf = any("gguf" in format_ for format_ in self.formats)
- if has_gguf:
- from auto_round.export.export_to_gguf.export import pack_gguf_layer
-
- output_dir = self.get_save_folder_name(self.formats[0])
- model_type = ModelType.MMPROJ if self.vlm else ModelType.TEXT
- pack_gguf_layer(
- tmp_m.tmp_name,
- self.model,
- self.formats[0],
- output_dir,
- self.layer_config,
- self.tokenizer,
- processor=self.processor if hasattr(self, "processor") else None,
- image_processor=self.image_processor if hasattr(self, "image_processor") else None,
- model_type=model_type,
- )
- else:
- PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.tmp_name, self.model, self.formats[0])
+ if not (hasattr(tmp_m, "bits") and check_to_quantized(tmp_m)):
+ continue
+ target_backend = self.formats[0].split(":")[0] if ":" in self.formats[0] else self.formats[0]
+ has_gguf = any("gguf" in format_ for format_ in self.formats)
+ if has_gguf:
+ from auto_round.export.export_to_gguf.export import pack_gguf_layer
+
+ output_dir = self._get_save_folder_name(self.formats[0])
+ model_type = ModelType.MMPROJ if self.vlm else ModelType.TEXT
+ pack_gguf_layer(
+ tmp_m.tmp_name,
+ self.model,
+ self.formats[0],
+ output_dir,
+ self.layer_config,
+ self.tokenizer,
+ processor=self.processor if hasattr(self, "processor") else None,
+ image_processor=self.image_processor if hasattr(self, "image_processor") else None,
+ model_type=model_type,
+ )
+ else:
+ PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.tmp_name, self.model, self.formats[0])
pbar.set_description("Quantizing done")
pbar.update(1)
pbar.close()
@@ -2739,7 +2764,9 @@ class AutoRound(object):
clear_memory()
- def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):
+ def save_quantized(
+ self, output_dir: str = None, format: str = "auto_round", inplace: bool = True, **kwargs
+ ) -> torch.nn.Module:
"""Save the quantized model to the specified output directory in the specified format.
Args:
@@ -2832,7 +2859,7 @@ class AutoRound(object):
)
return compressed_model
- def get_quantized_layer_names_outside_blocks(self):
+ def _get_quantized_layer_names_outside_blocks(self) -> list:
"""Gets the names of quantized layers outside blocks in the model.
Returns:
@@ -2856,7 +2883,8 @@ class AutoRound(object):
return layer_names
- def set_amp_dtype(self):
+ def _set_amp_dtype(self) -> None:
+ """Sets the automatic mixed precision (AMP) data type for the model based on the device and configuration."""
self.amp_dtype = torch.float16
if self.model.dtype != torch.float32:
self.amp_dtype = self.model.dtype
@@ -2876,7 +2904,7 @@ class AutoRound(object):
self.amp_dtype = torch.float32
self.model = self.model.to(torch.float32)
- def get_optimizer(self, optimizer):
+ def _get_optimizer(self, optimizer: Any):
"""Returns the specified optimizer. In SignRound, we fix the optimizer.
Args:
@@ -2887,11 +2915,11 @@ class AutoRound(object):
"""
return SignSGD
- def get_scaler(self):
+ def _get_scaler(self):
"""Returns scaler, in SignRound, no need to use scaler."""
return None
- def scale_loss_and_backward(self, scaler, loss):
+ def _scale_loss_and_backward(self, scaler: Any, loss: torch.Tensor) -> torch.Tensor:
"""Scales the loss and performs backward pass.
Args:
@@ -2907,7 +2935,7 @@ class AutoRound(object):
htcore.mark_step()
return scale_loss
- def step(self, scaler, optimizer, lr_schedule):
+ def _step(self, scaler: Any, optimizer: Any, lr_schedule: Any):
"""Performs a step in the optimization process.
Args:
@@ -2927,7 +2955,15 @@ class AutoRound(object):
@classmethod
@torch.no_grad()
- def sampling_inputs(cls, input_ids, input_others, indices, seqlen, batch_dim=0, share_cache_keys=()):
+ def _sampling_inputs(
+ cls,
+ input_ids: list[torch.Tensor],
+ input_others: dict,
+ indices: list[int],
+ seqlen: int,
+ batch_dim: int = 0,
+ share_cache_keys: tuple = (),
+ ):
"""Samples inputs based on the given indices and sequence length.
Args:
@@ -3105,9 +3141,9 @@ class AutoRoundAdam(AutoRound):
**kwargs,
)
- self.optimizer = self.get_optimizer(optimizer)
+ self.optimizer = self._get_optimizer(optimizer)
- def get_optimizer(self, optimizer):
+ def _get_optimizer(self, optimizer):
if optimizer is None:
optimizer = torch.optim.AdamW
elif isinstance(optimizer, str):
@@ -3116,7 +3152,7 @@ class AutoRoundAdam(AutoRound):
optimizer = optimizer
return optimizer
- def get_scaler(self):
+ def _get_scaler(self):
scaler = None
if self.amp and not check_is_cpu(self.device):
from torch.cuda.amp import GradScaler
@@ -3124,7 +3160,7 @@ class AutoRoundAdam(AutoRound):
scaler = GradScaler(init_scale=1024, growth_interval=100000)
return scaler
- def scale_loss_and_backward(self, scaler, loss):
+ def _scale_loss_and_backward(self, scaler, loss):
if scaler is not None:
loss = scaler.scale(loss)
@@ -3133,7 +3169,7 @@ class AutoRoundAdam(AutoRound):
htcore.mark_step()
return loss
- def step(self, scaler, optimizer, lr_schedule):
+ def _step(self, scaler, optimizer, lr_schedule):
if scaler is not None:
scaler.step(optimizer)
optimizer.zero_grad()
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index ea66a0389..1569e8005 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -816,8 +816,8 @@ def process_requirement(requirements: list, target_device="cuda", logger_level="
except:
missing_requirements.append(req)
- gptq_req = next((req for req in missing_requirements if "gptqmodel" in req), None)
- other_reqs = [req for req in missing_requirements if "gptqmodel" not in req]
+ gptq_req = next((f'"{req}"' for req in missing_requirements if "gptqmodel" in req), None)
+ other_reqs = [f'"{req}"' for req in missing_requirements if "gptqmodel" not in req]
pip_cmds = build_pip_commands(gptq_req, other_reqs)
if not pip_cmds:
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 4b7cc641c..3caefcb85 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -634,7 +634,11 @@ def tune(args):
eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
tmp_act_bits = infer_bits_by_data_type(args.act_data_type)
- if tmp_act_bits is not None and (args.act_bits <= 8 or tmp_act_bits <= 8 or eval_gguf_model):
+ if tmp_act_bits is not None:
+ act_bits = tmp_act_bits
+ else:
+ act_bits = args.act_bits
+ if act_bits <= 8 or eval_gguf_model:
if eval_gguf_model:
# for file in os.listdir(eval_folder):
# gguf_file = file
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index cf0bb90a5..e6d48567e 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -11,7 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
+from auto_round.utils import logger
mllms_with_limited_bs = ("llava", "qwen2_vl", "phi3_v", "mllama") # Limitations on batch_size
@@ -129,7 +129,7 @@ def _handle_moe_model(model):
parent = model.get_submodule(parent)
setattr(parent, child, new_module)
- print("warning, Llama4 experts are converted, the quantized model can not run on transformers.")
+ logger.warning("Llama4 experts are converted, the quantized model can not run on transformers.")
return model
@@ -192,7 +192,7 @@ def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):
for key in mllms_with_limited_bs:
if hasattr(model, "config") and key in model.config.model_type and batch_size != 1:
accumulate_steps = batch_size * gradient_accumulate_steps
- print(
+ logger.warning(
"To avoid the tensor concat mismatch problem, modified parameters to "
f"batch_size=1. As an alternative, set the gradient_accumulate_steps={accumulate_steps}"
)
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 74999c624..267f56005 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -24,7 +24,7 @@ import sys
from collections import UserDict
from enum import Enum
from functools import lru_cache
-from typing import Any, Tuple, Union
+from typing import Any, Callable, Tuple, Union
import cpuinfo
import torch
@@ -33,11 +33,9 @@ from packaging import version
from torch.amp import autocast
from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType
-from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK, SPECIAL_SHARED_CACHE_KEYS
SHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")
-
deepspeed_exists = False
if importlib.util.find_spec("deepspeed"): # check if deepspeed is installed
deepspeed_exists = True
@@ -421,6 +419,7 @@ def get_block_names(model, quant_vision=False):
Returns:
block_names: A list whose elements are list of block's layer names
"""
+ from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK
def _get_llm_block_names(model):
block_names = []
@@ -952,6 +951,7 @@ TORCH_VERSION_AT_LEAST_2_6 = torch_version_at_least("2.6.0")
TORCH_VERSION_AT_LEAST_2_5 = torch_version_at_least("2.5.0")
TORCH_VERSION_AT_LEAST_2_4 = torch_version_at_least("2.4.0")
+
# Note on HPU usage:
# There are two modes available for enabling auto-round on HPU:
# 1. Compile Mode
@@ -979,7 +979,10 @@ def compile_func_on_cuda_or_cpu(func):
return torch.compile(func)
-def compile_func(fun, device):
+def compile_func(
+ fun: Union[torch.nn.Module, Callable], device: Union[torch.nn.Module, Callable]
+) -> Union[torch.nn.Module, Callable]:
+ """Compile function on the specified device."""
if "hpu" in str(device):
return compile_func_on_hpu(fun) ## use auto by default
else:
@@ -1481,7 +1484,6 @@ def mllm_load_model(
model_dtype=None,
**kwargs,
):
-
import transformers
from huggingface_hub import HfApi, HfFileSystem, hf_hub_download
from transformers import AutoModel, AutoModelForCausalLM, AutoProcessor, AutoTokenizer
@@ -1668,6 +1670,8 @@ def get_shared_keys(model):
Returns:
tuple: tuple of shared keys.
"""
+ from auto_round.special_model_handler import SPECIAL_SHARED_CACHE_KEYS
+
shared_keys = SHARED_CACHE_KEYS
shared_keys += SPECIAL_SHARED_CACHE_KEYS.get(model.__class__.__name__, ())
return shared_keys
@@ -2201,7 +2205,6 @@ def unpad_weight(weight: torch.Tensor, original_M: int, original_N: int, keep_fi
def pad_block_fp8_weight_naive(
weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list
) -> Tuple[torch.Tensor, int, int]:
-
assert len(block_size) == 2
block_size_m, block_size_n = block_size
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 74efad368..9afc36fe9 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -145,6 +145,7 @@ class WrapperLinear(torch.nn.Module):
self._init_params("max_scale", p_dtype, shape, 1.0, (self.enable_minmax_tuning and self.orig_layer.bits < 16))
self.weight_quant_func, self.data_type = get_quant_func(orig_layer.data_type, orig_layer.bits, orig_layer.sym)
+
if self.enable_act_quant:
self.act_quant_func, self.act_data_type = get_quant_func(
orig_layer.act_data_type, orig_layer.act_bits, orig_layer.act_sym
diff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.py
index 1a1e0c8aa..f5e32507e 100644
--- a/auto_round_extension/torch/qlinear_torch.py
+++ b/auto_round_extension/torch/qlinear_torch.py
@@ -86,7 +86,8 @@ class QuantLinear(nn.Module):
def post_init(self):
pass
- def pack(self, linear, scales, zeros, g_idx=None):
+ # @torch.compile() ## cpu side has bug
+ def pack_248_bits(self, linear, scales, zeros, g_idx=None):
scales_t = scales.t().contiguous()
if linear.bias is not None:
self.bias = linear.bias.clone().half()
@@ -115,119 +116,166 @@ class QuantLinear(nn.Module):
del repeat_scales
- if self.bits in [2, 4, 8]:
- intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)
- order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits
- intweight = intweight.to(torch.int32)
- intweight = intweight << order_map
- intweight = torch.sum(intweight, dim=-1)
-
- intweight = intweight.t().contiguous().to(torch.int32)
- self.qweight = intweight.to("cpu")
- elif self.bits == 3:
- intweight = intweight.t().contiguous()
- intweight = intweight.cpu().numpy().astype(np.uint32)
- i = 0
- row = 0
- qweight = torch.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=torch.int32)
- while row < qweight.shape[0]:
- packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()
- shifts = torch.arange(0, 10) * self.bits
- shifted = packed_weight << shifts
- qweight[row] |= shifted.sum(dim=-1)
- i += 10
- qweight[row] |= intweight[i] << 30
- row += 1
- qweight[row] |= (intweight[i] >> 2) & 1
- i += 1
- packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()
- shifts = torch.arange(0, 10) * self.bits + 1
- shifted = packed_weight << shifts
- qweight[row] |= shifted.sum(dim=-1)
- i += 10
- qweight[row] |= intweight[i] << 31
- row += 1
- qweight[row] |= (intweight[i] >> 1) & 0x3
- i += 1
- packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()
- shifts = torch.arange(0, 10) * self.bits + 2
- shifted = packed_weight << shifts
- qweight[row] |= shifted.sum(dim=-1)
- i += 10
- row += 1
+ intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)
+ order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits
+ intweight = intweight.to(torch.int32)
+ intweight = intweight << order_map
+ intweight = torch.sum(intweight, dim=-1)
- self.qweight = qweight.cpu()
+ intweight = intweight.t().contiguous().to(torch.int32)
+ self.qweight = intweight.to("cpu")
if isinstance(zeros, torch.Tensor):
zeros = zeros.t().contiguous()
- zeros = zeros.numpy().astype(np.uint32)
- qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=torch.int32)
+ zeros -= 1
+ # zeros = zeros.numpy().astype(np.uint32)
+ qzeros = torch.zeros(
+ (zeros.shape[0], zeros.shape[1] // 32 * self.bits), device=self.device, dtype=torch.int32
+ )
i = 0
col = 0
while col < qzeros.shape[1]:
- if self.bits in [2, 4, 8]:
- packed_zeros = torch.tensor(zeros[:, i : i + (32 // self.bits)]).to(dtype=torch.int32)
- shifts = torch.arange(0, (32 // self.bits)) * self.bits
- shifted = packed_zeros << shifts
- qzeros[:, col] |= shifted.sum(dim=-1)
- i += 32 // self.bits
- col += 1
- elif self.bits == 3:
- packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)
- shifts = torch.arange(0, 10) * self.bits
- shifted = packed_zeros << shifts
- qzeros[:, col] = shifted.sum(dim=-1)
- i += 10
- qzeros[:, col] |= zeros[:, i] << 30
- col += 1
- qzeros[:, col] |= (zeros[:, i] >> 2) & 1
- i += 1
- packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)
- shifts = torch.arange(0, 10) * self.bits + 1
- shifted = packed_zeros << shifts
- qzeros[:, col] |= shifted.sum(dim=-1)
- i += 10
- qzeros[:, col] |= zeros[:, i] << 31
- col += 1
- qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3
- i += 1
- packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)
- shifts = torch.arange(0, 10) * self.bits + 2
- shifted = packed_zeros << shifts
- qzeros[:, col] |= shifted.sum(dim=-1)
- i += 10
- col += 1
+ packed_zeros = torch.tensor(zeros[:, i : i + (32 // self.bits)]).to(dtype=torch.int32)
+ shifts = torch.arange(0, (32 // self.bits)) * self.bits
+ shifted = packed_zeros << shifts
+ qzeros[:, col] |= shifted.sum(dim=-1)
+ i += 32 // self.bits
+ col += 1
+ self.qzeros = qzeros.cpu()
+ else:
+ zeros -= 1
+ shape = scales_t.shape
+ value = 0
+ for j in range(0, (32 // self.bits)):
+ value |= zeros << (self.bits * j)
+ qzeros = torch.ones((shape[0], shape[1] // 32 * self.bits), dtype=torch.int32) * value
+ self.qzeros = qzeros.cpu()
+
+ # @torch.compile()
+ def pack_3bits(self, linear, scales, zeros, g_idx=None):
+ scales_t = scales.t().contiguous()
+ if linear.bias is not None:
+ self.bias = linear.bias.clone().half()
+ self.scales = scales_t.clone().half()
+ device = "cpu"
+ if torch.cuda.is_available():
+ device = "cuda:0"
+ elif torch.xpu.is_available():
+ device = "xpu:0"
+ W = linear.weight.data.to(device).clone()
+ if isinstance(linear, nn.Conv2d):
+ W = W.flatten(1)
+ if isinstance(linear, transformers.pytorch_utils.Conv1D):
+ W = W.t()
+
+ repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)
+ if isinstance(zeros, torch.Tensor):
+ repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)
+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(
+ torch.int32
+ )
+ else:
+ repeat_zeros = zeros
+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)
+
+ del repeat_scales
+
+ intweight = intweight.t().contiguous().to(torch.int32)
+ i = 0
+ row = 0
+ qweight = torch.zeros(
+ (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=torch.int32, device=device
+ )
+ while row < qweight.shape[0]:
+ packed_weight = (intweight[i : i + 10]).to(dtype=torch.int32).t()
+ shifts = torch.arange(0, 10).to(device) * self.bits
+ shifted = packed_weight << shifts
+ qweight[row] |= shifted.sum(dim=-1)
+ i += 10
+ qweight[row] |= intweight[i] << 30
+ row += 1
+ qweight[row] |= (intweight[i] >> 2) & 1
+ i += 1
+ packed_weight = (intweight[i : i + 10]).to(dtype=torch.int32).t()
+ shifts = torch.arange(0, 10).to(device) * self.bits + 1
+ shifted = packed_weight << shifts
+ qweight[row] |= shifted.sum(dim=-1)
+ i += 10
+ qweight[row] |= intweight[i] << 31
+ row += 1
+ qweight[row] |= (intweight[i] >> 1) & 0x3
+ i += 1
+ packed_weight = (intweight[i : i + 10]).to(dtype=torch.int32).t()
+ shifts = torch.arange(0, 10).to(device) * self.bits + 2
+ shifted = packed_weight << shifts
+ qweight[row] |= shifted.sum(dim=-1)
+
+ i += 10
+ row += 1
+
+ self.qweight = qweight.cpu()
+
+ if isinstance(zeros, torch.Tensor):
+ zeros = zeros.t().contiguous().to(device).to(torch.int32)
+ # zeros = zeros.numpy().astype(np.uint32)
+ qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), device=device, dtype=torch.int32)
+ i = 0
+ col = 0
+ while col < qzeros.shape[1]:
+ packed_zeros = (zeros[:, i : i + 10]).to(dtype=torch.int32)
+ shifts = torch.arange(0, 10).to(device) * self.bits
+ shifted = packed_zeros << shifts
+ qzeros[:, col] = shifted.sum(dim=-1)
+ i += 10
+ qzeros[:, col] |= zeros[:, i] << 30
+ col += 1
+ qzeros[:, col] |= (zeros[:, i] >> 2) & 1
+ i += 1
+ # packed_zeros = (zeros[:, i: i + 10]).to(dtype=torch.int32)
+ shifts = torch.arange(0, 10).to(device) * self.bits + 1
+ shifted = packed_zeros << shifts
+ qzeros[:, col] |= shifted.sum(dim=-1)
+ i += 10
+ qzeros[:, col] |= zeros[:, i] << 31
+ col += 1
+ qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3
+ i += 1
+ # packed_zeros = (zeros[:, i: i + 10]).to(dtype=torch.int32)
+ shifts = torch.arange(0, 10).to(device) * self.bits + 2
+ shifted = packed_zeros << shifts
+ qzeros[:, col] |= shifted.sum(dim=-1)
+ i += 10
+ col += 1
+ self.qzeros = qzeros.cpu()
+ else:
+ shape = scales_t.shape[0], scales_t.shape[1] // 32 * self.bits
+ qzeros = torch.zeros(shape, dtype=torch.int32)
+ zero_val = zeros
+ total_cols = shape[1]
+ # Precompute shifts for 3-bit packing
+ shifts0 = torch.arange(0, 10, dtype=torch.int32) * self.bits
+ shifts1 = shifts0 + 1
+ shifts2 = shifts0 + 2
+ # Compute packed pattern parts
+ part0 = (zero_val << shifts0).sum().to(torch.int32) | (zero_val << 30)
+ part1 = (zero_val << shifts1).sum().to(torch.int32) | (zero_val << 31) | ((zero_val >> 2) & 1)
+ part2 = (zero_val << shifts2).sum().to(torch.int32) | ((zero_val >> 1) & 0x3)
+ pattern = torch.tensor([part0, part1, part2], dtype=torch.int32)
+ # Tile pattern across all columns
+ repeats = (total_cols + 2) // 3
+ full_row = pattern.repeat(repeats)[:total_cols]
+ # Broadcast across rows
+ qzeros[:] = full_row.unsqueeze(0)
self.qzeros = qzeros.cpu()
+
+ def pack(self, linear, scales, zeros, g_idx=None):
+ if self.bits in [2, 4, 8]:
+ return self.pack_248_bits(linear, scales, zeros, g_idx)
+ elif self.bits in [3]:
+ return self.pack_3bits(linear, scales, zeros, g_idx)
else:
- if self.bits in [2, 4, 8]:
- shape = scales_t.shape
- value = 0
- for j in range(0, (32 // self.bits)):
- value |= zeros << (self.bits * j)
- qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value
- qzeros = qzeros.astype(np.int32)
- self.qzeros = torch.from_numpy(qzeros)
- else: ## bits == 3
- shape = scales_t.shape[0], scales_t.shape[1] // 32 * self.bits
- qzeros = torch.zeros(shape, dtype=torch.int32)
- zero_val = zeros
- total_cols = shape[1]
- # Precompute shifts for 3-bit packing
- shifts0 = torch.arange(0, 10, dtype=torch.int32) * self.bits
- shifts1 = shifts0 + 1
- shifts2 = shifts0 + 2
- # Compute packed pattern parts
- part0 = (zero_val << shifts0).sum().to(torch.int32) | (zero_val << 30)
- part1 = (zero_val << shifts1).sum().to(torch.int32) | (zero_val << 31) | ((zero_val >> 2) & 1)
- part2 = (zero_val << shifts2).sum().to(torch.int32) | ((zero_val >> 1) & 0x3)
- pattern = torch.tensor([part0, part1, part2], dtype=torch.int32)
- # Tile pattern across all columns
- repeats = (total_cols + 2) // 3
- full_row = pattern.repeat(repeats)[:total_cols]
- # Broadcast across rows
- qzeros[:] = full_row.unsqueeze(0)
- self.qzeros = qzeros.cpu()
+ raise ValueError("Only 2,3,4,8 bits are supported.")
def forward(self, x):
out_shape = x.shape[:-1] + (self.outfeatures,)
diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.py
index 74485f93a..1ddbb363e 100644
--- a/auto_round_extension/torch/qlinear_torch_zp.py
+++ b/auto_round_extension/torch/qlinear_torch_zp.py
@@ -87,7 +87,8 @@ class QuantLinear(nn.Module):
def post_init(self):
pass
- def pack(self, linear, scales, zeros, g_idx=None):
+ # @torch.compile()
+ def pack_248_bits(self, linear, scales, zeros, g_idx=None):
scales_t = scales.t().contiguous()
if linear.bias is not None:
self.bias = linear.bias.clone().half()
@@ -116,120 +117,164 @@ class QuantLinear(nn.Module):
del repeat_scales
- if self.bits in [2, 4, 8]:
- intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)
- order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits
- intweight = intweight.to(torch.int32)
- intweight = intweight << order_map
- intweight = torch.sum(intweight, dim=-1)
-
- intweight = intweight.t().contiguous().to(torch.int32)
- self.qweight = intweight.to("cpu")
- else: ## bits == 3
- intweight = intweight.t().contiguous()
- intweight = intweight.cpu().numpy().astype(np.uint32)
+ intweight = intweight.reshape(-1, intweight.shape[1] // 32 * self.bits, 32 // self.bits)
+ order_map = torch.arange(0, 32 // self.bits, device=device) * self.bits
+ intweight = intweight.to(torch.int32)
+ intweight = intweight << order_map
+ intweight = torch.sum(intweight, dim=-1)
+
+ intweight = intweight.t().contiguous().to(torch.int32)
+ self.qweight = intweight.to("cpu")
+
+ if isinstance(zeros, torch.Tensor):
+ zeros = zeros.t().contiguous().to(device)
+ zeros -= 1
+ # zeros = zeros.numpy().astype(np.uint32)
+ qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), device=device, dtype=torch.int32)
i = 0
- row = 0
- qweight = torch.zeros((intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=torch.int32)
- while row < qweight.shape[0]:
- packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()
- shifts = torch.arange(0, 10) * self.bits
- shifted = packed_weight << shifts
- qweight[row] |= shifted.sum(dim=-1)
- i += 10
- qweight[row] |= intweight[i] << 30
- row += 1
- qweight[row] |= (intweight[i] >> 2) & 1
- i += 1
- packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()
- shifts = torch.arange(0, 10) * self.bits + 1
- shifted = packed_weight << shifts
- qweight[row] |= shifted.sum(dim=-1)
- i += 10
- qweight[row] |= intweight[i] << 31
- row += 1
- qweight[row] |= (intweight[i] >> 1) & 0x3
- i += 1
- packed_weight = torch.tensor(intweight[i : i + 10]).to(dtype=torch.int32).t()
- shifts = torch.arange(0, 10) * self.bits + 2
- shifted = packed_weight << shifts
- qweight[row] |= shifted.sum(dim=-1)
- i += 10
- row += 1
+ col = 0
+ while col < qzeros.shape[1]:
+ packed_zeros = torch.tensor(zeros[:, i : i + (32 // self.bits)]).to(dtype=torch.int32)
+ shifts = torch.arange(0, (32 // self.bits)) * self.bits
+ shifted = packed_zeros << shifts
+ qzeros[:, col] |= shifted.sum(dim=-1)
+ i += 32 // self.bits
+ col += 1
+ self.qzeros = qzeros.cpu()
+ else:
+ zeros -= 1
+ shape = scales_t.shape
+ value = 0
+ for j in range(0, (32 // self.bits)):
+ value |= zeros << (self.bits * j)
+ qzeros = torch.ones((shape[0], shape[1] // 32 * self.bits), dtype=torch.int32) * value
+ self.qzeros = qzeros.cpu()
- self.qweight = qweight.cpu()
+ def pack_3bits(self, linear, scales, zeros, g_idx=None):
+ scales_t = scales.t().contiguous()
+ if linear.bias is not None:
+ self.bias = linear.bias.clone().half()
+ self.scales = scales_t.clone().half()
+ device = "cpu"
+ if torch.cuda.is_available():
+ device = "cuda:0"
+ elif torch.xpu.is_available():
+ device = "xpu:0"
+ W = linear.weight.data.to(device).clone()
+ if isinstance(linear, nn.Conv2d):
+ W = W.flatten(1)
+ if isinstance(linear, transformers.pytorch_utils.Conv1D):
+ W = W.t()
+
+ repeat_scales = scales.to(device).repeat_interleave(self.group_size, 1)
if isinstance(zeros, torch.Tensor):
- zeros = zeros.t().contiguous()
+ repeat_zeros = zeros.to(device).repeat_interleave(self.group_size, 1)
+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros[:, : W.shape[1]]).to(
+ torch.int32
+ )
+ else:
+ repeat_zeros = zeros
+ intweight = torch.round(W.to(device) / repeat_scales[:, : W.shape[1]] + repeat_zeros).to(torch.int32)
+
+ del repeat_scales
+
+ intweight = intweight.t().contiguous().to(torch.int32)
+ i = 0
+ row = 0
+ qweight = torch.zeros(
+ (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=torch.int32, device=device
+ )
+ while row < qweight.shape[0]:
+ packed_weight = (intweight[i : i + 10]).to(dtype=torch.int32).t()
+ shifts = torch.arange(0, 10).to(device) * self.bits
+ shifted = packed_weight << shifts
+ qweight[row] |= shifted.sum(dim=-1)
+ i += 10
+ qweight[row] |= intweight[i] << 30
+ row += 1
+ qweight[row] |= (intweight[i] >> 2) & 1
+ i += 1
+ packed_weight = (intweight[i : i + 10]).to(dtype=torch.int32).t()
+ shifts = torch.arange(0, 10).to(device) * self.bits + 1
+ shifted = packed_weight << shifts
+ qweight[row] |= shifted.sum(dim=-1)
+ i += 10
+ qweight[row] |= intweight[i] << 31
+ row += 1
+ qweight[row] |= (intweight[i] >> 1) & 0x3
+ i += 1
+ packed_weight = (intweight[i : i + 10]).to(dtype=torch.int32).t()
+ shifts = torch.arange(0, 10).to(device) * self.bits + 2
+ shifted = packed_weight << shifts
+ qweight[row] |= shifted.sum(dim=-1)
+
+ i += 10
+ row += 1
+
+ self.qweight = qweight.cpu()
+
+ if isinstance(zeros, torch.Tensor):
+ zeros = zeros.t().contiguous().to(torch.int32).to(device)
zeros -= 1
- zeros = zeros.numpy().astype(np.uint32)
- qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=torch.int32)
+ qzeros = torch.zeros((zeros.shape[0], zeros.shape[1] // 32 * self.bits), device=device, dtype=torch.int32)
i = 0
col = 0
while col < qzeros.shape[1]:
- if self.bits in [2, 4, 8]:
- packed_zeros = torch.tensor(zeros[:, i : i + (32 // self.bits)]).to(dtype=torch.int32)
- shifts = torch.arange(0, (32 // self.bits)) * self.bits
- shifted = packed_zeros << shifts
- qzeros[:, col] |= shifted.sum(dim=-1)
- i += 32 // self.bits
- col += 1
- else: ## bits == 3
- packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)
- shifts = torch.arange(0, 10) * self.bits
- shifted = packed_zeros << shifts
- qzeros[:, col] = shifted.sum(dim=-1)
- i += 10
- qzeros[:, col] |= zeros[:, i] << 30
- col += 1
- qzeros[:, col] |= (zeros[:, i] >> 2) & 1
- i += 1
- packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)
- shifts = torch.arange(0, 10) * self.bits + 1
- shifted = packed_zeros << shifts
- qzeros[:, col] |= shifted.sum(dim=-1)
- i += 10
- qzeros[:, col] |= zeros[:, i] << 31
- col += 1
- qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3
- i += 1
- packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)
- shifts = torch.arange(0, 10) * self.bits + 2
- shifted = packed_zeros << shifts
- qzeros[:, col] |= shifted.sum(dim=-1)
- i += 10
- col += 1
+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)
+ shifts = torch.arange(0, 10).to(device) * self.bits
+ shifted = packed_zeros << shifts
+ qzeros[:, col] = shifted.sum(dim=-1)
+ i += 10
+ qzeros[:, col] |= zeros[:, i] << 30
+ col += 1
+ qzeros[:, col] |= (zeros[:, i] >> 2) & 1
+ i += 1
+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)
+ shifts = torch.arange(0, 10).to(device) * self.bits + 1
+ shifted = packed_zeros << shifts
+ qzeros[:, col] |= shifted.sum(dim=-1)
+ i += 10
+ qzeros[:, col] |= zeros[:, i] << 31
+ col += 1
+ qzeros[:, col] |= (zeros[:, i] >> 1) & 0x3
+ i += 1
+ packed_zeros = torch.tensor(zeros[:, i : i + 10]).to(dtype=torch.int32)
+ shifts = torch.arange(0, 10).to(device) * self.bits + 2
+ shifted = packed_zeros << shifts
+ qzeros[:, col] |= shifted.sum(dim=-1)
+ i += 10
+ col += 1
self.qzeros = qzeros.cpu()
else:
zeros -= 1
- if self.bits in [2, 4, 8]:
- shape = scales_t.shape
- value = 0
- for j in range(0, (32 // self.bits)):
- value |= zeros << (self.bits * j)
- qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * value
- qzeros = qzeros.astype(np.int32)
- self.qzeros = torch.from_numpy(qzeros)
- else: ## bits == 3
- shape = scales_t.shape[0], scales_t.shape[1] // 32 * self.bits
- qzeros = torch.zeros(shape, dtype=torch.int32)
- zero_val = zeros
- total_cols = shape[1]
- # Precompute shifts for 3-bit packing
- shifts0 = torch.arange(0, 10, dtype=torch.int32) * self.bits
- shifts1 = shifts0 + 1
- shifts2 = shifts0 + 2
- # Compute packed pattern parts
- part0 = (zero_val << shifts0).sum().to(torch.int32) | (zero_val << 30)
- part1 = (zero_val << shifts1).sum().to(torch.int32) | (zero_val << 31) | ((zero_val >> 2) & 1)
- part2 = (zero_val << shifts2).sum().to(torch.int32) | ((zero_val >> 1) & 0x3)
- pattern = torch.tensor([part0, part1, part2], dtype=torch.int32)
- # Tile pattern across all columns
- repeats = (total_cols + 2) // 3
- full_row = pattern.repeat(repeats)[:total_cols]
- # Broadcast across rows
- qzeros[:] = full_row.unsqueeze(0)
- self.qzeros = qzeros.cpu()
+ shape = scales_t.shape[0], scales_t.shape[1] // 32 * self.bits
+ qzeros = torch.zeros(shape, dtype=torch.int32)
+ zero_val = zeros
+ total_cols = shape[1]
+ # Precompute shifts for 3-bit packing
+ shifts0 = torch.arange(0, 10, dtype=torch.int32) * self.bits
+ shifts1 = shifts0 + 1
+ shifts2 = shifts0 + 2
+ # Compute packed pattern parts
+ part0 = (zero_val << shifts0).sum().to(torch.int32) | (zero_val << 30)
+ part1 = (zero_val << shifts1).sum().to(torch.int32) | (zero_val << 31) | ((zero_val >> 2) & 1)
+ part2 = (zero_val << shifts2).sum().to(torch.int32) | ((zero_val >> 1) & 0x3)
+ pattern = torch.tensor([part0, part1, part2], dtype=torch.int32)
+ # Tile pattern across all columns
+ repeats = (total_cols + 2) // 3
+ full_row = pattern.repeat(repeats)[:total_cols]
+ # Broadcast across rows
+ qzeros[:] = full_row.unsqueeze(0)
+ self.qzeros = qzeros.cpu()
+
+ def pack(self, linear, scales, zeros, g_idx=None):
+ if self.bits in [2, 4, 8]:
+ return self.pack_248_bits(linear, scales, zeros, g_idx)
+ elif self.bits in [3]:
+ return self.pack_3bits(linear, scales, zeros, g_idx)
+ else:
+ raise ValueError("Only 2,3,4,8 bits are supported.")
def forward(self, x):
out_shape = x.shape[:-1] + (self.outfeatures,)
diff --git a/test/test_cpu/test_auto_round_hpu_only.py b/test/test_cpu/test_auto_round_hpu_only.py
index a6fdb433a..fdcee80d6 100644
--- a/test/test_cpu/test_auto_round_hpu_only.py
+++ b/test/test_cpu/test_auto_round_hpu_only.py
@@ -74,6 +74,7 @@ def test_w4a8(data_type):
data_type=data_type,
act_data_type="fp8_sym",
act_bits=8,
+ nsamples=1,
act_dynamic=False,
)
q_model, qconfig = autoround.quantize()
diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.py
index 4d55869cd..86de61341 100644
--- a/test/test_cpu/test_autoround.py
+++ b/test/test_cpu/test_autoround.py
@@ -111,21 +111,6 @@ class TestAutoRound(unittest.TestCase):
)
autoround.quantize()
- def test_nsample(self):
- autoround = AutoRound(
- self.model,
- self.tokenizer,
- bits=4,
- group_size=128,
- seqlen=2,
- nsamples=3,
- batch_size=3,
- iters=2,
- dataset=self.llm_dataloader,
- gradient_accumulate_steps=4,
- )
- autoround.quantize()
-
def test_default(self):
bits, group_size, sym = 4, 128, False
autoround = AutoRound(
@@ -148,20 +133,6 @@ class TestAutoRound(unittest.TestCase):
if torch.cuda.is_available():
autoround.save_quantized(output_dir="./saved", inplace=False)
- def test_sym(self):
- bits, group_size, sym = 4, 128, True
- autoround = AutoRound(
- self.model,
- self.tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=2,
- seqlen=10,
- dataset=self.llm_dataloader,
- )
- autoround.quantize()
-
def test_w4g1(self):
bits, group_size, sym = 4, -1, True
autoround = AutoRound(
@@ -356,7 +327,6 @@ class TestAutoRound(unittest.TestCase):
autoround.quantize()
def test_tensor_reshape(self):
- model_name = "facebook/opt-125m"
bits, group_size, sym = 4, 100, False
autoround = AutoRound(
self.model,
diff --git a/test/test_cpu/test_autoround_export_to_itrex.py b/test/test_cpu/test_autoround_export_to_itrex.py
index 90415f240..09db9f6d4 100644
--- a/test/test_cpu/test_autoround_export_to_itrex.py
+++ b/test/test_cpu/test_autoround_export_to_itrex.py
@@ -56,7 +56,7 @@ class TestAutoroundExport(unittest.TestCase):
model = copy.deepcopy(self.gptj)
out1 = model(self.lm_input)
round = AutoRound
- optq_1 = round(model, self.tokenizer, nsamples=20, amp=False, seqlen=10, iters=10, enable_torch_compile=False)
+ optq_1 = round(model, self.tokenizer, nsamples=1, amp=False, seqlen=10, iters=10, enable_torch_compile=False)
q_model, layer_config1 = optq_1.quantize() ##compile model
from auto_round.export.export_to_itrex import pack_model
@@ -72,7 +72,7 @@ class TestAutoroundExport(unittest.TestCase):
model = copy.deepcopy(self.gptj)
out6 = model(self.lm_input)
- optq_2 = round(model, self.tokenizer, device="cpu", nsamples=20, seqlen=10)
+ optq_2 = round(model, self.tokenizer, device="cpu", nsamples=1, seqlen=10)
q_model, layer_config2 = optq_2.quantize()
compressed_model = pack_model(model=q_model, layer_config=layer_config2, inplace=False)
compressed_model = compressed_model.to(torch.float32)
@@ -96,7 +96,7 @@ class TestAutoroundExport(unittest.TestCase):
model = copy.deepcopy(self.gptj)
out1 = model(self.lm_input)
round = AutoRound
- optq_1 = round(model, self.tokenizer, nsamples=20, amp=False, seqlen=10, iters=10, enable_torch_compile=False)
+ optq_1 = round(model, self.tokenizer, nsamples=1, amp=False, seqlen=10, iters=10, enable_torch_compile=False)
q_model, layer_config1 = optq_1.quantize()
from auto_round.export.export_to_itrex import pack_model
diff --git a/test/test_cpu/test_basic_usage.py b/test/test_cpu/test_basic_usage.py
index 93266c2b7..02db860fe 100644
--- a/test/test_cpu/test_basic_usage.py
+++ b/test/test_cpu/test_basic_usage.py
@@ -56,13 +56,13 @@ class TestAutoRoundCmd(unittest.TestCase):
assert False, "cmd line test fail, please have a check"
res = os.system(
- f"cd ../.. && {python_path} -m auto_round --mllm --model Qwen/Qwen2-VL-2B-Instruct --iter 2 --nsamples 10 --seqlen 32 --format auto_round --output_dir ./saved"
+ f"cd ../.. && {python_path} -m auto_round --mllm --model Qwen/Qwen2-VL-2B-Instruct --iter 2 --nsamples 2 --seqlen 32 --format auto_round --output_dir ./saved"
)
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
res = os.system(
- f"cd ../.. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --model Qwen/Qwen2-VL-2B-Instruct --seqlen 256 --format auto_round"
+ f"cd ../.. && {python_path} -m auto_round --mllm --iter 2 --nsamples 2 --model Qwen/Qwen2-VL-2B-Instruct --seqlen 32 --format auto_round"
" --quant_nontext_module --output_dir ./saved "
)
if res > 0 or res == -1:
diff --git a/test/test_cpu/test_block_names.py b/test/test_cpu/test_block_names.py
index 403382580..cf3506816 100644
--- a/test/test_cpu/test_block_names.py
+++ b/test/test_cpu/test_block_names.py
@@ -217,24 +217,6 @@ class TestQuantizationBlocks(unittest.TestCase):
self.assertTrue(block_name == block_name_2)
self.assertTrue(len(block_name_2) == 1)
self.assertTrue("model.layers.23" == block_name_2[0][-1])
- ##tokenizer = AutoTokenizer.from_pretrained(model_name)
- # python_path = sys.executable
- # res = os.system(
- # f"cd ../.. && CUDA_VISIBLE_DEVICES=0 {python_path} -m auto_round --model {model_name} --iter 1 --nsamples 1 --format auto_round --output_dir test/saved --disable_eval")
- # if res > 0 or res == -1:
- # assert False, "cmd line test fail, please have a check"
- #
- # quantized_model_path = "../saved"
- #
- # from auto_round import AutoHfQuantizer
- # model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
- # tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- # text = "There is a girl who likes adventure,"
- # inputs = tokenizer(text, return_tensors="pt").to(model.device)
- # print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
- # shutil.rmtree("./saved", ignore_errors=True)
- # quant_config = model.config.quantization_config
- # assert quant_config.to_quant_block_names is not None
if __name__ == "__main__":
diff --git a/test/test_cpu/test_calib_dataset.py b/test/test_cpu/test_calib_dataset.py
index ce201048d..6746ff69c 100644
--- a/test/test_cpu/test_calib_dataset.py
+++ b/test/test_cpu/test_calib_dataset.py
@@ -77,7 +77,15 @@ class TestLocalCalibDataset(unittest.TestCase):
dataset = "NeelNanda/pile-10k:apply_chat_template:system_prompt=''"
bits, group_size, sym = 4, 128, True
autoround = AutoRound(
- model, tokenizer, bits=bits, group_size=group_size, sym=sym, iters=2, seqlen=128, dataset=dataset
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=128,
+ dataset=dataset,
+ nsamples=1,
)
autoround.quantize()
@@ -85,7 +93,15 @@ class TestLocalCalibDataset(unittest.TestCase):
dataset = "NeelNanda/pile-10k" + "," + "madao33/new-title-chinese" + "," + "mbpp"
bits, group_size, sym = 4, 128, True
autoround = AutoRound(
- self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=2, seqlen=128, dataset=dataset
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=128,
+ dataset=dataset,
+ nsamples=1,
)
autoround.quantize()
@@ -93,7 +109,15 @@ class TestLocalCalibDataset(unittest.TestCase):
dataset = "NeelNanda/pile-10k:num=256,mbpp:num=256"
bits, group_size, sym = 4, 128, True
autoround = AutoRound(
- self.model, self.tokenizer, bits=bits, group_size=group_size, sym=sym, iters=2, seqlen=128, dataset=dataset
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=128,
+ dataset=dataset,
+ nsamples=1,
)
autoround.quantize()