Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix some torch compile warnings (#755)

Wenhua Cheng1f1d0a9c433c
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index aedcd860d..c30d2280a 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -31,10 +31,12 @@ from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType
from auto_round.low_cpu_mem.utils import get_layers_before_block
+from auto_round.sign_sgd import SignSGD
from auto_round.special_model_handler import _handle_moe_model
from auto_round.utils import (
INNER_SUPPORTED_LAYER_TYPES,
SUPPORTED_DTYPES,
+ SUPPORTED_FORMATS,
SUPPORTED_LAYER_TYPES,
TORCH_VERSION_AT_LEAST_2_6,
CpuInfo,
@@ -75,6 +77,7 @@ from auto_round.utils import (
logger,
mv_module_from_gpu,
reset_params,
+ set_amax_for_all_moe_layers,
set_module,
to_device,
to_dtype,
@@ -629,7 +632,6 @@ class AutoRound(object):
_gguf_args_check(self, format, model_type=ModelType.MMPROJ)
formats = format.replace("q*_", f"q{self.bits}_").replace(" ", "").split(",")
- from auto_round.utils import SUPPORTED_FORMATS
for format_ in formats:
if format_ not in SUPPORTED_FORMATS:
@@ -926,7 +928,7 @@ class AutoRound(object):
logger.info("start to compute imatrix for GGUF quantization.")
# Load dataset
- from .calib_dataset import get_dataloader
+ from auto_round.calib_dataset import get_dataloader
if hasattr(self.model, "is_fp8"):
convert_fp8_model_to_16b_model(self.model, self.amp_dtype)
@@ -1004,8 +1006,6 @@ class AutoRound(object):
accelerate.hooks.remove_hook_from_submodules(model)
# Perform quantization using RTN
- from tqdm import tqdm
-
pbar = tqdm(all_to_quantized_module_names)
block_names_cnt = len(flatten_list(get_block_names(self.model, True)))
clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt
@@ -2224,12 +2224,14 @@ class AutoRound(object):
unwrapper_layer(self.model, wrapper_linear, layer_name, {})
mv_module_from_gpu(layer, self.low_cpu_mem_usage)
+ lr = torch.tensor(self.lr)
+ minmax_lr = torch.tensor(self.minmax_lr)
if self.enable_minmax_tuning:
optimizer = self.optimizer(
- [{"params": round_params}, {"params": minmax_params, "lr": self.minmax_lr}], lr=self.lr, weight_decay=0
+ [{"params": round_params}, {"params": minmax_params, "lr": minmax_lr}], lr=lr, weight_decay=0
)
else:
- optimizer = self.optimizer(round_params, lr=self.lr, weight_decay=0)
+ optimizer = self.optimizer(round_params, lr=lr, weight_decay=0)
if self.lr_scheduler is None:
lr_schedule = torch.optim.lr_scheduler.LinearLR(
@@ -2450,12 +2452,14 @@ class AutoRound(object):
else:
round_params.append(m.params[key])
+ lr = torch.tensor(self.lr)
+ minmax_lr = torch.tensor(self.minmax_lr)
if self.enable_minmax_tuning:
optimizer = self.optimizer(
- [{"params": round_params}, {"params": minmax_params, "lr": self.minmax_lr}], lr=self.lr, weight_decay=0
+ [{"params": round_params}, {"params": minmax_params, "lr": minmax_lr}], lr=lr, weight_decay=0
)
else:
- optimizer = self.optimizer(round_params, lr=self.lr, weight_decay=0)
+ optimizer = self.optimizer(round_params, lr=lr, weight_decay=0)
if len(round_params) + len(minmax_params) <= 0:
dump_info = (
@@ -2463,7 +2467,7 @@ class AutoRound(object):
f"layers in the block"
)
logger.info(dump_info)
- unwrapper_block(block, {}) ## TODO Quant layer should change
+ unwrapper_block(block, {}) # TODO Quant layer should change
mv_module_from_gpu(block, self.low_cpu_mem_usage)
return output, output
@@ -2490,15 +2494,15 @@ class AutoRound(object):
init_loss = None
best_params = {}
total_loss = 0
-
for i in range(self.iters):
total_loss = 0
if self.sampler == "rand":
whole_indices = torch.randperm(nsamples)[:pick_samples]
- ##we assume the block input and output shape is same
+ # We assume the block input and output shape is same
if self.gradient_accumulate_steps != 1:
current_input_ids = [input_ids[i] for i in whole_indices]
num_elm = sum(id.numel() for id in current_input_ids)
+
for tmp_step in range(self.gradient_accumulate_steps):
indices = whole_indices[tmp_step * self.batch_size : (tmp_step + 1) * self.batch_size]
current_input_ids, current_input_others = AutoRound.sampling_inputs(
@@ -2512,7 +2516,6 @@ class AutoRound(object):
current_output = [output[x] for x in indices]
current_output = torch.cat(current_output, dim=self.batch_dim)
-
current_output = to_device(current_output, device)
output_q = block_forward(
@@ -2561,11 +2564,10 @@ class AutoRound(object):
logger.info(f"{unquantized_layer_names} have not been quantized")
with torch.no_grad():
unwrapper_block(block, best_params)
+
if self.enable_quanted_input:
if is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats):
- from auto_round.utils import set_amax_for_all_moe_layers
-
- # enable moe experts act_max automatic generation for WrapperWALayer
+ # Enable moe experts act_max automatic generation for WrapperWALayer
set_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")
if self.low_cpu_mem_usage:
block = block.to(device)
@@ -2616,7 +2618,7 @@ class AutoRound(object):
clear_memory()
input_ids = to_device(input_ids, self.cache_device)
input_others = to_device(input_others, self.cache_device)
- ## as in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage
+ # As in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage
tmp_dtype = self.amp_dtype if self.amp else torch.float32
for i in range(len(input_ids)):
input_ids[i] = input_ids[i].to(tmp_dtype)
@@ -2871,8 +2873,6 @@ class AutoRound(object):
Returns:
The specified optimizer.
"""
- from auto_round.sign_sgd import SignSGD
-
return SignSGD
def get_scaler(self):
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 40fbcc5de..fec2177e2 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -172,6 +172,8 @@ auto_gptq = LazyImport("auto_gptq")
htcore = LazyImport("habana_frameworks.torch.core")
+@torch._dynamo.disable()
+@lru_cache(None)
def is_optimum_habana_available():
from transformers.utils.import_utils import is_optimum_available
@@ -920,6 +922,7 @@ def _clear_memory_for_cpu_and_cuda(tensor=None):
torch.xpu.empty_cache()
+@torch._dynamo.disable()
def clear_memory(tensor=None):
if is_hpu_supported():
# hpu does not have empty_cache
@@ -1117,8 +1120,6 @@ def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=Non
def get_device_and_parallelism(device: Union[str, torch.device, int]) -> Tuple[str, bool]:
- from auto_round.utils import detect_device
-
if isinstance(device, str):
devices = device.replace(" ", "").split(",")
elif isinstance(device, int):
@@ -2457,33 +2458,34 @@ def set_amax_for_all_moe_layers(model: torch.nn.Module, layer_name=None, attr_na
model = get_module(model, moe_name)
# Handle input quantizers of experts that are not calibrated
for name, sub_module in model.named_modules():
- if is_moe(sub_module) and hasattr(sub_module, "experts"):
- expert_linear_names = get_expert_linear_names(sub_module)
- for linear_name in expert_linear_names:
- if isinstance(sub_module.experts, collections.abc.Iterable):
- # For other MoE models (like Mixtral) with iterable experts
- try:
- set_amax_for_uncalibrated_experts(
- [getattr(expert, linear_name) for expert in sub_module.experts], attr_name=attr_name
- )
- except AttributeError as e:
- # Provide more helpful debugging information
- expert_types = list(set(type(expert).__name__ for expert in sub_module.experts))
- raise AttributeError(
- f"Failed to access attribute '{linear_name}' on experts. "
- f"MoE module type: {type(sub_module).__name__}, "
- f"Expert types: {expert_types}, "
- f"Expected linear names: {expert_linear_names}. "
- f"This suggests the get_expert_linear_names function may need "
- f"to be updated for this model architecture. "
- f"Original error: {e}"
- ) from e
- else:
- # Unsupported MoE model structure
- raise NotImplementedError(
- f"MoE model with experts type '{type(sub_module.experts).__name__}' is not supported in export."
- f"Please file an issue or add support for this model architecture."
+ if not (is_moe(sub_module) and hasattr(sub_module, "experts")):
+ continue
+ expert_linear_names = get_expert_linear_names(sub_module)
+ for linear_name in expert_linear_names:
+ if isinstance(sub_module.experts, collections.abc.Iterable):
+ # For other MoE models (like Mixtral) with iterable experts
+ try:
+ set_amax_for_uncalibrated_experts(
+ [getattr(expert, linear_name) for expert in sub_module.experts], attr_name=attr_name
)
+ except AttributeError as e:
+ # Provide more helpful debugging information
+ expert_types = list(set(type(expert).__name__ for expert in sub_module.experts))
+ raise AttributeError(
+ f"Failed to access attribute '{linear_name}' on experts. "
+ f"MoE module type: {type(sub_module).__name__}, "
+ f"Expert types: {expert_types}, "
+ f"Expected linear names: {expert_linear_names}. "
+ f"This suggests the get_expert_linear_names function may need "
+ f"to be updated for this model architecture. "
+ f"Original error: {e}"
+ ) from e
+ else:
+ # Unsupported MoE model structure
+ raise NotImplementedError(
+ f"MoE model with experts type '{type(sub_module.experts).__name__}' is not supported in export."
+ f"Please file an issue or add support for this model architecture."
+ )
class BackendDataType(str, Enum):