Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix bs!=1 for gemma and MiniMax-Text-01 (#515)

Wenhua Cheng23cd60dd541e
diff --git a/README.md b/README.md
index 33048db91..53a5b7171 100644
--- a/README.md
+++ b/README.md
@@ -44,7 +44,7 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).
```bash
# GPU
-pip install auto-round[gpu]
+pip install auto-round
# CPU
pip install auto-round[cpu]
@@ -58,7 +58,7 @@ pip install auto-round-lib
```bash
# GPU
- pip install .[gpu]
+ pip install .
# CPU
pip install .[cpu]
@@ -102,8 +102,8 @@ auto-round-best \
```
```bash
-auto-round-light \
## light accuracy, 2-3X speedup, slight accuracy drop at W4 and larger accuracy drop at W2
+auto-round-light \
--model facebook/opt-125m \
--bits 4 \
--group_size 128 \
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 4260e6720..af175fa18 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -25,12 +25,6 @@ from torch import autocast
from tqdm import tqdm
import accelerate
from .wrapper import WrapperMultiblock, wrapper_block, unwrapper_block, WrapperLinear, unwrapper_layer
-from .special_model_handler import (
- shareable_keywords,
- init_cache_for_special_model,
- reset_params,
- check_skippable_keywords
-)
from .utils import (
CpuInfo,
block_forward,
@@ -44,7 +38,6 @@ from .utils import (
htcore,
is_optimum_habana_available,
logger,
- sampling_inputs,
to_device,
to_dtype,
get_layer_names_in_block,
@@ -54,9 +47,11 @@ from .utils import (
find_matching_blocks, is_debug_mode,
TORCH_VERSION_AT_LEAST_2_6,
supported_layer_types,
- get_layer_features,
+ get_layer_features,
set_module,
- llm_load_model
+ llm_load_model,
+ reset_params,
+ init_cache, check_skippable_keywords, get_shared_keys,
)
from .low_cpu_mem.utils import get_layers_before_block
@@ -311,6 +306,7 @@ class AutoRound(object):
]
self.has_qlayer_outside_block = self.set_layerwise_config(self.layer_config) ##better place in the end
+ self.shared_cache_keys = get_shared_keys(self.model)
def set_device_map_in_blocks(self, device_map):
"""Sets the device map for specific blocks in the model.
@@ -773,12 +769,13 @@ class AutoRound(object):
for i in range(0, nsamples, bs):
end_index = min(nsamples, i + bs)
indices = torch.arange(i, end_index).to(torch.long)
- tmp_input_ids, tmp_input_others = sampling_inputs(
+ tmp_input_ids, tmp_input_others = AutoRound.sampling_inputs(
input_ids,
input_others,
indices,
self.seqlen,
- self.batch_dim
+ self.batch_dim,
+ share_cache_keys=self.shared_cache_keys
)
tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(
cache_device
@@ -1010,7 +1007,7 @@ class AutoRound(object):
new_data = data
if batch_size <= 1:
return new_data
- if data_name in shareable_keywords:
+ if data_name in self.shared_cache_keys:
return None
if "alibi" in data_name:
if isinstance(data, torch.Tensor):
@@ -1032,7 +1029,7 @@ class AutoRound(object):
"""
if name not in self.inputs:
self.inputs[name] = {}
- init_cache_for_special_model(self.model, positional_inputs, self.inputs[name])
+ init_cache(positional_inputs, self.inputs[name])
if self.batch_dim is None:
self.batch_dim = 0
@@ -1055,7 +1052,7 @@ class AutoRound(object):
or isinstance(kwargs[key], tuple):
if key not in self.inputs[name].keys(): # initialization
data = to_device(kwargs[key], device=torch.device("cpu"))
- if data is None or (self.batch_size > 1 and key in shareable_keywords):
+ if data is None or (self.batch_size > 1 and key in self.shared_cache_keys):
self.inputs[name][key] = data
continue
if self.batch_size <= 1:
@@ -1174,7 +1171,7 @@ class AutoRound(object):
if self.lr_scheduler is None:
lr_schedule = torch.optim.lr_scheduler.LinearLR(
- optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters, verbose=False
+ optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters
)
else:
lr_schedule = copy.deepcopy(self.lr_scheduler)
@@ -1357,7 +1354,7 @@ class AutoRound(object):
if self.lr_scheduler is None:
lr_schedule = torch.optim.lr_scheduler.LinearLR(
- optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters, verbose=False
+ optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters
)
else:
lr_schedule = copy.deepcopy(self.lr_scheduler)
@@ -1389,12 +1386,13 @@ class AutoRound(object):
num_elm = sum(id.numel() for id in current_input_ids)
for tmp_step in range(self.gradient_accumulate_steps):
indices = whole_indices[tmp_step * self.batch_size: (tmp_step + 1) * self.batch_size]
- current_input_ids, current_input_others = sampling_inputs(
+ current_input_ids, current_input_others = AutoRound.sampling_inputs(
input_ids,
input_others,
indices,
seqlen=self.seqlen,
batch_dim=self.batch_dim,
+ share_cache_keys=self.shared_cache_keys
)
current_output = [output[x] for x in indices]
@@ -1786,6 +1784,47 @@ class AutoRound(object):
optimizer.zero_grad()
lr_schedule.step()
+ @classmethod
+ @torch.no_grad()
+ def sampling_inputs(cls, input_ids, input_others, indices, seqlen,
+ batch_dim=0,share_cache_keys=()):
+ """Samples inputs based on the given indices and sequence length.
+
+ Args:
+ input_ids: The list of input tensor containing input_ids.
+ input_others: A dictionary containing other input data.
+ indices: The indices to sample from the input.
+ seqlen: The sequence length.
+
+ Returns:
+ current_input_ids: The sampled input IDs.
+ current_input_others: The sampled other input data.
+ """
+ current_input_ids = [input_ids[i] for i in indices]
+
+ current_input_ids = torch.cat(current_input_ids, dim=batch_dim)
+
+ current_input_others = {"positional_inputs": input_others["positional_inputs"]}
+ for key in input_others.keys():
+ if "positional_inputs" in key:
+ continue
+ if (key not in share_cache_keys or len(indices) == 1) \
+ and not isinstance(input_others[key], (str, bool, type(None))):
+ current_input_others[key] = None
+ if input_others[key] is not None:
+ current_input_others[key] = [input_others[key][i] for i in indices]
+ if len(indices) == 1:
+ current_input_others[key] = current_input_others[key][0]
+ else:
+ try:
+ current_input_others[key] = torch.cat(current_input_others[key], dim=0)
+ except TypeError as err:
+ logger.warning_once("Please check the model cache inputs or try setting batch_size to 1.")
+ else:
+ current_input_others[key] = input_others[key]
+
+ return current_input_ids, current_input_others
+
class AutoRoundOPT(AutoRound):
"""Class for automatic rounding-based quantization with optimizers like adamw of a PyTorch model.
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index de8eb4ea7..f602f9b3f 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -12,11 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import torch
-from collections import UserDict
-shareable_keywords = ("position_ids", "cache_position", "position_embeddings")
-mllms_with_limited_bs = ("llava", "qwen2_vl", "phi3_v", "mllama") # Limitations on batch_size
-skippable_cache_keys = ("past_key_value",)
+
+mllms_with_limited_bs = ("llava", "qwen2_vl", "phi3_v", "mllama") # Limitations on batch_size
SUPPORT_ONLY_TEXT_MODELS = [
"phi3_v",
@@ -28,9 +25,13 @@ SUPPORT_ONLY_TEXT_MODELS = [
"idefics3"
]
+SPECIAL_SHARED_CACHE_KEYS = {
+ "Gemma3ForConditionalGeneration": ("position_embeddings_global", "position_embeddings_local")}
+SPECIAL_SHARED_CACHE_KEYS["MiniMaxText01ForCausalLM"] = ("slope_rate",)
+
+
def _handle_special_model(model):
if model.config.model_type == "deepseek_vl_v2":
- from auto_round.special_model_handler import _deepseek_vl2_forward
from functools import partial
model.forward = partial(_deepseek_vl2_forward, model)
return model
@@ -45,31 +46,33 @@ def _get_deepseek_vl2_multimodal_block(model, quant_vision=False):
block_names.append([f"language.model.layers.{i}" for i in range(len(model.language.model.layers))])
return block_names
+
SPECIAL_MULTIMODAL_BLOCK = {
"deepseek_vl_v2": _get_deepseek_vl2_multimodal_block
}
+
def _deepseek_vl2_forward(
- model,
- input_ids = None,
-
- position_ids = None,
- attention_mask = None,
- past_key_values = None,
- inputs_embeds = None,
-
- images = None,
- images_seq_mask = None,
- images_spatial_crop = None,
-
- labels = None,
- use_cache = None,
- output_attentions = None,
- output_hidden_states = None,
- return_dict = None,
- cache_position = None,
- **kwargs
- ):
+ model,
+ input_ids=None,
+
+ position_ids=None,
+ attention_mask=None,
+ past_key_values=None,
+ inputs_embeds=None,
+
+ images=None,
+ images_seq_mask=None,
+ images_spatial_crop=None,
+
+ labels=None,
+ use_cache=None,
+ output_attentions=None,
+ output_hidden_states=None,
+ return_dict=None,
+ cache_position=None,
+ **kwargs
+):
inputs_embeds = model.prepare_inputs_embeds(
input_ids=input_ids,
images=images,
@@ -90,79 +93,6 @@ def _deepseek_vl2_forward(
cache_position=cache_position)
-def to_device(input, device=torch.device("cpu")):
- """Moves input data to the specified device.
-
- Args:
- input: The input data to be moved.
- device: The target device.
-
- Returns:
- The input data on the specified device.
- """
- if input is None:
- return None
- if isinstance(input, torch.Tensor):
- return input.to(device)
- if isinstance(input, dict) or isinstance(input, UserDict):
- for inp in input.keys():
- input[inp] = to_device(input[inp], device)
-
- elif isinstance(input, list) or isinstance(input, tuple):
- if len(input) == 0:
- return input
- input_res = []
- for inp in input:
- input_res.append(to_device(inp, device))
- if isinstance(input, tuple):
- input_res = tuple(input_res)
- input = input_res
-
- return input
-
-
-def init_cache_for_special_model(model, positional_inputs, inputs):
- """
- Initializes special model inputs by adding positional inputs if missing.
-
- Args:
- model: The model instance being initialized.
- positional_inputs (list): List of positional inputs to add to inputs.
- inputs (dict): Dictionary of model inputs.
-
- Modifies:
- inputs (dict): Adds "positional_inputs" key if not present.
- """
- if "positional_inputs" not in inputs: # for chatglm Series
- inputs["positional_inputs"] = []
- for idx, item in enumerate(positional_inputs):
- inputs["positional_inputs"] = to_device(positional_inputs)
-
-
-def reset_params(inputs):
- """
- Resets specific input parameters to avoid saving the key-value cache during fine-tuning.
-
- Args:
- inputs (dict): Dictionary of model inputs.
-
- Modifies:
- inputs (dict): Sets "use_cache" to False if the key is present.
- """
- if "use_cache" in inputs.keys(): # Not storing kv cache
- inputs['use_cache'] = False
-
-
-def check_skippable_keywords(key):
- """
- Prints a reminder if a key is not stored during quantization fine-tuning.
- """
- for cache_key in skippable_cache_keys:
- if cache_key not in key:
- return True
- return False
-
-
def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):
"""
Checks model configuration to determine if it's necessary to limit bs to avoid potential input shape mismatches.
@@ -171,6 +101,6 @@ def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):
if hasattr(model, "config") and key in model.config.model_type and batch_size != 1:
accumulate_steps = batch_size * gradient_accumulate_steps
print("To avoid the tensor concat mismatch problem, modified parameters to " \
- f"batch_size=1. As an alternative, set the gradient_accumulate_steps={accumulate_steps}")
+ f"batch_size=1. As an alternative, set the gradient_accumulate_steps={accumulate_steps}")
return 1, accumulate_steps
return batch_size, gradient_accumulate_steps
diff --git a/auto_round/utils.py b/auto_round/utils.py
index ea9779893..00db00aae 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -27,10 +27,12 @@ from torch.amp import autocast
from functools import lru_cache
from packaging import version
import gc
-from .special_model_handler import shareable_keywords, SPECIAL_MULTIMODAL_BLOCK
+from .special_model_handler import SPECIAL_MULTIMODAL_BLOCK, SPECIAL_SHARED_CACHE_KEYS
import transformers
from auto_round.export.export_to_gguf.config import GGUF_CONFIG
+shared_cache_keys = ("position_ids", "cache_position", "position_embeddings")
+
supported_formats = (
"auto_round", "auto_gptq", "auto_awq", "auto_round:auto_gptq", "auto_round:gptqmodel", "auto_round:auto_awq",
"itrex", "itrex_xpu", "fake"
@@ -389,6 +391,7 @@ def get_block_names(model, quant_vision=False):
Returns:
block_names: A list whose elements are list of block's layer names
"""
+
def _get_llm_block_names(model):
block_names = []
target_modules = []
@@ -438,47 +441,6 @@ def collect_best_params(block):
return params
-def sampling_inputs(input_ids, input_others, indices, seqlen,
- batch_dim=0):
- """Samples inputs based on the given indices and sequence length.
-
- Args:
- input_ids: The list of input tensor containing input_ids.
- input_others: A dictionary containing other input data.
- indices: The indices to sample from the input.
- seqlen: The sequence length.
-
- Returns:
- current_input_ids: The sampled input IDs.
- current_input_others: The sampled other input data.
- """
- current_input_ids = [input_ids[i] for i in indices]
-
- current_input_ids = torch.cat(current_input_ids, dim=batch_dim)
-
- current_input_others = {"positional_inputs": input_others["positional_inputs"]}
- for key in input_others.keys():
- if "positional_inputs" in key:
- continue
- if (key not in shareable_keywords or len(indices) == 1) \
- and not isinstance(input_others[key], (str, bool, type(None))):
- current_input_others[key] = None
- if input_others[key] is not None:
- current_input_others[key] = [input_others[key][i] for i in indices]
- if len(indices) == 1:
- current_input_others[key] = current_input_others[key][0]
- else:
- try:
- current_input_others[key] = torch.cat(current_input_others[key], dim=0)
- except TypeError as err:
- logger.warning_once("Please check the model cache inputs or try setting batch_size to 1.")
- else:
- current_input_others[key] = input_others[key]
-
- return current_input_ids, current_input_others
-
-
def block_forward(block, input_ids, input_others, amp=False, amp_dtype=torch.float16, device=torch.device("cpu")):
"""Performs a forward pass through a block with the given inputs.
@@ -1204,7 +1166,6 @@ def is_debug_mode():
return sys.gettrace() is not None or sys.flags.debug == 1
-
def get_layer_features(layer):
"""Extracts input and output feature dimensions for supported layers."""
if isinstance(layer, torch.nn.Linear):
@@ -1213,12 +1174,13 @@ def get_layer_features(layer):
return layer.weight.shape[0], layer.weight.shape[1]
return None, None # Unsupported layer type
+
def _gguf_args_check(args):
from auto_round.utils import logger
from auto_round.export.export_to_gguf.config import GGUF_CONFIG
formats = args.format.lower().replace(' ', '').split(",")
- formats = sorted(formats, key=lambda x:len(x))
+ formats = sorted(formats, key=lambda x: len(x))
pattern = re.compile("q\d_k")
pre_dq_format = ""
for format in GGUF_CONFIG:
@@ -1241,7 +1203,7 @@ def _gguf_args_check(args):
logger.error(f"Model {model_architecture} is not supported to export GGUF format")
sys.exit(1)
- if re.search(pattern, format) and ("hidden_size" in hparams and hparams["hidden_size"] % 256 !=0):
+ if re.search(pattern, format) and ("hidden_size" in hparams and hparams["hidden_size"] % 256 != 0):
model_name = args.model.split('/')
model_name = model_name[-1] if model_name[-1] else model_name[-2]
hidden_size = hparams["hidden_size"]
@@ -1267,7 +1229,7 @@ def _gguf_args_check(args):
def _to_model_dtype(model, model_dtype):
- if model_dtype != None:
+ if model_dtype is not None:
try:
if model_dtype == "float16" or model_dtype == "fp16":
model = model.to(torch.float16)
@@ -1280,6 +1242,7 @@ def _to_model_dtype(model, model_dtype):
exit()
return model
+
def llm_load_model(
pretrained_model_name_or_path,
torch_dtype="auto",
@@ -1288,7 +1251,7 @@ def llm_load_model(
model_dtype=None,
device="cpu",
low_cpu_mem_mode=0,
- low_cpu_mem_tmp_dir = None,
+ low_cpu_mem_tmp_dir=None,
**kwargs):
from transformers import AutoTokenizer, AutoModel, AutoModelForCausalLM
@@ -1322,7 +1285,6 @@ def llm_load_model(
torch_dtype=torch_dtype,
trust_remote_code=trust_remote_code)
else:
- from auto_round.utils import _use_hpu_compile_mode
if _use_hpu_compile_mode():
model = model_cls.from_pretrained(
pretrained_model_name_or_path, low_cpu_mem_usage=True, torch_dtype=torch_dtype,
@@ -1403,9 +1365,10 @@ def mllm_load_model(
return model, processor, tokenizer, image_processor
+
def is_pure_text_model(model):
"""verify on: phi-3.5, Mistral-Small-3.1, gemma-3, qwen2-vl, """
- if hasattr(model,"config") and hasattr(model.config, "vision_config"):
+ if hasattr(model, "config") and hasattr(model.config, "vision_config"):
return False
if hasattr(model.__class__, "main_input_name") and model.__class__.main_input_name != "input_ids":
return False
@@ -1419,3 +1382,60 @@ def is_pure_text_model(model):
if "img" in str(module.__class__).lower():
return False
return True
+
+
+def reset_params(inputs):
+ """
+ Resets specific input parameters to avoid saving the key-value cache during fine-tuning.
+
+ Args:
+ inputs (dict): Dictionary of model inputs.
+
+ Modifies:
+ inputs (dict): Sets "use_cache" to False if the key is present.
+ """
+ if "use_cache" in inputs.keys(): # Not storing kv cache
+ inputs['use_cache'] = False
+
+
+def check_skippable_keywords(key):
+ """
+ Prints a reminder if a key is not stored during quantization fine-tuning.
+ """
+ skippable_cache_keys = ("past_key_value",)
+ for cache_key in skippable_cache_keys:
+ if cache_key not in key:
+ return True
+ return False
+
+
+def init_cache(positional_inputs, inputs):
+ """
+ Initializes special model inputs by adding positional inputs if missing.
+
+ Args:
+ positional_inputs (list): List of positional inputs to add to inputs.
+ inputs (dict): Dictionary of model inputs.
+
+ Modifies:
+ inputs (dict): Adds "positional_inputs" key if not present.
+ """
+ if "positional_inputs" not in inputs: # for chatglm Series
+ inputs["positional_inputs"] = []
+ for idx, item in enumerate(positional_inputs):
+ inputs["positional_inputs"] = to_device(positional_inputs)
+
+
+def get_shared_keys(model):
+ """
+ Retrieves shared keys from the model's state dictionary.
+
+ Args:
+ model (torch.nn.Module): The model to retrieve shared keys from.
+
+ Returns:
+ tuple: tuple of shared keys.
+ """
+ shared_keys = shared_cache_keys
+ shared_keys += SPECIAL_SHARED_CACHE_KEYS.get(model.__class__.__name__, ())
+ return shared_keys
diff --git a/auto_round_extension/cuda/triton_utils/dequant.py b/auto_round_extension/cuda/triton_utils/dequant.py
index d971059d3..524612489 100644
--- a/auto_round_extension/cuda/triton_utils/dequant.py
+++ b/auto_round_extension/cuda/triton_utils/dequant.py
@@ -162,7 +162,6 @@ def quant_matmul_248(
class QuantLinearFunction(torch.autograd.Function):
@staticmethod
- @custom_fwd
def forward(ctx, input, qweight, scales, qzeros, g_idx, bits, maxq):
output = quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxq)
ctx.save_for_backward(qweight, scales, qzeros, g_idx)
@@ -170,7 +169,6 @@ class QuantLinearFunction(torch.autograd.Function):
return output
@staticmethod
- @custom_bwd
def backward(ctx, grad_output):
qweight, scales, qzeros, g_idx = ctx.saved_tensors
bits, maxq = ctx.bits, ctx.maxq
diff --git a/auto_round_extension/cuda/triton_utils_zp/dequant.py b/auto_round_extension/cuda/triton_utils_zp/dequant.py
index 4ef9936ce..416a406eb 100644
--- a/auto_round_extension/cuda/triton_utils_zp/dequant.py
+++ b/auto_round_extension/cuda/triton_utils_zp/dequant.py
@@ -162,7 +162,6 @@ def quant_matmul_248(
class QuantLinearFunction(torch.autograd.Function):
@staticmethod
- @custom_fwd
def forward(ctx, input, qweight, scales, qzeros, g_idx, bits, maxq):
output = quant_matmul_248(input, qweight, scales, qzeros, g_idx, bits, maxq)
ctx.save_for_backward(qweight, scales, qzeros, g_idx)
@@ -170,7 +169,6 @@ class QuantLinearFunction(torch.autograd.Function):
return output
@staticmethod
- @custom_bwd
def backward(ctx, grad_output):
qweight, scales, qzeros, g_idx = ctx.saved_tensors
bits, maxq = ctx.bits, ctx.maxq
diff --git a/requirements-gpu.txt b/requirements-gpu.txt
deleted file mode 100644
index 4a5350e73..000000000
--- a/requirements-gpu.txt
+++ /dev/null
@@ -1 +0,0 @@
-auto-gptq>=0.7.1
diff --git a/setup.py b/setup.py
index 3b8ec0222..a725e32e6 100644
--- a/setup.py
+++ b/setup.py
@@ -82,7 +82,6 @@ PKG_INSTALL_CFG = {
),
"install_requires": fetch_requirements("requirements.txt"),
"extras_require": {
- "gpu": fetch_requirements("requirements-gpu.txt"),
"cpu": fetch_requirements("requirements-cpu.txt"),
},
}