PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix gpt-j-6b RTN RuntimeError (#1848)
Liang Lv28f423896294
diff --git a/auto_round/calibration/hooks.py b/auto_round/calibration/hooks.pyindex 0d356ee9f..7ebbf1fe0 100644--- a/auto_round/calibration/hooks.py+++ b/auto_round/calibration/hooks.py@@ -59,7 +59,7 @@ def make_block_forward_func(state, name: str) -> Callable:new_data = alibireturn new_data- def forward(m, hidden_states=None, *positional_inputs, **kwargs):+ def forward_capture(m, hidden_states=None, *positional_inputs, **kwargs):if name not in state.inputs:state.inputs[name] = {}init_cache(positional_inputs, state.inputs[name])@@ -143,13 +143,19 @@ def make_block_forward_func(state, name: str) -> Callable:raise NotImplementedErrorelse:if hidden_states is not None:- kwargs.pop("hidden_states")- return m.orig_forward(hidden_states, *positional_inputs, **kwargs)+ kwargs.pop("hidden_states", None)+ if positional_inputs:+ return m.orig_forward(hidden_states=hidden_states, *positional_inputs, **kwargs)+ else:+ return m.orig_forward(hidden_states, **kwargs)else:# Currently only for Llama-3.2-Vision-Instruct Seriesreturn m.orig_forward(*positional_inputs, **kwargs)- return forward+ # Apply positional-to-kwargs conversion so positional_inputs get their proper parameter names.+ from auto_round.utils.model import wrap_block_forward_positional_to_kwargs++ return wrap_block_forward_positional_to_kwargs(forward_capture)def make_layer_cache_hook(state, name: str) -> Callable:diff --git a/auto_round/calibration/inputs.py b/auto_round/calibration/inputs.pyindex b397bdd05..c45fa33d3 100644--- a/auto_round/calibration/inputs.py+++ b/auto_round/calibration/inputs.py@@ -54,6 +54,16 @@ def split_inputs(return input_ids, input_others+def _unwrap_single_element(input_others):+ """Unwrap single-element list/tuple values from kwargs dict."""+ for key in list(input_others.keys()):+ if key == "positional_inputs":+ continue+ val = input_others[key]+ if isinstance(val, (list, tuple)) and len(val) == 1:+ input_others[key] = val[0]++def preprocess_block_inputs(inputs: dict,*,@@ -81,6 +91,8 @@ def preprocess_block_inputs(input_ids = to_dtype(input_ids, tmp_dtype)input_others = to_device(input_others, compress_context.cache_device)+ _unwrap_single_element(input_others)+for key in input_others.keys():if isinstance(input_others[key], torch.Tensor) and (input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16@@ -88,5 +100,8 @@ def preprocess_block_inputs(input_others[key] = input_others[key].to(tmp_dtype)elif isinstance(input_others[key], list):for i in range(len(input_others[key])):- to_dtype(input_others[key][i], tmp_dtype)+ v = input_others[key][i]+ if isinstance(v, torch.Tensor) and v.dtype in (torch.int32, torch.int64):+ continue+ input_others[key][i] = to_dtype(v, tmp_dtype)return input_ids, input_othersdiff --git a/auto_round/compressors/data_driven.py b/auto_round/compressors/data_driven.pyindex 10ac8332f..718558e7d 100644--- a/auto_round/compressors/data_driven.py+++ b/auto_round/compressors/data_driven.py@@ -964,11 +964,20 @@ class CalibratedRTNCompressor(DataDrivenCompressor):def process_input_others(input_others):input_others = to_device(input_others, self.compress_context.cache_device)+ # Unwrap single-element list/tuple so they are passed as bare values.+ for key in list(input_others.keys()):+ val = input_others[key]+ if isinstance(val, (list, tuple)) and len(val) == 1:+ input_others[key] = val[0]for key, val in input_others.items():if isinstance(val, torch.Tensor) and val.dtype in (torch.float16, torch.bfloat16):input_others[key] = val.to(tmp_dtype)elif isinstance(val, list):- input_others[key] = [to_dtype(v, tmp_dtype) for v in val]+ input_others[key] = [+ to_dtype(v, tmp_dtype)+ for v in val+ if not (isinstance(v, torch.Tensor) and v.dtype in (torch.int32, torch.int64))+ ]return input_othersinput_others = inputsdiff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 6cc00dd79..774c1083d 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -177,11 +177,51 @@ def block_forward(input_others, input_tuple = prepare_special_model_block_inputs(block, input_ids, input_others, input_tuple)+ # Use the block's actual parameter name for the first positional argument.+ import inspect as _inspect++ param_names = [p for p in _inspect.signature(block.forward).parameters.keys() if p != "self"]+ block_input_kwarg = param_names[0] if param_names else "hidden_states"+ if block_input_kwarg not in input_others:+ input_others[block_input_kwarg] = input_ids++ # Convert positional inputs to keyword args for any remaining positional parameters.+ positional_inputs = input_tuple or ()+ if positional_inputs:+ for i, val in enumerate(positional_inputs):+ param_idx = i + 1 # hidden_states is params[0]+ if param_idx < len(param_names):+ param_name = param_names[param_idx]+ if param_name not in input_others:+ input_others[param_name] = val+ positional_inputs = ()++ # Guard: ensure position_ids is a tensor, not a list or None.+ if "position_ids" in input_others:+ pid = input_others["position_ids"]+ if isinstance(pid, list):+ if len(pid) == 1:+ input_others["position_ids"] = pid[0]+ elif len(pid) == 0:+ # Generate position_ids from hidden_states shape when it's empty.+ input_others["position_ids"] = (+ torch.arange(input_ids.shape[1], device=input_ids.device, dtype=torch.long)+ .unsqueeze(0)+ .expand(input_ids.shape[0], -1)+ )+ elif pid is None:+ # Generate position_ids from hidden_states shape when it's None.+ input_others["position_ids"] = (+ torch.arange(input_ids.shape[1], device=input_ids.device, dtype=torch.long)+ .unsqueeze(0)+ .expand(input_ids.shape[0], -1)+ )+if amp:with autocast(device_type=str(device).split(":")[0], dtype=amp_dtype): # pragma: no cover- output = block(input_ids, *input_tuple, **input_others)+ output = block(**input_others)else:- output = block(input_ids, *input_tuple, **input_others)+ output = block(**input_others)if isinstance(output_return_id, int) and (isinstance(output, list) or isinstance(output, tuple)):output = output[output_return_id]return outputdiff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex b80a5ba6f..e4bc49765 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -634,7 +634,7 @@ class SupportedFormats:return self._support_list[key]-SHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")+SHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings", "cu_seqlens")deepspeed_exists = Falseif importlib.util.find_spec("deepspeed"): # check if deepspeed is installeddiff --git a/test/test_cpu/utils/test_calibration_inputs.py b/test/test_cpu/utils/test_calibration_inputs.pyindex 30e84b2fa..87009c253 100644--- a/test/test_cpu/utils/test_calibration_inputs.py+++ b/test/test_cpu/utils/test_calibration_inputs.py@@ -91,7 +91,6 @@ def test_preprocess_block_inputs_keeps_list_dtype():compress_context=compress_context,)- # The extracted helper intentionally preserves the legacy implementation,- # which iterates list entries through to_dtype() without writing the result- # back. Lock this down so the refactor does not change runtime behaviour.- assert input_others["past_key_values"][0].dtype == torch.float16+ # After Fix gpt-j-6b runtime error, list items are properly converted+ # to the AMP dtype. Previously the result of to_dtype() was not written back.+ assert input_others["past_key_values"][0].dtype == torch.bfloat16