Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix gpt-j-6b RTN RuntimeError (#1848)

Liang Lv28f423896294
diff --git a/auto_round/calibration/hooks.py b/auto_round/calibration/hooks.py
index 0d356ee9f..7ebbf1fe0 100644
--- a/auto_round/calibration/hooks.py
+++ b/auto_round/calibration/hooks.py
@@ -59,7 +59,7 @@ def make_block_forward_func(state, name: str) -> Callable:
new_data = alibi
return new_data
- def forward(m, hidden_states=None, *positional_inputs, **kwargs):
+ def forward_capture(m, hidden_states=None, *positional_inputs, **kwargs):
if name not in state.inputs:
state.inputs[name] = {}
init_cache(positional_inputs, state.inputs[name])
@@ -143,13 +143,19 @@ def make_block_forward_func(state, name: str) -> Callable:
raise NotImplementedError
else:
if hidden_states is not None:
- kwargs.pop("hidden_states")
- return m.orig_forward(hidden_states, *positional_inputs, **kwargs)
+ kwargs.pop("hidden_states", None)
+ if positional_inputs:
+ return m.orig_forward(hidden_states=hidden_states, *positional_inputs, **kwargs)
+ else:
+ return m.orig_forward(hidden_states, **kwargs)
else:
# Currently only for Llama-3.2-Vision-Instruct Series
return m.orig_forward(*positional_inputs, **kwargs)
- return forward
+ # Apply positional-to-kwargs conversion so positional_inputs get their proper parameter names.
+ from auto_round.utils.model import wrap_block_forward_positional_to_kwargs
+
+ return wrap_block_forward_positional_to_kwargs(forward_capture)
def make_layer_cache_hook(state, name: str) -> Callable:
diff --git a/auto_round/calibration/inputs.py b/auto_round/calibration/inputs.py
index b397bdd05..c45fa33d3 100644
--- a/auto_round/calibration/inputs.py
+++ b/auto_round/calibration/inputs.py
@@ -54,6 +54,16 @@ def split_inputs(
return input_ids, input_others
+def _unwrap_single_element(input_others):
+ """Unwrap single-element list/tuple values from kwargs dict."""
+ for key in list(input_others.keys()):
+ if key == "positional_inputs":
+ continue
+ val = input_others[key]
+ if isinstance(val, (list, tuple)) and len(val) == 1:
+ input_others[key] = val[0]
+
+
def preprocess_block_inputs(
inputs: dict,
*,
@@ -81,6 +91,8 @@ def preprocess_block_inputs(
input_ids = to_dtype(input_ids, tmp_dtype)
input_others = to_device(input_others, compress_context.cache_device)
+ _unwrap_single_element(input_others)
+
for key in input_others.keys():
if isinstance(input_others[key], torch.Tensor) and (
input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16
@@ -88,5 +100,8 @@ def preprocess_block_inputs(
input_others[key] = input_others[key].to(tmp_dtype)
elif isinstance(input_others[key], list):
for i in range(len(input_others[key])):
- to_dtype(input_others[key][i], tmp_dtype)
+ v = input_others[key][i]
+ if isinstance(v, torch.Tensor) and v.dtype in (torch.int32, torch.int64):
+ continue
+ input_others[key][i] = to_dtype(v, tmp_dtype)
return input_ids, input_others
diff --git a/auto_round/compressors/data_driven.py b/auto_round/compressors/data_driven.py
index 10ac8332f..718558e7d 100644
--- a/auto_round/compressors/data_driven.py
+++ b/auto_round/compressors/data_driven.py
@@ -964,11 +964,20 @@ class CalibratedRTNCompressor(DataDrivenCompressor):
def process_input_others(input_others):
input_others = to_device(input_others, self.compress_context.cache_device)
+ # Unwrap single-element list/tuple so they are passed as bare values.
+ for key in list(input_others.keys()):
+ val = input_others[key]
+ if isinstance(val, (list, tuple)) and len(val) == 1:
+ input_others[key] = val[0]
for key, val in input_others.items():
if isinstance(val, torch.Tensor) and val.dtype in (torch.float16, torch.bfloat16):
input_others[key] = val.to(tmp_dtype)
elif isinstance(val, list):
- input_others[key] = [to_dtype(v, tmp_dtype) for v in val]
+ input_others[key] = [
+ to_dtype(v, tmp_dtype)
+ for v in val
+ if not (isinstance(v, torch.Tensor) and v.dtype in (torch.int32, torch.int64))
+ ]
return input_others
input_others = inputs
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index 6cc00dd79..774c1083d 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -177,11 +177,51 @@ def block_forward(
input_others, input_tuple = prepare_special_model_block_inputs(block, input_ids, input_others, input_tuple)
+ # Use the block's actual parameter name for the first positional argument.
+ import inspect as _inspect
+
+ param_names = [p for p in _inspect.signature(block.forward).parameters.keys() if p != "self"]
+ block_input_kwarg = param_names[0] if param_names else "hidden_states"
+ if block_input_kwarg not in input_others:
+ input_others[block_input_kwarg] = input_ids
+
+ # Convert positional inputs to keyword args for any remaining positional parameters.
+ positional_inputs = input_tuple or ()
+ if positional_inputs:
+ for i, val in enumerate(positional_inputs):
+ param_idx = i + 1 # hidden_states is params[0]
+ if param_idx < len(param_names):
+ param_name = param_names[param_idx]
+ if param_name not in input_others:
+ input_others[param_name] = val
+ positional_inputs = ()
+
+ # Guard: ensure position_ids is a tensor, not a list or None.
+ if "position_ids" in input_others:
+ pid = input_others["position_ids"]
+ if isinstance(pid, list):
+ if len(pid) == 1:
+ input_others["position_ids"] = pid[0]
+ elif len(pid) == 0:
+ # Generate position_ids from hidden_states shape when it's empty.
+ input_others["position_ids"] = (
+ torch.arange(input_ids.shape[1], device=input_ids.device, dtype=torch.long)
+ .unsqueeze(0)
+ .expand(input_ids.shape[0], -1)
+ )
+ elif pid is None:
+ # Generate position_ids from hidden_states shape when it's None.
+ input_others["position_ids"] = (
+ torch.arange(input_ids.shape[1], device=input_ids.device, dtype=torch.long)
+ .unsqueeze(0)
+ .expand(input_ids.shape[0], -1)
+ )
+
if amp:
with autocast(device_type=str(device).split(":")[0], dtype=amp_dtype): # pragma: no cover
- output = block(input_ids, *input_tuple, **input_others)
+ output = block(**input_others)
else:
- output = block(input_ids, *input_tuple, **input_others)
+ output = block(**input_others)
if isinstance(output_return_id, int) and (isinstance(output, list) or isinstance(output, tuple)):
output = output[output_return_id]
return output
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.py
index b80a5ba6f..e4bc49765 100644
--- a/auto_round/utils/common.py
+++ b/auto_round/utils/common.py
@@ -634,7 +634,7 @@ class SupportedFormats:
return self._support_list[key]
-SHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")
+SHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings", "cu_seqlens")
deepspeed_exists = False
if importlib.util.find_spec("deepspeed"): # check if deepspeed is installed
diff --git a/test/test_cpu/utils/test_calibration_inputs.py b/test/test_cpu/utils/test_calibration_inputs.py
index 30e84b2fa..87009c253 100644
--- a/test/test_cpu/utils/test_calibration_inputs.py
+++ b/test/test_cpu/utils/test_calibration_inputs.py
@@ -91,7 +91,6 @@ def test_preprocess_block_inputs_keeps_list_dtype():
compress_context=compress_context,
)
- # The extracted helper intentionally preserves the legacy implementation,
- # which iterates list entries through to_dtype() without writing the result
- # back. Lock this down so the refactor does not change runtime behaviour.
- assert input_others["past_key_values"][0].dtype == torch.float16
+ # After Fix gpt-j-6b runtime error, list items are properly converted
+ # to the AMP dtype. Previously the result of to_dtype() was not written back.
+ assert input_others["past_key_values"][0].dtype == torch.bfloat16