Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

INC integration test bug fix (#2423)

chen, suyue5606837a3842
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index d9a7d4a0e..82373687c 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -113,6 +113,8 @@ def _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamp
def _preprocess_dataset_in_subprocess(result_queue, tokenizer, seqlen, dataset_name, seed, nsamples, output_path):
"""Run dataset preprocessing and report network failures to the parent process."""
+ # The parent's OpenMP thread pool does not survive fork; keep torch single-threaded here.
+ torch.set_num_threads(1)
try:
dataset = _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)
dataset.save_to_disk(output_path)
@@ -1007,7 +1009,6 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed
dataset_names = dataset_name.split(",")
def cast_dataset_columns(dataset):
- dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])
features = dict(dataset.features)
features["input_ids"] = Sequence(Value("int64"))
features["attention_mask"] = Sequence(Value("int8"))
@@ -1140,6 +1141,9 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed
dataset = dataset.filter(filter_func)
if name in data_lens:
dataset = select_dataset(dataset, range(data_lens[name]))
+ # Format last: tensorizing whole batches during filter/select would run torch ops that
+ # are not fork-safe (libgomp) in the preprocessing subprocess.
+ dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])
datasets.append(dataset)
if len(datasets) == 1:
diff --git a/auto_round/export/utils.py b/auto_round/export/utils.py
index 9ef56f57d..121f89e72 100644
--- a/auto_round/export/utils.py
+++ b/auto_round/export/utils.py
@@ -327,13 +327,22 @@ def _restore_original_layer_types(save_dir: str, source_dir: str) -> None:
json.dump(saved_config, f, indent=2)
+# Quantization metadata (weight/input/kv-cache scales and zero points) is part of the
+# checkpoint contract and must keep its own dtype, so it is never cast to the export dtype.
+_QUANT_PARAM_SUFFIXES = ("_scale", "_scales", "_scale_inv", "_zero_point", "_zeros", "_zp")
+
+
+def _is_quant_param(name: str) -> bool:
+ return name.endswith(_QUANT_PARAM_SUFFIXES)
+
+
def _get_state_dict_for_export_dtype(model: nn.Module, dtype) -> dict | None:
"""Return a state dict with float32 tensors cast to ``dtype``, or None if nothing needs casting.
Tuning may run the model in float32, e.g. when the device doesn't support bfloat16. The
exported config declares ``dtype``, so the saved tensors should use it too. Tensors of
modules the model keeps in float32 (``_keep_in_fp32_modules``) are left unchanged, as are
- quantized (non-float32) tensors.
+ quantized (non-float32) tensors and quantization scales/zero points.
"""
if dtype not in (torch.bfloat16, torch.float16):
return None
@@ -346,14 +355,12 @@ def _get_state_dict_for_export_dtype(model: nn.Module, dtype) -> dict | None:
names = getattr(model, attribute, None) or []
keep_in_fp32.update([names] if isinstance(names, str) else names)
- return {
- name: (
- tensor.to(dtype)
- if tensor.dtype == torch.float32 and not any(module_name in name for module_name in keep_in_fp32)
- else tensor
- )
- for name, tensor in state_dict.items()
- }
+ def _should_cast(name: str, tensor: torch.Tensor) -> bool:
+ if tensor.dtype != torch.float32 or _is_quant_param(name):
+ return False
+ return not any(module_name in name for module_name in keep_in_fp32)
+
+ return {name: (tensor.to(dtype) if _should_cast(name, tensor) else tensor) for name, tensor in state_dict.items()}
def apply_post_save_source_fixes(model: nn.Module, save_dir: str) -> None:
diff --git a/test/integration/test_cpu/requirements_inc.txt b/test/integration/test_cpu/requirements_inc.txt
index da8af977a..46c039c7d 100644
--- a/test/integration/test_cpu/requirements_inc.txt
+++ b/test/integration/test_cpu/requirements_inc.txt
@@ -1,2 +1,2 @@
-neural_compressor_pt
+git+https://github.com/intel/neural-compressor.git@main
compressed-tensors
diff --git a/test/unit/common/export/test_export_utils.py b/test/unit/common/export/test_export_utils.py
index 9099f0fe4..786bc87c8 100644
--- a/test/unit/common/export/test_export_utils.py
+++ b/test/unit/common/export/test_export_utils.py
@@ -378,6 +378,11 @@ class _ExportDtypeModel(nn.Module):
self.router = nn.Linear(4, 2)
self.register_buffer("qweight", torch.zeros(4, 4, dtype=torch.float8_e4m3fn))
self.register_buffer("indices", torch.zeros(4, dtype=torch.int32))
+ self.register_buffer("k_scale", torch.ones(1))
+ self.register_buffer("v_scale", torch.ones(1))
+ self.register_buffer("weight_zero_point", torch.zeros(1))
+ self.linear.register_buffer("input_scale", torch.ones(1))
+ self.linear.register_buffer("weight_scale", torch.ones(1))
class TestGetStateDictForExportDtype:
@@ -398,6 +403,12 @@ class TestGetStateDictForExportDtype:
assert state_dict["router.weight"].dtype == torch.float32
assert state_dict["linear.weight"].dtype == torch.float16
+ def test_keeps_quantization_scales_in_float32(self):
+ model = _ExportDtypeModel()
+ state_dict = _get_state_dict_for_export_dtype(model, torch.bfloat16)
+ for name in ("k_scale", "v_scale", "weight_zero_point", "linear.input_scale", "linear.weight_scale"):
+ assert state_dict[name].dtype == torch.float32, name
+
def test_does_not_modify_model(self):
model = _ExportDtypeModel()
_get_state_dict_for_export_dtype(model, torch.bfloat16)