PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
INC integration test bug fix (#2423)
chen, suyue5606837a3842
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex d9a7d4a0e..82373687c 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -113,6 +113,8 @@ def _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsampdef _preprocess_dataset_in_subprocess(result_queue, tokenizer, seqlen, dataset_name, seed, nsamples, output_path):"""Run dataset preprocessing and report network failures to the parent process."""+ # The parent's OpenMP thread pool does not survive fork; keep torch single-threaded here.+ torch.set_num_threads(1)try:dataset = _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)dataset.save_to_disk(output_path)@@ -1007,7 +1009,6 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seeddataset_names = dataset_name.split(",")def cast_dataset_columns(dataset):- dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])features = dict(dataset.features)features["input_ids"] = Sequence(Value("int64"))features["attention_mask"] = Sequence(Value("int8"))@@ -1140,6 +1141,9 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seeddataset = dataset.filter(filter_func)if name in data_lens:dataset = select_dataset(dataset, range(data_lens[name]))+ # Format last: tensorizing whole batches during filter/select would run torch ops that+ # are not fork-safe (libgomp) in the preprocessing subprocess.+ dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])datasets.append(dataset)if len(datasets) == 1:diff --git a/auto_round/export/utils.py b/auto_round/export/utils.pyindex 9ef56f57d..121f89e72 100644--- a/auto_round/export/utils.py+++ b/auto_round/export/utils.py@@ -327,13 +327,22 @@ def _restore_original_layer_types(save_dir: str, source_dir: str) -> None:json.dump(saved_config, f, indent=2)+# Quantization metadata (weight/input/kv-cache scales and zero points) is part of the+# checkpoint contract and must keep its own dtype, so it is never cast to the export dtype.+_QUANT_PARAM_SUFFIXES = ("_scale", "_scales", "_scale_inv", "_zero_point", "_zeros", "_zp")+++def _is_quant_param(name: str) -> bool:+ return name.endswith(_QUANT_PARAM_SUFFIXES)++def _get_state_dict_for_export_dtype(model: nn.Module, dtype) -> dict | None:"""Return a state dict with float32 tensors cast to ``dtype``, or None if nothing needs casting.Tuning may run the model in float32, e.g. when the device doesn't support bfloat16. Theexported config declares ``dtype``, so the saved tensors should use it too. Tensors ofmodules the model keeps in float32 (``_keep_in_fp32_modules``) are left unchanged, as are- quantized (non-float32) tensors.+ quantized (non-float32) tensors and quantization scales/zero points."""if dtype not in (torch.bfloat16, torch.float16):return None@@ -346,14 +355,12 @@ def _get_state_dict_for_export_dtype(model: nn.Module, dtype) -> dict | None:names = getattr(model, attribute, None) or []keep_in_fp32.update([names] if isinstance(names, str) else names)- return {- name: (- tensor.to(dtype)- if tensor.dtype == torch.float32 and not any(module_name in name for module_name in keep_in_fp32)- else tensor- )- for name, tensor in state_dict.items()- }+ def _should_cast(name: str, tensor: torch.Tensor) -> bool:+ if tensor.dtype != torch.float32 or _is_quant_param(name):+ return False+ return not any(module_name in name for module_name in keep_in_fp32)++ return {name: (tensor.to(dtype) if _should_cast(name, tensor) else tensor) for name, tensor in state_dict.items()}def apply_post_save_source_fixes(model: nn.Module, save_dir: str) -> None:diff --git a/test/integration/test_cpu/requirements_inc.txt b/test/integration/test_cpu/requirements_inc.txtindex da8af977a..46c039c7d 100644--- a/test/integration/test_cpu/requirements_inc.txt+++ b/test/integration/test_cpu/requirements_inc.txt@@ -1,2 +1,2 @@-neural_compressor_pt+git+https://github.com/intel/neural-compressor.git@maincompressed-tensorsdiff --git a/test/unit/common/export/test_export_utils.py b/test/unit/common/export/test_export_utils.pyindex 9099f0fe4..786bc87c8 100644--- a/test/unit/common/export/test_export_utils.py+++ b/test/unit/common/export/test_export_utils.py@@ -378,6 +378,11 @@ class _ExportDtypeModel(nn.Module):self.router = nn.Linear(4, 2)self.register_buffer("qweight", torch.zeros(4, 4, dtype=torch.float8_e4m3fn))self.register_buffer("indices", torch.zeros(4, dtype=torch.int32))+ self.register_buffer("k_scale", torch.ones(1))+ self.register_buffer("v_scale", torch.ones(1))+ self.register_buffer("weight_zero_point", torch.zeros(1))+ self.linear.register_buffer("input_scale", torch.ones(1))+ self.linear.register_buffer("weight_scale", torch.ones(1))class TestGetStateDictForExportDtype:@@ -398,6 +403,12 @@ class TestGetStateDictForExportDtype:assert state_dict["router.weight"].dtype == torch.float32assert state_dict["linear.weight"].dtype == torch.float16+ def test_keeps_quantization_scales_in_float32(self):+ model = _ExportDtypeModel()+ state_dict = _get_state_dict_for_export_dtype(model, torch.bfloat16)+ for name in ("k_scale", "v_scale", "weight_zero_point", "linear.input_scale", "linear.weight_scale"):+ assert state_dict[name].dtype == torch.float32, name+def test_does_not_modify_model(self):model = _ExportDtypeModel()_get_state_dict_for_export_dtype(model, torch.bfloat16)