PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
enhance attention module with original implementation retrieval and context restoration (#2408)
Xin He5eceba6ce6e8
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 1ced3542b..d9a7d4a0e 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -21,12 +21,22 @@ import queueimport randomimport sslimport sys+import tempfilefrom importlib.metadata import PackageNotFoundError, versionlogging.getLogger("datasets").setLevel(logging.WARNING)import torch-from datasets import Dataset, Features, IterableDataset, Sequence, Value, concatenate_datasets, load_dataset+from datasets import (+ Dataset,+ Features,+ IterableDataset,+ Sequence,+ Value,+ concatenate_datasets,+ load_dataset,+ load_from_disk,+)from packaging.version import Versionfrom torch.utils.data import DataLoader@@ -101,15 +111,16 @@ def _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsampreturn _get_dataset_impl(tokenizer, seqlen, _FINEWEB_EDU_MODELSCOPE_DATASET, seed, nsamples)-def _preprocess_dataset_in_subprocess(result_queue, tokenizer, seqlen, dataset_name, seed, nsamples):+def _preprocess_dataset_in_subprocess(result_queue, tokenizer, seqlen, dataset_name, seed, nsamples, output_path):"""Run dataset preprocessing and report network failures to the parent process."""try:- _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)+ dataset = _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)+ dataset.save_to_disk(output_path)except Exception as error:network_error = _get_dataset_network_error(error)result_queue.put((_DATASET_RESULT_ERROR, str(network_error) if network_error is not None else None))raise- result_queue.put((_DATASET_RESULT_SUCCESS, None))+ result_queue.put((_DATASET_RESULT_SUCCESS, output_path))def get_code_calibration_dataset(nsamples, datasets_version=None):@@ -1211,47 +1222,49 @@ def get_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, nreturn _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples)# Run preprocessing in a subprocess so all temporary memory is freed on exit.- # The HuggingFace datasets cache is warmed up as a side effect.logger.info("Preprocessing calibration dataset in a subprocess to avoid memory leaks...")subprocess_network_error = None- try:- if os.name == "nt":- raise OSError("fork is not available on Windows")-- # macOS crashes with SIGSEGV (exit code -11) when using "fork" after threads- # have been started (PyTorch, tokenizers, and HuggingFace datasets all use- # threads). Use "spawn" on macOS, which is safe but requires pickling args.- mp_context = "spawn" if sys.platform == "darwin" else "fork"- ctx = multiprocessing.get_context(mp_context)- result_queue = ctx.Queue()- p = ctx.Process(- target=_preprocess_dataset_in_subprocess,- args=(result_queue, tokenizer, seqlen, dataset_name, seed, nsamples),- )- p.start()- p.join()-+ with tempfile.TemporaryDirectory(prefix="auto_round_calib_") as temp_dir:try:- result_status, network_error_message = result_queue.get(timeout=1)- except queue.Empty:- result_status, network_error_message = None, None- result_queue.close()- result_queue.join_thread()- if p.exitcode != 0:- if result_status == _DATASET_RESULT_ERROR and network_error_message is not None:- subprocess_network_error = ConnectionError(network_error_message)+ if os.name == "nt":+ raise OSError("fork is not available on Windows")++ # macOS crashes with SIGSEGV (exit code -11) when using "fork" after threads+ # have been started (PyTorch, tokenizers, and HuggingFace datasets all use+ # threads). Use "spawn" on macOS, which is safe but requires pickling args.+ mp_context = "spawn" if sys.platform == "darwin" else "fork"+ ctx = multiprocessing.get_context(mp_context)+ result_queue = ctx.Queue()+ p = ctx.Process(+ target=_preprocess_dataset_in_subprocess,+ args=(result_queue, tokenizer, seqlen, dataset_name, seed, nsamples, os.path.join(temp_dir, "dataset")),+ )+ p.start()+ p.join()++ try:+ result_status, result_value = result_queue.get(timeout=1)+ except queue.Empty:+ result_status, result_value = None, None+ result_queue.close()+ result_queue.join_thread()+ if p.exitcode != 0:+ if result_status == _DATASET_RESULT_ERROR and result_value is not None:+ subprocess_network_error = ConnectionError(result_value)+ else:+ raise RuntimeError(f"Dataset preprocessing subprocess exited with code {p.exitcode}")+ elif result_status == _DATASET_RESULT_SUCCESS:+ return load_from_disk(result_value, keep_in_memory=True)else:- raise RuntimeError(f"Dataset preprocessing subprocess exited with code {p.exitcode}")+ raise RuntimeError("Dataset preprocessing subprocess returned no dataset")- except Exception as e:- logger.warning(f"Subprocess dataset preprocessing failed ({e}), falling back to in-process mode.")+ except Exception as e:+ logger.warning(f"Subprocess dataset preprocessing failed ({e}), falling back to in-process mode.")if subprocess_network_error is not None:return _fallback_to_fineweb_edu(subprocess_network_error, tokenizer, seqlen, dataset_name, seed, nsamples)- # (Re-)load the dataset in the main process. When the subprocess- # succeeded the HF datasets cache makes this almost instant.try:return _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)except Exception as error:diff --git a/auto_round/experimental/attention.py b/auto_round/experimental/attention.pyindex 49efce384..476159505 100644--- a/auto_round/experimental/attention.py+++ b/auto_round/experimental/attention.py@@ -81,6 +81,7 @@ class QuantizedAttentionImpl(torch.nn.Module):self.config = configself.granularity = normalize_fp8_granularity(granularity)self.attn_module = ref(attn_module) # avoid circular references+ self._original_impl = _get_original_attention_impl(config)# register query maxdevice = next(attn_module.parameters()).deviceinitial_max = torch.tensor([float("-inf")], device=device)@@ -129,6 +130,17 @@ def _ct_hooked_attention(module: Module, *args, **kwargs):return ALL_ATTENTION_FUNCTIONS[_original_impl](module, *args, **kwargs) # pylint: disable=E0601+def _get_attention_config(module: Module, fallback_config: PretrainedConfig) -> PretrainedConfig:+ module_config = getattr(module, "config", None)+ if module_config is not None and getattr(module_config, "_attn_implementation", None) is not None:+ return module_config+ return fallback_config+++def _get_original_attention_impl(config: PretrainedConfig) -> str:+ return getattr(config, "_auto_round_original_attn_impl", config._attn_implementation)++def init_hooked_attention(module: Module, config, granularity: str = "tensor"):"""Initialize `QuantizedAttentionImpl` and `QuantizedKVCache` instances@@ -137,15 +149,17 @@ def init_hooked_attention(module: Module, config, granularity: str = "tensor")::param model: parent model of attention module:param module: attention module to initialize with"""+ config = _get_attention_config(module, config)if not hasattr(module, ATTN_IMPL_ATTR_NAME):- module.register_module(ATTN_IMPL_ATTR_NAME, QuantizedAttentionImpl(config, module, granularity=granularity))if config._attn_implementation != HOOKED_ATTENTION_NAME:# assumes only one model at a timeglobal _original_impl_original_impl = config._attn_implementation+ config._auto_round_original_attn_impl = config._attn_implementation# Add new implementation to AttentionInterface(mapping)AttentionInterface.register(HOOKED_ATTENTION_NAME, _ct_hooked_attention)config._attn_implementation = HOOKED_ATTENTION_NAME+ module.register_module(ATTN_IMPL_ATTR_NAME, QuantizedAttentionImpl(config, module, granularity=granularity))# initialize_hooked_kv_cache(model, module)@@ -159,10 +173,10 @@ def prep_attention_module_for_calibration(module: torch.nn.Module, config, granudef clean_up_hooked_attention(module, model):if is_attention_module(module):clean_model_parameters_and_buffers_(module, (QUERY_MAX_NAME,))- # Cleanup phase: Restore the original attention implementation- if hasattr(model.config, "_attn_implementation") and hasattr(model, "_original_impl"):- model.config._attn_implementation = model._original_impl- del model._original_impl+ config = _get_attention_config(module, model.config)+ if hasattr(config, "_auto_round_original_attn_impl"):+ config._attn_implementation = config._auto_round_original_attn_impl+ del config._auto_round_original_attn_impl@contextlib.contextmanagerdiff --git a/test/integration/test_cpu/test_inc_static_attn.py b/test/integration/test_cpu/test_inc_static_attn.pyindex 1bfa10eef..7ba9924eb 100644--- a/test/integration/test_cpu/test_inc_static_attn.py+++ b/test/integration/test_cpu/test_inc_static_attn.py@@ -3,13 +3,51 @@ from test.helpers import get_model_pathimport pytestimport torch-from transformers import AutoModelForCausalLM, AutoTokenizer+from transformers import AutoModelForCausalLM, AutoTokenizer, PretrainedConfigfrom auto_round import AutoRound+from auto_round.experimental.attention import HOOKED_ATTENTION_NAME, QUERY_MAX_NAME, attention_quant_ctxdeepseekv3_model_name = get_model_path("tflsxyy/DeepSeek-V3-bf16-4layers")+def assert_valid_q_scale(model):+ q_scale = model.model.layers[0].self_attn.q_scale+ assert torch.isfinite(q_scale).all(), f"q_scale must be finite, got {q_scale}"+ assert (q_scale > 0).all(), f"q_scale must be positive, got {q_scale}"+++def test_attention_context_uses_module_config_and_restores_it():+ class TestAttention(torch.nn.Module):+ def __init__(self, config):+ super().__init__()+ self.config = config+ self.layer_idx = 0+ self.k_proj = torch.nn.Linear(4, 4, bias=False)++ model_config = PretrainedConfig()+ model_config._attn_implementation = "eager"+ attention_config = PretrainedConfig()+ attention_config._attn_implementation = "eager"+ attention = TestAttention(attention_config)+ model = torch.nn.Module()+ model.config = model_config+ model.attention = attention++ with attention_quant_ctx(model):+ assert model_config._attn_implementation == "eager"+ assert attention_config._attn_implementation == HOOKED_ATTENTION_NAME+ assert hasattr(attention, "impl")+ assert hasattr(attention, "q_scale")+ assert hasattr(attention, QUERY_MAX_NAME)++ assert model_config._attn_implementation == "eager"+ assert attention_config._attn_implementation == "eager"+ assert hasattr(attention, "q_scale")+ assert not hasattr(attention_config, "_auto_round_original_attn_impl")+ assert not hasattr(attention, QUERY_MAX_NAME)++def test_deepseek_v2(tiny_deepseek_v2_model_path):model_name = tiny_deepseek_v2_model_pathmodel = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=False)@@ -26,10 +64,7 @@ def test_deepseek_v2(tiny_deepseek_v2_model_path):)quantized_model, save_folder = autoround.quantize_and_save(format="llm_compressor", output_dir=output_dir)assert quantized_model is not None, "Expected quantized_model to be not None"- device = quantized_model.model.layers[0].self_attn.q_scale.device- assert not all(- quantized_model.model.layers[0].self_attn.q_scale == torch.tensor([0.0], device=device)- ), "q_scale is not collected"+ assert_valid_q_scale(quantized_model)# clean the output directory after testshutil.rmtree(output_dir, ignore_errors=True)@@ -63,10 +98,7 @@ def test_deepseek_v3(setup_deepseekv3):)quantized_model, save_folder = autoround.quantize_and_save(format="llm_compressor", output_dir=output_dir)assert quantized_model is not None, "Expected quantized_model to be not None"- device = quantized_model.model.layers[0].self_attn.q_scale.device- assert not all(- quantized_model.model.layers[0].self_attn.q_scale == torch.tensor([0.0], device=device)- ), "q_scale is not collected"+ assert_valid_q_scale(quantized_model)# clean the output directory after testshutil.rmtree(output_dir, ignore_errors=True)diff --git a/test/unit/common/core/test_calib_dataset_subprocess.py b/test/unit/common/core/test_calib_dataset_subprocess.pyindex 7c077cd3b..5b6363242 100644--- a/test/unit/common/core/test_calib_dataset_subprocess.py+++ b/test/unit/common/core/test_calib_dataset_subprocess.py@@ -138,3 +138,30 @@ def test_subprocess_network_error_falls_back_without_retrying_source(monkeypatchassert result is fallback_datasetassert isinstance(fallback[0][0], ConnectionError)assert fallback[0][1] == "source"++[email protected](os.name == "nt", reason="fork is unavailable on Windows")+def test_subprocess_returns_saved_dataset_without_reprocessing(monkeypatch, tmp_path):+ import auto_round.calib_dataset as cd++ parent_pid = os.getpid()+ output_path = tmp_path / "output_path"+ original_save = Dataset.save_to_disk++ def save_dataset(dataset, path):+ output_path.write_text(path)+ return original_save(dataset, path)++ def build_dataset(*args):+ assert os.getpid() != parent_pid, "dataset rebuilt in parent process"+ return Dataset.from_dict({"input_ids": [[1, 2]], "attention_mask": [[1, 1]]})++ monkeypatch.setattr(cd, "_get_dataset_impl", build_dataset)+ monkeypatch.setattr(cd.Dataset, "save_to_disk", save_dataset)+ monkeypatch.setattr(cd.envs, "AR_DISABLE_DATASET_SUBPROCESS", False)++ result = cd.get_dataset(tokenizer=None, seqlen=2)++ assert result[0]["input_ids"] == [1, 2]+ assert not os.path.exists(output_path.read_text())+ assert result[0]["attention_mask"] == [1, 1]