Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

enhance attention module with original implementation retrieval and context restoration (#2408)

Xin He5eceba6ce6e8
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index 1ced3542b..d9a7d4a0e 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -21,12 +21,22 @@ import queue
import random
import ssl
import sys
+import tempfile
from importlib.metadata import PackageNotFoundError, version
logging.getLogger("datasets").setLevel(logging.WARNING)
import torch
-from datasets import Dataset, Features, IterableDataset, Sequence, Value, concatenate_datasets, load_dataset
+from datasets import (
+ Dataset,
+ Features,
+ IterableDataset,
+ Sequence,
+ Value,
+ concatenate_datasets,
+ load_dataset,
+ load_from_disk,
+)
from packaging.version import Version
from torch.utils.data import DataLoader
@@ -101,15 +111,16 @@ def _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamp
return _get_dataset_impl(tokenizer, seqlen, _FINEWEB_EDU_MODELSCOPE_DATASET, seed, nsamples)
-def _preprocess_dataset_in_subprocess(result_queue, tokenizer, seqlen, dataset_name, seed, nsamples):
+def _preprocess_dataset_in_subprocess(result_queue, tokenizer, seqlen, dataset_name, seed, nsamples, output_path):
"""Run dataset preprocessing and report network failures to the parent process."""
try:
- _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)
+ dataset = _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)
+ dataset.save_to_disk(output_path)
except Exception as error:
network_error = _get_dataset_network_error(error)
result_queue.put((_DATASET_RESULT_ERROR, str(network_error) if network_error is not None else None))
raise
- result_queue.put((_DATASET_RESULT_SUCCESS, None))
+ result_queue.put((_DATASET_RESULT_SUCCESS, output_path))
def get_code_calibration_dataset(nsamples, datasets_version=None):
@@ -1211,47 +1222,49 @@ def get_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, n
return _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples)
# Run preprocessing in a subprocess so all temporary memory is freed on exit.
- # The HuggingFace datasets cache is warmed up as a side effect.
logger.info("Preprocessing calibration dataset in a subprocess to avoid memory leaks...")
subprocess_network_error = None
- try:
- if os.name == "nt":
- raise OSError("fork is not available on Windows")
-
- # macOS crashes with SIGSEGV (exit code -11) when using "fork" after threads
- # have been started (PyTorch, tokenizers, and HuggingFace datasets all use
- # threads). Use "spawn" on macOS, which is safe but requires pickling args.
- mp_context = "spawn" if sys.platform == "darwin" else "fork"
- ctx = multiprocessing.get_context(mp_context)
- result_queue = ctx.Queue()
- p = ctx.Process(
- target=_preprocess_dataset_in_subprocess,
- args=(result_queue, tokenizer, seqlen, dataset_name, seed, nsamples),
- )
- p.start()
- p.join()
-
+ with tempfile.TemporaryDirectory(prefix="auto_round_calib_") as temp_dir:
try:
- result_status, network_error_message = result_queue.get(timeout=1)
- except queue.Empty:
- result_status, network_error_message = None, None
- result_queue.close()
- result_queue.join_thread()
- if p.exitcode != 0:
- if result_status == _DATASET_RESULT_ERROR and network_error_message is not None:
- subprocess_network_error = ConnectionError(network_error_message)
+ if os.name == "nt":
+ raise OSError("fork is not available on Windows")
+
+ # macOS crashes with SIGSEGV (exit code -11) when using "fork" after threads
+ # have been started (PyTorch, tokenizers, and HuggingFace datasets all use
+ # threads). Use "spawn" on macOS, which is safe but requires pickling args.
+ mp_context = "spawn" if sys.platform == "darwin" else "fork"
+ ctx = multiprocessing.get_context(mp_context)
+ result_queue = ctx.Queue()
+ p = ctx.Process(
+ target=_preprocess_dataset_in_subprocess,
+ args=(result_queue, tokenizer, seqlen, dataset_name, seed, nsamples, os.path.join(temp_dir, "dataset")),
+ )
+ p.start()
+ p.join()
+
+ try:
+ result_status, result_value = result_queue.get(timeout=1)
+ except queue.Empty:
+ result_status, result_value = None, None
+ result_queue.close()
+ result_queue.join_thread()
+ if p.exitcode != 0:
+ if result_status == _DATASET_RESULT_ERROR and result_value is not None:
+ subprocess_network_error = ConnectionError(result_value)
+ else:
+ raise RuntimeError(f"Dataset preprocessing subprocess exited with code {p.exitcode}")
+ elif result_status == _DATASET_RESULT_SUCCESS:
+ return load_from_disk(result_value, keep_in_memory=True)
else:
- raise RuntimeError(f"Dataset preprocessing subprocess exited with code {p.exitcode}")
+ raise RuntimeError("Dataset preprocessing subprocess returned no dataset")
- except Exception as e:
- logger.warning(f"Subprocess dataset preprocessing failed ({e}), falling back to in-process mode.")
+ except Exception as e:
+ logger.warning(f"Subprocess dataset preprocessing failed ({e}), falling back to in-process mode.")
if subprocess_network_error is not None:
return _fallback_to_fineweb_edu(subprocess_network_error, tokenizer, seqlen, dataset_name, seed, nsamples)
- # (Re-)load the dataset in the main process. When the subprocess
- # succeeded the HF datasets cache makes this almost instant.
try:
return _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)
except Exception as error:
diff --git a/auto_round/experimental/attention.py b/auto_round/experimental/attention.py
index 49efce384..476159505 100644
--- a/auto_round/experimental/attention.py
+++ b/auto_round/experimental/attention.py
@@ -81,6 +81,7 @@ class QuantizedAttentionImpl(torch.nn.Module):
self.config = config
self.granularity = normalize_fp8_granularity(granularity)
self.attn_module = ref(attn_module) # avoid circular references
+ self._original_impl = _get_original_attention_impl(config)
# register query max
device = next(attn_module.parameters()).device
initial_max = torch.tensor([float("-inf")], device=device)
@@ -129,6 +130,17 @@ def _ct_hooked_attention(module: Module, *args, **kwargs):
return ALL_ATTENTION_FUNCTIONS[_original_impl](module, *args, **kwargs) # pylint: disable=E0601
+def _get_attention_config(module: Module, fallback_config: PretrainedConfig) -> PretrainedConfig:
+ module_config = getattr(module, "config", None)
+ if module_config is not None and getattr(module_config, "_attn_implementation", None) is not None:
+ return module_config
+ return fallback_config
+
+
+def _get_original_attention_impl(config: PretrainedConfig) -> str:
+ return getattr(config, "_auto_round_original_attn_impl", config._attn_implementation)
+
+
def init_hooked_attention(module: Module, config, granularity: str = "tensor"):
"""
Initialize `QuantizedAttentionImpl` and `QuantizedKVCache` instances
@@ -137,15 +149,17 @@ def init_hooked_attention(module: Module, config, granularity: str = "tensor"):
:param model: parent model of attention module
:param module: attention module to initialize with
"""
+ config = _get_attention_config(module, config)
if not hasattr(module, ATTN_IMPL_ATTR_NAME):
- module.register_module(ATTN_IMPL_ATTR_NAME, QuantizedAttentionImpl(config, module, granularity=granularity))
if config._attn_implementation != HOOKED_ATTENTION_NAME:
# assumes only one model at a time
global _original_impl
_original_impl = config._attn_implementation
+ config._auto_round_original_attn_impl = config._attn_implementation
# Add new implementation to AttentionInterface(mapping)
AttentionInterface.register(HOOKED_ATTENTION_NAME, _ct_hooked_attention)
config._attn_implementation = HOOKED_ATTENTION_NAME
+ module.register_module(ATTN_IMPL_ATTR_NAME, QuantizedAttentionImpl(config, module, granularity=granularity))
# initialize_hooked_kv_cache(model, module)
@@ -159,10 +173,10 @@ def prep_attention_module_for_calibration(module: torch.nn.Module, config, granu
def clean_up_hooked_attention(module, model):
if is_attention_module(module):
clean_model_parameters_and_buffers_(module, (QUERY_MAX_NAME,))
- # Cleanup phase: Restore the original attention implementation
- if hasattr(model.config, "_attn_implementation") and hasattr(model, "_original_impl"):
- model.config._attn_implementation = model._original_impl
- del model._original_impl
+ config = _get_attention_config(module, model.config)
+ if hasattr(config, "_auto_round_original_attn_impl"):
+ config._attn_implementation = config._auto_round_original_attn_impl
+ del config._auto_round_original_attn_impl
@contextlib.contextmanager
diff --git a/test/integration/test_cpu/test_inc_static_attn.py b/test/integration/test_cpu/test_inc_static_attn.py
index 1bfa10eef..7ba9924eb 100644
--- a/test/integration/test_cpu/test_inc_static_attn.py
+++ b/test/integration/test_cpu/test_inc_static_attn.py
@@ -3,13 +3,51 @@ from test.helpers import get_model_path
import pytest
import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, PretrainedConfig
from auto_round import AutoRound
+from auto_round.experimental.attention import HOOKED_ATTENTION_NAME, QUERY_MAX_NAME, attention_quant_ctx
deepseekv3_model_name = get_model_path("tflsxyy/DeepSeek-V3-bf16-4layers")
+def assert_valid_q_scale(model):
+ q_scale = model.model.layers[0].self_attn.q_scale
+ assert torch.isfinite(q_scale).all(), f"q_scale must be finite, got {q_scale}"
+ assert (q_scale > 0).all(), f"q_scale must be positive, got {q_scale}"
+
+
+def test_attention_context_uses_module_config_and_restores_it():
+ class TestAttention(torch.nn.Module):
+ def __init__(self, config):
+ super().__init__()
+ self.config = config
+ self.layer_idx = 0
+ self.k_proj = torch.nn.Linear(4, 4, bias=False)
+
+ model_config = PretrainedConfig()
+ model_config._attn_implementation = "eager"
+ attention_config = PretrainedConfig()
+ attention_config._attn_implementation = "eager"
+ attention = TestAttention(attention_config)
+ model = torch.nn.Module()
+ model.config = model_config
+ model.attention = attention
+
+ with attention_quant_ctx(model):
+ assert model_config._attn_implementation == "eager"
+ assert attention_config._attn_implementation == HOOKED_ATTENTION_NAME
+ assert hasattr(attention, "impl")
+ assert hasattr(attention, "q_scale")
+ assert hasattr(attention, QUERY_MAX_NAME)
+
+ assert model_config._attn_implementation == "eager"
+ assert attention_config._attn_implementation == "eager"
+ assert hasattr(attention, "q_scale")
+ assert not hasattr(attention_config, "_auto_round_original_attn_impl")
+ assert not hasattr(attention, QUERY_MAX_NAME)
+
+
def test_deepseek_v2(tiny_deepseek_v2_model_path):
model_name = tiny_deepseek_v2_model_path
model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=False)
@@ -26,10 +64,7 @@ def test_deepseek_v2(tiny_deepseek_v2_model_path):
)
quantized_model, save_folder = autoround.quantize_and_save(format="llm_compressor", output_dir=output_dir)
assert quantized_model is not None, "Expected quantized_model to be not None"
- device = quantized_model.model.layers[0].self_attn.q_scale.device
- assert not all(
- quantized_model.model.layers[0].self_attn.q_scale == torch.tensor([0.0], device=device)
- ), "q_scale is not collected"
+ assert_valid_q_scale(quantized_model)
# clean the output directory after test
shutil.rmtree(output_dir, ignore_errors=True)
@@ -63,10 +98,7 @@ def test_deepseek_v3(setup_deepseekv3):
)
quantized_model, save_folder = autoround.quantize_and_save(format="llm_compressor", output_dir=output_dir)
assert quantized_model is not None, "Expected quantized_model to be not None"
- device = quantized_model.model.layers[0].self_attn.q_scale.device
- assert not all(
- quantized_model.model.layers[0].self_attn.q_scale == torch.tensor([0.0], device=device)
- ), "q_scale is not collected"
+ assert_valid_q_scale(quantized_model)
# clean the output directory after test
shutil.rmtree(output_dir, ignore_errors=True)
diff --git a/test/unit/common/core/test_calib_dataset_subprocess.py b/test/unit/common/core/test_calib_dataset_subprocess.py
index 7c077cd3b..5b6363242 100644
--- a/test/unit/common/core/test_calib_dataset_subprocess.py
+++ b/test/unit/common/core/test_calib_dataset_subprocess.py
@@ -138,3 +138,30 @@ def test_subprocess_network_error_falls_back_without_retrying_source(monkeypatch
assert result is fallback_dataset
assert isinstance(fallback[0][0], ConnectionError)
assert fallback[0][1] == "source"
+
+
[email protected](os.name == "nt", reason="fork is unavailable on Windows")
+def test_subprocess_returns_saved_dataset_without_reprocessing(monkeypatch, tmp_path):
+ import auto_round.calib_dataset as cd
+
+ parent_pid = os.getpid()
+ output_path = tmp_path / "output_path"
+ original_save = Dataset.save_to_disk
+
+ def save_dataset(dataset, path):
+ output_path.write_text(path)
+ return original_save(dataset, path)
+
+ def build_dataset(*args):
+ assert os.getpid() != parent_pid, "dataset rebuilt in parent process"
+ return Dataset.from_dict({"input_ids": [[1, 2]], "attention_mask": [[1, 1]]})
+
+ monkeypatch.setattr(cd, "_get_dataset_impl", build_dataset)
+ monkeypatch.setattr(cd.Dataset, "save_to_disk", save_dataset)
+ monkeypatch.setattr(cd.envs, "AR_DISABLE_DATASET_SUBPROCESS", False)
+
+ result = cd.get_dataset(tokenizer=None, seqlen=2)
+
+ assert result[0]["input_ids"] == [1, 2]
+ assert not os.path.exists(output_path.read_text())
+ assert result[0]["attention_mask"] == [1, 1]