PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add fineweb-edu dataset as calibration bakeup (#2345)
Weiwei3938e152789e
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 9910b8d24..49f59a8c8 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -17,8 +17,11 @@ import jsonimport loggingimport multiprocessingimport os+import queueimport random+import sslimport sys+from importlib.metadata import PackageNotFoundError, versionlogging.getLogger("datasets").setLevel(logging.WARNING)@@ -32,6 +35,81 @@ from .utils import is_local_path, loggerCALIB_DATASETS = {}_GITHUB_CODE_CLEAN_MAX_DATASETS_VERSION = Version("3.6.0")+_FINEWEB_EDU_HF_DATASET = "HuggingFaceFW/fineweb-edu"+_FINEWEB_EDU_MODELSCOPE_DATASET = "AI-ModelScope/fineweb-edu"+_FINEWEB_EDU_CONFIG = "sample-10BT"+_FINEWEB_EDU_MIN_CANDIDATES = 10000+_DATASET_RESULT_SUCCESS = "success"+_DATASET_RESULT_ERROR = "error"+++def _get_dataset_network_error(error):+ """Return the network failure in an exception chain, if present."""+ visited = set()+ current = error+ while current is not None and id(current) not in visited:+ visited.add(id(current))+ error_name = type(current).__name__.lower()+ error_module = type(current).__module__.lower()+ error_message = str(current).lower()+ if isinstance(current, (ConnectionError, TimeoutError, ssl.SSLError)):+ return current+ if any(term in error_name for term in ("connection", "http", "proxy", "ssl", "timeout")) and any(+ package in error_module+ for package in ("datasets", "huggingface_hub", "httpcore", "httpx", "requests", "urllib3")+ ):+ return current+ if any(+ marker in error_message+ for marker in (+ "cannot send a request, as the client has been closed",+ "connection timed out",+ "max retries exceeded",+ "proxy error",+ "ssl error",+ )+ ):+ return current+ current = current.__cause__ or current.__context__+ return None+++def _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples):+ """Switch to ModelScope FineWeb-Edu after a calibration dataset network failure."""+ network_error = _get_dataset_network_error(error)+ if network_error is None:+ raise error+ try:+ version("modelscope")+ except PackageNotFoundError:+ raise RuntimeError(+ f"Failed to load calibration dataset {dataset_name!r} because of an HTTP/proxy network issue: "+ f"{network_error}. Install ModelScope with `pip install modelscope` to enable automatic fallback "+ "to FineWeb-Edu."+ ) from error++ if dataset_name == _FINEWEB_EDU_MODELSCOPE_DATASET:+ raise error++ logger.warning(+ "Failed to load calibration dataset %r because of an HTTP/proxy network issue: %s. "+ "Automatically switching to the ModelScope FineWeb-Edu mirror (%s).",+ dataset_name,+ network_error,+ _FINEWEB_EDU_MODELSCOPE_DATASET,+ )+ return _get_dataset_impl(tokenizer, seqlen, _FINEWEB_EDU_MODELSCOPE_DATASET, seed, nsamples)+++def _preprocess_dataset_in_subprocess(result_queue, tokenizer, seqlen, dataset_name, seed, nsamples):+ """Run dataset preprocessing and report network failures to the parent process."""+ try:+ _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)+ except Exception as error:+ network_error = _get_dataset_network_error(error)+ result_queue.put((_DATASET_RESULT_ERROR, str(network_error) if network_error is not None else None))+ raise+ result_queue.put((_DATASET_RESULT_SUCCESS, None))def get_code_calibration_dataset(nsamples, datasets_version=None):@@ -193,22 +271,7 @@ def get_pile_dataset(tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_chat_template=apply_chat_template, system_prompt=system_prompt)- try:- calib_dataset = load_dataset("NeelNanda/pile-10k", split=split)- except Exception as e:- import ssl-- error_message = str(e)- # Check for proxy or SSL error- if "proxy" in error_message.lower() or isinstance(e, ssl.SSLError) or "SSL" in error_message.upper():- logger.error(- f"Network error detected, please check proxy settings. "- f"Error: {error_message}. Or consider using a backup dataset by `pip install modelscope` "- f"and set '--dataset swift/pile-val-backup' in AutoRound API."- )- else:- logger.error(f"Failed to load the dataset: {error_message}")- sys.exit(1)+ calib_dataset = load_dataset("NeelNanda/pile-10k", split=split)calib_dataset = calib_dataset.shuffle(seed=seed)calib_dataset = calib_dataset.map(tokenizer_function,@@ -221,47 +284,72 @@ def get_pile_dataset(return calib_dataset-@register_dataset(["swift/pile-val-backup", "pile-val-backup"])-def get_pile_val_dataset(+@register_dataset([_FINEWEB_EDU_HF_DATASET, _FINEWEB_EDU_MODELSCOPE_DATASET, "fineweb-edu"])+def get_fineweb_edu_dataset(tokenizer,seqlen,- dataset_name="swift/pile-val-backup",+ dataset_name="fineweb-edu",split=None,seed=42,apply_chat_template=False,system_prompt=None,+ max_samples=_FINEWEB_EDU_MIN_CANDIDATES,):- """Returns a dataloader for the specified dataset and split.+ """Return a streaming FineWeb-Edu calibration dataset from either supported hub.Args:tokenizer: The tokenizer to be used for tokenization.seqlen: The maximum sequence length.- data_name: The name of the dataset.- split: The data split to be used (e.g., "train", "test", "validation").+ dataset_name: FineWeb-Edu alias or explicit Hugging Face/ModelScope repository.+ split: The data split to use. FineWeb-Edu currently provides ``train``.seed: The random seed for shuffling the dataset.apply_chat_template: Whether to apply chat template in tokenization.Returns:- A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.+ A tokenized streaming dataset using the provided tokenizer and sequence length."""-- split = "validation"+ split = "train" if split is None else split+ if isinstance(split, list):+ if len(split) != 1:+ raise ValueError("FineWeb-Edu supports only one split at a time.")+ split = split[0]+ if split != "train":+ raise ValueError("FineWeb-Edu supports only the train split.")tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_chat_template=apply_chat_template, system_prompt=system_prompt)- from transformers.utils.versions import require_version-- require_version(- "modelscope",- "Loading 'swift/pile-val-backup' dataset requires modelscope to be installed, " "`pip install modelscope`",+ use_modelscope = dataset_name == _FINEWEB_EDU_MODELSCOPE_DATASET or (+ dataset_name == "fineweb-edu" and envs.AR_USE_MODELSCOPE)- from modelscope import MsDataset # pylint: disable=E0401+ if use_modelscope:+ from transformers.utils.versions import require_version- calib_dataset = MsDataset.load(- "swift/pile-val-backup", "default", split=split- ).to_iterable_dataset() # , use_streaming=True- calib_dataset = calib_dataset.shuffle(seed=seed).take(10000)+ require_version(+ "modelscope",+ "Loading FineWeb-Edu from ModelScope requires `modelscope`; install it with `pip install modelscope`.",+ )+ from modelscope import MsDataset # pylint: disable=E0401++ calib_dataset = MsDataset.load(+ _FINEWEB_EDU_MODELSCOPE_DATASET,+ subset_name=_FINEWEB_EDU_CONFIG,+ split=split,+ use_streaming=True,+ )+ # ModelScope's streaming reader can keep background HTTP requests alive+ # after iteration. Materialize the bounded sample before tokenization so+ # downloads finish cleanly and shuffle remains deterministic.+ calib_dataset = Dataset.from_list(list(calib_dataset.take(max_samples)))+ calib_dataset = calib_dataset.shuffle(seed=seed)+ else:+ calib_dataset = load_dataset(+ _FINEWEB_EDU_HF_DATASET,+ name=_FINEWEB_EDU_CONFIG,+ split=split,+ streaming=True,+ )+ calib_dataset = calib_dataset.shuffle(seed=seed).take(max_samples)calib_dataset = calib_dataset.map(tokenizer_function, batched=True)return calib_dataset@@ -1000,15 +1088,21 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seedraise ValueError(f"Dataset '{name}' is not found. Please choose from the supported datasets: {filtered_keys}.")- dataset = get_dataset(- tokenizer,- seqlen,+ dataset_kwargs = dict(+ tokenizer=tokenizer,+ seqlen=seqlen,seed=seed,split=split,dataset_name=name,apply_chat_template=apply_chat_template,system_prompt=system_prompt,)+ if get_dataset is get_fineweb_edu_dataset:+ dataset_kwargs["max_samples"] = max(+ _FINEWEB_EDU_MIN_CANDIDATES,+ data_lens.get(name, nsamples),+ )+ dataset = get_dataset(**dataset_kwargs)if do_concat:dataset = concat_dataset_element(dataset)@@ -1109,12 +1203,16 @@ def get_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, n"""# Allow disabling subprocess mode via environment variableif envs.AR_DISABLE_DATASET_SUBPROCESS:- return _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)+ try:+ return _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)+ except Exception as error:+ return _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples)# Run preprocessing in a subprocess so all temporary memory is freed on exit.# The HuggingFace datasets cache is warmed up as a side effect.logger.info("Preprocessing calibration dataset in a subprocess to avoid memory leaks...")+ subprocess_network_error = Nonetry:if os.name == "nt":raise OSError("fork is not available on Windows")@@ -1124,22 +1222,38 @@ def get_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, n# threads). Use "spawn" on macOS, which is safe but requires pickling args.mp_context = "spawn" if sys.platform == "darwin" else "fork"ctx = multiprocessing.get_context(mp_context)+ result_queue = ctx.Queue()p = ctx.Process(- target=_get_dataset_impl,- args=(tokenizer, seqlen, dataset_name, seed, nsamples),+ target=_preprocess_dataset_in_subprocess,+ args=(result_queue, tokenizer, seqlen, dataset_name, seed, nsamples),)p.start()p.join()+ try:+ result_status, network_error_message = result_queue.get(timeout=1)+ except queue.Empty:+ result_status, network_error_message = None, None+ result_queue.close()+ result_queue.join_thread()if p.exitcode != 0:- raise RuntimeError(f"Dataset preprocessing subprocess exited with code {p.exitcode}")+ if result_status == _DATASET_RESULT_ERROR and network_error_message is not None:+ subprocess_network_error = ConnectionError(network_error_message)+ else:+ raise RuntimeError(f"Dataset preprocessing subprocess exited with code {p.exitcode}")except Exception as e:logger.warning(f"Subprocess dataset preprocessing failed ({e}), falling back to in-process mode.")+ if subprocess_network_error is not None:+ return _fallback_to_fineweb_edu(subprocess_network_error, tokenizer, seqlen, dataset_name, seed, nsamples)+# (Re-)load the dataset in the main process. When the subprocess# succeeded the HF datasets cache makes this almost instant.- return _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)+ try:+ return _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)+ except Exception as error:+ return _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples)def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, bs=8, nsamples=512):diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 2ff9edf2b..628b3e24e 100755--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -63,7 +63,9 @@ pip install auto-roundThe [NeelNanda/pile-10k](https://huggingface.co/datasets/NeelNanda/pile-10k) in huggingface is adopted as the defaultcalibration data and will be downloaded automatically from the datasets Hub. Other available datasets include:-- `swift/pile-val-backup` from modelscope for addressing HF network issue+- [`fineweb-edu`](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu) from Hugging Face. If Hugging Face is+ inaccessible, install `modelscope`, set `AR_USE_MODELSCOPE=1`, and use the same alias to load the+ [ModelScope mirror](https://modelscope.cn/datasets/AI-ModelScope/fineweb-edu)- `BAAI/CCI3-HQ` for Chinese- `codeparrot/github-code-clean` for code- `HuggingFaceH4/ultrachat_200k` for chat datadiff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex df4e660e1..6dde8eccf 100755--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -61,10 +61,11 @@ pip install auto-round## 2 准备标定数据集### 默认数据集-**对于中国大陆用户推荐使用 ModelScope 中的 swift/pile-val-backup 以解决 Huggingface 不能访问的问题**+**如果无法访问 Hugging Face,建议安装 `modelscope`、设置 `AR_USE_MODELSCOPE=1`,并使用 `fineweb-edu` 数据集。**默认标定数据集为 Hugging Face 上的 [NeelNanda/pile-10k](https://huggingface.co/datasets/NeelNanda/pile-10k) ,该数据集会自动从 Huggingface Hub 下载。同时也支持使用以下数据集:-- ModelScope 中的 `swift/pile-val-backup`:用于解决 HF 访问问题+- Hugging Face 上的 [`fineweb-edu`](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu)。如果无法访问 Hugging Face,+ 可安装 `modelscope`、设置 `AR_USE_MODELSCOPE=1`,并使用同一别名从 [ModelScope 镜像](https://modelscope.cn/datasets/AI-ModelScope/fineweb-edu) 加载- `BAAI/CCI3-HQ`:用于中文场景- `codeparrot/github-code-clean`:用于代码场景- `HuggingFaceH4/ultrachat_200k`:用于对话数据diff --git a/test/unit/common/core/test_calib_dataset_subprocess.py b/test/unit/common/core/test_calib_dataset_subprocess.pyindex b28a3c5e5..7c077cd3b 100644--- a/test/unit/common/core/test_calib_dataset_subprocess.py+++ b/test/unit/common/core/test_calib_dataset_subprocess.py@@ -7,6 +7,9 @@ The fix selects ``"spawn"`` on macOS and ``"fork"`` on Linux.import os+import pytest+from datasets import Dataset+class _FakeProcess:"""Minimal subprocess stub: starts, joins, and exits cleanly."""@@ -23,6 +26,17 @@ class _FakeProcess:exitcode = 0+class _FakeQueue:+ def get(self, timeout=None):+ raise __import__("queue").Empty++ def close(self):+ pass++ def join_thread(self):+ pass++def _fake_get_context(captured):"""Return a factory that records the requested multiprocessing context name."""@@ -31,6 +45,7 @@ def _fake_get_context(captured):class _FakeCtx:Process = _FakeProcess+ Queue = _FakeQueuereturn _FakeCtx()@@ -81,3 +96,45 @@ def test_windows_falls_back_to_inprocess(monkeypatch):cd.get_dataset(tokenizer=None, seqlen=512)assert inprocess_called, "in-process fallback should have been called on Windows"+++def test_subprocess_network_error_falls_back_without_retrying_source(monkeypatch):+ """A child-process network error must switch directly to FineWeb-Edu."""+ import auto_round.calib_dataset as cd++ class _NetworkErrorQueue:+ def get(self, timeout=None):+ return cd._DATASET_RESULT_ERROR, "simulated proxy failure"++ def close(self):+ pass++ def join_thread(self):+ pass++ class _NetworkErrorProcess(_FakeProcess):+ exitcode = 1++ class _NetworkErrorContext:+ Process = _NetworkErrorProcess+ Queue = _NetworkErrorQueue++ fallback_dataset = Dataset.from_dict({"input_ids": [[1]], "attention_mask": [[1]]})+ fallback = []++ def fallback_to_fineweb(error, tokenizer, seqlen, dataset_name, seed, nsamples):+ fallback.append((error, dataset_name))+ return fallback_dataset++ monkeypatch.setattr(cd.multiprocessing, "get_context", lambda method: _NetworkErrorContext())+ monkeypatch.setattr(cd.os, "name", "posix")+ monkeypatch.setattr(cd.sys, "platform", "linux")+ monkeypatch.setattr(cd.envs, "AR_DISABLE_DATASET_SUBPROCESS", False)+ monkeypatch.setattr(cd, "_fallback_to_fineweb_edu", fallback_to_fineweb)+ monkeypatch.setattr(cd, "_get_dataset_impl", lambda *args: pytest.fail("source retried"))++ result = cd.get_dataset(tokenizer=None, seqlen=128, dataset_name="source")++ assert result is fallback_dataset+ assert isinstance(fallback[0][0], ConnectionError)+ assert fallback[0][1] == "source"diff --git a/test/unit/common/utils/test_calib_dataset_helpers.py b/test/unit/common/utils/test_calib_dataset_helpers.pyindex c21088143..8b1abfebc 100644--- a/test/unit/common/utils/test_calib_dataset_helpers.py+++ b/test/unit/common/utils/test_calib_dataset_helpers.py@@ -13,6 +13,9 @@# limitations under the License."""Tests for the small pure helpers in ``auto_round/calib_dataset.py``."""+import ssl+import sys+import typesfrom unittest.mock import MagicMockimport pytest@@ -63,6 +66,125 @@ class TestRegisterDataset:CALIB_DATASETS.pop("_return_check_", None)+# ---------------------------------------------------------------------------+# network failures and FineWeb-Edu routing+# ---------------------------------------------------------------------------+class TestDatasetNetworkErrors:+ def test_network_error_without_modelscope_prompts_install(self, monkeypatch):+ import auto_round.calib_dataset as calib_dataset++ monkeypatch.setattr(calib_dataset, "version", MagicMock(side_effect=calib_dataset.PackageNotFoundError))++ with pytest.raises(RuntimeError, match="pip install modelscope"):+ calib_dataset._fallback_to_fineweb_edu(+ ssl.SSLError("proxy unavailable"), MagicMock(), 128, "dataset", 42, 1+ )++ def test_network_error_with_modelscope_switches_to_fineweb_edu(self, monkeypatch):+ import auto_round.calib_dataset as calib_dataset++ fallback_dataset = MagicMock()+ load_dataset = MagicMock(return_value=fallback_dataset)+ warning = MagicMock()+ monkeypatch.setattr(calib_dataset, "version", MagicMock(return_value="1.0"))+ monkeypatch.setattr(calib_dataset, "_get_dataset_impl", load_dataset)+ monkeypatch.setattr(calib_dataset.logger, "warning", warning)+ tokenizer = MagicMock()++ result = calib_dataset._fallback_to_fineweb_edu(+ ssl.SSLError("proxy unavailable"), tokenizer, 128, "dataset", 42, 2+ )++ assert result is fallback_dataset+ load_dataset.assert_called_once_with(tokenizer, 128, "AI-ModelScope/fineweb-edu", 42, 2)+ assert "Automatically switching" in warning.call_args.args[0]++ def test_closed_http_client_is_treated_as_network_error(self):+ import auto_round.calib_dataset as calib_dataset++ error = RuntimeError("Cannot send a request, as the client has been closed.")++ assert calib_dataset._get_dataset_network_error(error) is error++ def test_modelscope_fineweb_network_error_is_not_retried(self, monkeypatch):+ import auto_round.calib_dataset as calib_dataset++ monkeypatch.setattr(calib_dataset, "version", MagicMock(return_value="1.0"))+ load_dataset = MagicMock()+ monkeypatch.setattr(calib_dataset, "_get_dataset_impl", load_dataset)+ error = ConnectionError("ModelScope is unavailable")++ with pytest.raises(ConnectionError, match="ModelScope is unavailable"):+ calib_dataset._fallback_to_fineweb_edu(error, MagicMock(), 128, "AI-ModelScope/fineweb-edu", 42, 1)++ load_dataset.assert_not_called()+++class TestFineWebEduDataset:+ @staticmethod+ def _streaming_dataset_mock():+ dataset = MagicMock()+ dataset.n_shards = 1+ dataset.shuffle.return_value = dataset+ dataset.take.return_value = dataset+ dataset.map.return_value = dataset+ return dataset++ def test_alias_loads_huggingface_sample_without_modelscope(self, monkeypatch):+ import auto_round.calib_dataset as calib_dataset++ dataset = self._streaming_dataset_mock()+ load_dataset = MagicMock(return_value=dataset)+ monkeypatch.setattr(calib_dataset, "load_dataset", load_dataset)+ monkeypatch.delenv("AR_USE_MODELSCOPE", raising=False)+ monkeypatch.setitem(sys.modules, "modelscope", None)++ result = calib_dataset.get_fineweb_edu_dataset(MagicMock(), 128)++ assert result is dataset+ load_dataset.assert_called_once_with(+ "HuggingFaceFW/fineweb-edu", name="sample-10BT", split="train", streaming=True+ )+ dataset.shuffle.assert_called_once_with(seed=42)+ dataset.take.assert_called_once_with(10000)++ def test_alias_loads_modelscope_sample_when_enabled(self, monkeypatch):+ import transformers.utils.versions as transformers_versions++ import auto_round.calib_dataset as calib_dataset++ streamed_samples = [+ {+ "text": "sample text",+ "input_ids": [1, 2, 3],+ "attention_mask": [1, 1, 1],+ }+ ]+ dataset = self._streaming_dataset_mock()+ dataset.take.return_value = streamed_samples+ load = MagicMock(return_value=dataset)+ modelscope = types.ModuleType("modelscope")+ modelscope.MsDataset = types.SimpleNamespace(load=load)+ monkeypatch.setitem(sys.modules, "modelscope", modelscope)+ monkeypatch.setenv("AR_USE_MODELSCOPE", "1")+ monkeypatch.setattr(transformers_versions, "require_version", MagicMock())++ tokenizer = MagicMock(+ side_effect=lambda texts, **kwargs: {+ "input_ids": [[1, 2, 3] for _ in texts],+ "attention_mask": [[1, 1, 1] for _ in texts],+ }+ )+ result = calib_dataset.get_fineweb_edu_dataset(tokenizer, 128)++ assert isinstance(result, Dataset)+ assert len(result) == 1+ load.assert_called_once_with(+ "AI-ModelScope/fineweb-edu", subset_name="sample-10BT", split="train", use_streaming=True+ )+ dataset.take.assert_called_once_with(10000)++# ---------------------------------------------------------------------------# _make_map_fingerprint# ---------------------------------------------------------------------------@@ -299,6 +421,23 @@ class TestApplyChatTemplateToSamples:# _get_dataset_impl# ---------------------------------------------------------------------------class TestGetDatasetImpl:+ def test_fineweb_edu_loader_receives_requested_sample_limit(self, monkeypatch):+ import auto_round.calib_dataset as calib_dataset++ dataset = Dataset.from_dict(+ {+ "input_ids": [list(range(8))],+ "attention_mask": [[1] * 8],+ }+ )+ loader = MagicMock(return_value=dataset)+ monkeypatch.setattr(calib_dataset, "get_fineweb_edu_dataset", loader)+ monkeypatch.setattr(calib_dataset, "CALIB_DATASETS", {"fineweb-edu": loader})++ calib_dataset._get_dataset_impl(MagicMock(), 8, "fineweb-edu", nsamples=1)++ assert loader.call_args.kwargs["max_samples"] == calib_dataset._FINEWEB_EDU_MIN_CANDIDATES+def test_combines_sources_with_different_metadata_schemas(self, monkeypatch):import auto_round.calib_dataset as calib_dataset