PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
bug-fix: implement dedicated cache for streaming source shards (#2021)
Xin Heffd3dfe77508
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyindex e9b4e5437..242c000de 100644--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -306,6 +306,7 @@ def _download_single_shard(local_dir: str,) -> str:"""Download a single safetensors shard file. Returns the local path."""+ os.makedirs(local_dir, exist_ok=True)local_path = os.path.join(local_dir, shard_filename)if os.path.exists(local_path):logger.info(f"Shard '{shard_filename}' already exists at '{local_path}', skipping download.")@@ -1263,7 +1264,10 @@ def _prefetch_shard("""Return the local path of the next shard (download if needed)."""try:if streaming:- return _download_single_shard(model_name_or_path, shard_name, work_dir)+ # Keep source shards in a dedicated cache directory to avoid+ # colliding with quantized output shard names in output_dir.+ shard_cache_dir = os.path.join(work_dir, ".cache", "model_free_source_shards")+ return _download_single_shard(model_name_or_path, shard_name, shard_cache_dir)path = os.path.join(source_dir, shard_name)return path if os.path.exists(path) else Noneexcept Exception as e: # pragma: no cover@@ -2069,6 +2073,21 @@ class _ModelFreeCompressorCore:elif os.path.isfile(src) and not os.path.exists(dst):shutil.copy2(src, dst)+ def _cleanup_streaming_shard_cache(self) -> None:+ """Remove temporary streaming shard cache under output_dir/.cache."""+ if not self.is_streaming:+ return++ cache_dir = os.path.join(self.work_dir, ".cache", "model_free_source_shards")+ if os.path.isdir(cache_dir):+ shutil.rmtree(cache_dir, ignore_errors=True)++ # Best-effort prune for empty .cache directory created by this flow.+ try:+ os.rmdir(os.path.join(self.work_dir, ".cache"))+ except OSError:+ pass+def _log_summary(self, total_time: float) -> None:compressed_quantized = compress_layer_names(self.all_quantized_layers)compressed_ignored = compress_layer_names(list(dict.fromkeys(self.all_ignored_layers)))@@ -2152,6 +2171,7 @@ class _ModelFreeCompressorCore:self._write_index()self._write_config_files()self._copy_metadata_files()+ self._cleanup_streaming_shard_cache()self._log_summary(time.time() - start_time)return self.output_dirdiff --git a/test/test_cpu/quantization/test_model_free.py b/test/test_cpu/quantization/test_model_free.pyindex b4ab79c7d..bceede445 100644--- a/test/test_cpu/quantization/test_model_free.py+++ b/test/test_cpu/quantization/test_model_free.py@@ -36,6 +36,7 @@ from auto_round.compressors.model_free import (_PatternMatcher,_preprocess_model_type_source_tensors,_process_shard,+ _process_single_shard_task,_quantize_weight_mxfp,_validate_auto_scheme_options,get_predefined_ignore_layers_from_config,@@ -414,6 +415,59 @@ class TestModelFreeQuantize:qc = _read_qconfig(output_dir)assert qc["sym"] is False and qc["group_size"] == 64+ def test_streaming_uses_dedicated_source_shard_cache(self, tmp_path, monkeypatch):+ """Streaming source shards must not reuse same-named output shard files."""+ output_dir = str(tmp_path / "output")+ os.makedirs(output_dir, exist_ok=True)+ shard_name = "model-00001-of-00001.safetensors"+ stale_output_shard = os.path.join(output_dir, shard_name)++ # Simulate stale quantized output shard from a previous run.+ save_file({"lm_head.weight": torch.randn(8, 8)}, stale_output_shard)++ called_local_dirs = []++ def _fake_download_single_shard(model_name_or_path, shard_filename, local_dir):+ called_local_dirs.append(local_dir)+ os.makedirs(local_dir, exist_ok=True)+ source_path = os.path.join(local_dir, shard_filename)+ # Source shard contains quantizable linear weight.+ save_file({"layer.weight": torch.randn(8, 8)}, source_path)+ return source_path++ monkeypatch.setattr(+ "auto_round.compressors.model_free._download_single_shard",+ _fake_download_single_shard,+ )++ result = _process_single_shard_task(+ shard_idx=0,+ shard_name=shard_name,+ model_name_or_path="org/dummy-model",+ work_dir=output_dir,+ source_dir="",+ is_streaming=True,+ device="cpu",+ default_scheme=_DEFAULT_SCHEME,+ layer_config={},+ ignore_patterns=["lm_head"],+ fp8_block_size=None,+ model_type=None,+ quant_output_dir=output_dir,+ total_shards=1,+ )++ _, _, _, out_shard_name, _, quantized, _ = result+ assert out_shard_name == shard_name+ assert "layer" in quantized+ assert called_local_dirs+ assert called_local_dirs[0] != output_dir+ assert called_local_dirs[0].startswith(os.path.join(output_dir, ".cache"))++ with safe_open(stale_output_shard, framework="pt") as sf:+ out_keys = set(sf.keys())+ assert "layer.qweight" in out_keys+# ===========================================================================# MXFP4 / MXFP8 model-free quantization