Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

bug-fix: implement dedicated cache for streaming source shards (#2021)

Xin Heffd3dfe77508
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.py
index e9b4e5437..242c000de 100644
--- a/auto_round/compressors/model_free.py
+++ b/auto_round/compressors/model_free.py
@@ -306,6 +306,7 @@ def _download_single_shard(
local_dir: str,
) -> str:
"""Download a single safetensors shard file. Returns the local path."""
+ os.makedirs(local_dir, exist_ok=True)
local_path = os.path.join(local_dir, shard_filename)
if os.path.exists(local_path):
logger.info(f"Shard '{shard_filename}' already exists at '{local_path}', skipping download.")
@@ -1263,7 +1264,10 @@ def _prefetch_shard(
"""Return the local path of the next shard (download if needed)."""
try:
if streaming:
- return _download_single_shard(model_name_or_path, shard_name, work_dir)
+ # Keep source shards in a dedicated cache directory to avoid
+ # colliding with quantized output shard names in output_dir.
+ shard_cache_dir = os.path.join(work_dir, ".cache", "model_free_source_shards")
+ return _download_single_shard(model_name_or_path, shard_name, shard_cache_dir)
path = os.path.join(source_dir, shard_name)
return path if os.path.exists(path) else None
except Exception as e: # pragma: no cover
@@ -2069,6 +2073,21 @@ class _ModelFreeCompressorCore:
elif os.path.isfile(src) and not os.path.exists(dst):
shutil.copy2(src, dst)
+ def _cleanup_streaming_shard_cache(self) -> None:
+ """Remove temporary streaming shard cache under output_dir/.cache."""
+ if not self.is_streaming:
+ return
+
+ cache_dir = os.path.join(self.work_dir, ".cache", "model_free_source_shards")
+ if os.path.isdir(cache_dir):
+ shutil.rmtree(cache_dir, ignore_errors=True)
+
+ # Best-effort prune for empty .cache directory created by this flow.
+ try:
+ os.rmdir(os.path.join(self.work_dir, ".cache"))
+ except OSError:
+ pass
+
def _log_summary(self, total_time: float) -> None:
compressed_quantized = compress_layer_names(self.all_quantized_layers)
compressed_ignored = compress_layer_names(list(dict.fromkeys(self.all_ignored_layers)))
@@ -2152,6 +2171,7 @@ class _ModelFreeCompressorCore:
self._write_index()
self._write_config_files()
self._copy_metadata_files()
+ self._cleanup_streaming_shard_cache()
self._log_summary(time.time() - start_time)
return self.output_dir
diff --git a/test/test_cpu/quantization/test_model_free.py b/test/test_cpu/quantization/test_model_free.py
index b4ab79c7d..bceede445 100644
--- a/test/test_cpu/quantization/test_model_free.py
+++ b/test/test_cpu/quantization/test_model_free.py
@@ -36,6 +36,7 @@ from auto_round.compressors.model_free import (
_PatternMatcher,
_preprocess_model_type_source_tensors,
_process_shard,
+ _process_single_shard_task,
_quantize_weight_mxfp,
_validate_auto_scheme_options,
get_predefined_ignore_layers_from_config,
@@ -414,6 +415,59 @@ class TestModelFreeQuantize:
qc = _read_qconfig(output_dir)
assert qc["sym"] is False and qc["group_size"] == 64
+ def test_streaming_uses_dedicated_source_shard_cache(self, tmp_path, monkeypatch):
+ """Streaming source shards must not reuse same-named output shard files."""
+ output_dir = str(tmp_path / "output")
+ os.makedirs(output_dir, exist_ok=True)
+ shard_name = "model-00001-of-00001.safetensors"
+ stale_output_shard = os.path.join(output_dir, shard_name)
+
+ # Simulate stale quantized output shard from a previous run.
+ save_file({"lm_head.weight": torch.randn(8, 8)}, stale_output_shard)
+
+ called_local_dirs = []
+
+ def _fake_download_single_shard(model_name_or_path, shard_filename, local_dir):
+ called_local_dirs.append(local_dir)
+ os.makedirs(local_dir, exist_ok=True)
+ source_path = os.path.join(local_dir, shard_filename)
+ # Source shard contains quantizable linear weight.
+ save_file({"layer.weight": torch.randn(8, 8)}, source_path)
+ return source_path
+
+ monkeypatch.setattr(
+ "auto_round.compressors.model_free._download_single_shard",
+ _fake_download_single_shard,
+ )
+
+ result = _process_single_shard_task(
+ shard_idx=0,
+ shard_name=shard_name,
+ model_name_or_path="org/dummy-model",
+ work_dir=output_dir,
+ source_dir="",
+ is_streaming=True,
+ device="cpu",
+ default_scheme=_DEFAULT_SCHEME,
+ layer_config={},
+ ignore_patterns=["lm_head"],
+ fp8_block_size=None,
+ model_type=None,
+ quant_output_dir=output_dir,
+ total_shards=1,
+ )
+
+ _, _, _, out_shard_name, _, quantized, _ = result
+ assert out_shard_name == shard_name
+ assert "layer" in quantized
+ assert called_local_dirs
+ assert called_local_dirs[0] != output_dir
+ assert called_local_dirs[0].startswith(os.path.join(output_dir, ".cache"))
+
+ with safe_open(stale_output_shard, framework="pt") as sf:
+ out_keys = set(sf.keys())
+ assert "layer.qweight" in out_keys
+
# ===========================================================================
# MXFP4 / MXFP8 model-free quantization