Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support custom max shard size in CLI and model export (#2258)

Heng Guo5bc238f840f0
diff --git a/auto_round/cli/main.py b/auto_round/cli/main.py
index 167a634b2..59ffc447f 100644
--- a/auto_round/cli/main.py
+++ b/auto_round/cli/main.py
@@ -410,7 +410,9 @@ def tune(args):
)
model, folders = autoround.quantize_and_save( # pylint: disable=no-member
- args.output_dir, format=getattr(args, "_api_format", args.format)
+ args.output_dir,
+ format=getattr(args, "_api_format", args.format),
+ max_shard_size=args.max_shard_size,
)
tokenizer = autoround.tokenizer # pylint: disable=no-member
clear_memory()
diff --git a/auto_round/cli/parser.py b/auto_round/cli/parser.py
index d0926e8ee..def9707d7 100644
--- a/auto_round/cli/parser.py
+++ b/auto_round/cli/parser.py
@@ -120,6 +120,9 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Argu
rt.add_argument(
"--format", "--formats", default="auto_round", type=str, help="Output format for the quantized model."
)
+ rt.add_argument(
+ "--max_shard_size", default=None, type=str, help="Maximum size of each safetensors shard. Defaults to 5GB."
+ )
# TODO wenhuach need to add choice or verify the correctness
rt.add_argument(
"--algorithm",
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 94f6bab58..c858727b5 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -1851,7 +1851,7 @@ class BaseOrchestrator(object):
def _ensure_shard_writer(self):
"""Lazily create ShardWriter if it hasn't been created yet."""
if self.shard_writer is None and self.formats is not None:
- self.shard_writer = ShardWriter(self.model, bits=8)
+ self.shard_writer = ShardWriter(self.model, bits=8, max_shard_size=getattr(self, "max_shard_size", None))
def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:
"""Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.
@@ -1866,6 +1866,7 @@ class BaseOrchestrator(object):
format: Union[str, list[OutputFormat]] = None,
inplace: bool = True,
return_folders: bool = False,
+ max_shard_size: Union[int, str] = None,
**kwargs,
) -> torch.nn.Module:
"""Save the quantized model to the specified output directory in the specified format.
@@ -1874,11 +1875,14 @@ class BaseOrchestrator(object):
output_dir (str, optional): The directory to save the quantized model. Defaults to None.
format (str, optional): The format in which to save the model. Defaults to "auto_round".
inplace (bool, optional): Whether to modify the model in place. Defaults to True.
+ max_shard_size (int or str, optional): Maximum size of each safetensors shard. Defaults to 5GB.
**kwargs: Additional keyword arguments specific to the export format.
Returns:
object: The compressed model object.
"""
+ if max_shard_size is not None:
+ self.max_shard_size = max_shard_size
self.output_dir = output_dir
if output_dir is not None:
self.compress_context.output_dir = output_dir
@@ -2018,7 +2022,12 @@ class BaseOrchestrator(object):
)
def quantize_and_save(
- self, output_dir: str = "tmp_autoround", format: str = None, inplace: bool = True, **kwargs
+ self,
+ output_dir: str = "tmp_autoround",
+ format: str = None,
+ inplace: bool = True,
+ max_shard_size: Union[int, str] = None,
+ **kwargs,
) -> tuple[torch.nn.Module, dict[str, Any]]:
"""Quantizes the model and saves it in the specified format(s).
@@ -2033,6 +2042,7 @@ class BaseOrchestrator(object):
by commas if multiple. Defaults to "auto_round".
inplace (bool, optional): Whether to modify the model in place if only
one format is used. Defaults to True.
+ max_shard_size (int or str, optional): Maximum size of each safetensors shard. Defaults to 5GB.
**kwargs: Additional arguments for the quantization and saving process.
Returns:
@@ -2042,6 +2052,8 @@ class BaseOrchestrator(object):
Raises:
ValueError: If an unsupported format is specified.
"""
+ if max_shard_size is not None:
+ self.max_shard_size = max_shard_size
# Validate and process the specified formats
self.output_dir = output_dir
self.compress_context.output_dir = output_dir
diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.py
index f2bcf78bd..6ac0c8b06 100644
--- a/auto_round/compressors/shard_writer.py
+++ b/auto_round/compressors/shard_writer.py
@@ -33,6 +33,8 @@ from auto_round.utils import (
revert_checkpoint_conversion_mapping,
)
+DEFAULT_MAX_SHARD_SIZE = "5GB"
+
class ShardWriter:
"""
@@ -62,21 +64,7 @@ class ShardWriter:
return
self.model = model
self.lm_head_name = get_lm_head_name(self.model)
- total_params = sum(p.numel() for p in self.model.parameters())
- # Heuristic estimate of model size in GB used to choose a default max_shard_size:
- # - total_params * rounder.bits -> total number of bits in all parameters
- # - // 8 -> convert bits to bytes
- # - // 1e9 -> approx convert bytes to GB (1e9 bytes ~= 1 GB)
- # - final // 10 -> apply a safety margin so default shards are
- # smaller than the full model; this intentionally
- # underestimates size before clamping below.
- max_split_num = 10
- model_size = int(total_params * bits // 1e9 // 8 + max_split_num - 1) / max_split_num
- model_size = max(1, min(int(model_size), 5))
-
- # Configuration
- max_shard_size = max_shard_size or f"{model_size}GB"
- self.max_shard_size = self._parse_size(max_shard_size)
+ self.max_shard_size = self._parse_size(max_shard_size or DEFAULT_MAX_SHARD_SIZE)
self.safe_serialization = safe_serialization
# Internal State
@@ -297,11 +285,11 @@ class ShardWriter:
self.total_param_elems += tensor.numel()
self.total_param_size_bytes += t_size
tensor = tensor.detach().cpu()
- # If single tensor exceeds limit, flush current, save it solo, then continue
+ # Keep an oversized tensor with any buffered tensors so it does not
+ # leave a tiny shard immediately before its own shard.
if t_size > self.max_shard_size:
- self._flush_shard()
self.current_shard_tensors[name] = tensor
- self.current_shard_size = t_size
+ self.current_shard_size += t_size
self._flush_shard()
# If adding exceeds limit, flush first
elif self.current_shard_size + t_size > self.max_shard_size and self.current_shard_size > 0:
diff --git a/test/unit/common/utils/test_cli_usage.py b/test/unit/common/utils/test_cli_usage.py
index 9b009eb0e..158aaa491 100644
--- a/test/unit/common/utils/test_cli_usage.py
+++ b/test/unit/common/utils/test_cli_usage.py
@@ -472,3 +472,11 @@ def test_shared_layers_normalize():
p.parse_args(["--model", "dummy", "--shared_layers", "l1", "l2", "--shared_layers", "l3,l4"]).shared_layers
) == [["l1", "l2"], ["l3", "l4"]]
assert p.parse_args(["--model", "dummy"]).shared_layers is None
+
+
+def test_parse_max_shard_size():
+ from auto_round.cli.parser import build_quantize_parser
+
+ parser = build_quantize_parser()
+ assert parser.parse_args([]).max_shard_size is None
+ assert parser.parse_args(["--max_shard_size", "1GB"]).max_shard_size == "1GB"
diff --git a/test/unit/common/utils/test_shard_writer.py b/test/unit/common/utils/test_shard_writer.py
index dfd092b25..7c7a50c40 100644
--- a/test/unit/common/utils/test_shard_writer.py
+++ b/test/unit/common/utils/test_shard_writer.py
@@ -172,3 +172,28 @@ def test_finalize_offloads_module_with_tensor_in_parameters(tmp_path, monkeypatc
offloaded_weight = model.transformer_blocks[0].linear._parameters["weight"]
assert isinstance(offloaded_weight, torch.nn.Parameter)
assert offloaded_weight.device.type == "meta"
+
+
+def test_default_max_shard_size_is_fixed(tmp_path, monkeypatch):
+ writer = _make_writer(_DiffusionStyleModel(), str(tmp_path), monkeypatch)
+ assert writer.max_shard_size == 1 * 1024**2
+
+ ShardWriter.reset()
+ compress_context = SimpleNamespace(formats=[_FormatStub()], output_dir=str(tmp_path))
+ model_context = SimpleNamespace(is_diffusion=False)
+ monkeypatch.setattr(CompressContext, "get_context", classmethod(lambda cls: compress_context))
+ monkeypatch.setattr(ModelContext, "get_context", classmethod(lambda cls: model_context))
+ default_writer = ShardWriter(_DiffusionStyleModel(), bits=4, safe_serialization=False)
+ assert default_writer.max_shard_size == 5 * 1024**3
+
+
+def test_oversized_tensor_does_not_leave_tiny_preceding_shard(tmp_path, monkeypatch):
+ writer = _make_writer(_DiffusionStyleModel(), str(tmp_path), monkeypatch)
+ writer.max_shard_size = 1024
+
+ writer._add_tensor("small", torch.zeros(1, dtype=torch.uint8))
+ writer._add_tensor("large", torch.zeros(2048, dtype=torch.uint8))
+ writer._flush_shard()
+
+ assert writer.shard_counter == 1
+ assert set(writer.current_shard_tensors) == set()