PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support custom max shard size in CLI and model export (#2258)
Heng Guo5bc238f840f0
diff --git a/auto_round/cli/main.py b/auto_round/cli/main.pyindex 167a634b2..59ffc447f 100644--- a/auto_round/cli/main.py+++ b/auto_round/cli/main.py@@ -410,7 +410,9 @@ def tune(args):)model, folders = autoround.quantize_and_save( # pylint: disable=no-member- args.output_dir, format=getattr(args, "_api_format", args.format)+ args.output_dir,+ format=getattr(args, "_api_format", args.format),+ max_shard_size=args.max_shard_size,)tokenizer = autoround.tokenizer # pylint: disable=no-memberclear_memory()diff --git a/auto_round/cli/parser.py b/auto_round/cli/parser.pyindex d0926e8ee..def9707d7 100644--- a/auto_round/cli/parser.py+++ b/auto_round/cli/parser.py@@ -120,6 +120,9 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Argurt.add_argument("--format", "--formats", default="auto_round", type=str, help="Output format for the quantized model.")+ rt.add_argument(+ "--max_shard_size", default=None, type=str, help="Maximum size of each safetensors shard. Defaults to 5GB."+ )# TODO wenhuach need to add choice or verify the correctnessrt.add_argument("--algorithm",diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 94f6bab58..c858727b5 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1851,7 +1851,7 @@ class BaseOrchestrator(object):def _ensure_shard_writer(self):"""Lazily create ShardWriter if it hasn't been created yet."""if self.shard_writer is None and self.formats is not None:- self.shard_writer = ShardWriter(self.model, bits=8)+ self.shard_writer = ShardWriter(self.model, bits=8, max_shard_size=getattr(self, "max_shard_size", None))def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:"""Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.@@ -1866,6 +1866,7 @@ class BaseOrchestrator(object):format: Union[str, list[OutputFormat]] = None,inplace: bool = True,return_folders: bool = False,+ max_shard_size: Union[int, str] = None,**kwargs,) -> torch.nn.Module:"""Save the quantized model to the specified output directory in the specified format.@@ -1874,11 +1875,14 @@ class BaseOrchestrator(object):output_dir (str, optional): The directory to save the quantized model. Defaults to None.format (str, optional): The format in which to save the model. Defaults to "auto_round".inplace (bool, optional): Whether to modify the model in place. Defaults to True.+ max_shard_size (int or str, optional): Maximum size of each safetensors shard. Defaults to 5GB.**kwargs: Additional keyword arguments specific to the export format.Returns:object: The compressed model object."""+ if max_shard_size is not None:+ self.max_shard_size = max_shard_sizeself.output_dir = output_dirif output_dir is not None:self.compress_context.output_dir = output_dir@@ -2018,7 +2022,12 @@ class BaseOrchestrator(object):)def quantize_and_save(- self, output_dir: str = "tmp_autoround", format: str = None, inplace: bool = True, **kwargs+ self,+ output_dir: str = "tmp_autoround",+ format: str = None,+ inplace: bool = True,+ max_shard_size: Union[int, str] = None,+ **kwargs,) -> tuple[torch.nn.Module, dict[str, Any]]:"""Quantizes the model and saves it in the specified format(s).@@ -2033,6 +2042,7 @@ class BaseOrchestrator(object):by commas if multiple. Defaults to "auto_round".inplace (bool, optional): Whether to modify the model in place if onlyone format is used. Defaults to True.+ max_shard_size (int or str, optional): Maximum size of each safetensors shard. Defaults to 5GB.**kwargs: Additional arguments for the quantization and saving process.Returns:@@ -2042,6 +2052,8 @@ class BaseOrchestrator(object):Raises:ValueError: If an unsupported format is specified."""+ if max_shard_size is not None:+ self.max_shard_size = max_shard_size# Validate and process the specified formatsself.output_dir = output_dirself.compress_context.output_dir = output_dirdiff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.pyindex f2bcf78bd..6ac0c8b06 100644--- a/auto_round/compressors/shard_writer.py+++ b/auto_round/compressors/shard_writer.py@@ -33,6 +33,8 @@ from auto_round.utils import (revert_checkpoint_conversion_mapping,)+DEFAULT_MAX_SHARD_SIZE = "5GB"+class ShardWriter:"""@@ -62,21 +64,7 @@ class ShardWriter:returnself.model = modelself.lm_head_name = get_lm_head_name(self.model)- total_params = sum(p.numel() for p in self.model.parameters())- # Heuristic estimate of model size in GB used to choose a default max_shard_size:- # - total_params * rounder.bits -> total number of bits in all parameters- # - // 8 -> convert bits to bytes- # - // 1e9 -> approx convert bytes to GB (1e9 bytes ~= 1 GB)- # - final // 10 -> apply a safety margin so default shards are- # smaller than the full model; this intentionally- # underestimates size before clamping below.- max_split_num = 10- model_size = int(total_params * bits // 1e9 // 8 + max_split_num - 1) / max_split_num- model_size = max(1, min(int(model_size), 5))-- # Configuration- max_shard_size = max_shard_size or f"{model_size}GB"- self.max_shard_size = self._parse_size(max_shard_size)+ self.max_shard_size = self._parse_size(max_shard_size or DEFAULT_MAX_SHARD_SIZE)self.safe_serialization = safe_serialization# Internal State@@ -297,11 +285,11 @@ class ShardWriter:self.total_param_elems += tensor.numel()self.total_param_size_bytes += t_sizetensor = tensor.detach().cpu()- # If single tensor exceeds limit, flush current, save it solo, then continue+ # Keep an oversized tensor with any buffered tensors so it does not+ # leave a tiny shard immediately before its own shard.if t_size > self.max_shard_size:- self._flush_shard()self.current_shard_tensors[name] = tensor- self.current_shard_size = t_size+ self.current_shard_size += t_sizeself._flush_shard()# If adding exceeds limit, flush firstelif self.current_shard_size + t_size > self.max_shard_size and self.current_shard_size > 0:diff --git a/test/unit/common/utils/test_cli_usage.py b/test/unit/common/utils/test_cli_usage.pyindex 9b009eb0e..158aaa491 100644--- a/test/unit/common/utils/test_cli_usage.py+++ b/test/unit/common/utils/test_cli_usage.py@@ -472,3 +472,11 @@ def test_shared_layers_normalize():p.parse_args(["--model", "dummy", "--shared_layers", "l1", "l2", "--shared_layers", "l3,l4"]).shared_layers) == [["l1", "l2"], ["l3", "l4"]]assert p.parse_args(["--model", "dummy"]).shared_layers is None+++def test_parse_max_shard_size():+ from auto_round.cli.parser import build_quantize_parser++ parser = build_quantize_parser()+ assert parser.parse_args([]).max_shard_size is None+ assert parser.parse_args(["--max_shard_size", "1GB"]).max_shard_size == "1GB"diff --git a/test/unit/common/utils/test_shard_writer.py b/test/unit/common/utils/test_shard_writer.pyindex dfd092b25..7c7a50c40 100644--- a/test/unit/common/utils/test_shard_writer.py+++ b/test/unit/common/utils/test_shard_writer.py@@ -172,3 +172,28 @@ def test_finalize_offloads_module_with_tensor_in_parameters(tmp_path, monkeypatcoffloaded_weight = model.transformer_blocks[0].linear._parameters["weight"]assert isinstance(offloaded_weight, torch.nn.Parameter)assert offloaded_weight.device.type == "meta"+++def test_default_max_shard_size_is_fixed(tmp_path, monkeypatch):+ writer = _make_writer(_DiffusionStyleModel(), str(tmp_path), monkeypatch)+ assert writer.max_shard_size == 1 * 1024**2++ ShardWriter.reset()+ compress_context = SimpleNamespace(formats=[_FormatStub()], output_dir=str(tmp_path))+ model_context = SimpleNamespace(is_diffusion=False)+ monkeypatch.setattr(CompressContext, "get_context", classmethod(lambda cls: compress_context))+ monkeypatch.setattr(ModelContext, "get_context", classmethod(lambda cls: model_context))+ default_writer = ShardWriter(_DiffusionStyleModel(), bits=4, safe_serialization=False)+ assert default_writer.max_shard_size == 5 * 1024**3+++def test_oversized_tensor_does_not_leave_tiny_preceding_shard(tmp_path, monkeypatch):+ writer = _make_writer(_DiffusionStyleModel(), str(tmp_path), monkeypatch)+ writer.max_shard_size = 1024++ writer._add_tensor("small", torch.zeros(1, dtype=torch.uint8))+ writer._add_tensor("large", torch.zeros(2048, dtype=torch.uint8))+ writer._flush_shard()++ assert writer.shard_counter == 1+ assert set(writer.current_shard_tensors) == set()