Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix bug and support shared_layers in cli (#1138)

Xin He746f32b661b0
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index 554b62704..94e3ad163 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -286,6 +286,14 @@ class BasicArgumentParser(argparse.ArgumentParser):
scheme.add_argument(
"--disable_act_dynamic", action="store_true", help="Use static instead of dynamic activation quantization. "
)
+ scheme.add_argument(
+ "--shared_layers",
+ type=str,
+ nargs="+",
+ action="append",
+ default=None,
+ help="[mix-precision] ensure that listed layers are using same data type for quantization",
+ )
scheme.add_argument(
"--quant_lm_head",
action="store_true",
@@ -602,7 +610,10 @@ def tune(args):
if args.options is None:
raise ValueError("please set --options for auto scheme")
scheme = AutoScheme(
- options=args.options, avg_bits=args.avg_bits, ignore_scale_zp_bits=args.ignore_scale_zp_bits
+ options=args.options,
+ avg_bits=args.avg_bits,
+ shared_layers=args.shared_layers,
+ ignore_scale_zp_bits=args.ignore_scale_zp_bits,
)
autoround: BaseCompressor = AutoRound(
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 906e28ce0..c9a08479c 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -548,7 +548,8 @@ def is_mllm_model(model_or_path: Union[str, torch.nn.Module], platform: str = No
from auto_round.utils.common import MM_KEYS
model_path = model_or_path if isinstance(model_or_path, str) else model_or_path.name_or_path
- if not os.path.isdir(model_path):
+ # For dummy model, model_path could be "".
+ if model_path and not os.path.isdir(model_path):
model_path = download_or_get_path(model_path, platform=platform)
if isinstance(model_path, str):