PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
from types import SimpleNamespace
import pytest
from auto_round import AutoRound
from auto_round.algorithms.quantization.rtn.config import RTNConfig
from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
def test_builtin_recipes():
from auto_round.recipes import list_recipes, resolve_recipe
assert list_recipes() == ["auto-round", "auto-round-light", "auto-round-best", "rtn", "opt-rtn", "awq"]
for name, iters, nsamples in [
("auto-round", 200, 128),
("auto-round-light", 50, 128),
("auto-round-best", 1000, 512),
]:
plan = resolve_recipe(name)
assert plan.build_configs()[0].iters == iters
assert plan.calibration["nsamples"] == nsamples
assert resolve_recipe("light").build_configs()[0].lr == 0.005
assert resolve_recipe("rtn").build_configs()[0].disable_opt_rtn is True
assert resolve_recipe("opt_rtn").build_configs()[0].disable_opt_rtn is False
assert [type(config).__name__ for config in resolve_recipe("awq").build_configs()] == ["AWQConfig", "RTNConfig"]
def test_recipe_results_are_isolated():
from auto_round.recipes import get_recipe, resolve_recipe
definition = get_recipe("ar-best")
definition.calibration["nsamples"] = 1
plan = resolve_recipe("ar-best")
plan.calibration["nsamples"] = 2
plan.pipeline[0].params["iters"] = 3
assert resolve_recipe("ar-best").calibration["nsamples"] == 512
assert resolve_recipe("ar-best").build_configs()[0].iters == 1000
def test_recipe_function_selects_pipeline_and_parameters():
from auto_round.recipes import AlgorithmSpec, Recipe, resolve_recipe
def process(recipe, *, scheme, model_config):
if scheme == "MXFP4":
recipe.pipeline = (AlgorithmSpec("smooth", "awq"), AlgorithmSpec("quantizer", "auto_round", {"iters": 50}))
if getattr(model_config, "model_type", None) == "tiny":
recipe.pipeline[-1].params["iters"] = 7
recipe.calibration["nsamples"] = 16
recipe.layer_config["lm_head"] = "BF16"
recipe = Recipe(
name="contextual",
pipeline=(AlgorithmSpec("quantizer", "auto_round", {"iters": 200}),),
process=process,
)
plan = resolve_recipe(recipe, scheme="MXFP4", model_config=SimpleNamespace(model_type="tiny"))
assert [stage.algorithm for stage in plan.pipeline] == ["awq", "auto_round"]
assert plan.build_configs()[1].iters == 7
assert plan.calibration["nsamples"] == 16
assert plan.layer_config == {"lm_head": "BF16"}
assert len(resolve_recipe(recipe).pipeline) == 1
def test_recipe_validation():
from auto_round.recipes import Recipe, resolve_recipe
with pytest.raises(ValueError, match="Unknown recipe"):
resolve_recipe("ar-bset")
recipe = Recipe("broken", ())
with pytest.raises(ValueError, match="at least one stage"):
resolve_recipe(recipe)
@pytest.fixture
def capture_entry(monkeypatch):
from auto_round import autoround
def capture(model, scheme, configs, **kwargs):
return SimpleNamespace(model=model, scheme=scheme, configs=configs, **kwargs)
monkeypatch.setattr(autoround, "_CompressorBuilder", capture)
def test_default_entry_is_ar(capture_entry):
default = AutoRound("dummy")
explicit = AutoRound("dummy", recipe="ar")
assert default.configs[0].iters == explicit.configs[0].iters == 200
assert default.nsamples == explicit.nsamples == 128
def test_entry_recipe_and_explicit_calibration(capture_entry):
entry = AutoRound("dummy", recipe="ar-best", nsamples=128, batch_size=1, seqlen=32)
assert entry.configs[0].iters == 1000
assert (entry.nsamples, entry.batch_size, entry.seqlen) == (128, 1, 32)
assert AutoRound("dummy", recipe="ar-best").nsamples == 512
def test_explicit_pipeline_replaces_recipe_but_inherits_calibration(capture_entry):
config = SignRoundConfig(iters=3, lr=0.25)
entry = AutoRound("dummy", recipe="ar-best", alg_configs=config)
assert entry.configs[0].iters == 3
assert entry.configs[0].lr == 0.25
assert config.iters == 3
assert entry.nsamples == 512
rtn = AutoRound("dummy", recipe="ar-best", alg_configs=RTNConfig(disable_opt_rtn=True))
assert len(rtn.configs) == 1
assert isinstance(rtn.configs[0], RTNConfig)
assert not hasattr(rtn.configs[0], "iters")
def test_legacy_entry_selectors_override_recipe(capture_entry):
assert isinstance(AutoRound("dummy", iters=0).configs[0], RTNConfig)
awq = AutoRound("dummy", recipe="ar-best", algorithm="awq")
assert [type(config).__name__ for config in awq.configs] == ["AWQConfig", "RTNConfig"]
assert awq.nsamples == 512
def test_entry_applies_recipe_function_and_layer_overrides(capture_entry):
from auto_round.recipes import AlgorithmSpec, Recipe
def process(recipe, *, scheme, model_config):
if model_config.model_type == "tiny":
recipe.layer_config["proj"] = {"bits": 8}
recipe = Recipe(
"custom",
(AlgorithmSpec("quantizer", "auto_round"),),
process=process,
)
model = SimpleNamespace(config=SimpleNamespace(model_type="tiny"))
entry = AutoRound(model, recipe=recipe, layer_config={"proj": {"bits": 4}})
assert entry.layer_config["proj"]["bits"] == 4
@pytest.mark.parametrize(
"recipe,canonical",
[
("auto-round", "auto-round"),
("auto_round", "auto-round"),
("ar", "auto-round"),
("default", "auto-round"),
("auto-round-light", "auto-round-light"),
("auto_round_light", "auto-round-light"),
("ar-light", "auto-round-light"),
("ar_light", "auto-round-light"),
("light", "auto-round-light"),
("auto-round-best", "auto-round-best"),
("auto_round_best", "auto-round-best"),
("ar-best", "auto-round-best"),
("ar_best", "auto-round-best"),
("best", "auto-round-best"),
(" AR_LIGHT ", "auto-round-light"),
("awq", "awq"),
("rtn", "rtn"),
("opt-rtn", "opt-rtn"),
("opt_rtn", "opt-rtn"),
],
)
def test_cli_python_recipe_parity(recipe, canonical, capture_entry, monkeypatch):
from auto_round.cli import main
from auto_round.cli.algorithms import AlgorithmHandler
captured = {}
monkeypatch.setattr(main, "tune", lambda args: captured.setdefault("args", args))
main.start(argv=["dummy", "--recipe", recipe])
args = captured["args"]
configs = AlgorithmHandler.build_configs(args, main._extract_common_quantization_kwargs(args))
entry = AutoRound("dummy", recipe=recipe)
assert args.recipe == entry.resolved_recipe.name == canonical
assert [type(config) for config in configs] == [type(config) for config in entry.configs]
assert args.nsamples == entry.nsamples
for cli_config, api_config in zip(configs, entry.configs):
for field in ("iters", "lr", "disable_opt_rtn"):
assert getattr(cli_config, field, None) == getattr(api_config, field, None)
def test_cli_explicit_pipeline_and_shortcut_override(monkeypatch):
from auto_round.cli import main
from auto_round.cli.algorithms import AlgorithmHandler
captured = {}
monkeypatch.setattr(main, "tune", lambda args: captured.setdefault("args", args))
main.start("best", argv=["dummy", "--recipe", "ar-light", "--algorithm", "awq", "--nsamples", "512"])
args = captured["args"]
configs = AlgorithmHandler.build_configs(args, {})
assert args.recipe == "auto-round-light"
assert args.nsamples == 512
assert [type(config).__name__ for config in configs] == ["AWQConfig", "RTNConfig"]
def test_cli_explicit_values_override_recipe_stage_parameters(monkeypatch):
from auto_round.cli import main
from auto_round.cli.algorithms import AlgorithmHandler
captured = {}
monkeypatch.setattr(main, "tune", lambda args: captured.setdefault("args", args))
main.start(argv=["dummy", "--recipe", "ar-best", "--iters", "50", "--disable_minmax_tuning"])
config = AlgorithmHandler.build_configs(captured["args"], {})[0]
assert config.iters == 50
assert config.enable_minmax_tuning is False
def test_cli_recipe_listing(capsys):
from auto_round.cli.main import list_item
list_item(["recipes"])
output = capsys.readouterr().out
assert "auto-round-best" in output
assert "ar-best" not in output
list_item(["recipe", "awq"])
output = capsys.readouterr().out
assert "awq" in output and "rtn" in output
def test_cli_context_recipe_preserves_stage_defaults_and_layers(monkeypatch, capture_entry):
from auto_round.cli import main
from auto_round.cli.algorithms import AlgorithmHandler
from auto_round.recipes import AlgorithmSpec, Recipe, register_recipe, registry
def process(recipe, *, scheme, model_config):
if scheme == "MXFP4":
recipe.pipeline[0].params.update(enable_minmax_tuning=False, iters=7)
recipe.layer_config["proj"] = "BF16"
monkeypatch.setattr(registry, "_RECIPES", dict(registry._RECIPES))
monkeypatch.setattr(registry, "_ALIASES", dict(registry._ALIASES))
register_recipe(
Recipe(
"test-context",
(AlgorithmSpec("quantizer", "auto_round"),),
calibration={"nsamples": 32},
process=process,
)
)
captured = {}
monkeypatch.setattr(main, "tune", lambda args: captured.setdefault("args", args))
main.start(argv=["dummy", "--recipe", "test-context", "--scheme", "MXFP4"])
config = AlgorithmHandler.build_configs(captured["args"], {})[0]
assert config.enable_minmax_tuning is False
assert config.iters == 7
entry = AutoRound("dummy", recipe="test-context", scheme="MXFP4")
assert entry.layer_config == {"proj": "BF16"}
def test_entry_keeps_recipe_resolution_for_inspection(capture_entry):
entry = AutoRound("dummy", recipe="best")
assert entry.resolved_recipe.name == "auto-round-best"
def test_cli_zero_iters_retains_rtn_search_options(monkeypatch):
from auto_round.cli import main
from auto_round.cli.algorithms import AlgorithmHandler
captured = {}
monkeypatch.setattr(main, "tune", lambda args: captured.setdefault("args", args))
main.start(argv=["dummy", "--iters", "0", "--enable_neuqi", "--enable_opt_rtn"])
config = AlgorithmHandler.build_configs(captured["args"], {})[0]
assert isinstance(config, RTNConfig)
assert config.enable_neuqi is True
def test_cli_explicit_signround_uses_config_defaults(monkeypatch):
from auto_round.cli import main
from auto_round.cli.algorithms import AlgorithmHandler
captured = {}
monkeypatch.setattr(main, "tune", lambda args: captured.setdefault("args", args))
main.start(argv=["dummy", "--recipe", "ar-best", "--algorithm", "auto_round"])
configs = AlgorithmHandler.build_configs(captured["args"], {})
assert configs[0].iters == 200
assert captured["args"].nsamples == 512
def test_recipe_function_looks_up_metadata_only_when_needed(monkeypatch, capture_entry):
from transformers import AutoConfig
from auto_round.recipes import AlgorithmSpec, Recipe
calls = []
def load_config(name, **kwargs):
calls.append(name)
return SimpleNamespace(model_type="tiny")
monkeypatch.setattr(AutoConfig, "from_pretrained", load_config)
AutoRound("dummy")
assert calls == []
def process(recipe, *, scheme, model_config):
if model_config.model_type == "tiny":
recipe.pipeline[0].params["iters"] = 7
recipe = Recipe(
"model-aware",
(AlgorithmSpec("quantizer", "auto_round"),),
process=process,
)
assert AutoRound("dummy", recipe=recipe).configs[0].iters == 7
assert calls == ["dummy"]
def test_custom_recipe_inherits_shared_calibration_defaults():
from auto_round.recipes import AlgorithmSpec, Recipe, resolve_recipe
plan = resolve_recipe(Recipe("custom", (AlgorithmSpec("quantizer", "rtn"),)))
assert plan.calibration == {"batch_size": 8, "seqlen": 2048, "nsamples": 128}
def test_entry_rejects_invalid_recipe_type(capture_entry):
with pytest.raises(TypeError, match="recipe must"):
AutoRound("dummy", recipe=None)
@pytest.mark.parametrize("entry_name,recipe_name", [("run_best", "auto-round-best"), ("run_light", "auto-round-light")])
def test_legacy_shortcuts_select_named_recipe(monkeypatch, entry_name, recipe_name):
import sys
from auto_round.cli import main
captured = {}
monkeypatch.setattr(main, "tune", lambda args: captured.setdefault("args", args))
monkeypatch.setattr(sys, "argv", ["auto_round_shortcut", "dummy"])
getattr(main, entry_name)()
assert captured["args"].recipe == recipe_name
def test_callable_factory_defaults_survive_recipe_parameters(monkeypatch):
from auto_round.algorithms.registry import get_algorithm_entry
from auto_round.recipes import AlgorithmSpec, Recipe, resolve_recipe
monkeypatch.setattr(
get_algorithm_entry("auto_round"), "config_factory", lambda: SignRoundConfig(iters=17, momentum=0.25)
)
recipe = Recipe("factory", (AlgorithmSpec("quantizer", "auto_round", {"lr": 0.1}),))
config = resolve_recipe(recipe).build_configs()[0]
assert config.iters == 17
assert config.momentum == 0.25
assert config.lr == 0.1
def test_alias_factory_defaults_survive_recipe_and_cli_overrides(monkeypatch):
from auto_round.cli import main
from auto_round.cli.algorithms import AlgorithmHandler
from auto_round.recipes import AlgorithmSpec, Recipe, register_recipe, registry, resolve_recipe
monkeypatch.setattr(registry, "_RECIPES", dict(registry._RECIPES))
monkeypatch.setattr(registry, "_ALIASES", dict(registry._ALIASES))
recipe = Recipe(
"random-rotation",
(
AlgorithmSpec("rotation", "random_hadamard", {"block_size": 32}),
AlgorithmSpec("quantizer", "auto_round"),
),
)
register_recipe(recipe)
assert resolve_recipe(recipe).build_configs()[0].hadamard_type == "random_hadamard"
captured = {}
monkeypatch.setattr(main, "tune", lambda args: captured.setdefault("args", args))
main.start(argv=["dummy", "--recipe", recipe.name, "--rotation_block_size", "64", "--bits", "4"])
configs = AlgorithmHandler.build_configs(captured["args"], {"bits": 4})
assert configs[0].hadamard_type == "random_hadamard"
assert configs[0].block_size == 64
assert configs[1].bits == 4
def test_cli_recipe_respects_stage_common_argument_opt_out(monkeypatch):
import auto_round.algorithms.registry as algorithms
from auto_round.algorithms.config import AlgorithmConfig
from auto_round.algorithms.registry import register_algorithm
from auto_round.algorithms.transforms.base import BaseRotationConfig
from auto_round.cli import main
from auto_round.cli.algorithms import AlgorithmHandler
from auto_round.recipes import AlgorithmSpec, Recipe, register_recipe, registry
class StrictRotationConfig(AlgorithmConfig, BaseRotationConfig):
cli_include_common_args = False
def __init__(self):
pass
monkeypatch.setattr(registry, "_RECIPES", dict(registry._RECIPES))
monkeypatch.setattr(registry, "_ALIASES", dict(registry._ALIASES))
monkeypatch.setattr(algorithms, "_ALG_REGISTRY", dict(algorithms._ALG_REGISTRY))
monkeypatch.setattr(algorithms, "_ALIAS_TO_NAME", dict(algorithms._ALIAS_TO_NAME))
register_algorithm("strict-rotation", config_factory=StrictRotationConfig)
register_recipe(
Recipe(
"strict-pipeline",
(AlgorithmSpec("rotation", "strict-rotation"), AlgorithmSpec("quantizer", "auto_round")),
)
)
captured = {}
monkeypatch.setattr(main, "tune", lambda args: captured.setdefault("args", args))
main.start(argv=["dummy", "--recipe", "strict-pipeline", "--bits", "4", "--group_size", "32"])
configs = AlgorithmHandler.build_configs(captured["args"], {"bits": 4, "group_size": 32})
assert isinstance(configs[0], StrictRotationConfig)
assert configs[1].bits == 4
assert configs[1].group_size == 32
def test_direct_tune_applies_recipe_before_building_configs(monkeypatch):
from auto_round.cli import main
from auto_round.cli.algorithms import AlgorithmHandler
from auto_round.cli.parser import build_quantize_parser
args = build_quantize_parser().parse_args(["dummy", "--recipe", "ar-best", "--nsamples", "128"])
def check_configs(args, common_kwargs):
assert args.recipe == "auto-round-best"
assert args.nsamples == 128
assert args.batch_size == 8
assert args.seqlen == 2048
assert args._recipe_plan.build_configs()[0].iters == 1000
raise StopIteration("configuration inspected")
monkeypatch.setattr(AlgorithmHandler, "build_configs", check_configs)
with pytest.raises(StopIteration, match="configuration inspected"):
main.tune(args)
def test_dataset_omission_inherits_recipe_but_explicit_none_is_deferred(capture_entry):
from auto_round.recipes import AlgorithmSpec, Recipe
recipe = Recipe(
"dataset", (AlgorithmSpec("quantizer", "auto_round"),), calibration={"dataset": "custom-calibration"}
)
assert AutoRound("dummy", recipe=recipe).dataset == "custom-calibration"
assert AutoRound("dummy", recipe=recipe, dataset=None).dataset is None
assert AutoRound("dummy").dataset is None
def test_callable_factory_recipe_still_validates_overrides(monkeypatch):
from auto_round.algorithms.registry import get_algorithm_entry
from auto_round.recipes import AlgorithmSpec
entry = get_algorithm_entry("hadamard")
monkeypatch.setattr(entry, "config_factory", lambda: entry.alias_factories["random_hadamard"]())
with pytest.raises(ValueError):
AlgorithmSpec("rotation", "hadamard", {"backend": "invalid"}).build_config()
def test_exact_user_layer_wins_over_recipe_regex():
import torch.nn as nn
from auto_round.compressors.config_resolution import ResolvedScheme
from auto_round.compressors.layer_config_resolver import resolve_layer_config
from auto_round.recipes.resolver import merge_layer_defaults
from auto_round.schemes import QuantizationScheme
model = nn.Module()
model.a_proj = nn.Linear(32, 32)
model.b_proj = nn.Linear(32, 32)
layer_config = merge_layer_defaults({r".*proj$": {"bits": 8}}, {"a_proj": {"bits": 16}})
resolved = resolve_layer_config(
model=model,
scheme=ResolvedScheme.from_scheme(QuantizationScheme(act_bits=16, act_data_type="float")),
layer_config=layer_config,
supported_types=(nn.Linear,),
inner_supported_types=(),
)
assert resolved["a_proj"]["bits"] == 16
assert resolved["b_proj"]["bits"] == 8
def test_cli_resolves_contextual_recipe_once(monkeypatch, capture_entry):
from auto_round.cli import main
from auto_round.recipes import AlgorithmSpec, Recipe, register_recipe, registry
calls = []
def process(recipe, *, scheme, model_config):
calls.append(scheme)
recipe.calibration["nsamples"] = 16 + len(calls)
monkeypatch.setattr(registry, "_RECIPES", dict(registry._RECIPES))
monkeypatch.setattr(registry, "_ALIASES", dict(registry._ALIASES))
register_recipe(Recipe("test-once", (AlgorithmSpec("quantizer", "auto_round"),), process=process))
import auto_round
seen = {}
def fake_entry(*args, **kwargs):
entry = AutoRound(*args, **kwargs)
seen["entry"] = entry
raise StopIteration
monkeypatch.setattr(auto_round, "AutoRound", fake_entry)
with pytest.raises(StopIteration):
main.start(argv=["dummy", "--recipe", "test-once"])
assert calls == ["W4A16"]
assert seen["entry"].nsamples == 17
@pytest.mark.parametrize("option", ["--iter", "--iters", "--iters="])
def test_cli_option_spellings_override_recipe(monkeypatch, option):
from auto_round.cli import main
from auto_round.cli.algorithms import AlgorithmHandler
captured = {}
monkeypatch.setattr(main, "tune", lambda args: captured.setdefault("args", args))
argv = ["dummy", "--recipe", "ar-best", "--disable_minmax"]
argv += [option + "5"] if option.endswith("=") else [option, "5"]
main.start(argv=argv)
config = AlgorithmHandler.build_configs(captured["args"], {})[0]
assert config.iters == 5
assert config.enable_minmax_tuning is False
def test_tune_accepts_namespace_without_explicit_fields():
import argparse
from auto_round.cli import main
from auto_round.cli.parser import build_quantize_parser
args = argparse.Namespace(**vars(build_quantize_parser().parse_args(["dummy", "--iters", "5"])))
del args._explicit_fields
main._apply_recipe_defaults(args)
assert args.iters == 5
assert args._recipe_plan.name == "auto-round"