Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix: keep model obj evaluation for regular workflow (#2454)

Xin Heea35ee11fe20
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index 48ff1cfd5..f39ba47a4 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -516,39 +516,31 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, args):
if model is None:
return
else:
- # Evaluate the exported artifact for both regular and model-free flows.
- # The in-memory regular model still contains quantization wrappers,
- # while fake-format loading materializes FakeActQuantLinear modules.
- if model is not None:
- model_context = getattr(autoround, "model_context", None)
- if model_context is not None and getattr(model_context, "model", None) is model:
- model_context.model = None
- model = None
- from auto_round.utils import clear_memory
-
- clear_memory()
-
- eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
- if getattr(autoround, "mllm", False):
- from auto_round.utils.model import mllm_load_model
-
- model, _, loaded_tokenizer, _ = mllm_load_model(
- eval_folder,
- device=device_str,
- torch_dtype=eval_model_dtype,
- trust_remote_code=not args.disable_trust_remote_code,
- )
- if tokenizer is None:
- tokenizer = loaded_tokenizer
- else:
- from transformers import AutoModelForCausalLM, AutoTokenizer
-
- model = AutoModelForCausalLM.from_pretrained(
- eval_folder, device_map=device_str, torch_dtype=eval_model_dtype
- )
- model.eval()
- if tokenizer is None:
- tokenizer = AutoTokenizer.from_pretrained(eval_folder)
+ if model is None:
+ eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
+ if getattr(autoround, "mllm", False):
+ from auto_round.utils.model import mllm_load_model
+
+ model, _, loaded_tokenizer, _ = mllm_load_model(
+ eval_folder,
+ device=device_str,
+ torch_dtype=eval_model_dtype,
+ trust_remote_code=not args.disable_trust_remote_code,
+ )
+ if tokenizer is None:
+ tokenizer = loaded_tokenizer
+ else:
+ from auto_round.utils.model import llm_load_model
+
+ model, loaded_tokenizer = llm_load_model(
+ eval_folder,
+ device=device_str,
+ torch_dtype=eval_model_dtype,
+ trust_remote_code=not args.disable_trust_remote_code,
+ )
+ model.eval()
+ if tokenizer is None:
+ tokenizer = loaded_tokenizer
# Evaluate with model instance
evaluate_with_model_instance(model, tokenizer, device_str, args)
diff --git a/test/unit/common/advanced/test_evaluation_functions.py b/test/unit/common/advanced/test_evaluation_functions.py
index 570e66a3b..092681066 100644
--- a/test/unit/common/advanced/test_evaluation_functions.py
+++ b/test/unit/common/advanced/test_evaluation_functions.py
@@ -206,51 +206,42 @@ def _fake_lm_eval_modules(simple_evaluate_impl):
class TestEvalArgumentForwarding:
- def test_run_model_evaluation_fake_reloads_exported_artifact(self, tmp_path):
+ def test_run_model_evaluation_fake_loads_exported_artifact_with_llm_loader(self, tmp_path):
from auto_round.eval.evaluation import run_model_evaluation
args = _make_eval_args()
- in_memory_model = MagicMock()
loaded_model = MagicMock()
- tokenizer = MagicMock()
- model_context = SimpleNamespace(model=in_memory_model)
- autoround = SimpleNamespace(model_context=model_context)
+ loaded_tokenizer = MagicMock()
+ autoround = SimpleNamespace()
with patch("auto_round.utils.model.detect_model_type", return_value="llm"), patch(
"auto_round.utils.device_manager.get_device_and_parallelism", return_value=("cpu", False)
), patch("auto_round.utils.get_library_version", return_value="test"), patch(
- "auto_round.utils.clear_memory"
- ) as mock_clear_memory, patch(
- "transformers.AutoModelForCausalLM.from_pretrained", return_value=loaded_model
- ) as mock_from_pretrained, patch(
+ "auto_round.utils.model.llm_load_model", return_value=(loaded_model, loaded_tokenizer)
+ ) as mock_load_model, patch(
"auto_round.eval.evaluation.evaluate_with_model_instance"
) as mock_evaluate:
- run_model_evaluation(in_memory_model, tokenizer, autoround, str(tmp_path), ["fake"], args)
+ run_model_evaluation(None, None, autoround, str(tmp_path), ["fake"], args)
- mock_from_pretrained.assert_called_once_with(str(tmp_path), device_map="cpu", torch_dtype="auto")
- mock_clear_memory.assert_called_once()
- assert model_context.model is None
- mock_evaluate.assert_called_once_with(loaded_model, tokenizer, "cpu", args)
+ mock_load_model.assert_called_once_with(str(tmp_path), device="cpu", torch_dtype="auto", trust_remote_code=True)
+ mock_evaluate.assert_called_once_with(loaded_model, loaded_tokenizer, "cpu", args)
- def test_run_model_evaluation_fake_reloads_mllm_artifact(self, tmp_path):
+ def test_run_model_evaluation_fake_loads_mllm_artifact_when_model_missing(self, tmp_path):
from auto_round.eval.evaluation import run_model_evaluation
args = _make_eval_args(mllm=True, disable_trust_remote_code=True)
- in_memory_model = MagicMock()
loaded_model = MagicMock()
loaded_tokenizer = MagicMock()
- autoround = SimpleNamespace(mllm=True, model_context=SimpleNamespace(model=in_memory_model))
+ autoround = SimpleNamespace(mllm=True)
with patch("auto_round.utils.model.detect_model_type", return_value="mllm"), patch(
"auto_round.utils.device_manager.get_device_and_parallelism", return_value=("cpu", False)
), patch("auto_round.utils.get_library_version", return_value="test"), patch(
- "auto_round.utils.clear_memory"
- ), patch(
"auto_round.utils.model.mllm_load_model", return_value=(loaded_model, None, loaded_tokenizer, None)
) as mock_load, patch(
"auto_round.eval.evaluation.evaluate_with_model_instance"
) as mock_evaluate:
- run_model_evaluation(in_memory_model, None, autoround, str(tmp_path), ["fake"], args)
+ run_model_evaluation(None, None, autoround, str(tmp_path), ["fake"], args)
mock_load.assert_called_once_with(str(tmp_path), device="cpu", torch_dtype="auto", trust_remote_code=False)
mock_evaluate.assert_called_once_with(loaded_model, loaded_tokenizer, "cpu", args)