Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix missing trust_remote_code in unfused_moe (#1532)

Xin He837888ddccc2
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index afcc489cd..f2cfd8089 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -288,7 +288,9 @@ class BaseCompressor(object):
e,
)
- self.is_model_patched = apply_model_monkey_patches(model_name=model)
+ self.is_model_patched = apply_model_monkey_patches(
+ model_name=model, trust_remote_code=self.trust_remote_code
+ )
import transformers
if (
diff --git a/auto_round/modeling/unfused_moe/__init__.py b/auto_round/modeling/unfused_moe/__init__.py
index d00c21279..3b9511731 100644
--- a/auto_round/modeling/unfused_moe/__init__.py
+++ b/auto_round/modeling/unfused_moe/__init__.py
@@ -143,9 +143,9 @@ def get_file_path_via_model_name(model_or_path: str, file_name):
return index_path
-def pre_check_config(model_name: str | torch.nn.Module):
+def pre_check_config(model_name: str | torch.nn.Module, trust_remote_code: bool = True):
if isinstance(model_name, str):
- config = AutoConfig.from_pretrained(model_name)
+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=trust_remote_code)
elif isinstance(model_name, torch.nn.Module):
config = getattr(model_name, "config", None)
if config is None:
@@ -181,12 +181,12 @@ def pre_check_config(model_name: str | torch.nn.Module):
# This is for model checkpoint with linear definition
-def apply_model_monkey_patches(model_name: str) -> bool:
- res = pre_check_config(model_name)
+def apply_model_monkey_patches(model_name: str, trust_remote_code: bool = True) -> bool:
+ res = pre_check_config(model_name, trust_remote_code=trust_remote_code)
if not res:
return False
# patch blocks
- config = AutoConfig.from_pretrained(model_name)
+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=trust_remote_code)
model_type = getattr(config, "model_type")
cfg = MODEL_CONFIG[model_type]
diff --git a/test/test_cpu/quantization/test_mxfp_nvfp.py b/test/test_cpu/quantization/test_mxfp_nvfp.py
index 1d6bdcceb..54081839e 100644
--- a/test/test_cpu/quantization/test_mxfp_nvfp.py
+++ b/test/test_cpu/quantization/test_mxfp_nvfp.py
@@ -51,6 +51,7 @@ class TestAutoRoundFP:
nsamples=2,
dataset=dataloader,
layer_config=layer_config,
+ trust_remote_code=False,
)
compressed_model, _ = autoround.quantize()
moe = compressed_model.model.layers[1].mlp
@@ -75,6 +76,7 @@ class TestAutoRoundFP:
nsamples=2,
dataset=dataloader,
layer_config=layer_config,
+ trust_remote_code=False,
)
compressed_model, _ = autoround.quantize_and_save(output_dir=self.save_dir, inplace=True, format="auto_round")
lm_head = compressed_model.lm_head
@@ -107,6 +109,7 @@ class TestAutoRoundFP:
nsamples=2,
dataset=dataloader,
layer_config=layer_config,
+ trust_remote_code=False,
)
compressed_model, _ = autoround.quantize_and_save(output_dir=self.save_dir, inplace=True, format="auto_round")
lm_head = compressed_model.lm_head
diff --git a/test/test_cuda/quantization/test_mxfp_nvfp.py b/test/test_cuda/quantization/test_mxfp_nvfp.py
index b9263f5d9..f8258c706 100644
--- a/test/test_cuda/quantization/test_mxfp_nvfp.py
+++ b/test/test_cuda/quantization/test_mxfp_nvfp.py
@@ -118,6 +118,7 @@ class TestAutoRound:
seqlen=2,
nsamples=2,
dataset=dataloader,
+ trust_remote_code=False,
)
autoround.quantize()
quantized_model_path = self.save_dir
@@ -133,6 +134,7 @@ class TestAutoRound:
seqlen=2,
nsamples=2,
dataset=dataloader,
+ trust_remote_code=False,
)
autoround.quantize()
quantized_model_path = self.save_dir