PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix gguf fp8 input model and vram issue (#844)
Wenhua Cheng4562cf4774e9
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex a0df6b004..01546034d 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1195,9 +1195,6 @@ class BaseCompressor(object):# Load datasetfrom auto_round.calib_dataset import get_dataloader- if _is_fp8_model(self.model):- convert_fp8_model_to_16b_model(self.model, self.amp_dtype)-if isinstance(self.dataset, str):if self.tokenizer is None:raise ValueError("A tokenizer must be set for the model when using a dataset string.")@@ -1244,6 +1241,8 @@ class BaseCompressor(object):dispatch_model(self.model, self.model.hf_device_map)else:model = model.to(self.device)+ if _is_fp8_model(self.model):+ convert_fp8_model_to_16b_model(self.model, self.amp_dtype)cnt = 0# Run forward pass to accumulate imatrix@@ -1422,7 +1421,6 @@ class BaseCompressor(object):"""m = get_module(self.model, name)- # if m.__class__.__name__ == "FP8Linear":if _is_fp8_linear(m):m = convert_fp8_layer_to_linear(m, self.amp_dtype)set_module(self.model, name, m)diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 721b207df..3de99c357 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -394,7 +394,10 @@ def prepare_tensors(cls):clean_weight_list = []modify_name = _special_name_handle(cls, name)+ orig_device = data_torch.device+ data_torch = data_torch.to("cpu")for new_name, data_torch in cls.modify_tensors(data_torch, modify_name, bid):+ data_torch.to(orig_device)skip = Falsefor tensor_info in cls.gguf_writer.tensors:if new_name in tensor_info:@@ -545,7 +548,7 @@ def prepare_tensors(cls):attr_tensor = getattr(module, attr)else:attr_tensor = getattr(module, "w_" + attr)- if attr_tensor is None:+ if attr_tensor is None or not isinstance(attr_tensor, torch.Tensor):continuekv_b = attr_tensor.view(n_head_kv, v_head_dim + qk_nope_head_dim, -1)k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)diff --git a/auto_round/export/export_to_gguf/convert_hf_to_gguf.py b/auto_round/export/export_to_gguf/convert_hf_to_gguf.pyindex 2a468bd42..dab6926e2 100644--- a/auto_round/export/export_to_gguf/convert_hf_to_gguf.py+++ b/auto_round/export/export_to_gguf/convert_hf_to_gguf.py@@ -41,9 +41,11 @@ if TYPE_CHECKING:if "NO_LOCAL_GGUF" not in os.environ:sys.path.insert(1, str(Path(__file__).parent / "gguf-py"))+from auto_round.utils import LazyImportgguf = LazyImport("gguf")+MistralTokenizerType = LazyImport("gguf.vocab.MistralTokenizerType")MistralVocab = LazyImport("gguf.vocab.MistralVocab")DATASET_MEAN = LazyImport("mistral_common.tokens.tokenizers.multimodal.DATASET_MEAN")diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 004bfd9cc..9af09758e 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -79,8 +79,7 @@ SUPPORTED_LAYER_TYPES = (torch.nn.Linear, transformers.pytorch_utils.Conv1D)# Changed to str as it relies on triton or others lib to load thisINNER_SUPPORTED_LAYER_TYPES = ("FP8Linear",)-# INNER_SUPPORTED_LAYER_TYPES = (transformers.integrations.finegrained_fp8.FP8Linear,)-+# transformers.integrations.finegrained_fp8.FP8Linearif deepspeed_exists:from deepspeed.module_inject import LinearAllreduce, LinearLayer@@ -2298,10 +2297,14 @@ def convert_fp8_model_to_16b_model(model, dtype=torch.bfloat16):Convert a model with FP8 quantized layers to a model with 16-bit linear layers.This is useful for compatibility with other frameworks or for further processing."""+ cnt = 0for n, m in model.named_modules():if m.__class__.__name__ == "FP8Linear":new_module = convert_fp8_layer_to_linear(m, dtype=dtype)set_module(model, n, new_module)+ cnt += 1+ if cnt % 10 == 0: # Tricky setting+ clear_memory()return model@@ -2344,7 +2347,6 @@ def download_hf_model(repo_id, cache_dir=None, repo_type=None, revision=None):"""Download hugging face model from hf hub."""from huggingface_hub.constants import DEFAULT_REVISION, HUGGINGFACE_HUB_CACHEfrom huggingface_hub.file_download import REGEX_COMMIT_HASH, repo_folder_name- from huggingface_hub.utils import EntryNotFoundErrorif cache_dir is None:cache_dir = HUGGINGFACE_HUB_CACHE