PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix gguf packing device (#1105)
Heng Guo4f64730fefea
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex fd872288b..2fcc948fc 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1385,6 +1385,7 @@ class BaseCompressor(object):processor=self.processor if hasattr(self, "processor") else None,image_processor=self.image_processor if hasattr(self, "image_processor") else None,model_type=model_type,+ device=self.device,)else:PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0], device=self.device)diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 5f42535d3..ea080cc5d 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -84,7 +84,7 @@ def download_convert_file(redownload=False):f.write(response.text)-def wrapper_model_instance(model_instance, model, layer_config, low_cpu_mem_usage=False):+def wrapper_model_instance(model_instance, model, layer_config, low_cpu_mem_usage=False, device=None):if model_instance.model_arch == gguf.MODEL_ARCH.MMPROJ and model_instance.fname_out.is_dir():model_instance.fname_out = model_instance.fname_out / "mmproj-model.gguf"model_instance.model = model@@ -94,6 +94,8 @@ def wrapper_model_instance(model_instance, model, layer_config, low_cpu_mem_usagmodel_instance.get_tensors = partial(get_tensors, model_instance)model_instance.prepare_tensors = partial(prepare_tensors, model_instance)+ model_instance.device = device+return model_instance@@ -186,8 +188,10 @@ def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:yield name, tensor-def _quant_data_with_args(data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None, imatrix=None):- device = data_torch.device+def _quant_data_with_args(+ data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None, imatrix=None, device=None+):+ device = data_torch.device if device is None else devicedata_torch = data_torch.to(torch.float32)scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scalezp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp@@ -210,9 +214,9 @@ def _quant_data_with_args(data_torch, data_qtype, scale, zp, d_scale=None, wmin=return data-def _quant_data(cls, data_torch, data_qtype, name, modify_name, bid):+def _quant_data(cls, data_torch, data_qtype, name, modify_name, bid, device=None):suffix = ".weight"- device = data_torch.device+ device = data_torch.device if device is None else deviceif suffix in name:layer_name = name[: -len(suffix)]module = get_module(cls.model, layer_name)@@ -370,6 +374,7 @@ def _special_name_handle(cls, name):def prepare_tensors(cls):max_name_len = max(len(s) for _, s in cls.tensor_map.mapping.values()) + len(".weight,")+ device = get_packing_device(cls.device)for name, data_torch in chain(cls.generate_extra_tensors(), cls.get_tensors()):@@ -567,7 +572,7 @@ def prepare_tensors(cls):else:attr_list[attr] = v_battr_list["imatrix"] = module.imatrix if hasattr(module, "imatrix") else None- data = _quant_data_with_args(data_torch, data_qtype, **attr_list)+ data = _quant_data_with_args(data_torch, data_qtype, device=device, **attr_list)# for MOE modelelif len(data_torch.shape) == 3 and len(re.findall(r"\d+", name)) == 2:@@ -578,12 +583,12 @@ def prepare_tensors(cls):if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):arr_name[i] = str(idx)arr_name = ".".join(arr_name)- arr, data_qtype = _quant_data(cls, arr, data_qtype, arr_name, modify_name, bid)+ arr, data_qtype = _quant_data(cls, arr, data_qtype, arr_name, modify_name, bid, device=device)new_data.append(arr)data = np.array(new_data)del new_dataelse:- data, data_qtype = _quant_data(cls, data_torch, data_qtype, name, modify_name, bid)+ data, data_qtype = _quant_data(cls, data_torch, data_qtype, name, modify_name, bid, device=device)shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shapediff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex a01f660f1..3263ef63b 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -73,6 +73,7 @@ def create_model_class(backend="gguf:q4_0",low_cpu_mem_usage=False,model_type=convert_hf_to_gguf.ModelType.TEXT,+ device="cpu",):tmp_work_dir = model.name_or_pathos.makedirs(output_dir, exist_ok=True)@@ -113,7 +114,7 @@ def create_model_class(small_first_shard=False,)model_instance = wrapper_model_instance(- model_instance, model=model, layer_config=layer_config, low_cpu_mem_usage=low_cpu_mem_usage+ model_instance, model=model, layer_config=layer_config, low_cpu_mem_usage=low_cpu_mem_usage, device=device)model_instance = handle_special_model(model_instance, model_architecture)return model_instance@@ -130,6 +131,7 @@ def pack_gguf_layer(processor=None,image_processor=None,model_type=convert_hf_to_gguf.ModelType.TEXT,+ device="cpu",):"""Export the model to gguf format."""global gguf_model_instance_global@@ -157,6 +159,7 @@ def pack_gguf_layer(backend,low_cpu_mem_usage=True,model_type=convert_hf_to_gguf.ModelType.MMPROJ,+ device=device,))@@ -206,7 +209,7 @@ def pack_gguf_layer(@torch.inference_mode()-def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, vlm=False, **kwargs):+def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, vlm=False, device="cpu", **kwargs):"""Export the model to gguf format."""st = time.time()global gguf_model_instance_global@@ -214,12 +217,19 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, vmodel = kwargs["model"]if "gguf_model_instance_global" not in globals():gguf_model_instance_global = [- create_model_class(output_dir, model, layer_config, backend, model_type=convert_hf_to_gguf.ModelType.TEXT)+ create_model_class(+ output_dir, model, layer_config, backend, model_type=convert_hf_to_gguf.ModelType.TEXT, device=device+ )]if vlm:gguf_model_instance_global.append(create_model_class(- output_dir, model, layer_config, backend, model_type=convert_hf_to_gguf.ModelType.MMPROJ+ output_dir,+ model,+ layer_config,+ backend,+ model_type=convert_hf_to_gguf.ModelType.MMPROJ,+ device=device,))