Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix gguf packing device (#1105)

Heng Guo4f64730fefea
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index fd872288b..2fcc948fc 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -1385,6 +1385,7 @@ class BaseCompressor(object):
processor=self.processor if hasattr(self, "processor") else None,
image_processor=self.image_processor if hasattr(self, "image_processor") else None,
model_type=model_type,
+ device=self.device,
)
else:
PACKING_LAYER_WITH_FORMAT[target_backend](name, self.model, self.formats[0], device=self.device)
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 5f42535d3..ea080cc5d 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -84,7 +84,7 @@ def download_convert_file(redownload=False):
f.write(response.text)
-def wrapper_model_instance(model_instance, model, layer_config, low_cpu_mem_usage=False):
+def wrapper_model_instance(model_instance, model, layer_config, low_cpu_mem_usage=False, device=None):
if model_instance.model_arch == gguf.MODEL_ARCH.MMPROJ and model_instance.fname_out.is_dir():
model_instance.fname_out = model_instance.fname_out / "mmproj-model.gguf"
model_instance.model = model
@@ -94,6 +94,8 @@ def wrapper_model_instance(model_instance, model, layer_config, low_cpu_mem_usag
model_instance.get_tensors = partial(get_tensors, model_instance)
model_instance.prepare_tensors = partial(prepare_tensors, model_instance)
+ model_instance.device = device
+
return model_instance
@@ -186,8 +188,10 @@ def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:
yield name, tensor
-def _quant_data_with_args(data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None, imatrix=None):
- device = data_torch.device
+def _quant_data_with_args(
+ data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None, imatrix=None, device=None
+):
+ device = data_torch.device if device is None else device
data_torch = data_torch.to(torch.float32)
scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale
zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp
@@ -210,9 +214,9 @@ def _quant_data_with_args(data_torch, data_qtype, scale, zp, d_scale=None, wmin=
return data
-def _quant_data(cls, data_torch, data_qtype, name, modify_name, bid):
+def _quant_data(cls, data_torch, data_qtype, name, modify_name, bid, device=None):
suffix = ".weight"
- device = data_torch.device
+ device = data_torch.device if device is None else device
if suffix in name:
layer_name = name[: -len(suffix)]
module = get_module(cls.model, layer_name)
@@ -370,6 +374,7 @@ def _special_name_handle(cls, name):
def prepare_tensors(cls):
max_name_len = max(len(s) for _, s in cls.tensor_map.mapping.values()) + len(".weight,")
+ device = get_packing_device(cls.device)
for name, data_torch in chain(cls.generate_extra_tensors(), cls.get_tensors()):
@@ -567,7 +572,7 @@ def prepare_tensors(cls):
else:
attr_list[attr] = v_b
attr_list["imatrix"] = module.imatrix if hasattr(module, "imatrix") else None
- data = _quant_data_with_args(data_torch, data_qtype, **attr_list)
+ data = _quant_data_with_args(data_torch, data_qtype, device=device, **attr_list)
# for MOE model
elif len(data_torch.shape) == 3 and len(re.findall(r"\d+", name)) == 2:
@@ -578,12 +583,12 @@ def prepare_tensors(cls):
if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):
arr_name[i] = str(idx)
arr_name = ".".join(arr_name)
- arr, data_qtype = _quant_data(cls, arr, data_qtype, arr_name, modify_name, bid)
+ arr, data_qtype = _quant_data(cls, arr, data_qtype, arr_name, modify_name, bid, device=device)
new_data.append(arr)
data = np.array(new_data)
del new_data
else:
- data, data_qtype = _quant_data(cls, data_torch, data_qtype, name, modify_name, bid)
+ data, data_qtype = _quant_data(cls, data_torch, data_qtype, name, modify_name, bid, device=device)
shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index a01f660f1..3263ef63b 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -73,6 +73,7 @@ def create_model_class(
backend="gguf:q4_0",
low_cpu_mem_usage=False,
model_type=convert_hf_to_gguf.ModelType.TEXT,
+ device="cpu",
):
tmp_work_dir = model.name_or_path
os.makedirs(output_dir, exist_ok=True)
@@ -113,7 +114,7 @@ def create_model_class(
small_first_shard=False,
)
model_instance = wrapper_model_instance(
- model_instance, model=model, layer_config=layer_config, low_cpu_mem_usage=low_cpu_mem_usage
+ model_instance, model=model, layer_config=layer_config, low_cpu_mem_usage=low_cpu_mem_usage, device=device
)
model_instance = handle_special_model(model_instance, model_architecture)
return model_instance
@@ -130,6 +131,7 @@ def pack_gguf_layer(
processor=None,
image_processor=None,
model_type=convert_hf_to_gguf.ModelType.TEXT,
+ device="cpu",
):
"""Export the model to gguf format."""
global gguf_model_instance_global
@@ -157,6 +159,7 @@ def pack_gguf_layer(
backend,
low_cpu_mem_usage=True,
model_type=convert_hf_to_gguf.ModelType.MMPROJ,
+ device=device,
)
)
@@ -206,7 +209,7 @@ def pack_gguf_layer(
@torch.inference_mode()
-def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, vlm=False, **kwargs):
+def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, vlm=False, device="cpu", **kwargs):
"""Export the model to gguf format."""
st = time.time()
global gguf_model_instance_global
@@ -214,12 +217,19 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, v
model = kwargs["model"]
if "gguf_model_instance_global" not in globals():
gguf_model_instance_global = [
- create_model_class(output_dir, model, layer_config, backend, model_type=convert_hf_to_gguf.ModelType.TEXT)
+ create_model_class(
+ output_dir, model, layer_config, backend, model_type=convert_hf_to_gguf.ModelType.TEXT, device=device
+ )
]
if vlm:
gguf_model_instance_global.append(
create_model_class(
- output_dir, model, layer_config, backend, model_type=convert_hf_to_gguf.ModelType.MMPROJ
+ output_dir,
+ model,
+ layer_config,
+ backend,
+ model_type=convert_hf_to_gguf.ModelType.MMPROJ,
+ device=device,
)
)