Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

inc3 integration and refine export code (#28)

wenhuach21cb49e91e221f
diff --git a/README.md b/README.md
index 0f1c4c76a..4378d30d0 100644
--- a/README.md
+++ b/README.md
@@ -23,48 +23,27 @@ python setup.py install
```
## Usage of Tuning
-### On CPU/GPU
-```python
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from auto_round import AutoRound
-
-model_name = "meta-llama/Llama-2-7b-hf"
-model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
-tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-bits, group_size, scheme = 4, 128, "asym"
-tuning_device = "cuda:0" ## or "cpu"
-autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme, device=tuning_device)
-autoround.quantize()
-
-output_dir = "./tmp_autoround"
-deployment_device = "cpu" ## or gpu
-if deployment_device == "cpu":
- autoround.save_quantized(output_dir, format="itrex") ## export to itrex format
-else:
- autoround.save_quantized(output_dir, format="auto_gptq", use_triton=True) ##export to autogptq format
-```
-
-
-### On Intel Gaudi2
+### On CPU/Gaudi2/ GPU
```python
-from transformers import AutoModelForCausalLM, AutoTokenizer
import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+tuning_device = "cuda:0" ## or "cpu", "hpu"
+dtype = "auto" if tuning_device != "hpu" else torch.bfloat16
model_name = "meta-llama/Llama-2-7b-hf"
-model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=dtype, trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-bits, group_size, scheme = 4, 128, "asym"
-# need to load model first, then import
from auto_round import AutoRound
-tuning_device = "hpu"
+bits, group_size, scheme = 4, 128, "asym"
autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme, device=tuning_device)
autoround.quantize()
+output_dir = "./tmp_autoround"
+autoround.save_quantized(output_dir)
```
-
<details>
<summary>Detailed Hyperparameters</summary>
@@ -93,6 +72,8 @@ autoround.quantize()
- `seqlen (int)`: Data length of the sequence for tuning (default is 2048).
- `bs (int)`: Batch size for training (default is 8).
+
+- `scale_dtype (str)`: The data type of quantization scale to be used (default is "float32"), different kernels have different choices.
- `amp (bool)`: Whether to use automatic mixed precision (default is True).
@@ -110,26 +91,24 @@ autoround.quantize()
- `weight_config (dict)`: Configuration for weight quantization (default is an empty dictionary), mainly for mixed bits or mixed precision.
-- `device`: The device to be used for tuning (default is "cuda:0").
-
-- `scale_dtype (str)`: The data type of quantization scale to be used (default is "float32"), different kernels have different choices.
+- `device`: The device to be used for tuning. The default is set to None, allowing for automatic detection.
</details>
## Support List
-| Model | Supported |
-|--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Intel/neural-chat-7b-v3-3 | [HF-int4-model](https://huggingface.co/Intel/neural-chat-7b-v3-3-int4-inc), [accuracy](./docs/neural-chat-7b-v3-3-acc.md), [recipe](./examples/language-modeling/scripts/neural-chat-7b-v3-3.sh), [example](./examples/language-modeling/) |
-| Intel/neural-chat-7b-v3-1 | [HF-int4-model](https://huggingface.co/Intel/neural-chat-7b-v3-1-int4-inc), [accuracy](./docs/neural-chat-7b-v3-1-acc.md), [recipe](./examples/language-modeling/scripts/neural-chat-7b-v3-1.sh), [example](./examples/language-modeling/) |
-| mistralai/Mistral-7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mistral-7B-v0.1-int4-inc), [accuracy](./docs/Mistral-7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mistral-7B-v0.1.sh), [example](./examples/language-modeling/) |
-| google/gemma-7b | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-int4-inc) under review, [accuracy](./docs/gemma-7b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b.sh), [example](./examples/language-modeling/) |
-| google/gemma-7b-it | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-it-int4-inc) under review, [accuracy](./docs/gemma-7b-it-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b-it.sh), [example](./examples/language-modeling/) | |
+| Model | Supported |
+|-------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Intel/neural-chat-7b-v3-3 | [HF-int4-model](https://huggingface.co/Intel/neural-chat-7b-v3-3-int4-inc), [accuracy](./docs/neural-chat-7b-v3-3-acc.md), [recipe](./examples/language-modeling/scripts/neural-chat-7b-v3-3.sh), [example](./examples/language-modeling/) |
+| Intel/neural-chat-7b-v3-1 | [HF-int4-model](https://huggingface.co/Intel/neural-chat-7b-v3-1-int4-inc), [accuracy](./docs/neural-chat-7b-v3-1-acc.md), [recipe](./examples/language-modeling/scripts/neural-chat-7b-v3-1.sh), [example](./examples/language-modeling/) |
+| mistralai/Mistral-7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mistral-7B-v0.1-int4-inc), [accuracy](./docs/Mistral-7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mistral-7B-v0.1.sh), [example](./examples/language-modeling/) |
+| google/gemma-7b | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-int4-inc) under review, [accuracy](./docs/gemma-7b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b.sh), [example](./examples/language-modeling/) |
+| google/gemma-7b-it | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-it-int4-inc) under review, [accuracy](./docs/gemma-7b-it-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b-it.sh), [example](./examples/language-modeling/) | |
mistralai/Mixtral-8x7B-Instruct-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mistral-7B-v0.1-int4-inc) under review, [accuracy](./docs/Mixtral-8x7B-Instruct-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh), [example](./examples/language-modeling/) |
-| mistralai/Mixtral-8x7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mixtral-8x7B-v0.1-int4-inc) under review, [accuracy](./docs/Mixtral-8x7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh), [example](./examples/language-modeling/) |
-| microsoft/phi-2 | [HF-int4-model](https://huggingface.co/Intel/phi-2-int4-inc) under review, [accuracy](./docs/phi-2-acc.md), [recipe](./examples/language-modeling/scripts/phi-2.sh), [example](./examples/language-modeling/) |
-| Salesforce/codegen25-7b-multi | [example](./examples/code-generation) |
+| mistralai/Mixtral-8x7B-v0.1 | [HF-int4-model](https://huggingface.co/Intel/Mixtral-8x7B-v0.1-int4-inc) under review, [accuracy](./docs/Mixtral-8x7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh), [example](./examples/language-modeling/) |
+| microsoft/phi-2 | [HF-int4-model](https://huggingface.co/Intel/phi-2-int4-inc) under review, [accuracy](./docs/phi-2-acc.md), [recipe](./examples/language-modeling/scripts/phi-2.sh), [example](./examples/language-modeling/) |
+| Salesforce/codegen25-7b-multi | [example](./examples/code-generation) |
| EleutherAI/gpt-j-6b | [example](./examples/language-modeling/) |
| huggyllama/llama-7b | [example](./examples/language-modeling/) |
| meta-llama/Llama-2-7b-hf | [example](./examples/language-modeling/) |
@@ -173,7 +152,6 @@ outputs = model.generate(inputs, max_new_tokens=50)
```
### GPU
```python
-# save_quantized to autogptq format first and then follow transformers or auto-gptq to load the model and inference
from transformers import AutoModelForCausalLM, AutoTokenizer
quantized_model_path = "./tmp_autoround"
diff --git a/auto_round/__init__.py b/auto_round/__init__.py
index d26edcbf7..1b2514237 100644
--- a/auto_round/__init__.py
+++ b/auto_round/__init__.py
@@ -13,5 +13,3 @@
# limitations under the License.
from .autoround import AutoRound, AutoAdamRound, AutoOPTRound
from .version import __version__
-from auto_round.export.export_to_itrex import compress_model, QuantConfig
-from auto_round.export.export_to_autogptq import save_quantized_to_autogptq
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index e6a1f1a9d..b8f5f0733 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -19,11 +19,13 @@ import time
import torch
from .utils import (
+ CpuInfo,
block_forward,
check_is_cpu,
check_to_quantized,
collect_minmax_scale,
collect_round_v,
+ detect_device,
get_batch_dim,
get_block_names,
get_module,
@@ -42,151 +44,6 @@ if is_hpu_available:
import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401
-class SaveInputs:
- """Cache the inputs of the first block."""
-
- def __init__(self, model, dataloader, seqlen=256, block_name=None):
- """Initializes the SaveInputs class.
-
- Args:
- model: The model to be used.
- dataloader: The dataloader for the input data.
- seqlen (int): The sequence length.
- block_name (str): The name of the block.
- """
- self.model = model.eval()
- self.dataloader = dataloader
- self.inputs = {}
- self.block_name = block_name
- self.seqlen = seqlen
-
- @torch.no_grad()
- def get_forward_func(self, name):
- """Gets the forward function.
-
- Args:
- name (str): The name of the function.
-
- Returns:
- function: The forward function.
- """
-
- def forward(_, hidden_states, *positional_args, **kwargs):
- dim = int((hasattr(self.model, "config") and "chatglm" in self.model.config.model_type))
- if name in self.inputs:
- data = torch.cat([self.inputs[name]["input_ids"], hidden_states.to("cpu")], dim=dim)
- self.inputs[name]["input_ids"] = data
- else:
- self.inputs[name] = {}
- self.inputs[name]["input_ids"] = hidden_states.to("cpu")
-
- if "positional_inputs" not in self.inputs[name]:
- self.inputs[name]["positional_inputs"] = []
- for idx, item in enumerate(positional_args):
- self.inputs[name]["positional_inputs"] = move_input_to_device(positional_args)
-
- for key in kwargs.keys():
- if isinstance(kwargs[key], torch.Tensor) or isinstance(kwargs[key], list) or (key == "alibi"):
- if "attention_mask" in key:
- if key not in self.inputs[name].keys():
- self.inputs[name][key] = None
- if kwargs[key] is not None:
- if self.inputs[name][key] is not None:
- self.inputs[name][key] = torch.cat(
- [self.inputs[name][key], kwargs[key].to("cpu")], dim=0
- )
- else:
- self.inputs[name][key] = kwargs[key].to("cpu")
- elif "alibi" in key:
- if key not in self.inputs[name].keys():
- self.inputs[name][key] = None
- if isinstance(kwargs[key], torch.Tensor):
- alibi = kwargs[key]
- batch = kwargs["attention_mask"].shape[0]
- alibi = alibi.reshape(batch, -1, alibi.shape[1], alibi.shape[2])
- if self.inputs[name][key] is not None:
- self.inputs[name][key] = torch.cat([self.inputs[name][key], alibi.to("cpu")], dim=0)
- else:
- self.inputs[name][key] = alibi.to("cpu")
- elif key not in self.inputs[name].keys():
- self.inputs[name][key] = move_input_to_device(kwargs[key], device=torch.device("cpu"))
- raise NotImplementedError
-
- return forward
-
- @torch.no_grad()
- def get_inputs(self, n_samples=512):
- """Gets the inputs.
-
- Args:
- n_samples (int): The number of samples.
-
- Returns:
- dict: The inputs.
- """
- total_cnt = 0
- self._replace_forward()
- for data in self.dataloader:
- if data is None:
- continue
- if isinstance(data, torch.Tensor):
- data_new = data.to(self.model.device)
- input_ids = data_new
- else:
- data_new = {}
- for key in data.keys():
- data_new[key] = data[key].to(self.model.device)
- input_ids = data_new["input_ids"]
- # if input_ids.shape[-1] < self.seqlen:
- # continue
- if total_cnt + input_ids.shape[0] > n_samples:
- input_ids = input_ids[: n_samples - total_cnt, ...]
- try:
- if isinstance(data_new, torch.Tensor):
- self.model(data_new)
- elif isinstance(data_new, dict):
- self.model(**data_new)
- except NotImplementedError:
- pass
- except Exception as error:
- logger.error(error)
- total_cnt += input_ids.shape[0]
- if total_cnt >= n_samples:
- break
- self._recover_forward()
- if total_cnt == 0:
- logger.error(
- f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "
- f"dataloader or decease the sequence length"
- )
- exit()
- elif total_cnt < n_samples:
- logger.warning(
- f"Insufficient number of samples collected may affect the quantification. "
- f"Effective samples size:{total_cnt}, Target sample size:{n_samples}"
- )
- res = self.inputs[self.block_name]
- return res
-
- def _recover_forward(self):
- """Recovers the forward function."""
- for n, m in self.model.named_modules():
- if n == self.block_name:
- m.forward = m.orig_forward
- delattr(m, "orig_forward")
- break
-
- def _replace_forward(self):
- """Replaces the forward function."""
- from functools import partial
-
- for n, m in self.model.named_modules():
- if n == self.block_name:
- m.orig_forward = m.forward
- m.forward = partial(self.get_forward_func(n), m)
- break
-
-
class WrapperLinear(torch.nn.Module):
def __init__(self, orig_layer, enable_minmax_tuning=True):
"""A wrapper module for linear layers that enables quantization and min-max tuning of weights.
@@ -466,10 +323,10 @@ class AutoRound(object):
Args:
model: The PyTorch model to be quantized.
- tokenizer: Tokenizer for processing input data. Temporarily set as a mandatory parameter.
+ tokenizer: An optional tokenizer for processing input data. If none is provided, a dataloader must be supplied.
bits (int): Number of bits for quantization (default is 4).
group_size (int): Size of the quantization group (default is 128).
- scheme (str): The quantization scheme to be used (default is "asym").
+ scheme (str): The quantization scheme (sym/asym) to be used (default is "asym").
weight_config (dict): Configuration for weight quantization (default is an empty dictionary).
weight_config={
'layer1':##layer_name
@@ -489,13 +346,14 @@ class AutoRound(object):
dataloader: The dataloader for input data (to be supported in future).
dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").
dataset_split (str): The split of the dataset to be used (default is "train").
- use_quant_input (bool): Whether to use quantized input data (default is True).
- enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
+ use_quant_input (bool): Whether to use the output of the previous quantized block as the input for the current
+ block (default is True).
+ enable_minmax_tuning (bool): Whether to enable weight min-max tuning (default is True).
lr (float): The learning rate (default is None, will be set to 1.0/iters).
- minmax_lr (float): The learning rate for min-max tuning (default is None, will be set to 1.0/iters).
+ minmax_lr (float): The learning rate for min-max tuning (default is None, it will be set to lr automatically).
low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).
iters (int): Number of iterations (default is 200).
- seqlen (int): Length of the sequence.
+ seqlen (int): Data length of the sequence for tuning (default is 2048).
n_samples (int): Number of samples (default is 512).
sampler (str): The sampling method (default is "rand").
seed (int): The random seed (default is 42).
@@ -504,7 +362,8 @@ class AutoRound(object):
not_use_best_mse (bool): Whether to use mean squared error (default is False).
dynamic_max_gap (int): The dynamic maximum gap (default is -1).
data_type (str): The data type to be used (default is "int").
- scale_dtype (str): The data type of quantization scale to be used (default is "fp32")
+ scale_dtype (str): The data type of quantization scale to be used (default is "float32"), different kernels
+ have different choices.
**kwargs: Additional keyword arguments.
Returns:
@@ -522,7 +381,7 @@ class AutoRound(object):
enable_full_range: bool = False, ##for symmetric, TODO support later
bs: int = 8,
amp: bool = True,
- device="cuda:0",
+ device=None,
lr_scheduler=None,
dataloader=None, ## to support later
dataset_name: str = "NeelNanda/pile-10k",
@@ -545,11 +404,8 @@ class AutoRound(object):
scale_dtype: str = "fp32",
**kwargs,
):
- from .calib_dataset import CALIB_DATASETS
-
- self.model = model
self.model_orig_dtype = model.dtype
- self.model = self.model.to("cpu")
+ self.model = model.eval().to("cpu")
self.amp = amp
self.use_quant_input = use_quant_input
self.enable_minmax_tuning = enable_minmax_tuning
@@ -574,7 +430,7 @@ class AutoRound(object):
self.seqlen = seqlen
self.train_bs = bs
self.n_blocks = n_blocks
- self.device = device
+ self.device = detect_device(device)
if scale_dtype == "fp16" or scale_dtype == "float16":
self.scale_dtype = torch.float16
@@ -586,28 +442,21 @@ class AutoRound(object):
self.amp_dtype = torch.float16
if self.model.dtype != torch.float32:
self.amp_dtype = self.model.dtype
- if self.device == "cpu":
+ if self.device == "cpu" or self.device == "hpu":
self.amp_dtype = torch.bfloat16
if self.amp:
- self.model = self.model.to(self.amp_dtype)
- logger.info(f"using {self.amp_dtype} for quantization tuning")
+ if self.device == "cpu" and not CpuInfo().bf16:
+ self.amp = False
+ self.model = self.model.to(torch.float32)
+ logger.warning("amp is set to FALSE as the current" "device does not support the 'bf16' data type.")
+ else:
+ self.model = self.model.to(self.amp_dtype)
else:
self.model = self.model.to(torch.float32)
- logger.info(f"using {torch.float32} for quantization tuning")
+ logger.info(f"using {self.model.dtype} for quantization tuning")
self.dataset_name = dataset_name
- if dataloader is None:
- get_dataloader = CALIB_DATASETS.get(self.dataset_name, CALIB_DATASETS["NeelNanda/pile-10k"])
- self.dataloader = get_dataloader(
- self.tokenizer,
- self.seqlen,
- seed=self.seed,
- bs=self.train_bs,
- split=self.dataset_split,
- dataset_name=self.dataset_name,
- )
- else:
- self.dataloader = dataloader
+ self.dataloader = dataloader
self.iters = iters
if self.iters <= 0:
logger.warning("iters must be positive, reset it to 200")
@@ -695,6 +544,7 @@ class AutoRound(object):
assert self.seqlen > 0, "seqlen must be positive"
assert self.n_blocks > 0, "n_blocks must be positive"
assert self.gradient_accumulate_steps > 0, "gradient accumulate step must be positive"
+ # assert self.tokenizer != None or self.dataloader != None
def set_layerwise_config(self, weight_config):
"""Sets the layer-wise configuration based on the provided weight_config.
@@ -736,18 +586,6 @@ class AutoRound(object):
m.scheme = weight_config[n]["scheme"]
m.scale_dtype = weight_config[n]["scale_dtype"]
- def get_batch_dim(self, input_others):
- """Get the batch dimension of the input tensor.
-
- Args:
- input_others: A dictionary containing input data, including positional_inputs.
-
- Returns:
- The batch dimension of the input tensor.
- """
- dim = int(len(input_others["positional_inputs"]) > 0)
- return dim
-
@torch.no_grad()
def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_device, batch_dim):
"""Compute the output of a given block of the model for a given input.
@@ -778,6 +616,164 @@ class AutoRound(object):
torch.cuda.empty_cache()
return output
+ @torch.no_grad()
+ def calib(self, n_samples):
+ """Perform calibration for quantization.
+
+ This method calibrates the model for quantization by processing a specified
+ number of samples from the calibration dataset. It ensures that the data is
+ properly formatted and feeds it to the model. If the number of samples processed
+ is less than the specified number, it logs a warning. If no samples are processed,
+ it logs an error and exits.
+ Args:
+ n_samples (int): The number of samples to use for calibration.
+ """
+ if self.dataloader is None:
+ from .calib_dataset import CALIB_DATASETS
+
+ get_dataloader = CALIB_DATASETS.get(self.dataset_name, CALIB_DATASETS["NeelNanda/pile-10k"])
+ self.dataloader = get_dataloader(
+ self.tokenizer,
+ self.seqlen,
+ seed=self.seed,
+ bs=self.train_bs,
+ split=self.dataset_split,
+ dataset_name=self.dataset_name,
+ )
+ total_cnt = 0
+ for data in self.dataloader:
+ if data is None:
+ continue
+ if isinstance(data, torch.Tensor):
+ data_new = data.to(self.model.device)
+ input_ids = data_new
+ else:
+ data_new = {}
+ for key in data.keys():
+ data_new[key] = data[key].to(self.model.device)
+ input_ids = data_new["input_ids"]
+ # if input_ids.shape[-1] < self.seqlen:
+ # continue
+ if total_cnt + input_ids.shape[0] > n_samples:
+ input_ids = input_ids[: n_samples - total_cnt, ...]
+ try:
+ if isinstance(data_new, torch.Tensor):
+ self.model(data_new)
+ elif isinstance(data_new, dict):
+ self.model(**data_new)
+ except NotImplementedError:
+ pass
+ except Exception as error:
+ logger.error(error)
+ total_cnt += input_ids.shape[0]
+ if total_cnt >= n_samples:
+ break
+ if total_cnt == 0:
+ logger.error(
+ f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "
+ f"dataloader or decease the sequence length"
+ )
+ exit()
+ elif total_cnt < n_samples:
+ logger.warning(
+ f"Insufficient number of samples collected may affect the quantification. "
+ f"Effective samples size:{total_cnt}, Target sample size:{n_samples}"
+ )
+
+ @torch.no_grad()
+ def cache_block_input(self, block_name, n_samples):
+ """Save the inputs of the first block for calibration.
+
+ This method temporarily replaces the forward method of the model to capture
+ the inputs passing through the specified block. It then calibrates the model
+ using a specified number of samples. Finally, it restores the original forward
+ method and returns the inputs for the specified block.
+ Args:
+ block_name (str): The name of the block for which inputs are to be saved.
+ n_samples (int): The number of samples to use for calibration.
+ Returns:
+ dict: A dictionary containing the inputs for the specified block.
+ """
+ self.inputs = {}
+ self.tmp_block_name = block_name
+ self._replace_forward()
+ self.calib(n_samples)
+ self._recover_forward()
+ res = self.inputs[self.tmp_block_name]
+ del self.tmp_block_name
+ return res
+
+ @torch.no_grad()
+ def get_forward_func(self, name):
+ """Gets the forward function.
+
+ Args:
+ name (str): The name of the function.
+ Returns:
+ function: The forward function.
+ """
+
+ def forward(_, hidden_states, *positional_args, **kwargs):
+ dim = int((hasattr(self.model, "config") and "chatglm" in self.model.config.model_type))
+ if name in self.inputs:
+ data = torch.cat([self.inputs[name]["input_ids"], hidden_states.to("cpu")], dim=dim)
+ self.inputs[name]["input_ids"] = data
+ else:
+ self.inputs[name] = {}
+ self.inputs[name]["input_ids"] = hidden_states.to("cpu")
+
+ if "positional_inputs" not in self.inputs[name]:
+ self.inputs[name]["positional_inputs"] = []
+ for idx, item in enumerate(positional_args):
+ self.inputs[name]["positional_inputs"] = move_input_to_device(positional_args)
+
+ for key in kwargs.keys():
+ if isinstance(kwargs[key], torch.Tensor) or isinstance(kwargs[key], list) or (key == "alibi"):
+ if "attention_mask" in key:
+ if key not in self.inputs[name].keys():
+ self.inputs[name][key] = None
+ if kwargs[key] is not None:
+ if self.inputs[name][key] is not None:
+ self.inputs[name][key] = torch.cat(
+ [self.inputs[name][key], kwargs[key].to("cpu")], dim=0
+ )
+ else:
+ self.inputs[name][key] = kwargs[key].to("cpu")
+ elif "alibi" in key:
+ if key not in self.inputs[name].keys():
+ self.inputs[name][key] = None
+ if isinstance(kwargs[key], torch.Tensor):
+ alibi = kwargs[key]
+ batch = kwargs["attention_mask"].shape[0]
+ alibi = alibi.reshape(batch, -1, alibi.shape[1], alibi.shape[2])
+ if self.inputs[name][key] is not None:
+ self.inputs[name][key] = torch.cat([self.inputs[name][key], alibi.to("cpu")], dim=0)
+ else:
+ self.inputs[name][key] = alibi.to("cpu")
+ elif key not in self.inputs[name].keys():
+ self.inputs[name][key] = move_input_to_device(kwargs[key], device=torch.device("cpu"))
+ raise NotImplementedError
+
+ return forward
+
+ def _recover_forward(self):
+ """Recovers the forward function."""
+ for n, m in self.model.named_modules():
+ if n == self.tmp_block_name:
+ m.forward = m.orig_forward
+ delattr(m, "orig_forward")
+ break
+
+ def _replace_forward(self):
+ """Replaces the forward function."""
+ from functools import partial
+
+ for n, m in self.model.named_modules():
+ if n == self.tmp_block_name:
+ m.orig_forward = m.forward
+ m.forward = partial(self.get_forward_func(n), m)
+ break
+
def quant_block(self, block, input_ids, input_others, q_input=None, device=torch.device("cpu")):
"""Quantize the weights of a given block of the model.
@@ -961,37 +957,34 @@ class AutoRound(object):
torch.cuda.empty_cache()
- def save_quantized(self, output_dir=None, format="itrex", inplace=True, **kwargs):
- compressed_model = None
- if format == "itrex" and output_dir is None:
- from auto_round.export.export_to_itrex import compress_model
+ def save_quantized(self, output_dir=None, format="auto_gptq", inplace=True, **kwargs):
+ if not self.quantized:
+ logger.warning("please run autoround.quantize first")
+ return
+ from auto_round.export import EXPORT_FORMAT
- compressed_model = compress_model(self.model, self.weight_config, inplace=inplace)
- else:
- if not self.quantized:
- logger.warning("please run autoround.quantize first")
- return
- from .export import EXPORT_FORMAT
-
- save_quantized_as_format = EXPORT_FORMAT.get(format)
- save_quantized_as_format(
- output_dir,
- model=self.model,
- weight_config=self.weight_config,
- inplace=inplace,
- bits=self.bits,
- group_size=self.group_size,
- scheme=self.scheme,
- iters=self.iters,
- lr=self.lr,
- minmax_lr=self.minmax_lr,
- enable_minmax_tuning=self.enable_minmax_tuning,
- use_quant_input=self.use_quant_input,
- scale_dtype=self.scale_dtype,
- tokenizer=self.tokenizer,
- supported_types=self.supported_types,
- **kwargs,
- )
+ if format not in EXPORT_FORMAT:
+ logger.error(f"export format only supports {EXPORT_FORMAT.keys()}")
+ exit()
+ save_quantized_as_format = EXPORT_FORMAT.get(format)
+ compressed_model = save_quantized_as_format(
+ output_dir,
+ model=self.model,
+ weight_config=self.weight_config,
+ inplace=inplace,
+ bits=self.bits,
+ group_size=self.group_size,
+ scheme=self.scheme,
+ iters=self.iters,
+ lr=self.lr,
+ minmax_lr=self.minmax_lr,
+ enable_minmax_tuning=self.enable_minmax_tuning,
+ use_quant_input=self.use_quant_input,
+ scale_dtype=self.scale_dtype,
+ tokenizer=self.tokenizer,
+ supported_types=self.supported_types,
+ **kwargs,
+ )
return compressed_model
def quantize(self):
@@ -1011,9 +1004,9 @@ class AutoRound(object):
self.model = self.model.to(self.amp_dtype)
if not self.low_gpu_mem_usage:
self.model = self.model.to(self.device)
- save_input_actor = SaveInputs(self.model, self.dataloader, self.seqlen, block_names[0])
- inputs = save_input_actor.get_inputs(n_samples=self.n_samples)
- del save_input_actor
+ inputs = self.cache_block_input(block_names[0], self.n_samples)
+ del self.inputs
+
if "input_ids" in inputs.keys():
dim = int((hasattr(self.model, "config") and "chatglm" in self.model.config.model_type))
total_samples = inputs["input_ids"].shape[dim]
diff --git a/auto_round/export/__init__.py b/auto_round/export/__init__.py
index d8a3671e1..9b4384ff9 100644
--- a/auto_round/export/__init__.py
+++ b/auto_round/export/__init__.py
@@ -11,181 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-import copy
-import torch
-from ..utils import (
- logger,
- get_block_names,
- get_module,
- check_to_quantized,
-)
-EXPORT_FORMAT = {}
-
-
-def register_format(name):
- """Class decorator to register a EXPORT subclass to the registry.
-
- Decorator function used before a Pattern subclass.
-
- Args:
- cls (class): The subclass of register.
- name: A string. Define the export type.
-
- Returns:
- cls: The class of register.
- """
-
- def register(format):
- EXPORT_FORMAT[name] = format
- return format
-
- return register
-
-
-@register_format("auto_gptq")
-def save_quantized_as_autogptq(output_dir, use_triton=False, inplace=True, **kwargs):
- """Export the model to autogptq format to easily leverage cuda kernel."""
- model = kwargs["model"]
- weight_config = kwargs["weight_config"]
- sym = kwargs["scheme"] == "sym"
- bits = kwargs["bits"]
- group_size = kwargs["group_size"]
- iters = kwargs["iters"]
- lr = kwargs["lr"]
- minmax_lr = kwargs["minmax_lr"]
- enable_minmax_tuning = kwargs["enable_minmax_tuning"]
- use_quant_input = kwargs["use_quant_input"]
- scale_dtype = kwargs["scale_dtype"]
- tokenizer = kwargs["tokenizer"]
- supported_types = kwargs["supported_types"]
-
- logger.info("Saving quantized model to autogptq format, this may take a while...")
- if tokenizer is not None:
- tokenizer.save_pretrained(output_dir)
- ##check module quantized in block, this may have bug for mixed precision quantization
- block_name = get_block_names(model)[0]
- first_block = get_module(model, block_name)
- all_to_quantized = True
- modules_in_block_to_quantize = []
- for n, m in first_block.named_modules():
- is_supported_type = False
- for supported_type in supported_types:
- if isinstance(m, supported_type):
- is_supported_type = True
- break
- if not is_supported_type:
- continue
- if not check_to_quantized(m):
- all_to_quantized = False
- else:
- modules_in_block_to_quantize.append(n)
- modules_in_block_to_quantize = [modules_in_block_to_quantize] ##align with autogptq
- if all_to_quantized:
- modules_in_block_to_quantize = None
-
- if inplace:
- compressed_model = model.to("cpu")
- else:
- compressed_model = copy.deepcopy(model.to("cpu"))
-
- from auto_gptq.modeling._utils import pack_model
-
- if bits == 3 or use_triton is False:
- if bits == 3 and use_triton is True:
- logger.warning("triton does not support 3 bits, reset it to False")
- quantizers = {}
- for key in weight_config:
- info = weight_config[key]
- if not check_to_quantized(info):
- continue
- quantizers[key] = (None, info["scale"], info["zp"], info["g_idx"])
- pack_model(
- compressed_model,
- quantizers,
- bits,
- group_size,
- use_cuda_fp16=True,
- desc_act=False,
- force_layer_back_to_cpu=True,
- use_triton=False,
- )
- else:
- quantizers = {}
- for key in weight_config:
- info = weight_config[key]
- if not check_to_quantized(info):
- continue
- info["zp"] = info["zp"].to(torch.float32)
- quantizers[key] = (None, info["scale"].to(torch.float32), info["zp"], info["g_idx"])
- pack_model(
- compressed_model,
- quantizers,
- bits,
- group_size,
- use_cuda_fp16=True,
- desc_act=False,
- force_layer_back_to_cpu=True,
- use_triton=True,
- )
- from auto_round import save_quantized_to_autogptq
-
- save_quantized_to_autogptq(
- compressed_model,
- output_dir,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=iters,
- lr=lr,
- minmax_lr=minmax_lr,
- enable_minmax_tuning=enable_minmax_tuning,
- use_quant_input=use_quant_input,
- scale_dtype=scale_dtype,
- use_safetensors=True,
- modules_in_block_to_quantize=modules_in_block_to_quantize,
- )
-
-
-@register_format("itrex")
-def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):
- """Save configure file and weights for CPU backend inference."""
- model = kwargs["model"]
- weight_config = kwargs["weight_config"]
- sym = kwargs["scheme"] == "sym"
- bits = kwargs["bits"]
- group_size = kwargs["group_size"]
- iters = kwargs["iters"]
- lr = kwargs["lr"]
- minmax_lr = kwargs["minmax_lr"]
- enable_minmax_tuning = kwargs["enable_minmax_tuning"]
- use_quant_input = kwargs["use_quant_input"]
- scale_dtype = kwargs["scale_dtype"]
- tokenizer = kwargs["tokenizer"]
-
- from auto_round.export.export_to_itrex import compress_model, QuantConfig
-
- compressed_model = compress_model(model, weight_config, inplace=inplace)
- quantize_config = QuantConfig(
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=iters,
- lr=lr,
- minmax_lr=minmax_lr,
- enable_minmax_tuning=enable_minmax_tuning,
- use_quant_input=use_quant_input,
- scale_dtype=str(scale_dtype),
- )
- if quantize_config is not None:
- config = compressed_model.config
- setattr(config, "quantization_config", quantize_config.to_dict())
- config.save_pretrained(output_dir)
- quantize_config.save_pretrained(output_dir)
- try:
- compressed_model.save_pretrained(output_dir, safe_serialization=True)
- if tokenizer is not None:
- tokenizer.save_pretrained(output_dir)
- logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))
- except IOError as e: # pragma: no cover
- logger.error("Fail to save configure file and weights due to {}.".format(e))
+from .register import EXPORT_FORMAT
+from .export_to_autogptq import save_quantized_as_autogptq
+from .export_to_itrex import save_quantized_as_itrex, QuantConfig
diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.py
index 10619175d..d69d8404c 100644
--- a/auto_round/export/export_to_autogptq.py
+++ b/auto_round/export/export_to_autogptq.py
@@ -13,6 +13,7 @@
# limitations under the License.
+import copy
import json
import os
from os.path import isdir, isfile, join
@@ -42,10 +43,116 @@ from typing import Dict, List, Optional, Union
import torch
from safetensors.torch import save_file as safe_save
-from auto_round.utils import logger
+from auto_round.export.register import register_format
+from auto_round.utils import check_to_quantized, get_block_names, get_module, logger
-def save_quantized_to_autogptq(
+@register_format("auto_gptq")
+def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwargs):
+ """Export the model to autogptq format to easily leverage cuda kernel."""
+ model = kwargs["model"]
+ weight_config = kwargs["weight_config"]
+ sym = kwargs["scheme"] == "sym"
+ bits = kwargs["bits"]
+ group_size = kwargs["group_size"]
+ iters = kwargs["iters"]
+ lr = kwargs["lr"]
+ minmax_lr = kwargs["minmax_lr"]
+ enable_minmax_tuning = kwargs["enable_minmax_tuning"]
+ use_quant_input = kwargs["use_quant_input"]
+ scale_dtype = kwargs["scale_dtype"]
+ tokenizer = kwargs["tokenizer"]
+ supported_types = kwargs["supported_types"]
+
+ logger.info("Saving quantized model to autogptq format, this may take a while...")
+ if tokenizer is not None:
+ tokenizer.save_pretrained(output_dir)
+ ##check module quantized in block, this may have bug for mixed precision quantization
+ block_name = get_block_names(model)[0]
+ first_block = get_module(model, block_name)
+ all_to_quantized = True
+ modules_in_block_to_quantize = []
+ for n, m in first_block.named_modules():
+ is_supported_type = False
+ for supported_type in supported_types:
+ if isinstance(m, supported_type):
+ is_supported_type = True
+ break
+ if not is_supported_type:
+ continue
+ if not check_to_quantized(m):
+ all_to_quantized = False
+ else:
+ modules_in_block_to_quantize.append(n)
+ modules_in_block_to_quantize = [modules_in_block_to_quantize] ##align with autogptq
+ if all_to_quantized:
+ modules_in_block_to_quantize = None
+
+ if inplace:
+ compressed_model = model.to("cpu")
+ else:
+ compressed_model = copy.deepcopy(model.to("cpu"))
+
+ from auto_gptq.modeling._utils import pack_model
+
+ if bits == 3 or use_triton is False:
+ if bits == 3 and use_triton is True:
+ logger.warning("triton does not support 3 bits, reset it to False")
+ quantizers = {}
+ for key in weight_config:
+ info = weight_config[key]
+ if not check_to_quantized(info):
+ continue
+ quantizers[key] = (None, info["scale"], info["zp"], info["g_idx"])
+ pack_model(
+ compressed_model,
+ quantizers,
+ bits,
+ group_size,
+ use_cuda_fp16=True,
+ desc_act=False,
+ force_layer_back_to_cpu=True,
+ use_triton=False,
+ )
+ else:
+ quantizers = {}
+ for key in weight_config:
+ info = weight_config[key]
+ if not check_to_quantized(info):
+ continue
+ info["zp"] = info["zp"].to(torch.float32)
+ quantizers[key] = (None, info["scale"].to(torch.float32), info["zp"], info["g_idx"])
+ pack_model(
+ compressed_model,
+ quantizers,
+ bits,
+ group_size,
+ use_cuda_fp16=True,
+ desc_act=False,
+ force_layer_back_to_cpu=True,
+ use_triton=True,
+ )
+ if output_dir is None:
+ return compressed_model
+
+ _save_quantized_to_autogptq(
+ compressed_model,
+ output_dir,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=iters,
+ lr=lr,
+ minmax_lr=minmax_lr,
+ enable_minmax_tuning=enable_minmax_tuning,
+ use_quant_input=use_quant_input,
+ scale_dtype=scale_dtype,
+ use_safetensors=True,
+ modules_in_block_to_quantize=modules_in_block_to_quantize,
+ )
+
+
+def _save_quantized_to_autogptq(
model,
save_dir: str,
bits=4,
diff --git a/auto_round/export/export_to_itrex/__init__.py b/auto_round/export/export_to_itrex/__init__.py
index d627bc3b6..75fecd3b9 100644
--- a/auto_round/export/export_to_itrex/__init__.py
+++ b/auto_round/export/export_to_itrex/__init__.py
@@ -11,5 +11,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-from .export import compress_model
+from .export import save_quantized_as_itrex
from .config import QuantConfig
diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.py
index bf91960aa..a0f0ff024 100644
--- a/auto_round/export/export_to_itrex/export.py
+++ b/auto_round/export/export_to_itrex/export.py
@@ -21,13 +21,59 @@ from typing import Dict, List, Optional, Union
import torch
from safetensors.torch import save_file as safe_save
+from auto_round.export.register import register_format
from auto_round.utils import get_module, logger, quant_weight_w_scale, set_module
from .config import QuantConfig
from .model_wrapper import WeightOnlyLinear
-def compress_model(
+@register_format("itrex")
+def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):
+ """Save configure file and weights for CPU backend inference."""
+ model = kwargs["model"]
+ weight_config = kwargs["weight_config"]
+ sym = kwargs["scheme"] == "sym"
+ bits = kwargs["bits"]
+ group_size = kwargs["group_size"]
+ iters = kwargs["iters"]
+ lr = kwargs["lr"]
+ minmax_lr = kwargs["minmax_lr"]
+ enable_minmax_tuning = kwargs["enable_minmax_tuning"]
+ use_quant_input = kwargs["use_quant_input"]
+ scale_dtype = kwargs["scale_dtype"]
+ tokenizer = kwargs["tokenizer"]
+
+ compressed_model = _pack_model(model, weight_config, inplace=inplace)
+ if output_dir is None:
+ return compressed_model
+ quantize_config = QuantConfig(
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=iters,
+ lr=lr,
+ minmax_lr=minmax_lr,
+ enable_minmax_tuning=enable_minmax_tuning,
+ use_quant_input=use_quant_input,
+ scale_dtype=str(scale_dtype),
+ )
+ if quantize_config is not None:
+ config = compressed_model.config
+ setattr(config, "quantization_config", quantize_config.to_dict())
+ config.save_pretrained(output_dir)
+ quantize_config.save_pretrained(output_dir)
+ try:
+ compressed_model.save_pretrained(output_dir, safe_serialization=True)
+ if tokenizer is not None:
+ tokenizer.save_pretrained(output_dir)
+ logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))
+ except IOError as e: # pragma: no cover
+ logger.error("Fail to save configure file and weights due to {}.".format(e))
+ return compressed_model
+
+
+def _pack_model(
model,
weight_config: Union[str, dict],
enable_full_range=False,
diff --git a/auto_round/export/register.py b/auto_round/export/register.py
new file mode 100644
index 000000000..3947130c9
--- /dev/null
+++ b/auto_round/export/register.py
@@ -0,0 +1,36 @@
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+EXPORT_FORMAT = {}
+
+
+def register_format(name):
+ """Class decorator to register a EXPORT subclass to the registry.
+
+ Decorator function used before a Pattern subclass.
+
+ Args:
+ cls (class): The subclass of register.
+ name: A string. Define the export type.
+
+ Returns:
+ cls: The class of register.
+ """
+
+ def register(format):
+ EXPORT_FORMAT[name] = format
+ return format
+
+ return register
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 64eb0d6b9..2356214eb 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -14,8 +14,12 @@
import copy
import logging
+import subprocess
from collections import UserDict
+# for cpu usage
+import cpuinfo
+import psutil
import torch
from torch.amp import autocast
@@ -503,3 +507,87 @@ def check_to_quantized(config):
if config.bits > 8 or "fp" in config.data_type or "float" in config.data_type:
return False
return True
+
+
+def detect_device(device=None):
+ if device is None:
+ if torch.cuda.is_available():
+ device = torch.device("cuda:0")
+ logger.info("Using GPU device")
+ elif torch.hpu.is_available():
+ device = torch.device("hpu")
+ logger.info("Using HPU device")
+ # Use CPU as a fallback
+ else:
+ device = torch.device("cpu")
+ logger.info("Using CPU device")
+ return str(device)
+ elif isinstance(device, torch.device):
+ device = str(device)
+ return device
+
+
+class CpuInfo(object):
+ """Get CPU Info."""
+
+ def __init__(self):
+ """Get whether the cpu numerical format is bf16, the number of sockets, cores and cores per socket."""
+ self._bf16 = False
+ self._vnni = False
+ info = cpuinfo.get_cpu_info()
+ if "arch" in info and "X86" in info["arch"]:
+ cpuid = cpuinfo.CPUID()
+ max_extension_support = cpuid.get_max_extension_support()
+ if max_extension_support >= 7:
+ ecx = cpuid._run_asm(
+ b"\x31\xC9", # xor ecx, ecx
+ b"\xB8\x07\x00\x00\x00" b"\x0f\xa2" b"\x89\xC8" b"\xC3", # mov eax, 7 # cpuid # mov ax, cx # ret
+ )
+ self._vnni = bool(ecx & (1 << 11))
+ eax = cpuid._run_asm(
+ b"\xB9\x01\x00\x00\x00", # mov ecx, 1
+ b"\xB8\x07\x00\x00\x00" b"\x0f\xa2" b"\xC3", # mov eax, 7 # cpuid # ret
+ )
+ self._bf16 = bool(eax & (1 << 5))
+ if "arch" in info and "ARM" in info["arch"]: # pragma: no cover
+ self._sockets = 1
+ else:
+ self._sockets = self.get_number_of_sockets()
+ self._cores = psutil.cpu_count(logical=False)
+ self._cores_per_socket = int(self._cores / self._sockets)
+
+ @property
+ def bf16(self):
+ """Get whether it is bf16."""
+ return self._bf16
+
+ @property
+ def vnni(self):
+ """Get whether it is vnni."""
+ return self._vnni
+
+ @property
+ def cores_per_socket(self):
+ """Get the cores per socket."""
+ return self._cores_per_socket
+
+ def get_number_of_sockets(self) -> int:
+ """Get number of sockets in platform."""
+ cmd = "cat /proc/cpuinfo | grep 'physical id' | sort -u | wc -l"
+ if psutil.WINDOWS:
+ cmd = r'wmic cpu get DeviceID | C:\Windows\System32\find.exe /C "CPU"'
+ elif psutil.MACOS: # pragma: no cover
+ cmd = "sysctl -n machdep.cpu.core_count"
+
+ with subprocess.Popen(
+ args=cmd,
+ shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ universal_newlines=False,
+ ) as proc:
+ proc.wait()
+ if proc.stdout:
+ for line in proc.stdout:
+ return int(line.decode("utf-8", errors="ignore").strip())
+ return 0
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index ce5693d4e..41421c77d 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -1,7 +1,6 @@
import argparse
import sys
sys.path.insert(0, '../..')
-
parser = argparse.ArgumentParser()
import torch
import os
@@ -35,8 +34,9 @@ if __name__ == '__main__':
parser.add_argument("--eval_bs", default=4, type=int,
help="eval batch size")
- parser.add_argument("--device", default=0, type=str,
- help="device gpu int number, or 'cpu' ")
+ parser.add_argument("--device", default=None, type=str,
+ help="The device to be used for tuning. The default is set to None,"
+ "allowing for automatic detection. Currently, device settings support CPU, GPU, and HPU.")
parser.add_argument("--sym", action='store_true',
help=" sym quantization")
@@ -145,16 +145,14 @@ if __name__ == '__main__':
if model_name[-1] == "/":
model_name = model_name[:-1]
print(model_name, flush=True)
+
+ from auto_round.utils import detect_device
+ device_str = detect_device(args.device)
torch_dtype = "auto"
- if args.device == "cpu":
- device_str = "cpu"
- elif args.device == "hpu":
- device_str = "hpu"
+ if device_str == "hpu":
torch_dtype = torch.bfloat16
- else:
- device_str = f"cuda:{int(args.device)}"
-
torch_device = torch.device(device_str)
+
is_glm = bool(re.search("chatglm", model_name.lower()))
if is_glm:
model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
@@ -251,7 +249,7 @@ if __name__ == '__main__':
if 'gpu' in deployment_device:
autoround.save_quantized(f'{export_dir}-gpu', format="auto_gptq", use_triton=True, inplace=False)
if "cpu" in deployment_device:
- autoround.save_quantized(output_dir=f'{export_dir}-cpu', inplace=False)
+ autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=False)
if "fake" in deployment_device:
model = model.to("cpu")
model.save_pretrained(output_dir)
@@ -266,3 +264,4 @@ if __name__ == '__main__':
device=torch_device, excel_file=excel_name)
+
diff --git a/requirements.txt b/requirements.txt
index 865bb9f39..db836121a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,9 @@
accelerate
auto-gptq
datasets
+py-cpuinfo
sentencepiece
torch
transformers
+
+