Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

deprecate use_quant_inp arg (#90)

yintong-lu1bf1b486b524
diff --git a/README.md b/README.md
index dc4130f06..92d4a99c9 100644
--- a/README.md
+++ b/README.md
@@ -77,7 +77,7 @@ autoround.save_quantized(output_dir)
- `sym (bool)`: Whether to use symmetric quantization.
-- `use_quant_input (bool)`: Whether to use the output of the previous quantized block as the input for the current
+- `enable_quanted_input (bool)`: Whether to use the output of the previous quantized block as the input for the current
block (default is True).
- `enable_minmax_tuning (bool)`: Whether to enable weight min-max tuning (default is True).
@@ -193,7 +193,7 @@ average accuracies of 11 zero-shot tasks.
1 Consider increasing tuning steps to achieve better results, albeit with increased tuning time.
-2 Setting 'use_quant_input' to False has been observed to occasionally yield improved results.
+2 Setting 'enable_quanted_input' to False has been observed to occasionally yield improved results.
3 Setting 'minmax_lr' to 2.0/iters has been observed to occasionally yield improved results.
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index ef8dc6cbd..bff4e7c61 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -374,8 +374,8 @@ class AutoRound(object):
dataloader: The dataloader for input data (to be supported in future).
dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").
dataset_split (str): The split of the dataset to be used (default is "train").
- use_quant_input (bool): Whether to use the output of the previous quantized block as the input for the current
- block (default is True).
+ enable_quanted_input (bool): Whether to use the output of the previous quantized block as
+ the input for the current block (default is True).
enable_minmax_tuning (bool): Whether to enable weight min-max tuning (default is True).
lr (float): The learning rate (default is None, will be set to 1.0/iters).
minmax_lr (float): The learning rate for min-max tuning (default is None, it will be set to lr automatically).
@@ -413,7 +413,7 @@ class AutoRound(object):
dataloader=None, ## to support later
dataset: str = "NeelNanda/pile-10k",
dataset_split: str = "train",
- use_quant_input: bool = True,
+ enable_quanted_input: bool = True,
enable_minmax_tuning: bool = True,
lr: float = None,
minmax_lr: float = None,
@@ -435,7 +435,7 @@ class AutoRound(object):
self.model_orig_dtype = model.dtype
self.model = model.eval().to("cpu")
self.amp = amp
- self.use_quant_input = use_quant_input
+ self.enable_quanted_input = enable_quanted_input
self.enable_minmax_tuning = enable_minmax_tuning
self.n_samples = n_samples
self.n_blocks = n_blocks
@@ -1020,7 +1020,7 @@ class AutoRound(object):
device: The device for quantization.
Returns:
- Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output)
+ Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)
"""
cache_device = "cpu" ## force cache device to "cpu"
##change to block dtype:
@@ -1152,7 +1152,7 @@ class AutoRound(object):
logger.info(f"{unquantized_layer_names} have not been quantized")
with torch.no_grad():
unwrapper_block(block, best_v, best_min_scale, best_max_scale)
- if self.use_quant_input:
+ if self.enable_quanted_input:
q_outputs = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, cache_device)
return q_outputs, output
@@ -1251,7 +1251,7 @@ class AutoRound(object):
lr=self.lr,
minmax_lr=self.minmax_lr,
enable_minmax_tuning=self.enable_minmax_tuning,
- use_quant_input=self.use_quant_input,
+ enable_quanted_input=self.enable_quanted_input,
scale_dtype=self.scale_dtype,
tokenizer=self.tokenizer,
supported_types=self.supported_types,
@@ -1373,14 +1373,14 @@ class AutoRound(object):
torch.cuda.empty_cache()
layer_inputs = all_inputs
q_layer_inputs = None
- if self.use_quant_input:
+ if self.enable_quanted_input:
if not self.low_gpu_mem_usage:
self.model = self.model.to(self.device)
q_layer_inputs = self.cache_inter_data([], self.n_samples, layer_names=layer_names)
self.model = self.model.to("cpu")
torch.cuda.empty_cache()
for layer_name in layer_names:
- q_layer_input = q_layer_inputs[layer_name] if self.use_quant_input else None
+ q_layer_input = q_layer_inputs[layer_name] if self.enable_quanted_input else None
self.quant_layer(layer_name, layer_inputs[layer_name], q_layer_input, device=self.device)
torch.cuda.empty_cache()
@@ -1450,7 +1450,7 @@ class AutoOPTRound(AutoRound):
dataloader: The dataloader for input data (to be supported in future).
dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").
dataset_split (str): The split of the dataset to be used (default is "train").
- use_quant_input (bool): Whether to use quantized input data (default is True).
+ enable_quanted_input (bool): Whether to use quantized input data (default is True).
enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
lr (float): The learning rate (default is 0.005).
minmax_lr (float): The learning rate for min-max tuning (default is None).
@@ -1489,7 +1489,7 @@ class AutoOPTRound(AutoRound):
dataloader=None,
dataset: str = "NeelNanda/pile-10k",
dataset_split: str = "train",
- use_quant_input: bool = True,
+ enable_quanted_input: bool = True,
enable_minmax_tuning: bool = True,
lr: float = None,
minmax_lr: float = None,
@@ -1523,7 +1523,7 @@ class AutoOPTRound(AutoRound):
dataloader,
dataset,
dataset_split,
- use_quant_input,
+ enable_quanted_input,
enable_minmax_tuning,
lr,
minmax_lr,
@@ -1603,7 +1603,7 @@ class AutoAdamRound(AutoOPTRound):
dataloader: The dataloader for input data (to be supported in future).
dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").
dataset_split (str): The split of the dataset to be used (default is "train").
- use_quant_input (bool): Whether to use quantized input data (default is True).
+ enable_quanted_input (bool): Whether to use quantized input data (default is True).
enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
lr (float): The learning rate (default is 0.005).
minmax_lr (float): The learning rate for min-max tuning (default is None).
@@ -1642,7 +1642,7 @@ class AutoAdamRound(AutoOPTRound):
dataloader=None,
dataset: str = "NeelNanda/pile-10k",
dataset_split: str = "train",
- use_quant_input: bool = True,
+ enable_quanted_input: bool = True,
enable_minmax_tuning: bool = True,
lr: float = None,
minmax_lr: float = None,
@@ -1676,7 +1676,7 @@ class AutoAdamRound(AutoOPTRound):
dataloader,
dataset,
dataset_split,
- use_quant_input,
+ enable_quanted_input,
enable_minmax_tuning,
lr,
minmax_lr,
diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.py
index 17f6be1d1..5fc872cfe 100644
--- a/auto_round/export/export_to_autogptq.py
+++ b/auto_round/export/export_to_autogptq.py
@@ -59,7 +59,7 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwar
lr = kwargs["lr"]
minmax_lr = kwargs["minmax_lr"]
enable_minmax_tuning = kwargs["enable_minmax_tuning"]
- use_quant_input = kwargs["use_quant_input"]
+ enable_quanted_input = kwargs["enable_quanted_input"]
scale_dtype = kwargs["scale_dtype"]
tokenizer = kwargs["tokenizer"]
supported_types = kwargs["supported_types"]
@@ -145,7 +145,7 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwar
lr=lr,
minmax_lr=minmax_lr,
enable_minmax_tuning=enable_minmax_tuning,
- use_quant_input=use_quant_input,
+ enable_quanted_input=enable_quanted_input,
scale_dtype=scale_dtype,
use_safetensors=True,
modules_in_block_to_quantize=modules_in_block_to_quantize,
@@ -162,7 +162,7 @@ def _save_quantized_to_autogptq(
lr=5e-3,
minmax_lr=5e-3,
enable_minmax_tuning=True,
- use_quant_input=True,
+ enable_quanted_input=True,
use_safetensors: bool = True,
scale_dtype=torch.float32,
safetensors_metadata: Optional[Dict[str, str]] = None,
@@ -223,7 +223,7 @@ def _save_quantized_to_autogptq(
safetensors_metadata["lr"] = str(lr)
safetensors_metadata["minmax_lr"] = str(minmax_lr)
safetensors_metadata["enable_minmax_tuning"] = str(enable_minmax_tuning)
- safetensors_metadata["use_quant_input"] = str(use_quant_input)
+ safetensors_metadata["enable_quanted_input"] = str(enable_quanted_input)
safetensors_metadata["scale_dtype"] = str(scale_dtype)
safe_save(state_dict, join(save_dir, model_save_name), safetensors_metadata)
else:
@@ -250,7 +250,7 @@ def _save_quantized_to_autogptq(
config_dict["lr"] = lr
config_dict["minmax_lr"] = minmax_lr
config_dict["enable_minmax_tuning"] = enable_minmax_tuning
- config_dict["use_quant_input"] = use_quant_input
+ config_dict["enable_quanted_input"] = enable_quanted_input
config_dict["scale_dtype"] = str(scale_dtype)
if modules_in_block_to_quantize is not None:
config_dict["modules_in_block_to_quantize"] = modules_in_block_to_quantize
diff --git a/auto_round/export/export_to_itrex/config.py b/auto_round/export/export_to_itrex/config.py
index 76023b788..355b95a56 100644
--- a/auto_round/export/export_to_itrex/config.py
+++ b/auto_round/export/export_to_itrex/config.py
@@ -46,7 +46,7 @@ class QuantConfig(PretrainedConfig):
iters=1000,
lr=0.001,
minmax_lr=0.001,
- use_quant_input=True,
+ enable_quanted_input=True,
compute_dtype=None,
**kwargs,
):
@@ -61,7 +61,7 @@ class QuantConfig(PretrainedConfig):
self.iters = iters
self.lr = lr
self.minmax_lr = minmax_lr
- self.use_quant_input = use_quant_input
+ self.enable_quanted_input = enable_quanted_input
self.compute_dtype = convert_dtype_torch2str(compute_dtype)
if "export_to_xpu" not in kwargs or not kwargs["export_to_xpu"]:
@@ -287,13 +287,13 @@ class QuantConfig(PretrainedConfig):
"scheme",
"tokenizer",
"use_neural_speed",
- "use_quant_input",
+ "enable_quanted_input",
"layer_wise",
"nsamples",
"lr",
"minmax_lr",
"iters",
- "use_quant_input",
+ "enable_quanted_input",
"model_file_base_name",
"enable_minmax_tuning",
"model_name_or_path",
diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.py
index f56797a41..5c854822d 100644
--- a/auto_round/export/export_to_itrex/export.py
+++ b/auto_round/export/export_to_itrex/export.py
@@ -40,7 +40,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):
lr = kwargs["lr"]
minmax_lr = kwargs["minmax_lr"]
enable_minmax_tuning = kwargs["enable_minmax_tuning"]
- use_quant_input = kwargs["use_quant_input"]
+ enable_quanted_input = kwargs["enable_quanted_input"]
scale_dtype = kwargs["scale_dtype"]
tokenizer = kwargs["tokenizer"]
@@ -55,7 +55,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):
lr=lr,
minmax_lr=minmax_lr,
enable_minmax_tuning=enable_minmax_tuning,
- use_quant_input=use_quant_input,
+ enable_quanted_input=enable_quanted_input,
scale_dtype=scale_dtype,
)
if quantize_config is not None:
@@ -86,7 +86,7 @@ def save_quantized_as_itrex_xpu(output_dir, inplace=True, **kwargs):
lr = kwargs["lr"]
minmax_lr = kwargs["minmax_lr"]
enable_minmax_tuning = kwargs["enable_minmax_tuning"]
- use_quant_input = kwargs["use_quant_input"]
+ enable_quanted_input = kwargs["enable_quanted_input"]
scale_dtype = kwargs["scale_dtype"]
tokenizer = kwargs["tokenizer"]
@@ -101,7 +101,7 @@ def save_quantized_as_itrex_xpu(output_dir, inplace=True, **kwargs):
lr=lr,
minmax_lr=minmax_lr,
enable_minmax_tuning=enable_minmax_tuning,
- use_quant_input=use_quant_input,
+ enable_quanted_input=enable_quanted_input,
scale_dtype=scale_dtype,
export_to_xpu=True,
)
diff --git a/auto_round/version.py b/auto_round/version.py
index 952b12d4e..006ef5f4b 100644
--- a/auto_round/version.py
+++ b/auto_round/version.py
@@ -14,4 +14,4 @@
"""Intel® auto-round: An open-source Python library
supporting popular model weight only compression based on signround."""
-__version__ = "0.1"
+__version__ = "0.2.0.dev"
diff --git a/docs/acc.md b/docs/acc.md
index 48ea5bbce..1f113ee86 100644
--- a/docs/acc.md
+++ b/docs/acc.md
@@ -7,7 +7,7 @@ For GPTQ, we have enabled act-order and true-seqential, and also activated stati
For Omniquant, we adhere to the official settings, which include running for 20 epochs and disabling 'let'. We conducted calibration tests using sample sizes of 512 and 128, as well as a sample size of 512 with a batch size of 4. Our findings show that using a sample size of 512 typically results in comparable or slight higher performance for models <=13B. Therefore, we present the results based on the sample size of 512. For 70B models, due the the NAN loss issue and to reduce the tuning cost, we adopted 128 samples for calibration.
-For AutoRound, we used the default setting, iters 200, use_quant_input and enable_minmax_tuning, both the lr and minmax_lr are set to 1/iters,i.e. 5e-3.
+For AutoRound, we used the default setting, iters 200, enable_quanted_input and enable_minmax_tuning, both the lr and minmax_lr are set to 1/iters,i.e. 5e-3.
With these configurations, the tuning costs for GPTQ, AWQ, and ours are similar, while HQQ is much faster and Omniquant is noticebal slower.
@@ -3082,7 +3082,7 @@ With these configurations, the tuning costs for GPTQ, AWQ, and ours are similar,
</tr>
</tr>
- <th>Ours iters=1K,use_quant_input=False, minmax_lr=0.002</th>
+ <th>Ours iters=1K,disable_quanted_input, minmax_lr=0.002</th>
<td>67.70</td> <! acc avg -->
<td>60.57</td> <! MMLU -->
<td>73.74</td> <! Lambada_openai -->
@@ -3142,7 +3142,7 @@ With these configurations, the tuning costs for GPTQ, AWQ, and ours are similar,
<td>-</td>
</tr>
<tr>
- <th>Ours iters=1K,use_quant_input=False
+ <th>Ours iters=1K,disable_quanted_input
<td>66.78</td>
<td>68.68</td>
<td>78.61</td>
@@ -3203,7 +3203,7 @@ With these configurations, the tuning costs for GPTQ, AWQ, and ours are similar,
</tr>
</tr>
- <th>Ours iters=1K,use_quant_input=False </th>
+ <th>Ours iters=1K,disable_quanted_input </th>
<td>61.47</td> <! acc avg -->
<td>55.41</td> <! MMLU -->
<td>61.77</td> <! Lambada_openai -->
diff --git a/examples/code-generation/README.md b/examples/code-generation/README.md
index d69ab3a0f..312c00359 100644
--- a/examples/code-generation/README.md
+++ b/examples/code-generation/README.md
@@ -15,11 +15,11 @@ See more about loading [huggingface dataset](https://huggingface.co/docs/dataset
Enter into the examples folder
- **Default Settings:**
```bash
-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Salesforce/codegen25-7b-multi --amp --bits 4 --group_size -1 --enable_minmax_tuning --use_quant_input
+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Salesforce/codegen25-7b-multi --amp --bits 4 --group_size -1 --enable_minmax_tuning
```
- **Reduced GPU Memory Usage and Adjusted Training Batch Size:**
```bash
-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Salesforce/codegen25-7b-multi --amp --bits 4 --group_size -1 --low_gpu_mem_usage --train_bs 1 --gradient_accumulate_steps 8
+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Salesforce/codegen25-7b-multi --amp --bits 4 --group_size -1 --low_gpu_mem_usage --train_bs 1 --gradient_accumulate_steps 8
```
- **Utilizing the AdamW Optimizer:**
Include the flag `--adam`. Note that AdamW is less effective than Sign gradient descent in many scenarios we tested.
@@ -45,7 +45,7 @@ Please follow https://github.com/bigcode-project/bigcode-evaluation-harness to e
<th>0.2854</th>
</tr>
<tr>
- <th>AutoRound use_quant_input=False</th>
+ <th>AutoRound enable_quanted_input=False</th>
<th>0.2841</th>
</tr>
</table>
diff --git a/examples/code-generation/main.py b/examples/code-generation/main.py
index b04345006..7d20016d1 100644
--- a/examples/code-generation/main.py
+++ b/examples/code-generation/main.py
@@ -45,8 +45,8 @@ if __name__ == '__main__':
parser.add_argument("--iters", default=200, type=int,
help=" iters")
- parser.add_argument("--use_quant_input", action='store_true',
- help="whether to use the output of quantized block to tune the next block")
+ parser.add_argument("--enable_quanted_input", action='store_true',
+ help="enable_quanted_input is deprecated.")
parser.add_argument("--lr", default=None, type=float,
help="learning rate, if None, it will be set to 1.0/iters automatically")
@@ -104,6 +104,9 @@ if __name__ == '__main__':
parser.add_argument("--disable_minmax_tuning", action='store_true',
help="whether disable enable weight minmax tuning") # #not usage
+ parser.add_argument("--disable_quanted_input", action='store_true',
+ help="disuse the output of quantized block to tune the next block.")
+
args = parser.parse_args()
if args.low_gpu_mem_usage:
print(
@@ -114,6 +117,9 @@ if __name__ == '__main__':
if args.amp:
print(
"amp is deprecated, it has been set to the default, use disable_amp to turn it off")
+ if args.enable_quanted_input:
+ print(
+ "enable_quanted_input is deprecated. It has been set to the default; use disable_quanted_input to turn it off")
set_seed(args.seed)
tasks = args.tasks
@@ -177,7 +183,7 @@ if __name__ == '__main__':
round = AutoAdamRound
autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,
seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,
- minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input, device=device_str,
+ minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,
amp=not args.disable_amp, n_samples=args.n_samples,
low_gpu_mem_usage=not args.disable_low_gpu_mem_usage,
seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
diff --git a/examples/code-generation/run_autoround.sh b/examples/code-generation/run_autoround.sh
index 97880ef7e..473ebe942 100644
--- a/examples/code-generation/run_autoround.sh
+++ b/examples/code-generation/run_autoround.sh
@@ -13,4 +13,5 @@ python3 main.py \
--seqlen 128 \
--enable_minmax_tuning \
--output_dir "./tmp_signround" \
---amp
\ No newline at end of file
+--amp \
+--disable_quanted_input
\ No newline at end of file
diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index 9d04437ee..35e34d54a 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -55,7 +55,7 @@ pip install -r requirements.txt
- **Default Settings:**
```bash
-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size -1 --use_quant_input
+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size -1
```
- **Reduced GPU Memory Usage and Adjusted Training Batch Size:**
```bash
@@ -67,7 +67,7 @@ Include the flag `--adam`. Note that AdamW is less effective than sign gradient
- **Running the Original SignRound:**
```bash
-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size -1 --iters 400 --lr 0.0025 --disable_minmax_tuning
+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size -1 --iters 400 --lr 0.0025 --disable_minmax_tuning --disable_quanted_input
```
@@ -87,12 +87,12 @@ Due to the large size of the model, the quantization and evaluation processes ma
- You can set up multi-GPU cards for the quantization example, which will only use the first card for quantization and then evaluate with all GPU cards.
```bash
-CUDA_VISIBLE_DEVICES=1,2 python3 main.py --model_name facebook/opt-125m --amp --bits 4 --group_size -1 --deployment_device fake,cpu --output_dir /save_model_path/
+CUDA_VISIBLE_DEVICES=1,2 python3 main.py --model_name facebook/opt-125m --amp --bits 4 --group_size -1 --deployment_device fake,cpu --output_dir /save_model_path/
```
- Enable 'disable_eval' for the quantization example, save the qdq model by setting 'deployment_device=fake', and then set up multi-GPU cards for the evaluation script.
```bash
-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --amp --bits 4 --group_size -1 --disable_eval --deployment_device fake --output_dir /save_model_path/
+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --amp --bits 4 --group_size -1 --disable_eval --deployment_device fake --output_dir /save_model_path/
CUDA_VISIBLE_DEVICES=1,2 python3 eval/evaluation.py --model_name /save_model_path/ --eval_bs 8 --tasks mmlu,lambada_openai,ptb --excel_path /result_excel/save_path/
```
diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.py
index 4aadb5442..828984eb3 100644
--- a/examples/language-modeling/main.py
+++ b/examples/language-modeling/main.py
@@ -48,8 +48,8 @@ if __name__ == '__main__':
parser.add_argument("--dataset", default="NeelNanda/pile-10k", type=str,
help="The dataset for quantization training. It can be a custom one.")
- parser.add_argument("--use_quant_input", action='store_true',
- help="whether to use the output of quantized block to tune the next block")
+ parser.add_argument("--enable_quanted_input", action='store_true',
+ help="enable_quanted_input is deprecated.")
parser.add_argument("--lr", default=None, type=float,
help="learning rate, if None, it will be set to 1.0/iters automatically")
@@ -114,6 +114,9 @@ if __name__ == '__main__':
parser.add_argument("--disable_trust_remote_code", action='store_true',
help="Whether to disable trust_remote_code")
+
+ parser.add_argument("--disable_quanted_input", action='store_true',
+ help="whether to disuse the output of quantized block to tune the next block")
parser.add_argument("--quant_lm_head", action='store_true',
help="quant_lm_head")
@@ -128,6 +131,9 @@ if __name__ == '__main__':
if args.amp:
print(
"amp is deprecated, it has been set to the default, use disable_amp to turn it off")
+ if args.enable_quanted_input:
+ print(
+ "enable_quanted_input is deprecated. It has been set to the default; use disable_quanted_input to turn it off")
set_seed(args.seed)
tasks = args.tasks
@@ -265,7 +271,7 @@ if __name__ == '__main__':
weight_config['lm_head'] = {"data_type": "int"}
autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,
dataset=args.dataset, seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,
- minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input, device=device_str,
+ minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,
amp=not args.disable_amp, n_samples=args.n_samples,
low_gpu_mem_usage=not args.disable_low_gpu_mem_usage,
seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
diff --git a/examples/language-modeling/main_customized_data.py b/examples/language-modeling/main_customized_data.py
index 7ea020e73..021981cad 100644
--- a/examples/language-modeling/main_customized_data.py
+++ b/examples/language-modeling/main_customized_data.py
@@ -64,7 +64,7 @@ if __name__ == '__main__':
parser.add_argument("--iters", default=200, type=int,
help=" iters")
- parser.add_argument("--use_quant_input", action='store_true',
+ parser.add_argument("--enable_quanted_input", action='store_true',
help="whether to use the output of quantized block to tune the next block")
parser.add_argument("--lr", default=None, type=float,
@@ -248,7 +248,7 @@ if __name__ == '__main__':
autoround = round(model, tokenizer, args.bits, args.group_size, dataloader=data, sym=args.sym,
batch_size=args.train_bs,
seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,
- minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input, device=device_str,
+ minmax_lr=args.minmax_lr, enable_quanted_input=args.enable_quanted_input, device=device_str,
amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage,
seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
scale_dtype=args.scale_dtype, weight_config=weight_config) ##TODO args pass
diff --git a/examples/language-modeling/run_autoround.sh b/examples/language-modeling/run_autoround.sh
index 60c47f4ab..ddf13aa1a 100644
--- a/examples/language-modeling/run_autoround.sh
+++ b/examples/language-modeling/run_autoround.sh
@@ -9,7 +9,6 @@ python3 main.py \
--bits 4 \
--group_size 128 \
--enable_minmax_tuning \
---use_quant_input \
--amp \
--iters 200 \
--deployment_device 'fake,cpu' \
diff --git a/examples/language-modeling/run_autoround_on_gaudi.sh b/examples/language-modeling/run_autoround_on_gaudi.sh
index 1a063964a..ab862dc0c 100644
--- a/examples/language-modeling/run_autoround_on_gaudi.sh
+++ b/examples/language-modeling/run_autoround_on_gaudi.sh
@@ -10,7 +10,6 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--enable_minmax_tuning \
- --use_quant_input \
--eval_bs $eval_bs \
--output_dir "./tmp_autoround"
diff --git a/examples/language-modeling/run_xpu.sh b/examples/language-modeling/run_xpu.sh
index 12126ef72..dd651b8d6 100644
--- a/examples/language-modeling/run_xpu.sh
+++ b/examples/language-modeling/run_xpu.sh
@@ -12,7 +12,6 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 10 \
---use_quant_input \
--deployment_device 'xpu,fake' \
--quantize_layers_outside_blocks \
--disable_low_gpu_mem_usage \
diff --git a/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh b/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh
index 95616850e..7058a1983 100644
--- a/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh
+++ b/examples/language-modeling/scripts/Llama-2-7b-chat-hf.sh
@@ -5,6 +5,5 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
---use_quant_input \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
diff --git a/examples/language-modeling/scripts/Meta-Llama-3-8B-Instruct.sh b/examples/language-modeling/scripts/Meta-Llama-3-8B-Instruct.sh
index e83f608d1..7af9f6a7e 100644
--- a/examples/language-modeling/scripts/Meta-Llama-3-8B-Instruct.sh
+++ b/examples/language-modeling/scripts/Meta-Llama-3-8B-Instruct.sh
@@ -5,7 +5,6 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
---use_quant_input \
--quant_lm_head \
--deployment_device 'gpu' \
--disable_low_gpu_mem_usage \
@@ -20,7 +19,6 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
---use_quant_input \
--deployment_device 'gpu' \
--disable_low_gpu_mem_usage \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh b/examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh
index baa55cf31..762982acb 100644
--- a/examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh
+++ b/examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh
@@ -4,6 +4,5 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
---use_quant_input \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/Mistral-7B-v0.1.sh b/examples/language-modeling/scripts/Mistral-7B-v0.1.sh
index 34f2e0074..e433806d8 100644
--- a/examples/language-modeling/scripts/Mistral-7B-v0.1.sh
+++ b/examples/language-modeling/scripts/Mistral-7B-v0.1.sh
@@ -4,6 +4,5 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
---use_quant_input \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh b/examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh
index 59edc3366..018969dc5 100644
--- a/examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh
+++ b/examples/language-modeling/scripts/Mixtral-8x7B-Instruct-v0.1.sh
@@ -4,6 +4,5 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
---use_quant_input \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh b/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh
index 35439ad5c..a64143260 100644
--- a/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh
+++ b/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh
@@ -5,4 +5,5 @@ python3 main.py \
--bits 4 \
--iters 1000 \
--deployment_device 'gpu' \
---output_dir "./tmp_autoround"
\ No newline at end of file
+--output_dir "./tmp_autoround" \
+--disable_quanted_input \
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh
index 7b90de63d..3e1f75ffb 100644
--- a/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh
+++ b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh
@@ -6,5 +6,4 @@ python3 main.py \
--iters 1000 \
--deployment_device 'gpu' \
--minmax_lr 2e-3 \
---use_quant_input
diff --git a/examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.sh b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.sh
index cbe434bf2..32d8123d3 100644
--- a/examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.sh
+++ b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-sym.sh
@@ -3,7 +3,6 @@ python3 main.py \
--model_name Qwen/Qwen1.5-7B-Chat \
--bits 4 \
--group_size 128 \
---use_quant_input \
--iters 200 \
--deployment_device gpu \
--sym \
diff --git a/examples/language-modeling/scripts/Yi-6B-Chat.sh b/examples/language-modeling/scripts/Yi-6B-Chat.sh
index 514285b28..57fa0727f 100644
--- a/examples/language-modeling/scripts/Yi-6B-Chat.sh
+++ b/examples/language-modeling/scripts/Yi-6B-Chat.sh
@@ -6,4 +6,3 @@ python3 main.py \
--iters 1000 \
--deployment_device 'gpu' \
--minmax_lr 2e-3 \
---use_quant_input
diff --git a/examples/language-modeling/scripts/baichuan2-7b-chat.sh b/examples/language-modeling/scripts/baichuan2-7b-chat.sh
index 203fe2b67..7fb02dcfa 100644
--- a/examples/language-modeling/scripts/baichuan2-7b-chat.sh
+++ b/examples/language-modeling/scripts/baichuan2-7b-chat.sh
@@ -6,4 +6,3 @@ python3 main.py \
--iters 1000 \
--deployment_device 'gpu' \
--minmax_lr 2e-3 \
---use_quant_input
diff --git a/examples/language-modeling/scripts/bloom-3b.sh b/examples/language-modeling/scripts/bloom-3b.sh
index e4b30f5f6..a12ee2962 100644
--- a/examples/language-modeling/scripts/bloom-3b.sh
+++ b/examples/language-modeling/scripts/bloom-3b.sh
@@ -4,6 +4,5 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
---use_quant_input \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
diff --git a/examples/language-modeling/scripts/falcon-7b.sh b/examples/language-modeling/scripts/falcon-7b.sh
index d22c8f75a..20beca6c0 100644
--- a/examples/language-modeling/scripts/falcon-7b.sh
+++ b/examples/language-modeling/scripts/falcon-7b.sh
@@ -5,4 +5,5 @@ python3 main.py \
--bits 4 \
--iters 1000 \
--deployment_device 'gpu' \
---output_dir "./tmp_autoround"
\ No newline at end of file
+--output_dir "./tmp_autoround" \
+--disable_quanted_input
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/gemma-2b.sh b/examples/language-modeling/scripts/gemma-2b.sh
index 5ef8716a1..e562d55c1 100644
--- a/examples/language-modeling/scripts/gemma-2b.sh
+++ b/examples/language-modeling/scripts/gemma-2b.sh
@@ -7,6 +7,5 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 400 \
---use_quant_input \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/gemma-7b-it.sh b/examples/language-modeling/scripts/gemma-7b-it.sh
index 14bfb4bd6..4e19ef55d 100644
--- a/examples/language-modeling/scripts/gemma-7b-it.sh
+++ b/examples/language-modeling/scripts/gemma-7b-it.sh
@@ -4,7 +4,6 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
---use_quant_input \
--minmax_lr 2e-3 \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/gemma-7b.sh b/examples/language-modeling/scripts/gemma-7b.sh
index bf4daeabd..0ad3a3f16 100644
--- a/examples/language-modeling/scripts/gemma-7b.sh
+++ b/examples/language-modeling/scripts/gemma-7b.sh
@@ -4,7 +4,6 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
---use_quant_input \
--minmax_lr 2e-3 \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/gpt-j-6b.sh b/examples/language-modeling/scripts/gpt-j-6b.sh
index f80dddb06..e11f21ade 100644
--- a/examples/language-modeling/scripts/gpt-j-6b.sh
+++ b/examples/language-modeling/scripts/gpt-j-6b.sh
@@ -4,5 +4,6 @@ python3 main.py \
--group_size 128 \
--bits 4 \
--iters 1000 \
+--disable_quanted_input \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
diff --git a/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh b/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh
index 7d16a10db..d98afe8c4 100644
--- a/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh
+++ b/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh
@@ -7,4 +7,5 @@ python3 main.py \
--iters 1000 \
--minmax_lr 0.0002 \
--deployment_device 'gpu' \
---output_dir "./tmp_autoround"
\ No newline at end of file
+--output_dir "./tmp_autoround" \
+--disable_quanted_input \
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh b/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh
index 75dd31f2e..99ea7ccfe 100644
--- a/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh
+++ b/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh
@@ -6,4 +6,5 @@ python3 main.py \
--iters 1000 \
--minmax_lr 0.0002 \
--deployment_device 'gpu' \
---output_dir "./tmp_autoround"
\ No newline at end of file
+--output_dir "./tmp_autoround" \
+--disable_quanted_input \
\ No newline at end of file
diff --git a/examples/language-modeling/scripts/opt-2.7b.sh b/examples/language-modeling/scripts/opt-2.7b.sh
index 8d6200cde..bc9bc2ef5 100644
--- a/examples/language-modeling/scripts/opt-2.7b.sh
+++ b/examples/language-modeling/scripts/opt-2.7b.sh
@@ -5,5 +5,6 @@ python3 main.py \
--bits 4 \
--iters 1000 \
--minmax_lr 0.002 \
+--disable_quanted_input \
--deployment_device 'gpu' \
--output_dir "./tmp_autoround"
diff --git a/examples/language-modeling/scripts/phi-2.sh b/examples/language-modeling/scripts/phi-2.sh
index 69c44b24e..df702a886 100644
--- a/examples/language-modeling/scripts/phi-2.sh
+++ b/examples/language-modeling/scripts/phi-2.sh
@@ -6,4 +6,5 @@ python3 main.py \
--iters 1000 \
--deployment_device 'gpu' \
--disable_trust_remote_code \
---output_dir "./tmp_autoround"
\ No newline at end of file
+--output_dir "./tmp_autoround" \
+--disable_quanted_input \
\ No newline at end of file
diff --git a/test/test_autoround.py b/test/test_autoround.py
index cb03e7430..0ca68189f 100644
--- a/test/test_autoround.py
+++ b/test/test_autoround.py
@@ -106,7 +106,7 @@ class TestAutoRound(unittest.TestCase):
)
autoround.quantize()
- def test_disable_use_quant_input(self):
+ def test_disable_enable_quanted_input(self):
bits, group_size, sym = 4, -1, True
autoround = AutoRound(
self.model,
@@ -116,7 +116,7 @@ class TestAutoRound(unittest.TestCase):
sym=sym,
iters=2,
seqlen=10,
- use_quant_input=False,
+ enable_quanted_input=False,
dataloader=self.llm_dataloader,
)
autoround.quantize()
@@ -147,7 +147,7 @@ class TestAutoRound(unittest.TestCase):
iters=2,
seqlen=10,
enable_minmax_tuning=False,
- use_quant_input=False,
+ enable_quanted_input=False,
dataloader=self.llm_dataloader,
)
autoround.quantize()
@@ -164,7 +164,7 @@ class TestAutoRound(unittest.TestCase):
iters=2,
seqlen=10,
enable_minmax_tuning=False,
- use_quant_input=False,
+ enable_quanted_input=False,
dataloader=self.llm_dataloader,
weight_config=weight_config,
)