Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Consolidate dataloader&dataset_split to dataset (#105)

wenhuach211f9cb4f8b8e6
diff --git a/README.md b/README.md
index 8d2f5d14b..59fc584b7 100644
--- a/README.md
+++ b/README.md
@@ -105,11 +105,7 @@ autoround.save_quantized(output_dir)
- `low_gpu_mem_usage (bool)`: Whether to save GPU memory at the cost of a little tuning time (default is True).
-- `dataset (str)`: The default dataset name for tuning (default is "NeelNanda/pile-10k").
-
-- `dataset_split (str)`: The split of the dataset to be used for tuning (default is "train").
-
-- `dataloader`: The dataloader for tuning data.
+- `dataset Union[str, list, tuple, torch.utils.data.DataLoader]`: The dataset name for tuning (default is "NeelNanda/pile-10k"). Local json file and combination of datasets have been supported, e.g. "./tmp.json,NeelNanda/pile-10k:train, mbpp:train+validation+test"
- `weight_config (dict)`: Configuration for weight quantization (default is an empty dictionary), mainly for mixed bits
or mixed precision.
@@ -173,7 +169,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
| facebook/opt-2.7b | [accuracy](./docs/opt-2.7b-acc.md), [recipe](./examples/language-modeling/scripts/opt-2.7b.sh), [example](./examples/language-modeling/) |
| bigscience/bloom-3b | [accuracy](./docs/bloom-3B-acc.md), [recipe](./examples/language-modeling/scripts/bloom-3b.sh), [example](./examples/language-modeling/) |
| EleutherAI/gpt-j-6b | [accuracy](./docs/gpt-j-6B-acc.md), [recipe](./examples/language-modeling/scripts/gpt-j-6b.sh), [example](./examples/language-modeling/) |
-| Salesforce/codegen25-7b-multi | [example](./examples/code-generation) |
+| Salesforce/codegen25-7b-multi | [example](./examples/language-modeling/) |
| huggyllama/llama-7b | [example](./examples/language-modeling/) |
| mosaicml/mpt-7b | [example](./examples/language-modeling/) |
| THUDM/chatglm3-6b | [example](./examples/language-modeling/) |
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 933e34893..ba3b74a99 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -15,8 +15,10 @@
import copy
import time
+from typing import Optional, Union
import torch
+import transformers
from torch import autocast
from .calib_dataset import get_dataloader
@@ -25,7 +27,6 @@ from .utils import (
CpuInfo,
block_forward,
check_is_cpu,
- check_memory_availability,
check_to_quantized,
collect_minmax_scale,
collect_round_v,
@@ -34,7 +35,6 @@ from .utils import (
get_module,
get_scale_shape,
htcore,
- is_local_path,
is_optimum_habana_available,
logger,
move_input_to_device,
@@ -277,18 +277,14 @@ def wrapper_block(block, enable_minmax_tuning):
set_module(block, n, new_m)
quantized_layers.append(n)
- try:
- import transformers
+ if isinstance(m, transformers.modeling_utils.Conv1D):
+ if not check_to_quantized(m):
+ unquantized_layers.append(n)
+ continue
+ new_m = WrapperTransformerConv1d(m, enable_minmax_tuning=enable_minmax_tuning)
+ set_module(block, n, new_m)
+ quantized_layers.append(n)
- if isinstance(m, transformers.modeling_utils.Conv1D):
- if not check_to_quantized(m):
- unquantized_layers.append(n)
- continue
- new_m = WrapperTransformerConv1d(m, enable_minmax_tuning=enable_minmax_tuning)
- set_module(block, n, new_m)
- quantized_layers.append(n)
- except:
- pass
return quantized_layers, unquantized_layers
@@ -371,9 +367,7 @@ class AutoRound(object):
amp (bool): Whether to use automatic mixed precision (default is True).
device: The device to be used for tuning (default is "auto").
lr_scheduler: The learning rate scheduler to be used.
- dataloader: The dataloader for input data (to be supported in future).
- dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").
- dataset_split (str): The split of the dataset to be used (default is "train").
+ dataset (str): The default dataset name (default is "NeelNanda/pile-10k").
enable_quanted_input (bool): Whether to use the output of the previous quantized block as
the input for the current block (default is True).
enable_minmax_tuning (bool): Whether to enable weight min-max tuning (default is True).
@@ -410,9 +404,7 @@ class AutoRound(object):
amp: bool = True,
device=None,
lr_scheduler=None,
- dataloader=None, ## to support later
- dataset: str = "NeelNanda/pile-10k",
- dataset_split: str = "train",
+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
enable_quanted_input: bool = True,
enable_minmax_tuning: bool = True,
lr: float = None,
@@ -427,7 +419,7 @@ class AutoRound(object):
gradient_accumulate_steps: int = 1,
not_use_best_mse: bool = False,
dynamic_max_gap: int = -1,
- data_type: str = "int", ##only support data_type
+ data_type: str = "int", ##only support int for now
scale_dtype: str = "fp32",
**kwargs,
):
@@ -444,15 +436,8 @@ class AutoRound(object):
self.sym = sym
self.low_gpu_mem_usage = low_gpu_mem_usage
self.data_type = data_type
- self.supported_types = [torch.nn.Linear]
- try:
- import transformers
-
- self.supported_types.append(transformers.modeling_utils.Conv1D)
- except:
- pass
+ self.supported_types = [torch.nn.Linear, transformers.modeling_utils.Conv1D]
self.weight_config = weight_config
- self.dataset_split = dataset_split
self.seed = seed
self.tokenizer = tokenizer
self.seqlen = seqlen
@@ -476,15 +461,15 @@ class AutoRound(object):
if self.device == "cpu" and not CpuInfo().bf16:
self.amp = False
self.model = self.model.to(torch.float32)
- logger.warning("amp is set to FALSE as the current" "device does not support the 'bf16' data type.")
+ logger.warning(
+ f"amp is set to FALSE as the current {self.device} device does not support the 'bf16' data type."
+ )
else:
self.model = self.model.to(self.amp_dtype)
else:
self.model = self.model.to(torch.float32)
logger.info(f"using {self.model.dtype} for quantization tuning")
- self.dataset_name = dataset
-
- self.dataloader = dataloader
+ self.dataset = dataset
self.iters = iters
if self.iters <= 0:
logger.warning("iters must be positive, reset it to 200")
@@ -673,16 +658,18 @@ class AutoRound(object):
bs (int): The number of samples to use for calibration
"""
- if self.dataloader is None:
+ if isinstance(self.dataset, str):
+ dataset = self.dataset.replace(" ", "") ##remove all whitespaces
self.dataloader = get_dataloader(
self.tokenizer,
self.seqlen,
- self.dataset_name,
- self.dataset_split,
+ dataset,
self.seed,
bs,
self.n_samples,
)
+ else:
+ self.dataloader = self.dataset
total_cnt = 0
for data in self.dataloader:
if data is None:
@@ -693,7 +680,7 @@ class AutoRound(object):
elif isinstance(data, str):
if self.tokenizer is None:
- logger.error("for string input, please provide tokenizer")
+ logger.error("please provide tokenizer for string input")
exit()
data = self.tokenizer(data, truncation=True, max_length=self.seqlen, return_tensors="pt").data
data_new = {}
@@ -723,7 +710,7 @@ class AutoRound(object):
if total_cnt == 0:
logger.error(
f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "
- f"dataloader or decease the sequence length"
+ f"dataset or decease the sequence length"
)
exit()
elif total_cnt < n_samples:
@@ -1247,34 +1234,13 @@ class AutoRound(object):
)
return compressed_model
- #
- # @torch.no_grad()
- # def gets_layer_names_outside_blocks(self):
- # """Gets the names of layers outside blocks in the model.
- #
- # Returns:
- # list: List of layer names outside blocks.
- # """
- # all_layer_names = set()
- # for n, m in self.model.named_modules():
- # if isinstance(m, tuple(self.supported_types)):
- # m.tmp_name = n
- # all_layer_names.add(n)
- # block_names = get_block_names(self.model)
- # all_layer_names_in_block = set()
- # for block_name in block_names:
- # block = get_module(self.model, block_name)
- # for n, m in block.named_modules():
- # if isinstance(m, tuple(self.supported_types)):
- # all_layer_names_in_block.add(m.tmp_name)
- #
- # res = all_layer_names - all_layer_names_in_block
- # for n, m in self.model.named_modules():
- # if isinstance(m, tuple(self.supported_types)):
- # delattr(m, "tmp_name")
- # return list(res)
-
def get_layer_names_in_block(self):
+ """Retrieves the names of layers within each block of the model.
+
+ Returns:
+ list: A list of strings, where each string is the name of a layer
+ within a block of the model.
+ """
for n, m in self.model.named_modules():
if isinstance(m, tuple(self.supported_types)):
m.tmp_name = n
@@ -1434,9 +1400,7 @@ class AutoOPTRound(AutoRound):
amp (bool): Whether to use automatic mixed precision (default is True).
device: The device to be used for training (default is "auto").
lr_scheduler: The learning rate scheduler to be used.
- dataloader: The dataloader for input data (to be supported in future).
- dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").
- dataset_split (str): The split of the dataset to be used (default is "train").
+ dataset: The default dataset name (default is "NeelNanda/pile-10k").
enable_quanted_input (bool): Whether to use quantized input data (default is True).
enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
lr (float): The learning rate (default is 0.005).
@@ -1473,9 +1437,7 @@ class AutoOPTRound(AutoRound):
amp: bool = True,
device="auto",
lr_scheduler=None,
- dataloader=None,
- dataset: str = "NeelNanda/pile-10k",
- dataset_split: str = "train",
+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
enable_quanted_input: bool = True,
enable_minmax_tuning: bool = True,
lr: float = None,
@@ -1507,9 +1469,7 @@ class AutoOPTRound(AutoRound):
amp,
device,
lr_scheduler,
- dataloader,
dataset,
- dataset_split,
enable_quanted_input,
enable_minmax_tuning,
lr,
@@ -1587,9 +1547,8 @@ class AutoAdamRound(AutoOPTRound):
amp (bool): Whether to use automatic mixed precision (default is True).
device: The device to be used for training (default is "auto").
lr_scheduler: The learning rate scheduler to be used.
- dataloader: The dataloader for input data (to be supported in future).
- dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").
- dataset_split (str): The split of the dataset to be used (default is "train").
+ dataset (Union[str, list, tuple, torch.utils.data.DataLoader]):
+ The default dataset name (default is "NeelNanda/pile-10k").
enable_quanted_input (bool): Whether to use quantized input data (default is True).
enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
lr (float): The learning rate (default is 0.005).
@@ -1626,9 +1585,7 @@ class AutoAdamRound(AutoOPTRound):
amp: bool = True,
device="auto",
lr_scheduler=None,
- dataloader=None,
- dataset: str = "NeelNanda/pile-10k",
- dataset_split: str = "train",
+ dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
enable_quanted_input: bool = True,
enable_minmax_tuning: bool = True,
lr: float = None,
@@ -1660,9 +1617,7 @@ class AutoAdamRound(AutoOPTRound):
amp,
device,
lr_scheduler,
- dataloader,
dataset,
- dataset_split,
enable_quanted_input,
enable_minmax_tuning,
lr,
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index 242604fbb..7b25eef45 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -47,7 +47,7 @@ def get_tokenizer_function(tokenizer, seqlen):
@register_dataset("NeelNanda/pile-10k")
-def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=None, seed=42, bs=4):
+def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=None, seed=42):
"""Returns a dataloader for the specified dataset and split.
Args:
@@ -56,7 +56,6 @@ def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split
data_name: The name of the dataset.
split: The data split to be used (e.g., "train", "test").
seed: The random seed for shuffling the dataset.
- bs: The batch size for the dataloader.
Returns:
A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.
@@ -74,9 +73,7 @@ def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split
@register_dataset("madao33/new-title-chinese")
-def get_new_chinese_title_dataset(
- tokenizer, seqlen, dataset_name="madao33/new-title-chinese", split=None, seed=42, bs=4
-):
+def get_new_chinese_title_dataset(tokenizer, seqlen, dataset_name="madao33/new-title-chinese", split=None, seed=42):
"""Returns a dataloader for the specified dataset and split.
Args:
@@ -85,7 +82,6 @@ def get_new_chinese_title_dataset(
data_name: The name of the dataset.
split: The data split to be used (e.g., "train", "test").
seed: The random seed for shuffling the dataset.
- bs: The batch size for the dataloader.
Returns:
A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.
@@ -121,7 +117,7 @@ def get_new_chinese_title_dataset(
@register_dataset("mbpp")
-def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42, bs=4):
+def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42):
"""Returns a dataloader for the specified dataset and split.
Args:
@@ -130,7 +126,6 @@ def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42
data_name: The name of the dataset.
split: The data split to be used (e.g., "train", "test").
seed: The random seed for shuffling the dataset.
- bs: The batch size for the dataloader.
Returns:
A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.
@@ -144,7 +139,7 @@ def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42
if splits is None:
splits = ["train", "validation", "test"]
if isinstance(splits, str):
- splits = [split]
+ splits = splits.split("+")
for split in splits:
dataset = load_dataset(dataset_name, split=split)
@@ -160,7 +155,7 @@ def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42
@register_dataset("local")
-def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None, seed=42, bs=4):
+def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None, seed=42):
"""Returns a dataloader for a custom dataset and split.
We allow the input of a json or text file containing a processed text sample each line.
@@ -170,7 +165,6 @@ def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None,
data_name: The name or path of the dataset, which is a jsonl file.
split: The data split to be used (e.g., "train", "test").
seed: The random seed for shuffling the dataset.
- bs: The batch size for the dataloader.
Returns:
A dataloader for a custom dataset and split, using the provided tokenizer and sequence length.
@@ -214,7 +208,7 @@ def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None,
return calib_dataset
-def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=None, seed=42, bs=4, n_samples=512):
+def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, bs=8, n_samples=512):
"""Generate a DataLoader for calibration using specified parameters.
Args:
@@ -234,17 +228,6 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=N
dataset_names = dataset_name.split(",")
- # if len(dataset_names) == 1 and dataset_names[0] == "NeelNanda/pile-10k": ##to guarantee the reproducibility.
- # get_dataloader = CALIB_DATASETS.get("legacy-NeelNanda/pile-10k")
- # dataloader = get_dataloader(
- # tokenizer,
- # seqlen,
- # seed=seed,
- # bs=bs,
- # split=split,
- # dataset_name=dataset_name,
- # )
- # return dataloader
def filter_func(example):
if example["input_ids"].shape[-1] < seqlen:
return False
@@ -256,6 +239,9 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=N
datasets = []
for name in dataset_names:
+ split = None
+ if ":" in name:
+ name, split = name.split(":")
if is_local_path(name):
get_dataset = CALIB_DATASETS.get("local")
else:
@@ -264,7 +250,6 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=N
tokenizer,
seqlen,
seed=seed,
- bs=bs,
split=split,
dataset_name=name,
)
@@ -316,50 +301,3 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=N
calib_dataloader = DataLoader(dataset_final, batch_size=bs, shuffle=False, collate_fn=collate_batch)
return calib_dataloader
-
-
-# ##we keep this for reproduce issue, do not delete, keep it for a while
-# @register_dataset("legacy-NeelNanda/pile-10k")
-# def get_pile_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split="train", seed=42, bs=4):
-# """Returns a dataloader for the specified dataset and split.
-#
-# Args:
-# tokenizer: The tokenizer to be used for tokenization.
-# seqlen: The maximum sequence length.
-# data_name: The name of the dataset.
-# split: The data split to be used (e.g., "train", "test").
-# seed: The random seed for shuffling the dataset.
-# bs: The batch size for the dataloader.
-#
-# Returns:
-# A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.
-# """
-# from datasets import load_dataset
-# from torch.utils.data import DataLoader
-#
-# tokenizer_function = get_tokenizer_function(tokenizer, seqlen)
-#
-# @torch.no_grad()
-# def collate_batch(batch):
-# input_ids_new = []
-# for text in batch:
-# input_ids = text["input_ids"]
-# if input_ids.shape[0] < seqlen:
-# continue
-# input_ids = input_ids[:seqlen]
-# input_ids_list = input_ids.tolist()
-# if input_ids_list.count(input_ids_list[-1]) > seqlen // 2:
-# continue
-# input_ids_new.append(input_ids)
-# if len(input_ids_new) == 0:
-# return None
-# tmp = torch.vstack(input_ids_new)
-# res = {"input_ids": tmp}
-# return res
-#
-# calib_dataset = load_dataset(dataset_name, split=split)
-# calib_dataset = calib_dataset.shuffle(seed=seed)
-# calib_dataset = calib_dataset.map(tokenizer_function, batched=True)
-# calib_dataset.set_format(type="torch", columns=["input_ids"])
-# calib_dataloader = DataLoader(calib_dataset, batch_size=bs, shuffle=False, collate_fn=collate_batch)
-# return calib_dataloader
diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.py
index aaf40eef7..73fcc50a9 100644
--- a/auto_round/export/export_to_autogptq.py
+++ b/auto_round/export/export_to_autogptq.py
@@ -86,7 +86,7 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwar
all_to_quantized = False
else:
modules_in_block_to_quantize.append(n)
- modules_in_block_to_quantize = [modules_in_block_to_quantize] ##align with autogptq
+ modules_in_block_to_quantize = [modules_in_block_to_quantize]
if all_to_quantized:
modules_in_block_to_quantize = None
@@ -102,6 +102,8 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwar
logger.warning("triton does not support 3 bits, reset it to False")
quantizers = {}
for key in weight_config:
+ if key == "lm_head": ##TODO remove this after pr 87 is merged
+ continue
info = weight_config[key]
if not check_to_quantized(info):
continue
@@ -119,6 +121,8 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwar
else:
quantizers = {}
for key in weight_config:
+ if key == "lm_head": ##TODO remove this after pr 87 is merged
+ continue
info = weight_config[key]
if not check_to_quantized(info):
continue
diff --git a/examples/code-generation/README.md b/examples/code-generation/README.md
deleted file mode 100644
index 312c00359..000000000
--- a/examples/code-generation/README.md
+++ /dev/null
@@ -1,65 +0,0 @@
-Step-by-Step
-============
-
-This document presents step-by-step instructions for auto-round.
-
-
-## 2. Prepare Dataset
-
-The mbpp in huggingface is adopted as the default calibration data and will be downloaded automatically from the datasets Hub. To customize a dataset, please kindly follow our dataset code.
-See more about loading [huggingface dataset](https://huggingface.co/docs/datasets/loading_datasets.html)
-
-<br />
-
-## 3. Run Examples
-Enter into the examples folder
-- **Default Settings:**
-```bash
-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Salesforce/codegen25-7b-multi --amp --bits 4 --group_size -1 --enable_minmax_tuning
-```
-- **Reduced GPU Memory Usage and Adjusted Training Batch Size:**
-```bash
-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Salesforce/codegen25-7b-multi --amp --bits 4 --group_size -1 --low_gpu_mem_usage --train_bs 1 --gradient_accumulate_steps 8
-```
-- **Utilizing the AdamW Optimizer:**
-Include the flag `--adam`. Note that AdamW is less effective than Sign gradient descent in many scenarios we tested.
-
-- **Running the Original SignRound:**
-```bash
-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Salesforce/codegen25-7b-multi --amp --bits 4 --group_size -1 --iters 400 --lr 0.0025 --minmax_lr 0.0025
-```
- `--enable_minmax_tuning` is strongly recommended
-
-
-## 4. Evaluation
-Please follow https://github.com/bigcode-project/bigcode-evaluation-harness to eval the model, currently we only support fake model evaluation.
-<table border="1">
- <tr>
- <th>Model</th>
- <th>Method</th>
- <th>HumanEval top1 t=0.2 n_samples=20</th>
- </tr>
- <tr>
- <th rowspan="2">Salesforce/codegen25-7b-multi</th>
- <th>FP16 </th>
- <th>0.2854</th>
- </tr>
- <tr>
- <th>AutoRound enable_quanted_input=False</th>
- <th>0.2841</th>
- </tr>
-</table>
-
-
-
-## Reference
-If you find SignRound useful for your research, please cite our paper:
-```bash
-@article{cheng2023optimize,
- title={Optimize Weight Rounding via Signed Gradient Descent for the Quantization of LLMs},
- author={Cheng, Wenhua and Zhang, Weiwei and Shen, Haihao and Cai, Yiyang and He, Xin and Lv, Kaokao},
- journal={arXiv preprint arXiv:2309.05516},
- year={2023}
-}
-```
-
diff --git a/examples/code-generation/main.py b/examples/code-generation/main.py
deleted file mode 100644
index 7d20016d1..000000000
--- a/examples/code-generation/main.py
+++ /dev/null
@@ -1,208 +0,0 @@
-import argparse
-import random
-import sys
-
-sys.path.insert(0, '../..')
-parser = argparse.ArgumentParser()
-import torch
-import os
-
-os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
-torch.use_deterministic_algorithms(True, warn_only=True)
-from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel
-
-from transformers import set_seed
-
-import re
-
-os.environ["TOKENIZERS_PARALLELISM"] = "false"
-
-if __name__ == '__main__':
-
- parser.add_argument(
- "--model_name", default="facebook/opt-125m"
- )
-
- parser.add_argument("--bits", default=4, type=int,
- help="number of bits")
-
- parser.add_argument("--group_size", default=128, type=int,
- help="group size")
-
- parser.add_argument("--train_bs", default=8, type=int,
- help="train batch size")
-
- parser.add_argument("--eval_bs", default=4, type=int,
- help="eval batch size")
-
- parser.add_argument("--device", default="auto", type=str,
- help="The device to be used for tuning. The default is set to auto/None,"
- "allowing for automatic detection. Currently, device settings support CPU, GPU, and HPU.")
-
- parser.add_argument("--sym", action='store_true',
- help=" sym quantization")
-
- parser.add_argument("--iters", default=200, type=int,
- help=" iters")
-
- parser.add_argument("--enable_quanted_input", action='store_true',
- help="enable_quanted_input is deprecated.")
-
- parser.add_argument("--lr", default=None, type=float,
- help="learning rate, if None, it will be set to 1.0/iters automatically")
-
- parser.add_argument("--minmax_lr", default=None, type=float,
- help="minmax learning rate, if None,it will beset to be the same with lr")
-
- parser.add_argument("--seed", default=42, type=int,
- help="seed")
-
- parser.add_argument("--amp", action='store_true',
- help="amp is deprecated")
-
- parser.add_argument("--adam", action='store_true',
- help="adam")
-
- parser.add_argument("--seqlen", default=128, type=int,
- help="sequence length")
-
- parser.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")
-
- parser.add_argument("--n_blocks", default=1, type=int, help="num of blocks to tune together")
-
- parser.add_argument("--n_samples", default=512, type=int,
- help="number of samples")
-
- parser.add_argument("--low_gpu_mem_usage", action='store_true',
- help="low_gpu_mem_usage is deprecated")
-
- parser.add_argument("--enable_minmax_tuning", action='store_true',
- help="enable_minmax_tuning is deprecated")
-
- parser.add_argument("--deployment_device", default='fake', type=str,
- help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu'."
- "default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")
-
- parser.add_argument("--scale_dtype", default='fp32',
- help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")
-
- parser.add_argument("--tasks",
- default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
- "hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte",
- "arc_easy", "arc_challenge"],
- help="lm-eval tasks")
-
- parser.add_argument("--output_dir", default="./tmp_autoround", type=str,
- help="Where to store the final model.")
-
- parser.add_argument("--disable_amp", action='store_true',
- help="disable amp")
-
- parser.add_argument("--disable_low_gpu_mem_usage", action='store_true',
- help="disable low_gpu_mem_usage")
-
- parser.add_argument("--disable_minmax_tuning", action='store_true',
- help="whether disable enable weight minmax tuning") # #not usage
-
- parser.add_argument("--disable_quanted_input", action='store_true',
- help="disuse the output of quantized block to tune the next block.")
-
- args = parser.parse_args()
- if args.low_gpu_mem_usage:
- print(
- "low_gpu_mem_usage is deprecated, it has been set to the default, use disable_low_gpu_mem_usage to turn it off")
- if args.enable_minmax_tuning:
- print(
- "enable_minmax_tuning is deprecated, it has been set to the default, use disable_minmax_tuning to turn it off")
- if args.amp:
- print(
- "amp is deprecated, it has been set to the default, use disable_amp to turn it off")
- if args.enable_quanted_input:
- print(
- "enable_quanted_input is deprecated. It has been set to the default; use disable_quanted_input to turn it off")
-
- set_seed(args.seed)
- tasks = args.tasks
-
- model_name = args.model_name
- if model_name[-1] == "/":
- model_name = model_name[:-1]
- print(model_name, flush=True)
-
- from auto_round.utils import detect_device
-
- device_str = detect_device(args.device)
- torch_dtype = "auto"
- if "hpu" in device_str:
- torch_dtype = torch.bfloat16
- torch_device = torch.device(device_str)
- is_glm = bool(re.search("chatglm", model_name.lower()))
-
- if is_glm:
- model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
- else:
- model = AutoModelForCausalLM.from_pretrained(
- model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype, trust_remote_code=True
- )
-
- from auto_round import (AutoRound,
- AutoAdamRound)
-
- model = model.eval()
- # align wigh GPTQ to eval ppl
- if "opt" in model_name:
- seqlen = model.config.max_position_embeddings
- model.seqlen = model.config.max_position_embeddings
- else:
- seqlen = 2048
- model.seqlen = seqlen
- seqlen = args.seqlen
-
- if "llama" in model_name:
- from transformers import LlamaTokenizer
-
- tokenizer = LlamaTokenizer.from_pretrained(model_name)
- if tokenizer.pad_token is None:
- tokenizer.add_special_tokens({'pad_token': '[PAD]'})
- else:
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
- if hasattr(tokenizer, "model_max_length"):
- if tokenizer.model_max_length < seqlen:
- print(f"change sequence length to {tokenizer.model_max_length} due to the limitation of model_max_length",
- flush=True)
- seqlen = min(seqlen, tokenizer.model_max_length)
- args.seqlen = seqlen
-
- excel_name = f"{model_name}_{args.bits}_{args.group_size}"
-
- if args.disable_low_gpu_mem_usage:
- model = model.to(torch_device)
-
- round = AutoRound
- if args.adam:
- round = AutoAdamRound
- autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,
- seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,
- minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,
- amp=not args.disable_amp, n_samples=args.n_samples,
- low_gpu_mem_usage=not args.disable_low_gpu_mem_usage,
- seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
- scale_dtype=args.scale_dtype, dataset="mbpp", dataset_split=['train', 'validation', 'test'],
- enable_minmax_tuning=not args.disable_minmax_tuning) ##TODO args pass
- model, q_config = autoround.quantize()
- model_name = args.model_name.rstrip("/")
- model.eval()
- if args.device != "cpu":
- torch.cuda.empty_cache()
-
- export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"
- output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"
- deployment_device = args.deployment_device.split(',')
- if 'gpu' in deployment_device:
- autoround.save_quantized(f'{export_dir}-gpu', format="auto_gptq", use_triton=True, inplace=False)
- if "cpu" in deployment_device:
- autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=False)
- if "fake" in deployment_device:
- model = model.to("cpu")
- model.save_pretrained(output_dir)
- tokenizer.save_pretrained(output_dir)
diff --git a/examples/code-generation/run_autoround.sh b/examples/code-generation/run_autoround.sh
deleted file mode 100644
index 473ebe942..000000000
--- a/examples/code-generation/run_autoround.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/bash
-set -x
-device=0
-model_name="Salesforce/codegen25-7b-multi"
-
-CUDA_VISIBLE_DEVICES=$device \
-python3 main.py \
---model_name $model_name \
---device $device \
---group_size 128 \
---bits 4 \
---iters 200 \
---seqlen 128 \
---enable_minmax_tuning \
---output_dir "./tmp_signround" \
---amp \
---disable_quanted_input
\ No newline at end of file
diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index df370963f..92faa77c4 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -44,10 +44,28 @@ See more about loading [huggingface dataset](https://huggingface.co/docs/dataset
### Customized Dataset
- Option 1: Pass a local json file path to dataset argument
-- Option 2: Register your dataset following the [code](../../auto_round/calib_dataset.py) and pass the new dataset&split args to initialize AutoRound object.
-- Option 3: Following the [code](./main_customized_data.py) to pass list of string or list of inputs to dataloader.
-
-Combination of different datasets has been supported, --dataset "./tmp.json,NeelNanda/pile-10k". Please note that samples with sequence length < args.seq will be dropped.
+- Option 2: Register your dataset following the [code](../../auto_round/calib_dataset.py) and pass the new dataset&split args to initialize AutoRound object,e.g. autoround=Autoround(dataset="NeelNanda/pile-10k:train",...)
+- Option 3: pass list of string or list of input_ids to dataset.
+~~~python
+def customized_data():
+ ##Important Notice!!! Autoround will drop data < args.seqlen and truncate data to args.seqlen
+ data = ["AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference" * 240]
+ data.append("AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference")
+ return data
+
+
+def customized_data_with_tokenizer(tokenizer, seqlen=2048):
+ ##Import notice!!! Autoround will drop data < args.seqlen
+ data = ["AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference" * 240]
+ data.append("AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference")
+ tokens = []
+ for d in data:
+ token = tokenizer(d, truncation=True, max_length=seqlen, return_tensors="pt").data
+ tokens.append(token)
+ return tokens
+~~~
+
+Combination of different datasets has been supported, --dataset "./tmp.json,NeelNanda/pile-10k:train, mbpp:train+validation+test". Please note that samples with sequence length < args.seq will be dropped.
<br />
@@ -59,18 +77,18 @@ pip install -r requirements.txt
- **Default Settings:**
```bash
-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size -1
+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128
```
- **Reduced GPU Memory Usage:**
```bash
-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size -1 --train_bs 1 --gradient_accumulate_steps 8
+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --train_bs 1 --gradient_accumulate_steps 8
```
- **Enable quantized lm-head:**
--disable_low_gpu_mem_usage is strongly recommended if the whole model could be loaded to the device, otherwise it will be quite slow to cache the inputs of lm-head. Another way is reducing n_samples,e.g. 128, to alleviate the issue.
```bash
-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size -1 --quant_lm_head --disable_low_gpu_mem_usage
+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --quant_lm_head --disable_low_gpu_mem_usage
```
- **Utilizing the AdamW Optimizer:**
@@ -79,9 +97,15 @@ Include the flag `--adam`. Note that AdamW is less effective than sign gradient
- **Running the Original SignRound:**
```bash
-CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size -1 --iters 400 --lr 0.0025 --disable_minmax_tuning --disable_quanted_input
+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name facebook/opt-125m --bits 4 --group_size 128 --iters 400 --lr 0.0025 --disable_minmax_tuning --disable_quanted_input
```
+- **Code generation LLM:**
+
+We utilized mbpp for calibration, but your own training dataset is highly recommended. Please note that samples with seqlen < args.seqlen will be dropped in current version.
+```bash
+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name Salesforce/codegen25-7b-multi --bits 4 --group_size 128 --dataset "mbpp" --seqlen 128 "
+```
- **Running on Intel Gaudi2**
```bash
bash run_autoround_on_gaudi.sh
diff --git a/examples/language-modeling/main_customized_data.py b/examples/language-modeling/main_customized_data.py
deleted file mode 100644
index 021981cad..000000000
--- a/examples/language-modeling/main_customized_data.py
+++ /dev/null
@@ -1,280 +0,0 @@
-import argparse
-import sys
-
-sys.path.insert(0, '../..')
-parser = argparse.ArgumentParser()
-import torch
-import os
-import transformers
-
-os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
-torch.use_deterministic_algorithms(True, warn_only=True)
-from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel
-
-from transformers import set_seed
-
-import re
-
-os.environ["TOKENIZERS_PARALLELISM"] = "false"
-
-
-def customized_data():
- ##Important Notice!!! Autoround will drop data < args.seqlen and truncate data to args.seqlen
- data = ["AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference" * 240]
- data.append("AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference")
- return data
-
-
-def customized_data_with_tokenizer(tokenizer, seqlen=2048):
- ##Import notice!!! Autoround will dump data < args.seqlen
- data = ["AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference" * 240]
- data.append("AutoRound is an advanced weight-only quantization algorithm for low-bits LLM inference")
- tokens = []
- for d in data:
- token = tokenizer(d, truncation=True, max_length=seqlen, return_tensors="pt").data
- tokens.append(token)
- return tokens
-
-
-if __name__ == '__main__':
-
- parser.add_argument(
- "--model_name", default="facebook/opt-125m"
- )
-
- parser.add_argument("--bits", default=4, type=int,
- help="number of bits")
-
- parser.add_argument("--group_size", default=128, type=int,
- help="group size")
-
- parser.add_argument("--train_bs", default=8, type=int,
- help="train batch size")
-
- parser.add_argument("--eval_bs", default=4, type=int,
- help="eval batch size")
-
- parser.add_argument("--device", default="auto", type=str,
- help="The device to be used for tuning. The default is set to auto/None,"
- "allowing for automatic detection. Currently, device settings support CPU, GPU, and HPU.")
-
- parser.add_argument("--sym", action='store_true',
- help=" sym quantization")
-
- parser.add_argument("--iters", default=200, type=int,
- help=" iters")
-
- parser.add_argument("--enable_quanted_input", action='store_true',
- help="whether to use the output of quantized block to tune the next block")
-
- parser.add_argument("--lr", default=None, type=float,
- help="learning rate, if None, it will be set to 1.0/iters automatically")
-
- parser.add_argument("--minmax_lr", default=None, type=float,
- help="minmax learning rate, if None,it will beset to be the same with lr")
-
- parser.add_argument("--seed", default=42, type=int,
- help="seed")
-
- parser.add_argument("--eval_fp16_baseline", action='store_true',
- help="whether to eval FP16 baseline")
-
- parser.add_argument("--amp", action='store_true',
- help="amp")
-
- parser.add_argument("--adam", action='store_true',
- help="adam")
-
- parser.add_argument("--seqlen", default=2048, type=int,
- help="sequence length")
-
- parser.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")
-
- parser.add_argument("--n_blocks", default=1, type=int, help="num of blocks to tune together")
-
- parser.add_argument("--n_samples", default=512, type=int,
- help="number of samples")
-
- parser.add_argument("--low_gpu_mem_usage", action='store_true',
- help="low_gpu_mem_usage")
-
- parser.add_argument("--enable_minmax_tuning", action='store_true',
- help="whether enable weight minmax tuning")
-
- parser.add_argument("--deployment_device", default='fake', type=str,
- help="targeted inference acceleration platform,The options are 'fake', 'cpu' and 'gpu'."
- "default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")
-
- parser.add_argument("--scale_dtype", default='fp32',
- help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")
-
- parser.add_argument("--tasks",
- default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
- "mmlu", "wikitext", "truthfulqa_mc1", "truthfulqa_mc2", "openbookqa", "boolq", "rte",
- "arc_easy", "arc_challenge"],
- help="lm-eval tasks for lm_eval version 0.4")
-
- parser.add_argument("--output_dir", default="./tmp_autoround", type=str,
- help="Where to store the final model.")
-
- parser.add_argument("--disable_eval", action='store_true',
- help="Whether to do lmeval evaluation.")
-
- args = parser.parse_args()
- set_seed(args.seed)
- tasks = args.tasks
- use_eval_legacy = False
- import subprocess
-
-
- def get_library_version(library_name):
- try:
- version = subprocess.check_output(['pip', 'show', library_name]).decode().split('\n')[1].split(': ')[1]
- return version
- except subprocess.CalledProcessError:
- return "Library not found"
-
-
- res = get_library_version("lm-eval")
- if res == "0.3.0":
- use_eval_legacy = True
- if not use_eval_legacy:
- from eval import eval_model
- else:
- from eval_legacy import eval_model
-
- if isinstance(tasks, str):
- tasks = tasks.split(',')
- if isinstance(tasks, list):
- if "mmlu" in tasks:
- tmp_tasks = tasks
- tasks = ["hendrycksTest-*" if x == "mmlu" else x for x in tmp_tasks]
- if "truthfulqa_mc1" in tasks or "truthfulqa_mc2" in tasks:
- tmp_tasks = tasks
- tasks = ["truthfulqa_mc" if "truthfulqa_mc" in x else x for x in tmp_tasks]
- tasks = list(set(tasks))
- if isinstance(args.tasks, str):
- tasks = ','.join(tasks)
-
- model_name = args.model_name
- if model_name[-1] == "/":
- model_name = model_name[:-1]
- print(model_name, flush=True)
-
- from auto_round.utils import detect_device
-
- device_str = detect_device(args.device)
- torch_dtype = "auto"
- if "hpu" in device_str:
- torch_dtype = torch.bfloat16
- torch_device = torch.device(device_str)
-
- is_glm = bool(re.search("chatglm", model_name.lower()))
- if is_glm:
- model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
- else:
- model = AutoModelForCausalLM.from_pretrained(
- model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype, trust_remote_code=True
- )
-
- from auto_round import (AutoRound,
- AutoAdamRound)
-
- model = model.eval()
- # align with GPTQ to eval ppl
- if "opt" in model_name:
- seqlen = model.config.max_position_embeddings
- model.seqlen = model.config.max_position_embeddings
- else:
- seqlen = 2048
- model.seqlen = seqlen
- seqlen = args.seqlen
-
- if "llama" in model_name:
- from transformers import LlamaTokenizer
-
- tokenizer = LlamaTokenizer.from_pretrained(model_name)
- if tokenizer.pad_token is None:
- tokenizer.add_special_tokens({'pad_token': '[PAD]'})
- else:
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-
- if hasattr(tokenizer, "model_max_length"):
- if tokenizer.model_max_length < seqlen:
- print(f"change sequence length to {tokenizer.model_max_length} due to the limitation of model_max_length",
- flush=True)
- seqlen = min(seqlen, tokenizer.model_max_length)
- args.seqlen = seqlen
-
- pt_dtype = torch.float16
- if (hasattr(model, 'config') and (model.dtype is torch.bfloat16 or model.config.torch_dtype is torch.bfloat16)):
- dtype = 'bfloat16'
- pt_dtype = torch.bfloat16
- else:
- if str(args.device) != "cpu":
- pt_dtype = torch.float16
- dtype = 'float16'
- else:
- pt_dtype = torch.float32
- dtype = 'float32'
-
- excel_name = f"{model_name}_{args.bits}_{args.group_size}"
- if args.eval_fp16_baseline:
- if not args.low_gpu_mem_usage:
- model = model.to(torch_device)
- excel_name += "_fp16.xlsx"
- eval_model(model_path=model_name, tasks=tasks, dtype=dtype, \
- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,
- device=torch_device, excel_file=excel_name)
- exit()
-
- if not args.low_gpu_mem_usage:
- model = model.to(torch_device)
-
- round = AutoRound
- if args.adam:
- round = AutoAdamRound
-
- weight_config = {}
- for n, m in model.named_modules():
- if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):
- if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:
- weight_config[n] = {"data_type": "fp"}
- print(
- f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")
-
- data = customized_data()
-
- autoround = round(model, tokenizer, args.bits, args.group_size, dataloader=data, sym=args.sym,
- batch_size=args.train_bs,
- seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,
- minmax_lr=args.minmax_lr, enable_quanted_input=args.enable_quanted_input, device=device_str,
- amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage,
- seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
- scale_dtype=args.scale_dtype, weight_config=weight_config) ##TODO args pass
- model, _ = autoround.quantize()
- model_name = args.model_name.rstrip("/")
-
- model.eval()
- if args.device != "cpu":
- torch.cuda.empty_cache()
-
- export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"
- output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"
- deployment_device = args.deployment_device.split(',')
- if 'gpu' in deployment_device:
- autoround.save_quantized(f'{export_dir}-gpu', format="auto_gptq", use_triton=True, inplace=False)
- if "cpu" in deployment_device:
- autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=False)
- if "fake" in deployment_device:
- model = model.to("cpu")
- model.save_pretrained(output_dir)
- tokenizer.save_pretrained(output_dir)
-
- if not args.disable_eval and "fake" in deployment_device: ##support autogptq real eval later
- excel_name = f"{output_dir}_result.xlsx"
- output_dir += "/"
- print(excel_name, flush=True)
- eval_model(model_path=output_dir, tasks=tasks, dtype=dtype, limit=None,
- eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage,
- device=torch_device, excel_file=excel_name)
diff --git a/test/test_autoopt.py b/test/test_autoopt.py
index d52556bbf..2f500162a 100644
--- a/test/test_autoopt.py
+++ b/test/test_autoopt.py
@@ -43,7 +43,7 @@ class TestAutoRound(unittest.TestCase):
sym=sym,
iters=2,
seqlen=10,
- dataloader=self.llm_dataloader,
+ dataset=self.llm_dataloader,
)
autoround.quantize()
diff --git a/test/test_autoround.py b/test/test_autoround.py
index 0ca68189f..d871705f0 100644
--- a/test/test_autoround.py
+++ b/test/test_autoround.py
@@ -43,7 +43,7 @@ class TestAutoRound(unittest.TestCase):
sym=sym,
iters=2,
seqlen=2,
- dataloader=self.llm_dataloader,
+ dataset=self.llm_dataloader,
)
autoround.quantize()
if torch.cuda.is_available():
@@ -60,7 +60,7 @@ class TestAutoRound(unittest.TestCase):
sym=sym,
iters=2,
seqlen=10,
- dataloader=self.llm_dataloader,
+ dataset=self.llm_dataloader,
)
autoround.quantize()
@@ -74,7 +74,7 @@ class TestAutoRound(unittest.TestCase):
sym=sym,
iters=2,
seqlen=10,
- dataloader=self.llm_dataloader,
+ dataset=self.llm_dataloader,
)
autoround.quantize()
@@ -88,7 +88,7 @@ class TestAutoRound(unittest.TestCase):
sym=sym,
iters=2,
seqlen=10,
- dataloader=self.llm_dataloader,
+ dataset=self.llm_dataloader,
)
autoround.quantize()
@@ -102,7 +102,7 @@ class TestAutoRound(unittest.TestCase):
sym=sym,
iters=2,
seqlen=10,
- dataloader=self.llm_dataloader,
+ dataset=self.llm_dataloader,
)
autoround.quantize()
@@ -117,7 +117,7 @@ class TestAutoRound(unittest.TestCase):
iters=2,
seqlen=10,
enable_quanted_input=False,
- dataloader=self.llm_dataloader,
+ dataset=self.llm_dataloader,
)
autoround.quantize()
@@ -132,7 +132,7 @@ class TestAutoRound(unittest.TestCase):
iters=2,
seqlen=10,
enable_minmax_tuning=False,
- dataloader=self.llm_dataloader,
+ dataset=self.llm_dataloader,
)
autoround.quantize()
@@ -148,7 +148,7 @@ class TestAutoRound(unittest.TestCase):
seqlen=10,
enable_minmax_tuning=False,
enable_quanted_input=False,
- dataloader=self.llm_dataloader,
+ dataset=self.llm_dataloader,
)
autoround.quantize()
@@ -165,7 +165,7 @@ class TestAutoRound(unittest.TestCase):
seqlen=10,
enable_minmax_tuning=False,
enable_quanted_input=False,
- dataloader=self.llm_dataloader,
+ dataset=self.llm_dataloader,
weight_config=weight_config,
)
autoround.quantize()