Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add chat template in calib tokenization (#171)

yintong-lu735dfc9e59c3
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index fe8a1be6e..b71301f4b 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -42,26 +42,40 @@ def register_dataset(name):
return register
-def get_tokenizer_function(tokenizer, seqlen):
+def get_tokenizer_function(tokenizer, seqlen, apply_template=False):
"""Returns a default tokenizer function.
Args:
tokenizer: The tokenizer to be used for tokenization.
seqlen: The maximum sequence length.
+ apply_template: Whether to apply chat template in tokenization.
Returns: A default tokenizer function that applies the provided tokenizer with truncation and a maximum length of
seqlen to the "text" field of examples.
"""
- def default_tokenizer_function(examples):
- example = tokenizer(examples["text"], truncation=True, max_length=seqlen)
+ def default_tokenizer_function(examples, apply_template=apply_template):
+ if not apply_template:
+ example = tokenizer(examples["text"], truncation=True, max_length=seqlen)
+ else:
+ from jinja2 import Template # pylint: disable=E0401
+ chat_template = tokenizer.chat_template if tokenizer.chat_template is not None \
+ else tokenizer.default_chat_template
+ template = Template(chat_template)
+ rendered_messages = []
+ for text in examples["text"]:
+ message = [{"role": "user", "content": text}]
+ rendered_message = template.render(messages=message, add_generation_prompt=True, \
+ bos_token=tokenizer.bos_token)
+ rendered_messages.append(rendered_message)
+ example = tokenizer(rendered_messages, truncation=True, max_length=seqlen)
return example
return default_tokenizer_function
@register_dataset("NeelNanda/pile-10k")
-def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=None, seed=42):
+def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=None, seed=42, apply_template=False):
"""Returns a dataloader for the specified dataset and split.
Args:
@@ -70,6 +84,7 @@ def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split
data_name: The name of the dataset.
split: The data split to be used (e.g., "train", "test").
seed: The random seed for shuffling the dataset.
+ apply_template: Whether to apply chat template in tokenization.
Returns:
A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.
@@ -77,7 +92,7 @@ def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split
from datasets import load_dataset
split = "train"
- tokenizer_function = get_tokenizer_function(tokenizer, seqlen)
+ tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_template=apply_template)
calib_dataset = load_dataset(dataset_name, split=split)
calib_dataset = calib_dataset.shuffle(seed=seed)
@@ -87,7 +102,14 @@ def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split
@register_dataset("madao33/new-title-chinese")
-def get_new_chinese_title_dataset(tokenizer, seqlen, dataset_name="madao33/new-title-chinese", split=None, seed=42):
+def get_new_chinese_title_dataset(
+ tokenizer,
+ seqlen,
+ dataset_name="madao33/new-title-chinese",
+ split=None,
+ seed=42,
+ apply_template=False
+):
"""Returns a dataloader for the specified dataset and split.
Args:
@@ -96,24 +118,39 @@ def get_new_chinese_title_dataset(tokenizer, seqlen, dataset_name="madao33/new-t
data_name: The name of the dataset.
split: The data split to be used (e.g., "train", "test").
seed: The random seed for shuffling the dataset.
+ apply_template: Whether to apply chat template in tokenization.
Returns:
A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.
"""
- def get_tokenizer_function(tokenizer, seqlen):
+ def get_tokenizer_function(tokenizer, seqlen, apply_template=apply_template):
"""Returns a default tokenizer function.
Args:
tokenizer: The tokenizer to be used for tokenization.
seqlen: The maximum sequence length.
+ apply_template: Whether to apply chat template in tokenization.
Returns: A default tokenizer function that applies the provided tokenizer with truncation and a maximum length
of seqlen to the "text" field of examples.
"""
- def default_tokenizer_function(examples):
- example = tokenizer(examples["content"], truncation=True, max_length=seqlen)
+ def default_tokenizer_function(examples, apply_template=apply_template):
+ if not apply_template:
+ example = tokenizer(examples["content"], truncation=True, max_length=seqlen)
+ else:
+ from jinja2 import Template
+ chat_template = tokenizer.chat_template if tokenizer.chat_template is not None \
+ else tokenizer.default_chat_template
+ template = Template(chat_template)
+ rendered_messages = []
+ for text in examples["text"]:
+ message = [{"role": "user", "content": text}]
+ rendered_message = template.render(messages=message, add_generation_prompt=True, \
+ bos_token=tokenizer.bos_token)
+ rendered_messages.append(rendered_message)
+ example = tokenizer(rendered_messages, truncation=True, max_length=seqlen)
return example
return default_tokenizer_function
@@ -121,7 +158,7 @@ def get_new_chinese_title_dataset(tokenizer, seqlen, dataset_name="madao33/new-t
split = "train"
from datasets import load_dataset
- tokenizer_function = get_tokenizer_function(tokenizer, seqlen)
+ tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_template=apply_template)
calib_dataset = load_dataset(dataset_name, split=split)
calib_dataset = calib_dataset.shuffle(seed=seed)
@@ -131,7 +168,7 @@ def get_new_chinese_title_dataset(tokenizer, seqlen, dataset_name="madao33/new-t
@register_dataset("mbpp")
-def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42):
+def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42, apply_template=False):
"""Returns a dataloader for the specified dataset and split.
Args:
@@ -140,13 +177,14 @@ def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42
data_name: The name of the dataset.
split: The data split to be used (e.g., "train", "test").
seed: The random seed for shuffling the dataset.
+ apply_template: Whether to apply chat template in tokenization.
Returns:
A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.
"""
from datasets import load_dataset
- tokenizer_function = get_tokenizer_function(tokenizer, seqlen)
+ tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_template=apply_template)
samples = []
splits = split
@@ -169,7 +207,7 @@ def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42
@register_dataset("local")
-def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None, seed=42):
+def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None, seed=42, apply_template=False):
"""Returns a dataloader for a custom dataset and split.
We allow the input of a json or text file containing a processed text sample each line.
@@ -179,11 +217,12 @@ def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None,
data_name: The name or path of the dataset, which is a jsonl file.
split: The data split to be used (e.g., "train", "test").
seed: The random seed for shuffling the dataset.
+ apply_template: Whether to apply chat template in tokenization.
Returns:
A dataloader for a custom dataset and split, using the provided tokenizer and sequence length.
"""
- tokenizer_function = get_tokenizer_function(tokenizer, seqlen)
+ tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_template=apply_template)
def load_local_data(data_path):
if data_path.endswith(".json"):
@@ -227,7 +266,14 @@ def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None,
return calib_dataset
-def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, bs=8, nsamples=512):
+def get_dataloader(
+ tokenizer,
+ seqlen,
+ dataset_name="NeelNanda/pile-10k",
+ seed=42,
+ bs=8,
+ nsamples=512,
+):
"""Generate a DataLoader for calibration using specified parameters.
Args:
@@ -240,6 +286,7 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42
seed (int, optional): The random seed for reproducibility. Defaults to 42.
bs (int, optional): The batch size. Defaults to 4.
nsamples (int, optional): The total number of samples to include. Defaults to 512.
+ apply_template: Whether to apply chat template in tokenization.
Returns:
DataLoader: The DataLoader for the calibrated dataset.
@@ -304,6 +351,7 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42
for name in dataset_names:
split = None
do_concat = False
+ apply_template = False
if ":" in name:
split_list = name.split(":")
name, split_list = name.split(":")[0], name.split(":")[1:]
@@ -314,7 +362,9 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42
if key == "num":
data_lens[name] = int(values[0])
if key == "concat":
- do_concat = True if (len(values) > 0 and values[0].lower == 'false') else True
+ do_concat = False if (len(values) > 0 and values[0].lower() == 'false') else True
+ if key == "apply_template":
+ apply_template = False if (len(values) > 0 and values[0].lower() == 'false') else True
if is_local_path(name):
get_dataset = CALIB_DATASETS.get("local")
else:
@@ -325,6 +375,7 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42
seed=seed,
split=split,
dataset_name=name,
+ apply_template=apply_template,
)
dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])
if do_concat:
diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.md
index bfb5a9fd5..02ae14ae7 100644
--- a/examples/language-modeling/README.md
+++ b/examples/language-modeling/README.md
@@ -35,7 +35,7 @@ def customized_data_with_tokenizer(tokenizer, seqlen=2048):
return tokens
~~~
-We support combination of different datasets and parametrization of calibration datasets by using "--dataset ./tmp.json:concat,NeelNanda/pile-10k:split=train+val:num=256,mbpp:concat=True:num=128". Both local calibration file and huggingface dataset are supported. Through parametrization, users could specify splits of a dataset by setting "split=split1+split2". A concatenation option could enable users to merge calibration samples, a process commonly used to enhance calibration reliability. Please note that samples shorter than args.seqlen will be dropped when concatenation option is not enabled.
+We support combination of different datasets and parametrization of calibration datasets by using "--dataset ./tmp.json:concat,NeelNanda/pile-10k:split=train+val:num=256,mbpp:concat=True:num=128:apply_template". Both local calibration file and huggingface dataset are supported. Through parametrization, users could specify splits of a dataset by setting "split=split1+split2". A concatenation option could enable users to merge calibration samples, a process commonly used to enhance calibration reliability. An 'apply_template' option would enable users to apply chat_template to calibration data before tokenization and is widely used by instruct-models in generation. Please note that samples shorter than args.seqlen will be dropped when concatenation option is not enabled.
Please use ',' to split datasets, ':' to split parameters of a dataset and '+' to add values for one targeted parameter.