PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add chat template in calib tokenization (#171)
yintong-lu735dfc9e59c3
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex fe8a1be6e..b71301f4b 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -42,26 +42,40 @@ def register_dataset(name):return register-def get_tokenizer_function(tokenizer, seqlen):+def get_tokenizer_function(tokenizer, seqlen, apply_template=False):"""Returns a default tokenizer function.Args:tokenizer: The tokenizer to be used for tokenization.seqlen: The maximum sequence length.+ apply_template: Whether to apply chat template in tokenization.Returns: A default tokenizer function that applies the provided tokenizer with truncation and a maximum length ofseqlen to the "text" field of examples."""- def default_tokenizer_function(examples):- example = tokenizer(examples["text"], truncation=True, max_length=seqlen)+ def default_tokenizer_function(examples, apply_template=apply_template):+ if not apply_template:+ example = tokenizer(examples["text"], truncation=True, max_length=seqlen)+ else:+ from jinja2 import Template # pylint: disable=E0401+ chat_template = tokenizer.chat_template if tokenizer.chat_template is not None \+ else tokenizer.default_chat_template+ template = Template(chat_template)+ rendered_messages = []+ for text in examples["text"]:+ message = [{"role": "user", "content": text}]+ rendered_message = template.render(messages=message, add_generation_prompt=True, \+ bos_token=tokenizer.bos_token)+ rendered_messages.append(rendered_message)+ example = tokenizer(rendered_messages, truncation=True, max_length=seqlen)return examplereturn default_tokenizer_function@register_dataset("NeelNanda/pile-10k")-def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=None, seed=42):+def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split=None, seed=42, apply_template=False):"""Returns a dataloader for the specified dataset and split.Args:@@ -70,6 +84,7 @@ def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", splitdata_name: The name of the dataset.split: The data split to be used (e.g., "train", "test").seed: The random seed for shuffling the dataset.+ apply_template: Whether to apply chat template in tokenization.Returns:A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.@@ -77,7 +92,7 @@ def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", splitfrom datasets import load_datasetsplit = "train"- tokenizer_function = get_tokenizer_function(tokenizer, seqlen)+ tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_template=apply_template)calib_dataset = load_dataset(dataset_name, split=split)calib_dataset = calib_dataset.shuffle(seed=seed)@@ -87,7 +102,14 @@ def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split@register_dataset("madao33/new-title-chinese")-def get_new_chinese_title_dataset(tokenizer, seqlen, dataset_name="madao33/new-title-chinese", split=None, seed=42):+def get_new_chinese_title_dataset(+ tokenizer,+ seqlen,+ dataset_name="madao33/new-title-chinese",+ split=None,+ seed=42,+ apply_template=False+):"""Returns a dataloader for the specified dataset and split.Args:@@ -96,24 +118,39 @@ def get_new_chinese_title_dataset(tokenizer, seqlen, dataset_name="madao33/new-tdata_name: The name of the dataset.split: The data split to be used (e.g., "train", "test").seed: The random seed for shuffling the dataset.+ apply_template: Whether to apply chat template in tokenization.Returns:A dataloader for the specified dataset and split, using the provided tokenizer and sequence length."""- def get_tokenizer_function(tokenizer, seqlen):+ def get_tokenizer_function(tokenizer, seqlen, apply_template=apply_template):"""Returns a default tokenizer function.Args:tokenizer: The tokenizer to be used for tokenization.seqlen: The maximum sequence length.+ apply_template: Whether to apply chat template in tokenization.Returns: A default tokenizer function that applies the provided tokenizer with truncation and a maximum lengthof seqlen to the "text" field of examples."""- def default_tokenizer_function(examples):- example = tokenizer(examples["content"], truncation=True, max_length=seqlen)+ def default_tokenizer_function(examples, apply_template=apply_template):+ if not apply_template:+ example = tokenizer(examples["content"], truncation=True, max_length=seqlen)+ else:+ from jinja2 import Template+ chat_template = tokenizer.chat_template if tokenizer.chat_template is not None \+ else tokenizer.default_chat_template+ template = Template(chat_template)+ rendered_messages = []+ for text in examples["text"]:+ message = [{"role": "user", "content": text}]+ rendered_message = template.render(messages=message, add_generation_prompt=True, \+ bos_token=tokenizer.bos_token)+ rendered_messages.append(rendered_message)+ example = tokenizer(rendered_messages, truncation=True, max_length=seqlen)return examplereturn default_tokenizer_function@@ -121,7 +158,7 @@ def get_new_chinese_title_dataset(tokenizer, seqlen, dataset_name="madao33/new-tsplit = "train"from datasets import load_dataset- tokenizer_function = get_tokenizer_function(tokenizer, seqlen)+ tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_template=apply_template)calib_dataset = load_dataset(dataset_name, split=split)calib_dataset = calib_dataset.shuffle(seed=seed)@@ -131,7 +168,7 @@ def get_new_chinese_title_dataset(tokenizer, seqlen, dataset_name="madao33/new-t@register_dataset("mbpp")-def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42):+def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42, apply_template=False):"""Returns a dataloader for the specified dataset and split.Args:@@ -140,13 +177,14 @@ def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42data_name: The name of the dataset.split: The data split to be used (e.g., "train", "test").seed: The random seed for shuffling the dataset.+ apply_template: Whether to apply chat template in tokenization.Returns:A dataloader for the specified dataset and split, using the provided tokenizer and sequence length."""from datasets import load_dataset- tokenizer_function = get_tokenizer_function(tokenizer, seqlen)+ tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_template=apply_template)samples = []splits = split@@ -169,7 +207,7 @@ def get_mbpp_dataset(tokenizer, seqlen, dataset_name="mbpp", split=None, seed=42@register_dataset("local")-def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None, seed=42):+def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None, seed=42, apply_template=False):"""Returns a dataloader for a custom dataset and split.We allow the input of a json or text file containing a processed text sample each line.@@ -179,11 +217,12 @@ def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None,data_name: The name or path of the dataset, which is a jsonl file.split: The data split to be used (e.g., "train", "test").seed: The random seed for shuffling the dataset.+ apply_template: Whether to apply chat template in tokenization.Returns:A dataloader for a custom dataset and split, using the provided tokenizer and sequence length."""- tokenizer_function = get_tokenizer_function(tokenizer, seqlen)+ tokenizer_function = get_tokenizer_function(tokenizer, seqlen, apply_template=apply_template)def load_local_data(data_path):if data_path.endswith(".json"):@@ -227,7 +266,14 @@ def get_local_dataset(tokenizer, seqlen, dataset_name="./tmp.json", split=None,return calib_dataset-def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, bs=8, nsamples=512):+def get_dataloader(+ tokenizer,+ seqlen,+ dataset_name="NeelNanda/pile-10k",+ seed=42,+ bs=8,+ nsamples=512,+):"""Generate a DataLoader for calibration using specified parameters.Args:@@ -240,6 +286,7 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42seed (int, optional): The random seed for reproducibility. Defaults to 42.bs (int, optional): The batch size. Defaults to 4.nsamples (int, optional): The total number of samples to include. Defaults to 512.+ apply_template: Whether to apply chat template in tokenization.Returns:DataLoader: The DataLoader for the calibrated dataset.@@ -304,6 +351,7 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42for name in dataset_names:split = Nonedo_concat = False+ apply_template = Falseif ":" in name:split_list = name.split(":")name, split_list = name.split(":")[0], name.split(":")[1:]@@ -314,7 +362,9 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42if key == "num":data_lens[name] = int(values[0])if key == "concat":- do_concat = True if (len(values) > 0 and values[0].lower == 'false') else True+ do_concat = False if (len(values) > 0 and values[0].lower() == 'false') else True+ if key == "apply_template":+ apply_template = False if (len(values) > 0 and values[0].lower() == 'false') else Trueif is_local_path(name):get_dataset = CALIB_DATASETS.get("local")else:@@ -325,6 +375,7 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42seed=seed,split=split,dataset_name=name,+ apply_template=apply_template,)dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])if do_concat:diff --git a/examples/language-modeling/README.md b/examples/language-modeling/README.mdindex bfb5a9fd5..02ae14ae7 100644--- a/examples/language-modeling/README.md+++ b/examples/language-modeling/README.md@@ -35,7 +35,7 @@ def customized_data_with_tokenizer(tokenizer, seqlen=2048):return tokens~~~-We support combination of different datasets and parametrization of calibration datasets by using "--dataset ./tmp.json:concat,NeelNanda/pile-10k:split=train+val:num=256,mbpp:concat=True:num=128". Both local calibration file and huggingface dataset are supported. Through parametrization, users could specify splits of a dataset by setting "split=split1+split2". A concatenation option could enable users to merge calibration samples, a process commonly used to enhance calibration reliability. Please note that samples shorter than args.seqlen will be dropped when concatenation option is not enabled.+We support combination of different datasets and parametrization of calibration datasets by using "--dataset ./tmp.json:concat,NeelNanda/pile-10k:split=train+val:num=256,mbpp:concat=True:num=128:apply_template". Both local calibration file and huggingface dataset are supported. Through parametrization, users could specify splits of a dataset by setting "split=split1+split2". A concatenation option could enable users to merge calibration samples, a process commonly used to enhance calibration reliability. An 'apply_template' option would enable users to apply chat_template to calibration data before tokenization and is widely used by instruct-models in generation. Please note that samples shorter than args.seqlen will be dropped when concatenation option is not enabled.Please use ',' to split datasets, ':' to split parameters of a dataset and '+' to add values for one targeted parameter.