PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
feat: add OpenCodeInstruct calibration dataset (#2096)
Wang, Changca9524aeccec
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex e46e95032..94ae3aa7c 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -338,6 +338,108 @@ def get_github_code_clean_dataset(return calib_dataset+@register_dataset(["nvidia/OpenCodeInstruct", "opencode-instruct"])+def get_opencode_instruct_dataset(+ tokenizer,+ seqlen,+ dataset_name="nvidia/OpenCodeInstruct",+ split=None,+ seed=42,+ apply_chat_template=False,+ system_prompt=None,+):+ """Return tokenized coding instructions and responses from OpenCodeInstruct."""+ split = "train" if split is None else split+ if isinstance(split, list):+ if len(split) != 1:+ raise ValueError("OpenCodeInstruct supports only the train split.")+ split = split[0]+ if split != "train":+ raise ValueError("OpenCodeInstruct supports only the train split.")++ tokenizer_function = get_tokenizer_function(+ tokenizer, seqlen, apply_chat_template=apply_chat_template, system_prompt=system_prompt+ )++ dataset = load_dataset("nvidia/OpenCodeInstruct", split=split, streaming=True)+ dataset = dataset.shuffle(seed=seed).take(10000)+ samples = []+ for data in dataset:+ if apply_chat_template:+ messages = []+ if system_prompt is not None and system_prompt != "":+ messages.append({"role": "system", "content": system_prompt})+ messages.extend(+ [+ {"role": "user", "content": data["input"]},+ {"role": "assistant", "content": data["output"]},+ ]+ )+ try:+ text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)+ except:+ logger.warning("Failed to apply chat template. removing the system role in chat history.")+ messages = [message for message in messages if message["role"] != "system"]+ text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)+ samples.append({"text": text})+ else:+ samples.append({"text": f"{data['input']}\n\n{data['output']}"})+ random.Random(seed).shuffle(samples)++ calib_dataset = Dataset.from_list(samples)+ calib_dataset = calib_dataset.map(+ tokenizer_function,+ batched=True,+ new_fingerprint=_make_map_fingerprint(+ calib_dataset, tokenizer, seqlen, apply_chat_template, system_prompt, "text"+ ),+ )++ calib_dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])+ input_ids, concat_input_ids = [eg["input_ids"] for eg in calib_dataset], []+ attention_mask_list, attention_mask = [], torch.ones([seqlen]).to(torch.int64)+ buffer_input_id = torch.Tensor().to(torch.int64)+ bos_token_id, eos_token_id = tokenizer.bos_token_id, tokenizer.eos_token_id+ os_cnt, have_bos, have_eos = 0, False, False++ for input_id in input_ids:+ if input_id[0] == bos_token_id:+ input_id = input_id[1:]+ os_cnt, have_bos = os_cnt + 1, True+ if input_id[-1] == eos_token_id:+ input_id = input_id[:-1]+ os_cnt, have_eos = os_cnt + 1, True++ if buffer_input_id.shape[-1] + input_id.shape[-1] + os_cnt > seqlen:+ idx_keep = seqlen - buffer_input_id.shape[-1] - os_cnt+ input_id_to_append = [buffer_input_id, input_id[:idx_keep]]+ if have_bos:+ input_id_to_append = [torch.tensor([bos_token_id])] + input_id_to_append+ if have_eos:+ input_id_to_append.append(torch.tensor([eos_token_id]))++ concat_input_ids.append(torch.cat(input_id_to_append).to(torch.int64).tolist())+ attention_mask_list.append(attention_mask.tolist())+ buffer_input_id = input_id[idx_keep:]+ else:+ buffer_input_id = torch.cat([buffer_input_id, input_id])++ if buffer_input_id.shape[-1] + os_cnt == seqlen:+ input_id_to_append = [buffer_input_id]+ if have_bos:+ input_id_to_append = [torch.tensor([bos_token_id])] + input_id_to_append+ if have_eos:+ input_id_to_append.append(torch.tensor([eos_token_id]))+ concat_input_ids.append(torch.cat(input_id_to_append).to(torch.int64).tolist())+ attention_mask_list.append(attention_mask.tolist())+ buffer_input_id = torch.Tensor().to(torch.int64)++ data = [{"input_ids": a, "attention_mask": b} for a, b in zip(concat_input_ids, attention_mask_list)]+ calib_dataset = Dataset.from_list(data)++ return calib_dataset++@register_dataset(["HuggingFaceH4/ultrachat_200k", "ultrachat_200k"])def get_ultrachat_dataset(tokenizer,diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 1c9f38706..74a178834 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -68,6 +68,7 @@ calibration data and will be downloaded automatically from the datasets Hub. Oth- `HuggingFaceH4/ultrachat_200k` for chat data- `madao33/new-title-chinese` for Chinese- `mbpp` for code+- `nvidia/OpenCodeInstruct` for code instruction data- `openbmb/Ultra-FineWeb`### Customized Datasetdiff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex 1d9dda39f..9163b93d8 100644--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -69,6 +69,7 @@ pip install auto-round- `HuggingFaceH4/ultrachat_200k`:用于对话数据- `madao33/new-title-chinese`:用于中文场景- `mbpp`:用于代码场景+- `nvidia/OpenCodeInstruct`:用于代码指令数据- `openbmb/Ultra-FineWeb`### 自定义数据集diff --git a/test/test_cpu/utils/test_calib_dataset.py b/test/test_cpu/utils/test_calib_dataset.pyindex adc129f8d..8be78742e 100644--- a/test/test_cpu/utils/test_calib_dataset.py+++ b/test/test_cpu/utils/test_calib_dataset.py@@ -82,7 +82,7 @@ class TestLocalCalibDataset:autoround.quantize()def test_combine_dataset(self, tiny_qwen_model_path):- dataset = "NeelNanda/pile-10k" + "," + "madao33/new-title-chinese" + "," + "mbpp"+ dataset = "NeelNanda/pile-10k" + "," + "madao33/new-title-chinese" + "," + "mbpp" + "," + "opencode-instruct"bits, group_size, sym = 4, 128, Trueautoround = AutoRound(tiny_qwen_model_path,