Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

feat: add OpenCodeInstruct calibration dataset (#2096)

Wang, Changca9524aeccec
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index e46e95032..94ae3aa7c 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -338,6 +338,108 @@ def get_github_code_clean_dataset(
return calib_dataset
+@register_dataset(["nvidia/OpenCodeInstruct", "opencode-instruct"])
+def get_opencode_instruct_dataset(
+ tokenizer,
+ seqlen,
+ dataset_name="nvidia/OpenCodeInstruct",
+ split=None,
+ seed=42,
+ apply_chat_template=False,
+ system_prompt=None,
+):
+ """Return tokenized coding instructions and responses from OpenCodeInstruct."""
+ split = "train" if split is None else split
+ if isinstance(split, list):
+ if len(split) != 1:
+ raise ValueError("OpenCodeInstruct supports only the train split.")
+ split = split[0]
+ if split != "train":
+ raise ValueError("OpenCodeInstruct supports only the train split.")
+
+ tokenizer_function = get_tokenizer_function(
+ tokenizer, seqlen, apply_chat_template=apply_chat_template, system_prompt=system_prompt
+ )
+
+ dataset = load_dataset("nvidia/OpenCodeInstruct", split=split, streaming=True)
+ dataset = dataset.shuffle(seed=seed).take(10000)
+ samples = []
+ for data in dataset:
+ if apply_chat_template:
+ messages = []
+ if system_prompt is not None and system_prompt != "":
+ messages.append({"role": "system", "content": system_prompt})
+ messages.extend(
+ [
+ {"role": "user", "content": data["input"]},
+ {"role": "assistant", "content": data["output"]},
+ ]
+ )
+ try:
+ text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
+ except:
+ logger.warning("Failed to apply chat template. removing the system role in chat history.")
+ messages = [message for message in messages if message["role"] != "system"]
+ text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
+ samples.append({"text": text})
+ else:
+ samples.append({"text": f"{data['input']}\n\n{data['output']}"})
+ random.Random(seed).shuffle(samples)
+
+ calib_dataset = Dataset.from_list(samples)
+ calib_dataset = calib_dataset.map(
+ tokenizer_function,
+ batched=True,
+ new_fingerprint=_make_map_fingerprint(
+ calib_dataset, tokenizer, seqlen, apply_chat_template, system_prompt, "text"
+ ),
+ )
+
+ calib_dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])
+ input_ids, concat_input_ids = [eg["input_ids"] for eg in calib_dataset], []
+ attention_mask_list, attention_mask = [], torch.ones([seqlen]).to(torch.int64)
+ buffer_input_id = torch.Tensor().to(torch.int64)
+ bos_token_id, eos_token_id = tokenizer.bos_token_id, tokenizer.eos_token_id
+ os_cnt, have_bos, have_eos = 0, False, False
+
+ for input_id in input_ids:
+ if input_id[0] == bos_token_id:
+ input_id = input_id[1:]
+ os_cnt, have_bos = os_cnt + 1, True
+ if input_id[-1] == eos_token_id:
+ input_id = input_id[:-1]
+ os_cnt, have_eos = os_cnt + 1, True
+
+ if buffer_input_id.shape[-1] + input_id.shape[-1] + os_cnt > seqlen:
+ idx_keep = seqlen - buffer_input_id.shape[-1] - os_cnt
+ input_id_to_append = [buffer_input_id, input_id[:idx_keep]]
+ if have_bos:
+ input_id_to_append = [torch.tensor([bos_token_id])] + input_id_to_append
+ if have_eos:
+ input_id_to_append.append(torch.tensor([eos_token_id]))
+
+ concat_input_ids.append(torch.cat(input_id_to_append).to(torch.int64).tolist())
+ attention_mask_list.append(attention_mask.tolist())
+ buffer_input_id = input_id[idx_keep:]
+ else:
+ buffer_input_id = torch.cat([buffer_input_id, input_id])
+
+ if buffer_input_id.shape[-1] + os_cnt == seqlen:
+ input_id_to_append = [buffer_input_id]
+ if have_bos:
+ input_id_to_append = [torch.tensor([bos_token_id])] + input_id_to_append
+ if have_eos:
+ input_id_to_append.append(torch.tensor([eos_token_id]))
+ concat_input_ids.append(torch.cat(input_id_to_append).to(torch.int64).tolist())
+ attention_mask_list.append(attention_mask.tolist())
+ buffer_input_id = torch.Tensor().to(torch.int64)
+
+ data = [{"input_ids": a, "attention_mask": b} for a, b in zip(concat_input_ids, attention_mask_list)]
+ calib_dataset = Dataset.from_list(data)
+
+ return calib_dataset
+
+
@register_dataset(["HuggingFaceH4/ultrachat_200k", "ultrachat_200k"])
def get_ultrachat_dataset(
tokenizer,
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 1c9f38706..74a178834 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -68,6 +68,7 @@ calibration data and will be downloaded automatically from the datasets Hub. Oth
- `HuggingFaceH4/ultrachat_200k` for chat data
- `madao33/new-title-chinese` for Chinese
- `mbpp` for code
+- `nvidia/OpenCodeInstruct` for code instruction data
- `openbmb/Ultra-FineWeb`
### Customized Dataset
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index 1d9dda39f..9163b93d8 100644
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -69,6 +69,7 @@ pip install auto-round
- `HuggingFaceH4/ultrachat_200k`:用于对话数据
- `madao33/new-title-chinese`:用于中文场景
- `mbpp`:用于代码场景
+- `nvidia/OpenCodeInstruct`:用于代码指令数据
- `openbmb/Ultra-FineWeb`
### 自定义数据集
diff --git a/test/test_cpu/utils/test_calib_dataset.py b/test/test_cpu/utils/test_calib_dataset.py
index adc129f8d..8be78742e 100644
--- a/test/test_cpu/utils/test_calib_dataset.py
+++ b/test/test_cpu/utils/test_calib_dataset.py
@@ -82,7 +82,7 @@ class TestLocalCalibDataset:
autoround.quantize()
def test_combine_dataset(self, tiny_qwen_model_path):
- dataset = "NeelNanda/pile-10k" + "," + "madao33/new-title-chinese" + "," + "mbpp"
+ dataset = "NeelNanda/pile-10k" + "," + "madao33/new-title-chinese" + "," + "mbpp" + "," + "opencode-instruct"
bits, group_size, sym = 4, 128, True
autoround = AutoRound(
tiny_qwen_model_path,