PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix: avoid excessive memory during calibration dataset casting (#2405)
Wang, Chang22d6e28c0133
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 49f59a8c8..1ced3542b 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -995,6 +995,13 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed"""dataset_names = dataset_name.split(",")+ def cast_dataset_columns(dataset):+ dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])+ features = dict(dataset.features)+ features["input_ids"] = Sequence(Value("int64"))+ features["attention_mask"] = Sequence(Value("int8"))+ return dataset.cast(Features(features))+def filter_func(example):if isinstance(example["input_ids"], list):example["input_ids"] = torch.tensor(example["input_ids"])@@ -1106,27 +1113,22 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seedif do_concat:dataset = concat_dataset_element(dataset)- dataset = dataset.filter(filter_func)- if name in data_lens:- dataset = select_dataset(dataset, range(data_lens[name]))if isinstance(dataset, IterableDataset):- # A single dataset source never contributes more than `nsamples` rows to the- # final combined dataset, so cap materialization here instead of fully consuming- # the (much larger) internal `.take(...)` pool used by streaming dataset loaders.- # This avoids needless downloads/shard resolution for large remote datasets- # (e.g. BAAI/CCI3-HQ) when only a small subset of samples is actually needed.- dataset = Dataset.from_list(list(itertools.islice(dataset, nsamples)))- dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])- new_features = {}- for k, v in dataset.features.items():- if k == "input_ids":- new_features[k] = Sequence(Value("int64"))- elif k == "attention_mask":- new_features[k] = Sequence(Value("int8"))- else:- new_features[k] = v-- dataset = dataset.cast(Features(new_features))+ # Filter and limit the stream before materializing it in memory.+ dataset = dataset.filter(filter_func)+ if name in data_lens:+ dataset = select_dataset(dataset, range(data_lens[name]))+ # select_dataset may have already materialized the requested rows.+ if isinstance(dataset, IterableDataset):+ dataset = Dataset.from_list(list(itertools.islice(dataset, nsamples)))+ dataset = cast_dataset_columns(dataset)+ else:+ # Cast before filter/select creates indices, avoiding a full values+ # buffer conversion for each indexed Arrow row.+ dataset = cast_dataset_columns(dataset)+ dataset = dataset.filter(filter_func)+ if name in data_lens:+ dataset = select_dataset(dataset, range(data_lens[name]))datasets.append(dataset)if len(datasets) == 1: