PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[Experimental]Add generic HF dataset support and two more datasets (#2363)
Wenhua Chenge31b77284e8b
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 604a28ae3..472706173 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -16,7 +16,7 @@ from __future__ import annotationsimport functoolsimport inspect-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, Optional, Unionimport torch@@ -30,13 +30,14 @@ from auto_round.scheme_entry import (resolve_entry_scheme,)from auto_round.schemes import QuantizationScheme+from auto_round.utils.dataset_utils import CalibDataset, normalize_dataset_specfrom auto_round.utils.device_manager import normalize_default_device_mapif TYPE_CHECKING:from auto_round.algorithms.quantization.config import QuantizationConfigfrom auto_round.algorithms.quantization.rtn.config import RTNConfigfrom auto_round.auto_scheme.gen_auto_scheme import AutoScheme- from auto_round.compressors.base import BaseOrchestrator as BaseCompressor+ from auto_round.compressors.base import BaseCompressor# ---------------------------------------------------------------------------@@ -541,7 +542,7 @@ class _CompressorBuilder:tokenizer=None,platform="hf",format=None,- dataset="NeelNanda/pile-10k",+ dataset: str | CalibDataset | list | tuple | torch.utils.data.DataLoader | None = None,low_gpu_mem_usage: bool = False,device_map: str | torch.device | int | dict = 0,iters: int | None = None,@@ -558,8 +559,15 @@ class _CompressorBuilder:from auto_round.algorithms.registry import normalize_algorithm_configfrom auto_round.compressors.orchestrator import CompressionOrchestrator as Compressorfrom auto_round.compressors.utils import check_need_act_calibration+ from auto_round.utils.dataset_utils import CalibDatasetfrom auto_round.utils.model import is_model_free_route+ # Normalize CalibDataset objects to spec strings+ if isinstance(dataset, CalibDataset):+ dataset = dataset.to_spec_string()+ elif isinstance(dataset, (list, tuple)):+ dataset = normalize_dataset_spec(dataset)+if alg_configs is None:alg_configs = "signround"# TODO wenhuach if key in kwargs could override scheme and alg_config, we should pop and override,@@ -714,7 +722,8 @@ class AutoRound:Use config instances to provide algorithm-specific options, suchas ``SignRoundConfig(iters=50)`` or ``AWQConfig(apply_clip=True)``.layer_config: Optional per-layer quantization overrides.- dataset: Calibration dataset name, samples, or dataloader.+ dataset: Calibration dataset name, samples, dataloader, or a+ :class:`~auto_round.utils.dataset_utils.CalibDataset` object.seqlen: Calibration sequence length.nsamples: Number of calibration samples.batch_size: Calibration batch size.diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 499e630eb..09df68453 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -29,8 +29,10 @@ logging.getLogger("datasets").setLevel(logging.WARNING)import torchfrom datasets import (Dataset,+ DatasetDict,Features,IterableDataset,+ IterableDatasetDict,Sequence,Value,concatenate_datasets,@@ -42,6 +44,15 @@ from torch.utils.data import DataLoaderfrom . import envsfrom .utils import is_local_path, logger+from .utils.dataset_utils import (+ CalibDataset,+ auto_detect_text_field,+ auto_detect_text_field_from_sample,+ extract_text_from_sample,+ normalize_dataset_spec,+ split_unescaped,+ unescape_spec_value,+)CALIB_DATASETS = {}_GITHUB_CODE_CLEAN_MAX_DATASETS_VERSION = Version("3.6.0")@@ -84,7 +95,7 @@ def _get_dataset_network_error(error):return None-def _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples):+def _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code=False):"""Switch to ModelScope FineWeb-Edu after a calibration dataset network failure."""network_error = _get_dataset_network_error(error)if network_error is None:@@ -108,15 +119,17 @@ def _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsampnetwork_error,_FINEWEB_EDU_MODELSCOPE_DATASET,)- return _get_dataset_impl(tokenizer, seqlen, _FINEWEB_EDU_MODELSCOPE_DATASET, seed, nsamples)+ return _get_dataset_impl(tokenizer, seqlen, _FINEWEB_EDU_MODELSCOPE_DATASET, seed, nsamples, trust_remote_code)-def _preprocess_dataset_in_subprocess(result_queue, tokenizer, seqlen, dataset_name, seed, nsamples, output_path):+def _preprocess_dataset_in_subprocess(+ result_queue, tokenizer, seqlen, dataset_name, seed, nsamples, output_path, trust_remote_code=False+):"""Run dataset preprocessing and report network failures to the parent process."""# The parent's OpenMP thread pool does not survive fork; keep torch single-threaded here.torch.set_num_threads(1)try:- dataset = _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)+ dataset = _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code)dataset.save_to_disk(output_path)except Exception as error:network_error = _get_dataset_network_error(error)@@ -175,35 +188,83 @@ def register_dataset(name):return register-def apply_chat_template_to_samples(samples, tokenizer, seqlen, system_prompt=None):- rendered_messages = []- # if system_prompt is None: ## remove system prompt as models like deepseek don't recommend using it- # system_prompt = "You are a helpful assistant."- for text in samples:- message = []- if system_prompt is not None and system_prompt != "":- message.append({"role": "system", "content": system_prompt})+def render_chat_sample(tokenizer, text, system_prompt=None, tools=None):+ """Render a single conversation to a string with the chat template.- if isinstance(text, list) and isinstance(text[0], dict):- message += text- else:- message.append({"role": "user", "content": text})+ Args:+ tokenizer: The tokenizer providing ``apply_chat_template``.+ text: Either a plain user string or a list of chat messages (dicts with+ ``role``/``content``).+ system_prompt: Optional global system prompt. It is only prepended when+ the conversation does not already start with its own system turn.+ tools: Optional tool definitions for this conversation, passed to+ ``apply_chat_template(..., tools=...)``.++ Returns:+ The rendered chat string.+ """+ message = []+ is_full_conversation = isinstance(text, list) and len(text) > 0 and isinstance(text[0], dict)++ # Respect a per-sample system prompt: if the conversation already starts+ # with its own system message, do not prepend the global system_prompt+ # (otherwise the rendered text would contain two system turns).+ has_own_system = is_full_conversation and text[0].get("role") == "system"+ if system_prompt is not None and system_prompt != "" and not has_own_system:+ message.append({"role": "system", "content": system_prompt})++ if is_full_conversation:+ message += text+ else:+ message.append({"role": "user", "content": text})++ sample_tools = tools or None++ # For full multi-turn conversations ending in an assistant reply we want+ # the whole trace as calibration text (including tool_calls / tool+ # responses), so we do not append a generation prompt.+ add_generation_prompt = not (is_full_conversation and message[-1].get("role") == "assistant")++ template_kwargs = {"tokenize": False, "add_generation_prompt": add_generation_prompt}+ if sample_tools is not None:+ template_kwargs["tools"] = sample_tools++ try:+ return tokenizer.apply_chat_template(message, **template_kwargs)+ except Exception:+ logger.warning("Failed to apply chat template. removing the system role in chat history.")+ message_modified = [msg for msg in message if msg["role"] != "system"]try:- chat_templated = tokenizer.apply_chat_template(- message,- tokenize=False,- add_generation_prompt=True,- )+ return tokenizer.apply_chat_template(message_modified, **template_kwargs)except Exception:- logger.warning("Failed to apply chat template. removing the system role in chat history.")- message_modified = [msg for msg in message if msg["role"] != "system"]- chat_templated = tokenizer.apply_chat_template(- message_modified,- tokenize=False,- add_generation_prompt=True,- )+ # Last resort: drop tools in case the template does not support them.+ template_kwargs.pop("tools", None)+ return tokenizer.apply_chat_template(message_modified, **template_kwargs)+++def apply_chat_template_to_samples(samples, tokenizer, seqlen, system_prompt=None, tools=None):+ """Render samples with the tokenizer chat template.- rendered_messages.append(chat_templated)+ Args:+ samples: A list where each item is either a plain user string or a list+ of chat messages (dicts with ``role``/``content``).+ tokenizer: The tokenizer providing ``apply_chat_template``.+ seqlen: Maximum sequence length for tokenization.+ system_prompt: Optional system prompt prepended to each conversation.+ tools: Optional per-sample tool definitions aligned with ``samples``+ (a list the same length as ``samples``; each entry is the ``tools``+ list for that conversation, or ``None``). Passed to+ ``apply_chat_template(..., tools=...)`` so agent/tool-calling data+ keeps its tool schema. When the conversation already contains the+ final assistant turn (full multi-turn agent traces), the generation+ prompt is not appended.+ """+ rendered_messages = []+ # if system_prompt is None: ## remove system prompt as models like deepseek don't recommend using it+ # system_prompt = "You are a helpful assistant."+ for idx, text in enumerate(samples):+ sample_tools = tools[idx] if (tools is not None and idx < len(tools)) else None+ rendered_messages.append(render_chat_sample(tokenizer, text, system_prompt, sample_tools))example = tokenizer(rendered_messages, truncation=True, max_length=seqlen)return example@@ -370,7 +431,14 @@ def get_fineweb_edu_dataset(@register_dataset(["BAAI/CCI3-HQ", "CCI3-HQ"])def get_cci3_hq_dataset(- tokenizer, seqlen, dataset_name="BAAI/CCI3-HQ", split=None, seed=42, apply_chat_template=False, system_prompt=None+ tokenizer,+ seqlen,+ dataset_name="BAAI/CCI3-HQ",+ split=None,+ seed=42,+ apply_chat_template=False,+ system_prompt=None,+ nsamples=512,):"""Returns a dataloader for the specified dataset and split.@@ -392,7 +460,7 @@ def get_cci3_hq_dataset()calib_dataset = load_dataset("BAAI/CCI3-HQ", split="train", streaming=True)- calib_dataset = calib_dataset.shuffle(seed=seed).take(10000)+ calib_dataset = calib_dataset.shuffle(seed=seed).take(nsamples * envs.AR_CALIB_DATA_MULTIPLIER)calib_dataset = calib_dataset.map(tokenizer_function, batched=True)return calib_dataset@@ -407,6 +475,7 @@ def get_github_code_clean_dataset(seed=42,apply_chat_template=False,system_prompt=None,+ nsamples=512,):"""Returns a dataloader for the specified dataset and split.@@ -462,7 +531,9 @@ def get_github_code_clean_dataset(else:raisecalib_dataset = concatenate_datasets([dataset_mit, dataset_apache])- calib_dataset = calib_dataset.shuffle(seed=seed).take(10000) ##TODO concat data'shuffle may have bugs+ calib_dataset = calib_dataset.shuffle(seed=seed).take(+ nsamples * envs.AR_CALIB_DATA_MULTIPLIER+ ) ##TODO concat data'shuffle may have bugscalib_dataset = calib_dataset.map(tokenizer_function, batched=True)return calib_dataset@@ -477,6 +548,7 @@ def get_opencode_instruct_dataset(seed=42,apply_chat_template=False,system_prompt=None,+ nsamples=512,):"""Return tokenized coding instructions and responses from OpenCodeInstruct."""split = "train" if split is None else split@@ -492,7 +564,7 @@ def get_opencode_instruct_dataset()dataset = load_dataset("nvidia/OpenCodeInstruct", split=split, streaming=True)- dataset = dataset.shuffle(seed=seed).take(10000)+ dataset = dataset.shuffle(seed=seed).take(nsamples * envs.AR_CALIB_DATA_MULTIPLIER)samples = []for data in dataset:if apply_chat_template:@@ -570,6 +642,103 @@ def get_opencode_instruct_dataset(return calib_dataset+@register_dataset(["qwedsacf/competition_math", "competition_math"])+def get_competition_math_dataset(+ tokenizer,+ seqlen,+ dataset_name="qwedsacf/competition_math",+ split=None,+ seed=42,+ apply_chat_template=False,+ system_prompt=None,+ nsamples=512,+):+ """Return a calibration dataset built from the competition_math corpus.++ Each sample concatenates the ``problem`` and ``solution`` fields. Because the+ individual problem/solution pairs are much shorter than typical calibration+ sequence lengths (e.g. 2048), the tokenized samples are packed together into+ full ``seqlen``-length sequences instead of being dropped by the downstream+ length filter.++ Args:+ tokenizer: The tokenizer to be used for tokenization.+ seqlen: The maximum/target sequence length.+ dataset_name: Dataset identifier (defaults to ``qwedsacf/competition_math``).+ split: The data split to use. Defaults to ``train``.+ seed: The random seed for shuffling.+ apply_chat_template: Whether to apply the tokenizer's chat template.+ system_prompt: Optional system prompt for the chat template.+ nsamples: Number of calibration samples requested. The candidate pool is+ ``nsamples * AR_CALIB_DATA_MULTIPLIER`` before packing.++ Returns:+ A HuggingFace ``Dataset`` of packed, tokenized calibration sequences.+ """+ split = "train" if split is None else split+ if isinstance(split, list):+ if len(split) != 1:+ raise ValueError("competition_math supports only one split at a time.")+ split = split[0]++ # The chat template (when requested) is already rendered per-sample in the+ # loop below, so tokenize the resulting "text" plainly here to avoid applying+ # the template a second time (which would duplicate role markers/prompts).+ tokenizer_function = get_tokenizer_function(tokenizer, seqlen)++ # competition_math is small; load it fully instead of streaming.+ # Always use the canonical Hub repo id (the short "competition_math" alias+ # is only a registry key, not a valid Hub dataset name).+ dataset = load_dataset("qwedsacf/competition_math", split=split, trust_remote_code=False)+ dataset = dataset.shuffle(seed=seed)+ max_candidates = nsamples * envs.AR_CALIB_DATA_MULTIPLIER+ if len(dataset) > max_candidates:+ dataset = dataset.select(range(max_candidates))++ samples = []+ for data in dataset:+ problem = (data.get("problem") or "").strip()+ solution = (data.get("solution") or "").strip()+ if not problem and not solution:+ continue+ if apply_chat_template:+ messages = []+ if system_prompt is not None and system_prompt != "":+ messages.append({"role": "system", "content": system_prompt})+ messages.extend(+ [+ {"role": "user", "content": problem},+ {"role": "assistant", "content": solution},+ ]+ )+ try:+ text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)+ except Exception:+ logger.warning("Failed to apply chat template. removing the system role in chat history.")+ messages = [message for message in messages if message["role"] != "system"]+ text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)+ samples.append({"text": text})+ else:+ samples.append({"text": f"{problem}\n\n{solution}"})+ random.Random(seed).shuffle(samples)++ calib_dataset = Dataset.from_list(samples)+ calib_dataset = calib_dataset.map(+ tokenizer_function,+ batched=True,+ new_fingerprint=_make_map_fingerprint(+ calib_dataset, tokenizer, seqlen, apply_chat_template, system_prompt, "text"+ ),+ )++ # Sample-level concat by default: pack whole samples until each reaches+ # seqlen, then reset, so short problem/solution pairs are not dropped by the+ # downstream length filter.+ calib_dataset = _concat_samples_to_seqlen(calib_dataset, seqlen)++ return calib_dataset++@register_dataset(["HuggingFaceH4/ultrachat_200k", "ultrachat_200k"])def get_ultrachat_dataset(tokenizer,@@ -579,6 +748,7 @@ def get_ultrachat_dataset(seed=42,apply_chat_template=True,system_prompt=None,+ nsamples=512,):if split is None:split = "train_sft"@@ -587,7 +757,7 @@ def get_ultrachat_dataset(raise ValueError(f"split must be one of {all_splits} for ultrachat_200k ")dataset = load_dataset("HuggingFaceH4/ultrachat_200k", split=split, streaming=True, trust_remote_code=True)- dataset = dataset.shuffle(seed=seed).take(20000)+ dataset = dataset.shuffle(seed=seed).take(nsamples * envs.AR_CALIB_DATA_MULTIPLIER)def is_instruct_tokenizer(tokenizer):try:@@ -598,12 +768,9 @@ def get_ultrachat_dataset(is_instruct = is_instruct_tokenizer(tokenizer)- if is_instruct and not apply_chat_template:- logger.info("Tokenizer looks like an instruct/chat model, but apply_chat_template=False. Setting to True.")- apply_chat_template = True- elif not is_instruct and apply_chat_template:+ if not is_instruct and apply_chat_template:logger.info("Tokenizer is not an instruct/chat model, but apply_chat_template=True. Setting to False.")- apply_chat_template = False+ apply_chat_template = Falsedef tokenize_example_batch(examples):if not apply_chat_template:@@ -619,6 +786,250 @@ def get_ultrachat_dataset(return dataset+@register_dataset(["openbmb/UltraData-SFT-Agent-2609", "UltraData-SFT-Agent-2609"])+def get_ultradata_agent_dataset(+ tokenizer,+ seqlen,+ dataset_name="openbmb/UltraData-SFT-Agent-2609",+ split=None,+ seed=42,+ apply_chat_template=True,+ system_prompt=None,+ nsamples=512,+):+ """Agent/tool-calling calibration dataset.++ Each sample carries a ``messages`` conversation (system/user/assistant with+ ``tool_calls``/tool turns) and a ``tools`` schema list. The chat template is+ rendered with ``tools=`` so the quantized model sees full tool-calling+ traces. ``apply_chat_template`` is forced on because the raw ``messages``+ structure is only meaningful once templated.++ The dataset ships multiple configs (``Code-Agent``, ``Search-Agent``,+ ``General-Agent``, ``Tool-Use``); all of them are loaded and interleaved so+ calibration covers the full range of agent behaviors.++ Length-aware sampling: conversations in this dataset can be very long. A+ sample whose *messages alone* already exceed ``seqlen`` would have its actual+ conversation content truncated, so such samples are deprioritized. We first+ collect samples whose rendered messages fit within ``seqlen`` (even if+ ``messages + tools`` exceeds it, since only the tool schema overflows); only+ if there are not enough of those do we fall back to the oversized ones.+ """+ if split is None:+ split = "train"++ # Always load from the canonical Hub repo id, regardless of whether the user+ # passed the full name or the short alias ("UltraData-SFT-Agent-2609").+ hub_repo_id = "openbmb/UltraData-SFT-Agent-2609"++ # This dataset is split into multiple configs; mix them all so the quantized+ # model sees the full range of agent behaviors (code / search / general /+ # tool-use). We avoid ``interleave_datasets`` on purpose: the configs have+ # slightly different ``tools`` feature schemas (e.g. an extra ``title`` key),+ # which makes HF feature alignment fail. Since we only read ``messages`` and+ # ``tools`` and render them ourselves, we scan each config independently and+ # then balance the final selection across configs.+ configs = ["Code-Agent", "Search-Agent", "General-Agent", "Tool-Use"]+ sub_datasets = []+ for cfg in configs:+ try:+ d = load_dataset(hub_repo_id, cfg, split=split, streaming=True)+ d = d.shuffle(seed=seed, buffer_size=1000)+ sub_datasets.append(d)+ except Exception as e: # noqa: BLE001+ logger.warning("Failed to load config '%s' of %s: %s", cfg, hub_repo_id, e)++ if not sub_datasets:+ raise ValueError(f"Could not load any config of {hub_repo_id} (tried {configs}).")++ if not apply_chat_template:+ logger.info(+ "apply_chat_template=False is ignored for %s; agent/tool data requires chat templating.", hub_repo_id+ )++ # target = nsamples * envs.AR_CALIB_DATA_MULTIPLIER+ target = nsamples * 2+ n_cfg = len(sub_datasets)+ # Per-config quota so every config contributes a balanced share. Collect a+ # bit extra per config so that, if one config is short, others can cover the+ # shortfall during the balanced round-robin selection below.+ per_cfg_quota = -(-target // n_cfg) # ceil(target / n_cfg)+ # Safety bound per config: don't scan a whole (huge) config looking for+ # short-enough samples when they are scarce.+ max_scan_per_cfg = target++ # The per-sample system prompt in this dataset is very long and pushes most+ # samples past ``seqlen``. Drop it with high probability so the bulk of the+ # calibration data reflects the actual conversation/tool content, while a+ # small fraction still keeps the system prompt so the model sees that format.+ drop_system_prob = 0.99+ rng = random.Random(seed)++ def _maybe_drop_system(messages):+ """Drop a leading system message with probability ``drop_system_prob``."""+ if (+ messages+ and isinstance(messages[0], dict)+ and messages[0].get("role") == "system"+ and rng.random() < drop_system_prob+ ):+ return messages[1:]+ return messages++ def _token_len(text):+ return len(tokenizer(text, add_special_tokens=False)["input_ids"])++ def _approx_word_count(messages):+ """Cheap whitespace-split length estimate of the raw conversation.++ Token count is almost always >= whitespace word count (subword / code /+ JSON only increase it), and the chat template only adds more tokens. So+ if this estimate already exceeds ``seqlen`` the templated sample is+ definitely longer than ``seqlen`` and we can skip the expensive+ ``apply_chat_template`` + tokenization entirely.+ """+ count = 0+ for msg in messages:+ content = msg.get("content")+ if content:+ count += len(content.split())+ tool_calls = msg.get("tool_calls")+ if tool_calls:+ count += len(str(tool_calls).split())+ return count++ def _build_length_conversation(messages):+ """Build the message list used for the messages-only length check.++ Mirrors the system-prompt handling of ``render_chat_sample`` (tools are+ excluded here on purpose: we filter on the messages-only length).+ """+ conv = []+ has_own_system = len(messages) > 0 and isinstance(messages[0], dict) and messages[0].get("role") == "system"+ if system_prompt is not None and system_prompt != "" and not has_own_system:+ conv.append({"role": "system", "content": system_prompt})+ conv += messages+ return conv++ def _batch_msg_lengths(conversations):+ """Batch-render conversations and return their token lengths.++ We render to strings first (``tokenize=False`` on a batch reliably+ returns a list[str]) and then tokenize the strings in one batched call.+ This avoids the version-dependent return type of+ ``apply_chat_template(batch, tokenize=True)`` (which can be a dict-like+ ``BatchEncoding`` and make per-sample length extraction wrong).+ ``add_generation_prompt=False`` because these are full traces and the few+ generation-prompt tokens are negligible for a length filter.+ """+ if not conversations:+ return []+ try:+ rendered = tokenizer.apply_chat_template(conversations, tokenize=False, add_generation_prompt=False)+ except Exception:+ # Fallback: per-sample, dropping the system role if the template fails.+ rendered = []+ for conv in conversations:+ try:+ rendered.append(tokenizer.apply_chat_template(conv, tokenize=False, add_generation_prompt=False))+ except Exception:+ conv2 = [m for m in conv if m.get("role") != "system"]+ rendered.append(tokenizer.apply_chat_template(conv2, tokenize=False, add_generation_prompt=False))+ enc = tokenizer(rendered, add_special_tokens=False)+ return [len(ids) for ids in enc["input_ids"]]++ # Scan each config separately. Store raw (messages, tools) and only render+ # the finally selected samples with tools, so we don't pay the cost of fully+ # rendering the (large, mostly unused) fallback pool. The accurate length+ # check is done in a single batched template call per config.+ preferred_by_cfg = [[] for _ in sub_datasets] # messages fit within seqlen+ fallback_by_cfg = [[] for _ in sub_datasets] # messages exceed seqlen (gets truncated)++ for i, d in enumerate(sub_datasets):+ scanned = 0+ candidates = [] # (messages, tools) that passed the cheap word-count filter+ for sample in d:+ if len(candidates) >= per_cfg_quota or scanned >= max_scan_per_cfg:+ break+ scanned += 1+ messages = sample["messages"]+ tools = sample.get("tools") if isinstance(sample, dict) else sample["tools"]++ # Decide once (at read time) whether to drop the long system prompt,+ # so word-count, length check and final render all stay consistent.+ messages = _maybe_drop_system(messages)++ # Fast path: if even the raw whitespace word count exceeds seqlen, the+ # sample is definitely too long -> skip rendering/tokenizing.+ if _approx_word_count(messages) > seqlen:+ if len(fallback_by_cfg[i]) < per_cfg_quota:+ fallback_by_cfg[i].append((messages, tools))+ continue+ candidates.append((messages, tools))++ # Batched accurate length check for this config's candidates.+ lengths = _batch_msg_lengths([_build_length_conversation(m) for m, _ in candidates])+ for (messages, tools), length in zip(candidates, lengths):+ if length <= seqlen:+ preferred_by_cfg[i].append((messages, tools))+ elif len(fallback_by_cfg[i]) < per_cfg_quota:+ fallback_by_cfg[i].append((messages, tools))++ def _round_robin_take(pools, limit, already):+ """Pull items round-robin across per-config pools until ``limit``."""+ out = []+ idx = [0] * len(pools)+ while already + len(out) < limit and any(idx[i] < len(pools[i]) for i in range(len(pools))):+ for i in range(len(pools)):+ if already + len(out) >= limit:+ break+ if idx[i] < len(pools[i]):+ out.append(pools[i][idx[i]])+ idx[i] += 1+ return out++ # Balanced selection: round-robin across configs' preferred pools first,+ # then top up (still round-robin) from the fallback pools if short.+ selected = _round_robin_take(preferred_by_cfg, target, 0)+ n_preferred = len(selected)+ if len(selected) < target:+ selected += _round_robin_take(fallback_by_cfg, target, len(selected))+ if len(selected) > n_preferred:+ logger.info(+ "%s: only %d samples fit messages<=seqlen; topping up with %d oversized samples.",+ hub_repo_id,+ n_preferred,+ len(selected) - n_preferred,+ )++ # Global shuffle so configs/tiers are mixed across the calibration set.+ random.Random(seed).shuffle(selected)++ # Render the final selection with tools included.+ texts = [render_chat_sample(tokenizer, messages, system_prompt, tools=tools) for messages, tools in selected]++ import datasets++ calib_dataset = datasets.Dataset.from_list([{"text": t} for t in texts])+ calib_dataset = calib_dataset.map(+ lambda examples: tokenizer(examples["text"], truncation=True, max_length=seqlen),+ batched=True,+ remove_columns=["text"],+ )++ # Pack whole traces up to seqlen before returning. "Preferred" traces are+ # chosen on their messages-only length (<= seqlen) and can remain shorter+ # than seqlen even after rendering tools, so returning them unpacked would+ # let _get_dataset_impl.filter_func drop them -- leaving calibration+ # undersized despite available oversized fallback data. Sample-level packing+ # concatenates short traces into full-length sequences, while oversized+ # traces (already truncated to seqlen) each emit a single full-length+ # sequence, so no usable data is lost.+ calib_dataset = _concat_samples_to_seqlen(calib_dataset, seqlen)+ return calib_dataset++@register_dataset("openbmb/Ultra-FineWeb")def get_ultrafinweb_dataset(tokenizer,@@ -628,19 +1039,20 @@ def get_ultrafinweb_dataset(seed=42,apply_chat_template=True,system_prompt=None,+ nsamples=512,):if split is not None:if split not in ["en", "zh"]:raise ValueError("split must be one of ['en', 'zh'] for Ultra-FineWeb dataset")- calib_dataset = load_dataset("openbmb/Ultra-FineWeb", split=split, streaming=True, trust_remote_code=True)+ calib_dataset = load_dataset("openbmb/Ultra-FineWeb", split=split, streaming=True)else:- calib_dataset = load_dataset("openbmb/Ultra-FineWeb", split="en", streaming=True, trust_remote_code=True)+ calib_dataset = load_dataset("openbmb/Ultra-FineWeb", split="en", streaming=True)# dataset_ch = load_dataset("openbmb/Ultra-FineWeb", split='zh',# streaming=True, trust_remote_code=True).shuffle(seed=seed).take(2000)# calib_dataset = concatenate_datasets([dataset_en, dataset_ch]) ##concat dasetset could not shuffle- calib_dataset = calib_dataset.shuffle(seed=seed).take(20000)+ calib_dataset = calib_dataset.shuffle(seed=seed).take(nsamples * envs.AR_CALIB_DATA_MULTIPLIER)def get_default_tokenizer_function():def default_tokenizer_function(examples):@@ -923,6 +1335,330 @@ def get_local_dataset(return calib_dataset+def _concat_samples_to_seqlen(calib_dataset, seqlen):+ """Pack whole samples together until each reaches ``seqlen`` tokens.++ Unlike the continuous packing in :func:`concat_dataset_element` (where a+ sample's leftover tokens carry over into the next sequence, splitting+ samples across boundaries), this packs *whole* samples into a buffer and,+ once the buffer reaches ``seqlen``, emits a single ``seqlen``-length+ sequence and resets the buffer. The next sample therefore starts a fresh+ concatenation; any trailing buffer shorter than ``seqlen`` is dropped.++ Args:+ calib_dataset: A tokenized ``Dataset`` with an ``input_ids`` column.+ seqlen: Target sequence length for each packed sample.++ Returns:+ A new ``Dataset`` with packed ``input_ids`` / ``attention_mask`` columns.+ """+ packed = []+ buffer_ids = []+ for example in calib_dataset:+ ids = example["input_ids"]+ if isinstance(ids, torch.Tensor):+ ids = ids.tolist()+ buffer_ids.extend(ids)+ if len(buffer_ids) >= seqlen:+ trimmed = buffer_ids[:seqlen]+ packed.append({"input_ids": trimmed, "attention_mask": [1] * seqlen})+ # Reset: the next sample starts a fresh concatenation.+ buffer_ids = []++ if not packed:+ logger.warning(+ "Sample-level concat produced no full-length sequences (all samples "+ "shorter than seqlen=%d combined did not reach it). Returning the "+ "original dataset unpacked.",+ seqlen,+ )+ return calib_dataset++ return Dataset.from_list(packed)+++def _normalize_split(split):+ """Reduce a requested split spec to a single ``load_dataset`` split string.++ The dispatcher passes splits as lists (e.g. ``["train", "test"]``); combine+ every requested split into HuggingFace's ``"train+test"`` form so they are+ loaded together instead of only the first one being used. Returns ``None``+ when nothing was requested, so the caller can pick an available default from+ the resulting split dictionary.+ """+ if split is None:+ return None+ if isinstance(split, (list, tuple)):+ parts = [str(part) for part in split if part]+ return "+".join(parts) if parts else None+ return split+++def _reduce_to_single_dataset(loaded):+ """Reduce a ``load_dataset`` result to a single (Iterable)Dataset.++ When a concrete split string is passed, ``load_dataset`` already returns a+ single dataset and this is a no-op. When no split is requested it returns a+ split dictionary; in that case pick a sensible default (prefer ``train``,+ otherwise the first available split) so downstream ``.map()`` / column+ access receive a single dataset rather than a dict or list.+ """+ if isinstance(loaded, (DatasetDict, IterableDatasetDict)):+ keys = list(loaded.keys())+ if not keys:+ raise ValueError("Loaded dataset contains no splits to select from.")+ key = "train" if "train" in keys else keys[0]+ return loaded[key]+ # Defensive: a list is returned if a list split ever reaches load_dataset;+ # combine the splits so callers always receive a single dataset.+ if isinstance(loaded, list):+ if not loaded:+ raise ValueError("Loaded dataset contains no splits to combine.")+ if len(loaded) == 1:+ return loaded[0]+ if any(isinstance(part, IterableDataset) for part in loaded):+ from datasets import interleave_datasets++ return interleave_datasets(loaded)+ return concatenate_datasets(loaded)+ return loaded+++def _get_generic_dataset(+ tokenizer,+ seqlen,+ dataset_name,+ split=None,+ seed=42,+ apply_chat_template=False,+ system_prompt=None,+ fields=None,+ template=None,+ separator="\n\n",+ streaming=True,+ nsamples=None,+ timeout=300,+ concat=False,+ trust_remote_code=False,+):+ """Generic dataset loader that works with any HuggingFace dataset.++ Automatically detects the text field if not specified, or uses the+ user-provided fields/template to extract text.++ By default uses streaming mode to avoid downloading the entire dataset.+ Samples are collected until ``nsamples`` valid samples (meeting ``seqlen``)+ are gathered, or the ``timeout`` is reached.++ Args:+ tokenizer: The tokenizer to use for tokenization.+ seqlen: The maximum sequence length.+ dataset_name: HuggingFace dataset name or local path.+ split: The data split to use.+ seed: Random seed for shuffling.+ apply_chat_template: Whether to apply chat template.+ system_prompt: Optional system prompt.+ fields: Text field(s) to extract. A single string for one column,+ or a list of strings for multiple columns (concatenated with+ *separator*). If ``None``, the text field is auto-detected.+ template: Template string with {field} placeholders.+ separator: Separator for multi-field concatenation.+ streaming: Whether to use streaming mode (default True for remote+ datasets to avoid downloading the full dataset).+ nsamples: Maximum number of samples to collect (for streaming mode).+ If None, collects up to 10000 samples.+ timeout: Timeout in seconds for streaming collection. If the dataset+ cannot provide enough samples within this time, a warning is+ logged and the collected samples are used.+ concat: When True, pack whole samples together until each reaches+ ``seqlen`` tokens (resetting after every full sequence), so that+ datasets whose individual samples are shorter than ``seqlen`` are+ not dropped by the downstream length filter.+ trust_remote_code: Passed through to ``datasets.load_dataset``. Controlled+ by the AutoRound interface so the user decides whether to execute+ remote dataset-loading scripts. Defaults to ``False``.++ Returns:+ A tokenized HuggingFace Dataset.+ """+ import time++ # Determine if we should use streaming+ # Local files are always loaded fully (they're already on disk)+ is_local = is_local_path(dataset_name)+ use_streaming = streaming and not is_local++ # Normalize the requested split(s) once so local, streaming and+ # non-streaming branches behave consistently. The dispatcher passes splits+ # as lists; ``_normalize_split`` combines them into HF's "train+test" form+ # (or ``None`` when nothing was requested).+ split_arg = _normalize_split(split)++ # Load the dataset+ if is_local:+ # Local file: use json or text loader (always full load)+ if dataset_name.endswith((".json", ".jsonl")):+ loaded = load_dataset("json", data_files=dataset_name, split=split_arg)+ elif dataset_name.endswith(".txt"):+ loaded = load_dataset("text", data_files=dataset_name, split=split_arg)+ else:+ loaded = load_dataset(dataset_name, split=split_arg)+ else:+ # HuggingFace dataset. ``trust_remote_code`` is passed through from the+ # AutoRound interface so the user controls whether remote code runs.+ loaded = load_dataset(+ dataset_name,+ split=split_arg,+ trust_remote_code=trust_remote_code,+ streaming=use_streaming,+ )++ # When no split was requested, ``load_dataset`` returns a split dictionary;+ # reduce it (and any list result) to a single dataset for extraction below.+ calib_dataset = _reduce_to_single_dataset(loaded)++ # Determine the text field+ if fields is None and template is None:+ # Auto-detect. Streaming IterableDatasets may expose ``column_names`` as+ # None before the schema is resolved, so check for non-None metadata and+ # otherwise detect from a single sample.+ if getattr(calib_dataset, "column_names", None) is not None:+ fields = auto_detect_text_field(calib_dataset)+ else:+ # IterableDataset / unresolved schema – grab first sample to detect+ first_sample = next(iter(calib_dataset))+ fields = auto_detect_text_field_from_sample(first_sample)+ logger.info(f"Auto-detected text field '{fields}' for dataset '{dataset_name}'")++ # Build the text extraction function+ def _extract_text_from_sample(sample):+ """Extract text from a single sample dict."""+ try:+ return extract_text_from_sample(+ sample,+ fields=fields,+ template=template,+ separator=separator,+ )+ except (KeyError, ValueError) as e:+ logger.warning(f"Skipping sample due to field extraction error: {e}")+ return None++ # Tokenizer function+ tokenizer_function = get_tokenizer_function(+ tokenizer, seqlen, apply_chat_template=apply_chat_template, system_prompt=system_prompt+ )++ if use_streaming:+ # --- Streaming mode: collect samples until nsamples or timeout ---+ # Over-collect by AR_CALIB_DATA_MULTIPLIER (same as registered datasets)+ # so that enough samples survive the downstream seqlen filter in+ # ``_get_dataset_impl`` (short samples < seqlen are dropped there).+ target = nsamples * envs.AR_CALIB_DATA_MULTIPLIER if nsamples is not None else 10000+ start_time = time.time()+ collected_texts = []+ total_scanned = 0+ skipped = 0++ logger.info(+ f"Streaming dataset '{dataset_name}': collecting up to {target} samples " f"(timeout={timeout}s)..."+ )++ # Shuffle the iterable dataset for random sampling+ iter_ds = calib_dataset.shuffle(seed=seed)++ for sample in iter_ds:+ elapsed = time.time() - start_time+ if elapsed > timeout:+ logger.warning(+ f"Dataset '{dataset_name}' streaming timeout reached ({timeout}s). "+ f"Collected {len(collected_texts)}/{target} samples after scanning "+ f"{total_scanned} samples ({skipped} skipped). "+ f"Using available samples for calibration."+ )+ break++ total_scanned += 1+ text = _extract_text_from_sample(sample)+ if text is None or not text.strip():+ skipped += 1+ continue++ collected_texts.append(text)++ if len(collected_texts) >= target:+ break++ # Progress logging every 1000 samples+ if total_scanned % 1000 == 0:+ logger.info(+ f" Streaming '{dataset_name}': {len(collected_texts)}/{target} "+ f"valid samples collected ({total_scanned} scanned, {skipped} skipped, "+ f"{elapsed:.1f}s elapsed)"+ )++ if len(collected_texts) == 0:+ raise ValueError(+ f"Dataset '{dataset_name}' yielded no valid samples within {timeout}s. "+ f"Scanned {total_scanned} samples. Check the dataset format or "+ f"specify 'fields' explicitly."+ )++ if len(collected_texts) < target:+ logger.warning(+ f"Dataset '{dataset_name}': only {len(collected_texts)}/{target} samples "+ f"collected (scanned {total_scanned} total, {skipped} skipped). "+ f"Calibration will use {len(collected_texts)} samples."+ )++ # Build a Dataset from collected texts+ calib_dataset = Dataset.from_dict({"text": collected_texts})++ else:+ # --- Non-streaming mode: full dataset ---+ def _extract_text_batched(examples):+ """Extract text from batched examples."""+ texts = []+ for i in range(len(next(iter(examples.values())))):+ sample = {k: v[i] for k, v in examples.items()}+ text = _extract_text_from_sample(sample)+ if text is not None:+ texts.append(text)+ else:+ texts.append("") # placeholder, will be filtered later+ return {"text": texts}++ if fields is not None or template is not None:+ calib_dataset = calib_dataset.map(+ _extract_text_batched,+ batched=True,+ remove_columns=calib_dataset.column_names if hasattr(calib_dataset, "column_names") else None,+ )+ else:+ # Auto-detected field: rename to "text" if needed+ if fields != "text":+ calib_dataset = calib_dataset.rename_column(fields, "text")++ calib_dataset = calib_dataset.shuffle(seed=seed)++ # Filter out empty/short texts and tokenize+ calib_dataset = calib_dataset.filter(lambda x: len(x["text"].strip()) > 0)++ calib_dataset = calib_dataset.map(+ tokenizer_function,+ batched=True,+ new_fingerprint=_make_map_fingerprint(+ calib_dataset, tokenizer, seqlen, apply_chat_template, system_prompt, "text"+ ),+ )++ # Pack short samples into full seqlen-length sequences when requested.+ if concat:+ calib_dataset = _concat_samples_to_seqlen(calib_dataset, seqlen)++ return calib_dataset++def get_dataset_len(dataset):"""Calculates the length of a dataset.@@ -989,7 +1725,9 @@ def select_dataset(dataset, indices):return dataset-def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, nsamples=512):+def _get_dataset_impl(+ tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, nsamples=512, trust_remote_code=False+):"""Internal implementation: generate a dataset for calibration.Args:@@ -1006,7 +1744,15 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seedReturns:Dataset: The processed dataset ready for calibration."""- dataset_names = dataset_name.split(",")+ # Normalize CalibDataset objects to spec strings+ if isinstance(dataset_name, CalibDataset):+ dataset_name = dataset_name.to_spec_string()+ elif isinstance(dataset_name, (list, tuple)):+ dataset_name = normalize_dataset_spec(dataset_name)++ # Split on unescaped commas so that commas inside option values (e.g. a+ # system_prompt or template) are not mistaken for dataset separators.+ dataset_names = split_unescaped(dataset_name, ",")def cast_dataset_columns(dataset):features = dict(dataset.features)@@ -1073,24 +1819,88 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seedsplit = Nonedo_concat = Falseapply_chat_template = False+ fields = None+ template = None+ separator = "\n\n"+ timeout = 300if ":" in name:- name, split_list = name.split(":")[0], name.split(":")[1:]- for ele in split_list:- key, values = ele.split("=")[0], ele.split("=")[1:]+ # Split on unescaped ":" so values containing ":" (e.g. templates)+ # are not truncated; the first segment is the dataset name.+ segments = split_unescaped(name, ":")+ name = segments[0]+ for ele in segments[1:]:+ if "=" in ele:+ key, raw_value = ele.split("=", 1)+ elif ele.strip() in ("concat", "apply_chat_template"):+ # Preserve the bare boolean syntax (e.g. "pile-10k:apply_chat_template").+ key, raw_value = ele, "true"+ else:+ # Ignore incomplete non-boolean options.+ continue+ key = key.strip()+ # Restore escaped delimiters / whitespace; keep separator spaces.+ value = unescape_spec_value(raw_value if key == "separator" else raw_value.strip())if key == "split":- split = values[0].split("+")- if key == "num":- data_lens[name] = int(values[0])- if key == "concat":- do_concat = not (len(values) > 0 and values[0].lower() == "false")- if key == "apply_chat_template":- apply_chat_template = not (len(values) > 0 and values[0].lower() == "false")- if key == "system_prompt":- system_prompt = values[0]+ split = value.split("+")+ elif key == "num":+ data_lens[name] = int(value)+ elif key == "concat":+ do_concat = value.lower() != "false"+ elif key == "apply_chat_template":+ apply_chat_template = value.lower() != "false"+ elif key == "system_prompt":+ system_prompt = valueapply_chat_template = True- if is_local_path(name):- get_dataset = CALIB_DATASETS.get("local")+ elif key == "fields":+ # Single field: "fields=text" -> "text"+ # Multiple fields: "fields=q+a" -> ["q", "a"]+ if "+" in value:+ fields = value.split("+")+ else:+ fields = value+ elif key == "template":+ template = value+ elif key == "separator":+ separator = value+ elif key == "timeout":+ timeout = int(value)+ # If user explicitly specified fields/template, use the generic+ # loader regardless of whether the dataset is registered.+ use_generic = fields is not None or template is not None++ if use_generic or is_local_path(name):+ if is_local_path(name) and not use_generic:+ # Local file without explicit field spec: use the local handler+ dataset = CALIB_DATASETS["local"](+ tokenizer,+ seqlen,+ seed=seed,+ split=split,+ dataset_name=name,+ apply_chat_template=apply_chat_template,+ system_prompt=system_prompt,+ )+ else:+ # Generic loader (for local files with field spec, or any HF dataset)+ dataset = _get_generic_dataset(+ tokenizer,+ seqlen,+ dataset_name=name,+ split=split,+ seed=seed,+ apply_chat_template=apply_chat_template,+ system_prompt=system_prompt,+ fields=fields,+ template=template,+ separator=separator,+ nsamples=nsamples,+ timeout=timeout,+ concat=do_concat,+ trust_remote_code=trust_remote_code,+ )+ # Packing already handled inside the generic loader.+ do_concat = Falseelse:calib_name = nameif name not in CALIB_DATASETS:@@ -1100,26 +1910,59 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seedcalib_name = keybreakget_dataset = CALIB_DATASETS.get(calib_name)- if get_dataset is None:- filtered_keys = [k for k in CALIB_DATASETS if "/" not in k]- raise ValueError(- f"Dataset '{name}' is not found. Please choose from the supported datasets: {filtered_keys}."- )- dataset_kwargs = dict(- tokenizer=tokenizer,- seqlen=seqlen,- seed=seed,- split=split,- dataset_name=name,- apply_chat_template=apply_chat_template,- system_prompt=system_prompt,- )- if get_dataset is get_fineweb_edu_dataset:- dataset_kwargs["max_samples"] = max(- _FINEWEB_EDU_MIN_CANDIDATES,- data_lens.get(name, nsamples),- )- dataset = get_dataset(**dataset_kwargs)+ if get_dataset is None:+ # Fallback: use generic loader for any HuggingFace dataset+ logger.info(f"Dataset '{name}' not in registry, using generic loader with auto field detection.")+ dataset = _get_generic_dataset(+ tokenizer,+ seqlen,+ dataset_name=name,+ split=split,+ seed=seed,+ apply_chat_template=apply_chat_template,+ system_prompt=system_prompt,+ fields=fields,+ template=template,+ separator=separator,+ nsamples=nsamples,+ timeout=timeout,+ concat=do_concat,+ trust_remote_code=trust_remote_code,+ )+ # Packing already handled inside the generic loader.+ do_concat = False+ else:+ if get_dataset is None:+ filtered_keys = [k for k in CALIB_DATASETS.keys() if "/" not in k]+ raise ValueError(+ f"Dataset '{name}' is not found. Please choose from the supported datasets: {filtered_keys}."+ )+ dataset_kwargs = dict(+ tokenizer=tokenizer,+ seqlen=seqlen,+ seed=seed,+ split=split,+ dataset_name=name,+ apply_chat_template=apply_chat_template,+ system_prompt=system_prompt,+ )+ if get_dataset is get_fineweb_edu_dataset:+ dataset_kwargs["max_samples"] = max(+ _FINEWEB_EDU_MIN_CANDIDATES,+ data_lens.get(name, nsamples),+ )+ elif get_dataset in (+ get_cci3_hq_dataset,+ get_github_code_clean_dataset,+ get_opencode_instruct_dataset,+ get_competition_math_dataset,+ get_ultrachat_dataset,+ get_ultrafinweb_dataset,+ get_ultradata_agent_dataset,+ ):+ dataset_kwargs["nsamples"] = nsamples+ dataset = get_dataset(**dataset_kwargs)+if do_concat:dataset = concat_dataset_element(dataset)@@ -1194,7 +2037,7 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seedreturn dataset_final-def get_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, nsamples=512):+def get_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, nsamples=512, trust_remote_code=False):"""Generate a dataset for calibration.Uses a subprocess for preprocessing to ensure all temporary memory is fully@@ -1209,19 +2052,26 @@ def get_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, nArgs:tokenizer: The tokenizer to use for tokenization.seqlen (int): The exact sequence length.- dataset_name (str, optional): Dataset name(s) separated by commas.+ dataset_name (str, optional): Dataset name(s) separated by commas, or a+ ``CalibDataset`` object. Defaults to "NeelNanda/pile-10k".seed (int, optional): Random seed for reproducibility. Defaults to 42.nsamples (int, optional): Total number of samples to include. Defaults to 512.Returns:Dataset: The processed dataset ready for calibration."""+ # Normalize CalibDataset objects to spec strings+ if isinstance(dataset_name, CalibDataset):+ dataset_name = dataset_name.to_spec_string()+ elif isinstance(dataset_name, (list, tuple)):+ dataset_name = normalize_dataset_spec(dataset_name)+# Allow disabling subprocess mode via environment variableif envs.AR_DISABLE_DATASET_SUBPROCESS:try:- return _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)+ return _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code)except Exception as error:- return _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples)+ return _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code)# Run preprocessing in a subprocess so all temporary memory is freed on exit.logger.info("Preprocessing calibration dataset in a subprocess to avoid memory leaks...")@@ -1240,7 +2090,16 @@ def get_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, nresult_queue = ctx.Queue()p = ctx.Process(target=_preprocess_dataset_in_subprocess,- args=(result_queue, tokenizer, seqlen, dataset_name, seed, nsamples, os.path.join(temp_dir, "dataset")),+ args=(+ result_queue,+ tokenizer,+ seqlen,+ dataset_name,+ seed,+ nsamples,+ os.path.join(temp_dir, "dataset"),+ trust_remote_code,+ ),)p.start()p.join()@@ -1265,15 +2124,19 @@ def get_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, nlogger.warning(f"Subprocess dataset preprocessing failed ({e}), falling back to in-process mode.")if subprocess_network_error is not None:- return _fallback_to_fineweb_edu(subprocess_network_error, tokenizer, seqlen, dataset_name, seed, nsamples)+ return _fallback_to_fineweb_edu(+ subprocess_network_error, tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code+ )try:- return _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)+ return _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code)except Exception as error:- return _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples)+ return _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code)-def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, bs=8, nsamples=512):+def get_dataloader(+ tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, bs=8, nsamples=512, trust_remote_code=False+):"""Generate a DataLoader for calibration using specified parameters.Args:@@ -1316,6 +2179,6 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42res = {"input_ids": input_ids_new, "attention_mask": attention_mask_new}return res- dataset_final = get_dataset(tokenizer, seqlen, dataset_name, seed, nsamples)+ dataset_final = get_dataset(tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code)calib_dataloader = DataLoader(dataset_final, batch_size=bs, shuffle=False, collate_fn=collate_batch)return calib_dataloaderdiff --git a/auto_round/calibration/base.py b/auto_round/calibration/base.pyindex 539d9f418..90afb9040 100644--- a/auto_round/calibration/base.py+++ b/auto_round/calibration/base.py@@ -41,6 +41,7 @@ class Calibrator(ABC):def __init__(self, compressor: "BaseCompressor", **kwargs) -> None:self.model = compressor.model_context.modelself.tokenizer = compressor.model_context.tokenizer+ self.trust_remote_code = getattr(compressor.model_context, "trust_remote_code", False)self.dataset = compressor.datasetself.seed = compressor.seedself.low_gpu_mem_usage = compressor.low_gpu_mem_usagediff --git a/auto_round/calibration/diffusion.py b/auto_round/calibration/diffusion.pyindex c278872bc..0b2270bfe 100644--- a/auto_round/calibration/diffusion.py+++ b/auto_round/calibration/diffusion.py@@ -114,7 +114,11 @@ class DiffusionCalibrator(LLMCalibrator):)requires_image = self._requires_calibration_image()if isinstance(self.dataset, str):- dataset = self.dataset.replace(" ", "")+ from auto_round.utils.dataset_utils import strip_dataset_spec_whitespace++ # Strip only cosmetic whitespace; spaces inside option values+ # (e.g. separator / template / system_prompt) are preserved.+ dataset = strip_dataset_spec_whitespace(self.dataset)self.dataloader, self.batch_size = get_diffusion_dataloader(dataset=dataset, bs=bs, seed=self.seed, nsamples=nsamples, image_required=requires_image)diff --git a/auto_round/calibration/llm.py b/auto_round/calibration/llm.pyindex 287d8c769..41ed11322 100644--- a/auto_round/calibration/llm.py+++ b/auto_round/calibration/llm.py@@ -288,11 +288,14 @@ class LLMCalibrator(Calibrator):Verbatim port of the legacy ``Compressor.calib`` (LLM path only)."""from auto_round.calib_dataset import get_dataloader+ from auto_round.utils.dataset_utils import strip_dataset_spec_whitespaceneed_attention_mask = Trueif isinstance(self.dataset, str):need_attention_mask = False # all supported datasets do not use pad- dataset = self.dataset.replace(" ", "") # remove all whitespaces+ # Strip only cosmetic whitespace; spaces inside option values+ # (e.g. separator / template / system_prompt) are preserved.+ dataset = strip_dataset_spec_whitespace(self.dataset)# slow hereself.dataloader = get_dataloader(@@ -302,6 +305,7 @@ class LLMCalibrator(Calibrator):self.seed,bs,nsamples,+ trust_remote_code=self.trust_remote_code,)else:self.dataloader = self.datasetdiff --git a/auto_round/calibration/mllm.py b/auto_round/calibration/mllm.pyindex 24493aa20..e2e8d2ab2 100644--- a/auto_round/calibration/mllm.py+++ b/auto_round/calibration/mllm.py@@ -87,12 +87,16 @@ class MLLMCalibrator(LLMCalibrator):logger.info(f"Using MLLM template: {template_name}")- dataset = self.dataset.replace(" ", "") if isinstance(self.dataset, str) else self.dataset+ from auto_round.utils.dataset_utils import strip_dataset_spec_whitespace++ # Strip only cosmetic whitespace; spaces inside option values+ # (e.g. separator / template / system_prompt) are preserved.+ dataset = strip_dataset_spec_whitespace(self.dataset) if isinstance(self.dataset, str) else self.datasetif dataset is None:dataset = template_obj.default_datasetif isinstance(dataset, str):- dataset = dataset.replace(" ", "")+ dataset = strip_dataset_spec_whitespace(dataset)# Switch text-only dataset to MLLM dataset when quant_nontext_module=True,# as text datasets cannot calibrate vision modules.from auto_round.calib_dataset import CALIB_DATASETS@@ -130,7 +134,9 @@ class MLLMCalibrator(LLMCalibrator):"(vision/audio towers are not being quantized, so text-only ""calibration through the full-model forward is sufficient).")- self.dataloader = get_dataloader(tokenizer, self.seqlen, dataset, self.seed, bs, nsamples)+ self.dataloader = get_dataloader(+ tokenizer, self.seqlen, dataset, self.seed, bs, nsamples, trust_remote_code=self.trust_remote_code+ )else:orig_bs = self.batch_size(diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 5c3d3de4d..bf973f581 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -75,6 +75,7 @@ from auto_round.utils import (revert_checkpoint_conversion_mapping,revert_name_with_weight_transforms,)+from auto_round.utils.dataset_utils import CalibDatasetfrom auto_round.utils.device import (_force_trim_malloc,patch_xpu_sdpa_drop_causal_mask,@@ -247,7 +248,7 @@ class BaseOrchestrator:ignore_layers: str = "",quant_lm_head: bool = False,to_quant_block_names: str | list[str] | None = None,- dataset: str | list | tuple | torch.utils.data.DataLoader | None = None,+ dataset: str | list | tuple | torch.utils.data.DataLoader | CalibDataset | None = None,**kwargs,) -> None:# ``CalibrationContext`` is the single source of truth for calibration@@ -256,6 +257,13 @@ class BaseOrchestrator:# via property forwarders. ``_resolve_scheme`` later wires this same# instance onto the quantizer so the two share state.from auto_round.calibration.state import CalibrationContext+ from auto_round.utils.dataset_utils import CalibDataset, normalize_dataset_spec++ # Normalize CalibDataset objects to spec strings+ if isinstance(dataset, CalibDataset):+ dataset = dataset.to_spec_string()+ elif isinstance(dataset, (list, tuple)):+ dataset = normalize_dataset_spec(dataset)dataset_was_explicitly_set = dataset is not Noneself.dataset = dataset if dataset_was_explicitly_set else "NeelNanda/pile-10k"diff --git a/auto_round/envs.py b/auto_round/envs.pyindex c0337fb75..f0140b89e 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -39,6 +39,7 @@ if TYPE_CHECKING:AR_QUANTIZE_BAGEL_MOE_GEN: bool = FalseAR_NVFP4_FUSED_LAYER_GLOBAL_SCALE: bool = TrueAR_ALLOW_W8_ASYM: bool = False+ AR_CALIB_DATA_MULTIPLIER: int = 100def _get_optional_positive_int_env(name: str) -> int | None:@@ -66,6 +67,9 @@ environment_variables: dict[str, Callable[[], Any]] = {"AR_OMP_NUM_THREADS": lambda: os.getenv("AR_OMP_NUM_THREADS", None),"AR_DISABLE_OFFLOAD": lambda: os.getenv("AR_DISABLE_OFFLOAD", "0").lower() in ("1", "true", "yes"),"AR_DISABLE_DATASET_SUBPROCESS": lambda: os.getenv("AR_DISABLE_DATASET_SUBPROCESS", "0").lower() in ("1", "true"),+ # Multiplier applied to nsamples to determine the candidate pool size for+ # streaming calibration datasets. The pool is nsamples * AR_CALIB_DATA_MULTIPLIER.+ "AR_CALIB_DATA_MULTIPLIER": lambda: int(os.getenv("AR_CALIB_DATA_MULTIPLIER", "100")),"AR_DISABLE_COPY_MTP_WEIGHTS": lambda: os.getenv("AR_DISABLE_COPY_MTP_WEIGHTS", "0").lower()in ("1", "true", "yes"),# Device for the disk-streamed calibration forward pass indiff --git a/auto_round/utils/__init__.py b/auto_round/utils/__init__.pyindex 62fcc1506..1775c9a85 100644--- a/auto_round/utils/__init__.py+++ b/auto_round/utils/__init__.py@@ -21,6 +21,15 @@ from auto_round.utils.weight_handler import (detect_weight_type,is_quantized_input_module,)+from auto_round.utils.dataset_utils import (+ CalibDataset,+ auto_detect_text_field,+ auto_detect_text_field_from_sample,+ build_dataset_spec,+ extract_text_from_sample,+ normalize_dataset_spec,+ parse_dataset_spec,+)def copy_missing_tensors_from_source(*args, **kwargs):diff --git a/auto_round/utils/dataset_utils.py b/auto_round/utils/dataset_utils.pynew file mode 100644index 000000000..83892e617--- /dev/null+++ b/auto_round/utils/dataset_utils.py@@ -0,0 +1,906 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Utilities for flexible calibration dataset loading and field extraction.++This module provides:++- **CalibDataset** dataclass for a clean, typed API to describe calibration datasets.+- **Auto-detection** of the text field in a HuggingFace ``Dataset`` by inspecting+ column names and sampling values.+- **Field extraction** with support for single fields, multiple fields, and+ user-defined templates (``{field}`` placeholders).+- **Spec-string building and parsing** for CLI / string-based usage.++Example::++ from auto_round.utils.dataset_utils import CalibDataset++ # Single field+ spec = CalibDataset("my-org/my-dataset", split="train", num=1000, fields="text")++ # Multiple fields concatenated+ spec = CalibDataset("my-org/my-dataset", fields=["question", "answer"], separator="\\n")++ # Template+ spec = CalibDataset("my-org/my-dataset", template="{question} {answer}")++ # Pass directly to AutoRound+ from auto_round import AutoRound+ model = AutoRound(model, dataset=spec)+"""++import re+from dataclasses import dataclass+from typing import Any, Union++# ---------------------------------------------------------------------------+# CalibDataset dataclass+# ---------------------------------------------------------------------------+++@dataclass+class CalibDataset:+ """Structured specification for a calibration dataset.++ This is the preferred way to specify calibration data programmatically.+ It can be passed directly to ``AutoRound(dataset=...)`` or converted to a+ spec string via :meth:`to_spec_string`.++ Args:+ name: Dataset name (HuggingFace ID) or local file path.+ split: Split name(s). A single string or a list of strings.+ num: Number of samples to take from this dataset.+ concat: Whether to concatenate samples into full-length sequences.+ fields: Text field(s) to extract. A single string for one column,+ or a list of strings for multiple columns (concatenated with+ *separator*). If ``None``, the text field is auto-detected.+ template: A format string with ``{field_name}`` placeholders.+ Takes precedence over *fields* when both are set.+ separator: Separator used when concatenating multiple fields.+ Defaults to ``"\\n\\n"``.+ apply_chat_template: Whether to apply the tokenizer's chat template.+ system_prompt: Optional system prompt for chat template.+ timeout: Seconds to wait for data collection before falling back to+ whatever has been collected so far (default: 300).++ Example::++ >>> spec = CalibDataset("my-org/my-ds", split="train", num=500, fields="text")+ >>> spec = CalibDataset("my-org/my-ds", fields=["question", "answer"], separator="\\n")+ >>> spec = CalibDataset("my-org/my-ds", template="{question} {answer}")+ """++ name: str+ split: str | list[str] | None = None+ num: int | None = None+ concat: bool = False+ fields: str | list[str] | None = None+ template: str | None = None+ separator: str = "\n\n"+ apply_chat_template: bool = False+ system_prompt: str | None = None+ timeout: int = 300++ def to_spec_string(self) -> str:+ """Convert to a spec string (the format used in CLI / ``dataset`` param).++ Returns:+ A string like ``"my/ds:split=train:num=100:fields=q+a"``.+ """+ parts = [self.name]++ if self.split is not None:+ if isinstance(self.split, list):+ parts.append(f"split={'+'.join(escape_spec_value(s) for s in self.split)}")+ else:+ parts.append(f"split={escape_spec_value(self.split)}")++ if self.num is not None:+ parts.append(f"num={self.num}")++ if self.concat:+ parts.append("concat=true")++ if self.template is not None:+ parts.append(f"template={escape_spec_value(self.template)}")+ elif self.fields is not None:+ if isinstance(self.fields, str):+ parts.append(f"fields={escape_spec_value(self.fields)}")+ else:+ parts.append(f"fields={'+'.join(escape_spec_value(f) for f in self.fields)}")++ # Only include separator if it differs from the default and fields/template is used+ if self.separator != "\n\n" and (self.template is not None or self.fields is not None):+ parts.append(f"separator={escape_spec_value(self.separator)}")++ if self.apply_chat_template:+ parts.append("apply_chat_template=true")++ if self.system_prompt is not None:+ parts.append(f"system_prompt={escape_spec_value(self.system_prompt)}")++ if self.timeout != 300:+ parts.append(f"timeout={self.timeout}")++ return ":".join(parts)++ @classmethod+ def from_spec_string(cls, spec: str) -> "CalibDataset":+ """Parse a spec string into a ``CalibDataset`` instance.++ Args:+ spec: A spec string like ``"my/ds:split=train:num=100:fields=q+a"``.++ Returns:+ A ``CalibDataset`` instance.+ """+ return cls(**parse_dataset_spec(spec))+++# ---------------------------------------------------------------------------+# Auto-detection of text fields+# ---------------------------------------------------------------------------++#: Priority-ordered list of column names commonly used for text content in+#: HuggingFace datasets. The first match (in this order) wins when multiple+#: candidates are present.+TEXT_FIELD_PRIORITY: list[str] = [+ "text",+ "content",+ "prompt",+ "question",+ "instruction",+ "input",+ "code",+ "conversations",+ "messages",+ "dialog",+ "utterance",+ "utterances",+ "sentence",+ "paragraph",+ "article",+ "body",+ "abstract",+ "title",+ "caption",+ "description",+ "label",+ "answer",+ "response",+ "output",+ "completion",+ "translation",+ "summary",+ "context",+ "passage",+ "document",+ "doc",+ "raw_text",+ "raw",+]++#: Regex that matches ``{field_name}`` placeholders in a template string.+_TEMPLATE_PLACEHOLDER_RE = re.compile(r"\{(\w+)\}")+++def _is_text_value(value: Any) -> bool:+ """Return ``True`` if *value* looks like a text string (non-trivial length)."""+ if isinstance(value, str):+ return len(value.strip()) > 0+ return False+++def _is_text_list(value: Any) -> bool:+ """Return ``True`` if *value* is a list of non-empty strings (e.g. messages)."""+ if not isinstance(value, (list, tuple)):+ return False+ if len(value) == 0:+ return False+ # At least half of the elements should be non-empty strings+ text_count = sum(1 for v in value if isinstance(v, str) and len(v.strip()) > 0)+ return text_count >= len(value) // 2+++#: Keys commonly used for the role of a chat message across dataset schemas+#: (HF ``role``, ShareGPT ``from``, etc.).+_MESSAGE_ROLE_KEYS: tuple[str, ...] = ("role", "from", "speaker")+#: Keys commonly used for the textual content of a chat message.+_MESSAGE_CONTENT_KEYS: tuple[str, ...] = ("content", "value", "text", "message")+++def _message_content(message: dict[str, Any]) -> str:+ """Extract the textual content from a single chat-message dict.++ Handles both plain-string content and structured (e.g. multimodal) content+ that is itself a list of parts such as ``[{"type": "text", "text": ...}]``.+ """+ for key in _MESSAGE_CONTENT_KEYS:+ if key in message and message[key] is not None:+ content = message[key]+ if isinstance(content, (list, tuple)):+ parts = []+ for part in content:+ if isinstance(part, dict):+ parts.append(str(part.get("text", part.get("content", ""))))+ elif isinstance(part, str):+ parts.append(part)+ return " ".join(p for p in parts if p)+ return str(content)+ return ""+++def _message_role(message: dict[str, Any]) -> str | None:+ """Extract the role/speaker of a single chat-message dict, if present."""+ for key in _MESSAGE_ROLE_KEYS:+ if message.get(key):+ return str(message[key])+ return None+++def is_conversation_value(value: Any) -> bool:+ """Return ``True`` if *value* is a list of chat-message dicts.++ Recognizes structured conversations stored under fields like ``messages`` or+ ``conversations`` where each element is a dict carrying a content field+ (``content``/``value``/...). Such values would otherwise flatten to an empty+ string in :func:`extract_text_from_sample`, causing every row to be skipped.+ """+ if not isinstance(value, (list, tuple)) or len(value) == 0:+ return False+ dict_messages = [item for item in value if isinstance(item, dict)]+ # Require the list to be predominantly message dicts.+ if len(dict_messages) < max(1, len(value) // 2):+ return False+ # At least one message must expose a recognizable content field.+ return any(any(k in msg for k in _MESSAGE_CONTENT_KEYS) for msg in dict_messages)+++def extract_conversation_text(value: Any, separator: str = "\n") -> str:+ """Flatten a list of chat-message dicts into plain text for calibration.++ Role markers (``role: content``) are preserved so text-only calibration+ retains the conversational structure. Plain-string elements are kept as-is.+ """+ parts: list[str] = []+ for message in value:+ if isinstance(message, dict):+ content = _message_content(message)+ if not content:+ continue+ role = _message_role(message)+ parts.append(f"{role}: {content}" if role else content)+ elif isinstance(message, str) and message.strip():+ parts.append(message)+ return separator.join(parts)+++def auto_detect_text_field(dataset, sample_size: int = 10) -> str:+ """Auto-detect the most likely text field in a HuggingFace ``Dataset``.++ Strategy (in order of priority):++ 1. If the dataset has exactly **one** column, use it.+ 2. Check for well-known text field names (``text``, ``content``, ``prompt``,+ ``question``, …) in :data:`TEXT_FIELD_PRIORITY` order.+ 3. Among all remaining string columns, pick the one with the longest average+ text length across the first *sample_size* rows.++ Args:+ dataset: A HuggingFace ``Dataset`` or ``IterableDataset``.+ sample_size: Number of rows to sample for heuristic scoring. Defaults+ to 10.++ Returns:+ The name of the detected text field.++ Raises:+ ValueError: If no suitable text field can be found.+ """+ # --- 1. Single-column shortcut ------------------------------------------+ # Note: HF ``IterableDataset`` can expose ``column_names``/``features`` as+ # ``None`` before the schema is resolved (streaming is the generic loader's+ # default), so check for non-None metadata rather than mere attribute+ # presence; otherwise ``list(None)`` would raise ``TypeError``.+ if getattr(dataset, "column_names", None) is not None:+ columns = list(dataset.column_names)+ elif getattr(dataset, "features", None) is not None:+ columns = list(dataset.features.keys())+ else:+ # Schema not resolved (e.g. streaming IterableDataset) – grab one sample.+ columns = list(next(iter(dataset)).keys())++ if len(columns) == 1:+ return columns[0]++ # --- 2. Well-known names ------------------------------------------------+ for candidate in TEXT_FIELD_PRIORITY:+ if candidate in columns:+ return candidate++ # --- 3. Heuristic: longest average string length ------------------------+ # Sample a few rows to estimate average text length per column.+ samples = []+ for i, row in enumerate(dataset):+ if i >= sample_size:+ break+ samples.append(row)++ if not samples:+ raise ValueError(f"Cannot auto-detect text field: dataset is empty. " f"Available columns: {columns}")++ best_field = None+ best_avg_len = 0.0+ for col in columns:+ total_len = 0+ count = 0+ for row in samples:+ val = row.get(col)+ if isinstance(val, str):+ total_len += len(val)+ count += 1+ elif isinstance(val, (list, tuple)):+ # Sum lengths of string elements (e.g. messages / conversations)+ for item in val:+ if isinstance(item, str):+ total_len += len(item)+ elif isinstance(item, dict) and "content" in item:+ total_len += len(str(item["content"]))+ count += 1+ if count > 0:+ avg_len = total_len / count+ if avg_len > best_avg_len:+ best_avg_len = avg_len+ best_field = col++ if best_field is None:+ raise ValueError(+ f"Cannot auto-detect a text field in dataset. "+ f"Available columns: {columns}. "+ f"Please specify the field explicitly via the 'field' or 'fields' parameter."+ )+ return best_field+++def auto_detect_text_field_from_sample(sample: dict[str, Any]) -> str:+ """Auto-detect the most likely text field from a single sample dict.++ Uses the same priority logic as :func:`auto_detect_text_field` but operates+ on a single row instead of a full dataset. Useful for streaming /+ ``IterableDataset`` where column names may not be available upfront.++ Args:+ sample: A single row (dict) from the dataset.++ Returns:+ The name of the detected text field.++ Raises:+ ValueError: If no suitable text field can be found.+ """+ if len(sample) == 1:+ return next(iter(sample.keys()))++ for candidate in TEXT_FIELD_PRIORITY:+ if candidate in sample:+ return candidate++ best_field = None+ best_len = 0+ for key, val in sample.items():+ if isinstance(val, str):+ if len(val) > best_len:+ best_len = len(val)+ best_field = key+ elif isinstance(val, (list, tuple)):+ total_len = sum(len(str(v)) for v in val if isinstance(v, str))+ if total_len > best_len:+ best_len = total_len+ best_field = key++ if best_field is None:+ raise ValueError(+ f"Cannot auto-detect a text field from sample. "+ f"Available fields: {list(sample.keys())}. "+ f"Please specify the field explicitly via the 'field' or 'fields' parameter."+ )+ return best_field+++# ---------------------------------------------------------------------------+# Text extraction from a single sample+# ---------------------------------------------------------------------------+++def extract_text_from_sample(+ sample: dict[str, Any],+ fields: str | list[str] | None = None,+ template: str | None = None,+ separator: str = "\n\n",+) -> str:+ """Extract a text string from a single dataset sample.++ Three modes are supported (checked in order):++ 1. **Template** – if *template* is given, ``{field_name}`` placeholders are+ replaced with the corresponding values from *sample*.+ 2. **Multiple fields** – if *fields* is a list with more than one entry,+ the values are concatenated with *separator*.+ 3. **Single field** – if *fields* is a string (or a single-element list),+ the value of that column is returned as-is.++ If none of the above are provided, the first string-valued column is used.++ Args:+ sample: A single row (dict) from the dataset.+ fields: A single field name (str) or a list of field names to+ concatenate. If ``None``, auto-detection is used.+ template: A format string with ``{field_name}`` placeholders.+ Takes precedence over *fields*.+ separator: Separator used when concatenating multiple fields.+ Defaults to ``"\\n\\n"``.++ Returns:+ The extracted text string.++ Raises:+ KeyError: If a referenced field is not present in *sample*.+ ValueError: If no field can be determined.+ """+ # --- Template mode ------------------------------------------------------+ if template is not None:++ def _replace(match: re.Match) -> str:+ key = match.group(1)+ if key not in sample:+ raise KeyError(+ f"Template references field '{key}' which is not in the sample. "+ f"Available fields: {list(sample.keys())}"+ )+ val = sample[key]+ if isinstance(val, (list, tuple)):+ # Flatten list-of-dicts (e.g. messages) or list-of-strings+ parts = []+ for item in val:+ if isinstance(item, dict):+ parts.append(str(item.get("content", item)))+ else:+ parts.append(str(item))+ return separator.join(parts)+ return str(val)++ return _TEMPLATE_PLACEHOLDER_RE.sub(_replace, template)++ # --- Normalize fields to a list ------------------------------------------+ if fields is None:+ field_list: list[str] | None = None+ elif isinstance(fields, str):+ field_list = [fields]+ else:+ field_list = list(fields)++ # --- Multi-field mode ---------------------------------------------------+ if field_list is not None and len(field_list) > 1:+ parts = []+ for f in field_list:+ if f not in sample:+ raise KeyError(f"Field '{f}' not found in sample. Available fields: {list(sample.keys())}")+ val = sample[f]+ if is_conversation_value(val):+ parts.append(extract_conversation_text(val, separator))+ elif isinstance(val, (list, tuple)):+ parts.append(separator.join(str(v) for v in val if isinstance(v, str)))+ else:+ parts.append(str(val))+ return separator.join(parts)++ # --- Single-field mode --------------------------------------------------+ if field_list is not None and len(field_list) == 1:+ f = field_list[0]+ if f not in sample:+ raise KeyError(f"Field '{f}' not found in sample. Available fields: {list(sample.keys())}")+ val = sample[f]+ # Structured conversation (list of role/content dicts): preserve roles ++ # content instead of flattening to an empty string (which would make the+ # downstream loader skip every row, even with apply_chat_template=True).+ if is_conversation_value(val):+ return extract_conversation_text(val, separator)+ if isinstance(val, (list, tuple)):+ return separator.join(str(v) for v in val if isinstance(v, str))+ return str(val)++ # --- Fallback: first string column ---------------------------------------+ for val in sample.values():+ if isinstance(val, str) and len(val.strip()) > 0:+ return val+ if is_conversation_value(val):+ return extract_conversation_text(val, separator)+ if isinstance(val, (list, tuple)) and _is_text_list(val):+ return separator.join(str(v) for v in val if isinstance(v, str))++ raise ValueError(+ f"Cannot extract text from sample. Available fields: {list(sample.keys())}. "+ f"Please specify 'fields' or 'template' explicitly."+ )+++# ---------------------------------------------------------------------------# Dataset normalization+# ---------------------------------------------------------------------------+++def normalize_dataset_spec(dataset: Union[str, "CalibDataset", list, tuple]) -> str | list | tuple:+ """Normalize a dataset specification to a spec string or pass through raw data.++ Accepts a ``CalibDataset`` instance, a plain string, or a list/tuple.+ ``CalibDataset`` objects are converted to their spec-string representation+ via :meth:`CalibDataset.to_spec_string`.++ A list/tuple is treated as **raw calibration data** (returned unchanged)+ unless it consists solely of ``CalibDataset`` objects, in which case the+ specs are joined into a comma-separated spec string. A list of plain+ strings is therefore raw calibration text (each sample is tokenized on the+ fly), not a list of dataset names. To combine multiple *named* datasets,+ pass a single comma-separated string (e.g. ``"ds1,ds2"``) or a list of+ ``CalibDataset`` objects.++ Args:+ dataset: A dataset specification — a string, a ``CalibDataset``, or a+ list/tuple of raw samples / ``CalibDataset`` objects.++ Returns:+ A comma-separated spec string suitable for ``_get_dataset_impl``, or+ the original list/tuple when it holds raw calibration data.++ Raises:+ TypeError: If *dataset* is ``None``.+ """+ if dataset is None:+ raise TypeError(+ f"dataset must be a str, CalibDataset, or list of str/CalibDataset, got {type(dataset).__name__}"+ )+ if isinstance(dataset, CalibDataset):+ return dataset.to_spec_string()+ if isinstance(dataset, str):+ return dataset+ if isinstance(dataset, (list, tuple)):+ # A list/tuple is raw calibration data (returned unchanged) unless it+ # is a list of CalibDataset specs. A list of plain strings is raw+ # calibration text (each sample tokenized on the fly), not a list of+ # dataset names. Multiple named datasets use a comma-separated string+ # or a list of CalibDataset objects.+ if not dataset:+ return ""+ if all(isinstance(item, CalibDataset) for item in dataset):+ return ",".join(item.to_spec_string() for item in dataset)+ return dataset+ # Non-spec types (DataLoader, BatchEncoding, etc.) are passed through.+ return dataset+++# ---------------------------------------------------------------------------# Spec-string building and parsing+# ---------------------------------------------------------------------------++#: Backslash-escape sequences used to protect structural delimiters (and+#: whitespace control chars) inside option values of a dataset spec string.+#: ``:`` separates key=value options, ``,`` separates datasets, and ``\`` is the+#: escape introducer; ``\n`` / ``\t`` keep newline / tab round-trippable.+_SPEC_ESCAPE_MAP = {+ "\\": "\\\\",+ ":": "\\:",+ ",": "\\,",+ "\n": "\\n",+ "\t": "\\t",+}+#: Reverse mapping (escape character -> literal) used when unescaping.+_SPEC_UNESCAPE_MAP = {"\\": "\\", ":": ":", ",": ",", "n": "\n", "t": "\t"}+++def escape_spec_value(value: str) -> str:+ """Escape structural delimiters inside a spec option value.++ Protects ``:``, ``,`` and ``\\`` (plus newline/tab) so that values such as+ ``template='Question: {q}\\nAnswer: {a}'`` or ``system_prompt='a, b'`` survive+ a serialize/parse round-trip instead of being split into extra options or+ extra datasets.+ """+ return "".join(_SPEC_ESCAPE_MAP.get(ch, ch) for ch in value)+++def unescape_spec_value(value: str) -> str:+ """Reverse :func:`escape_spec_value` (also accepts legacy ``\\n`` / ``\\t``)."""+ out: list[str] = []+ i = 0+ n = len(value)+ while i < n:+ ch = value[i]+ if ch == "\\" and i + 1 < n and value[i + 1] in _SPEC_UNESCAPE_MAP:+ out.append(_SPEC_UNESCAPE_MAP[value[i + 1]])+ i += 2+ continue+ out.append(ch)+ i += 1+ return "".join(out)+++def split_unescaped(spec: str, delim: str, maxsplit: int = -1) -> list[str]:+ """Split *spec* on *delim* occurrences that are not backslash-escaped.++ Escape sequences (``\\:`` / ``\\,`` / ``\\\\`` / ...) are preserved verbatim in+ the returned parts so the caller can :func:`unescape_spec_value` them.+ """+ parts: list[str] = []+ current: list[str] = []+ i = 0+ n = len(spec)+ splits = 0+ while i < n:+ ch = spec[i]+ if ch == "\\" and i + 1 < n:+ # Keep the escape sequence intact (two chars) for later unescaping.+ current.append(ch)+ current.append(spec[i + 1])+ i += 2+ continue+ if ch == delim and (maxsplit < 0 or splits < maxsplit):+ parts.append("".join(current))+ current = []+ splits += 1+ i += 1+ continue+ current.append(ch)+ i += 1+ parts.append("".join(current))+ return parts+++def strip_dataset_spec_whitespace(spec: str) -> str:+ """Remove cosmetic whitespace from a calibration dataset spec string.++ A spec has the form ``name[:key=value[:key=value]][,name2:...]``. Users may+ add cosmetic spaces around the comma separators, dataset names, and option+ keys; those are stripped. Spaces *inside* option values are meaningful and+ kept intact — e.g. ``separator= `` or ``template={question} {answer}`` —+ so serialized :class:`CalibDataset` options are not silently altered the way+ a blanket ``spec.replace(" ", "")`` would (turning ``separator=' '`` into an+ empty separator and ``{question} {answer}`` into ``{question}{answer}``).++ Args:+ spec: The dataset spec string.++ Returns:+ The spec with cosmetic whitespace removed and value whitespace preserved.+ """+ entries = []+ for entry in split_unescaped(spec, ","):+ segments = split_unescaped(entry, ":")+ if len(segments) == 1:+ # Plain dataset name (no options): safe to trim surrounding spaces.+ entries.append(segments[0].strip())+ continue+ normalized = [segments[0].strip()]+ for option in segments[1:]:+ if "=" in option:+ key, value = option.split("=", 1)+ # Trim the key only; preserve all whitespace inside the value.+ normalized.append(f"{key.strip()}={value}")+ else:+ normalized.append(option.strip())+ entries.append(":".join(normalized))+ return ",".join(entries)+++#: Keys that are recognised in a dataset spec string.+_SPEC_KEYS = {+ "split",+ "num",+ "concat",+ "apply_chat_template",+ "system_prompt",+ "fields",+ "template",+ "separator",+ "timeout",+}+++def build_dataset_spec(+ name: str,+ split: str | list[str] | None = None,+ num: int | None = None,+ concat: bool = False,+ fields: str | list[str] | None = None,+ template: str | None = None,+ separator: str = "\n\n",+ apply_chat_template: bool = False,+ system_prompt: str | None = None,+ timeout: int = 300,+) -> str:+ """Build a dataset spec string from individual parameters.++ This is the inverse of :func:`parse_dataset_spec` and is useful for+ programmatically constructing dataset specifications.++ Args:+ name: Dataset name (HuggingFace ID or local path).+ split: Split name(s). A single string or a list of strings.+ num: Number of samples to take from this dataset.+ concat: Whether to concatenate samples into full-length sequences.+ fields: Text field(s) to extract. A single string for one column,+ or a list of strings for multiple columns (concatenated with+ *separator*). If ``None``, the text field is auto-detected.+ template: A format string with ``{field_name}`` placeholders.+ Takes precedence over *fields* when both are set.+ separator: Separator used when concatenating multiple fields.+ Defaults to ``"\\n\\n"``.+ apply_chat_template: Whether to apply the tokenizer's chat template.+ system_prompt: Optional system prompt for chat template.+ timeout: Timeout in seconds for streaming data collection (default: 300).++ Returns:+ A spec string like ``"dataset:split=train:num=100:fields=q+a"``.++ Example::++ >>> build_dataset_spec("my/ds", split="train", num=500, fields=["q", "a"])+ 'my/ds:split=train:num=500:fields=q+a'++ >>> build_dataset_spec("my/ds", fields="text")+ 'my/ds:fields=text'++ >>> build_dataset_spec("my/ds", template="{question} {answer}", separator=" ")+ 'my/ds:template={question} {answer}:separator= '+ """+ parts = [name]++ if split is not None:+ if isinstance(split, list):+ parts.append(f"split={'+'.join(escape_spec_value(s) for s in split)}")+ else:+ parts.append(f"split={escape_spec_value(split)}")++ if num is not None:+ parts.append(f"num={num}")++ if concat:+ parts.append("concat=true")++ if template is not None:+ parts.append(f"template={escape_spec_value(template)}")+ elif fields is not None:+ if isinstance(fields, str):+ parts.append(f"fields={escape_spec_value(fields)}")+ else:+ parts.append(f"fields={'+'.join(escape_spec_value(f) for f in fields)}")++ # Only include separator if it differs from the default and fields/template is used+ if separator != "\n\n" and (template is not None or fields is not None):+ parts.append(f"separator={escape_spec_value(separator)}")++ if apply_chat_template:+ parts.append("apply_chat_template=true")++ if system_prompt is not None:+ parts.append(f"system_prompt={escape_spec_value(system_prompt)}")++ if timeout != 300:+ parts.append(f"timeout={timeout}")++ return ":".join(parts)+++def parse_dataset_spec(spec: str) -> dict[str, Any]:+ """Parse a dataset spec string into a structured dict.++ The spec string format is::++ dataset_name[:key=value[:key=value[...]]]++ Recognised keys: ``split``, ``num``, ``concat``, ``apply_chat_template``,+ ``system_prompt``, ``fields``, ``template``, ``separator``, ``timeout``.++ Multi-value keys (``split``, ``fields``) use ``+`` as the intra-value+ separator.++ Args:+ spec: The spec string to parse.++ Returns:+ A dict with the following keys (all optional except ``name``):++ - ``name`` (str): Dataset name / path.+ - ``split`` (list[str] | None): Split name(s).+ - ``num`` (int | None): Number of samples.+ - ``concat`` (bool): Whether to concatenate.+ - ``fields`` (str | list[str] | None): Text field(s) to extract.+ - ``template`` (str | None): Template string with ``{field}`` placeholders.+ - ``separator`` (str): Separator for multi-field concatenation.+ - ``apply_chat_template`` (bool): Whether to apply chat template.+ - ``system_prompt`` (str | None): System prompt.++ Example::++ >>> parse_dataset_spec("my/ds:split=train+test:num=100:fields=q+a")+ {'name': 'my/ds', 'split': ['train', 'test'], 'num': 100,+ 'concat': False, 'fields': ['q', 'a'],+ 'template': None, 'separator': '\\n\\n',+ 'apply_chat_template': False, 'system_prompt': None}+ """+ result: dict[str, Any] = {+ "name": spec,+ "split": None,+ "num": None,+ "concat": False,+ "fields": None,+ "template": None,+ "separator": "\n\n",+ "apply_chat_template": False,+ "system_prompt": None,+ "timeout": 300,+ }++ if ":" not in spec:+ return result++ # Split on unescaped ":" so that ":" inside option values (e.g. a template+ # like "Question: {q}") does not truncate the value. The first segment is+ # the dataset name; the rest are key=value options.+ segments = split_unescaped(spec, ":")+ result["name"] = unescape_spec_value(segments[0]).strip()++ for part in segments[1:]:+ if not part:+ continue+ if "=" in part:+ key, value = part.split("=", 1)+ elif part.strip() in ("concat", "apply_chat_template"):+ # Preserve the bare boolean syntax (e.g. "pile-10k:apply_chat_template").+ key, value = part, "true"+ else:+ # Ignore incomplete non-boolean options.+ continue+ key = key.strip()+ # Don't strip value for separator (whitespace is meaningful)+ if key != "separator":+ value = value.strip()+ # Restore escaped delimiters / whitespace control chars in the value.+ value = unescape_spec_value(value)++ if key == "split":+ result["split"] = value.split("+") if value else None+ elif key == "num":+ result["num"] = int(value)+ elif key == "concat":+ result["concat"] = value.lower() != "false"+ elif key == "apply_chat_template":+ result["apply_chat_template"] = value.lower() != "false"+ elif key == "system_prompt":+ result["system_prompt"] = value+ elif key == "fields":+ # Single field: "fields=text" -> "text"+ # Multiple fields: "fields=q+a" -> ["q", "a"]+ if "+" in value:+ result["fields"] = value.split("+")+ else:+ result["fields"] = value+ elif key == "template":+ result["template"] = value+ elif key == "separator":+ result["separator"] = value+ elif key == "timeout":+ result["timeout"] = int(value)+ else:+ # Unknown key – store it for forward compatibility+ result[key] = value++ return resultdiff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 31e0baedf..86349fda5 100755--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -73,6 +73,12 @@ calibration data and will be downloaded automatically from the datasets Hub. Oth- `mbpp` for code- `nvidia/OpenCodeInstruct` for code instruction data- `openbmb/Ultra-FineWeb`+- `openbmb/UltraData-SFT-Agent-2609` (experimental) for agent/tool-calling data. Each sample carries a `messages` conversation+ (with `tool_calls`/tool turns) and a `tools` schema that is rendered through the chat template (`apply_chat_template`+ is always enabled), so the quantized model sees full tool-calling traces. All configs (`Code-Agent`, `Search-Agent`,+ `General-Agent`, `Tool-Use`) are loaded and interleaved+- `qwedsacf/competition_math` (experimental) for math (the `problem` and `solution` fields are concatenated, and short+ samples are packed into full `seqlen` sequences by default)### Customized Dataset@@ -107,12 +113,59 @@ and huggingface dataset are supported. You could specify splits of a dataset by**Samples concatenation**: An optional setting allows users to concatenate calibration samplesusing `--dataset NeelNanda/pile-10k:concat=True`.All samples will be concatenated first, then split into chunks of seqlen length.+For the generic loader (any HuggingFace dataset, or one specified with `fields`/`template`),+`concat=True` performs sample-level packing instead: whole samples are accumulated until the buffer+reaches `seqlen`, a full-length sequence is emitted, and the buffer is reset so the next sample starts+a fresh concatenation. This prevents datasets whose individual samples are shorter than `seqlen`+(e.g. `qwedsacf/competition_math`) from being dropped by the length filter.**Apply chat template**: Using `--dataset NeelNanda/pile-10k:apply_chat_template` enables application of a chat templateto the calibration data before tokenization. This is commonly used for instruct-style models during generation. Tocustomize the system prompt,use:`--dataset 'NeelNanda/pile-10k:apply_chat_template:system_prompt="You are a helpful assistant."'`+**Generic dataset loading** (experimental): Any HuggingFace dataset can be used directly without prior registration. The text field+is auto-detected by inspecting column names and sampling values. For example:+`--dataset my-org/my-dataset:split=train:num=512`++> **Note**: Generic dataset loading (including `fields`, `template`, `separator`, `timeout`, auto field detection, and the+> `CalibDataset` API below) is **experimental** and its behavior/interface may change in future releases.++**Field selection**: Use `fields` to explicitly specify the text column(s). A single field:+`--dataset my-org/my-dataset:fields=text`. Multiple fields are concatenated with a separator:+`--dataset my-org/my-dataset:fields=question+answer`++**Template**: Use `template` with `{field}` placeholders to build custom text from multiple columns.+Takes precedence over `fields` when both are set.+`--dataset 'my-org/my-dataset:template={question} {answer}'`++**Separator**: When concatenating multiple fields, customize the separator (default `"\n\n"`):+`--dataset 'my-org/my-dataset:fields=q+a:separator=\n'`++**Timeout**: For streaming datasets, set `timeout` (seconds) to bound data collection. If the dataset+cannot provide enough samples within the timeout, a warning is logged and the collected samples are used.+`--dataset my-org/my-dataset:timeout=600`++**CalibDataset (API)**: For programmatic usage, the `CalibDataset` dataclass provides a typed interface:++ ~~~python+ from auto_round.utils.dataset_utils import CalibDataset+ from auto_round import AutoRound++ # Single field+ spec = CalibDataset("my-org/my-dataset", split="train", num=1000, fields="text")++ # Multiple fields concatenated+ spec = CalibDataset("my-org/my-dataset", fields=["question", "answer"], separator="\n")++ # Template+ spec = CalibDataset("my-org/my-dataset", template="{question} {answer}")++ # Pass directly to AutoRound+ ar = AutoRound(model="Qwen/Qwen3-8B", dataset=spec, scheme="W4A16")+ ar.quantize_and_save()+ ~~~+Note: If the concatenation option is not enabled, samples shorter than args.seqlen will be dropped.Please use ',' to split datasets, ':' to split parameters of a dataset and '+' to add values for one targeted parameter.diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex b40b40ccb..2d78a0df5 100755--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -73,6 +73,10 @@ pip install auto-round- `mbpp`:用于代码场景- `nvidia/OpenCodeInstruct`:用于代码指令数据- `openbmb/Ultra-FineWeb`+- `openbmb/UltraData-SFT-Agent-2609`(实验性):用于智能体/工具调用(tool-calling)数据。每个样本都包含一个 `messages` 对话+ (含 `tool_calls`/tool 轮次)以及 `tools` 工具 schema,并通过聊天模板渲染(始终启用 `apply_chat_template`),+ 因此量化模型可以看到完整的工具调用轨迹。该数据集的所有配置(`Code-Agent`、`Search-Agent`、`General-Agent`、`Tool-Use`)都会被加载并交错混合+- `qwedsacf/competition_math`(实验性):用于数学场景(会拼接 `problem` 与 `solution` 字段,并默认将较短的样本打包成完整的 `seqlen` 序列)### 自定义数据集**建议用户还是尽量不使用 padding 的数据**。虽然对于 padding 过的数据我们有做特殊处理,但是目前验证的比较少。@@ -102,10 +106,48 @@ pip install auto-round**数据集组合**:可使用 `--dataset` 参数组合不同数据集并设置其他的参数。示例代码 `--dataset ./tmp.json,NeelNanda/pile-10k:num=256,mbpp:num=128`。此参数同时支持本地标定文件和 Hugging Face 数据集,同时还可用 `split = split1+split2` 指定某个数据集的多个拆分子集。-**样本拼接**:可使用 `--dataset NeelNanda/pile-10k:concat=True` 拼接标定样本。在该模式下,会先将所有样本拼接成完整文本,再按 seqlen 长度切分出来。+**样本拼接**:可使用 `--dataset NeelNanda/pile-10k:concat=True` 拼接标定样本。在该模式下,会先将所有样本拼接成完整文本,再按 seqlen 长度切分出来。对于通用加载器(任意 HuggingFace 数据集,或通过 `fields`/`template` 指定的数据集),`concat=True` 会改为执行“样本级拼接”:不断累积整条样本直到缓冲区达到 `seqlen`,输出一条完整长度的序列后清空缓冲区,下一条样本重新开始拼接。这样可以避免单条样本短于 `seqlen` 的数据集(例如 `qwedsacf/competition_math`)被长度过滤丢弃。**启用对话模板**:可使用 `--dataset NeelNanda/pile-10k:apply_chat_template` 在分词前为标定数据应用对话模板,这在指令式模型的生成任务中比较常用。若需自定义系统提示词,可使用 `--dataset 'NeelNanda/pile-10k:apply_chat_template:system_prompt="你是一个乐于助人的智能助手。"'`+**通用数据集加载**(实验性):任何 HuggingFace 数据集都可以直接使用,无需预先注册。文本字段会通过检查列名和采样值自动检测。例如:+`--dataset my-org/my-dataset:split=train:num=512`++> **注意**:通用数据集加载(包括 `fields`、`template`、`separator`、`timeout`、自动字段检测,以及下文的 `CalibDataset` API)为**实验性**功能,其行为和接口在未来版本中可能发生变化。++**字段选择**:使用 `fields` 参数显式指定文本列。单个字段:+`--dataset my-org/my-dataset:fields=text`。多个字段会用分隔符拼接:+`--dataset my-org/my-dataset:fields=question+answer`++**模板**:使用 `template` 配合 `{field}` 占位符从多列构建自定义文本。同时设置时优先于 `fields`。+`--dataset 'my-org/my-dataset:template={question} {answer}'`++**分隔符**:拼接多个字段时可自定义分隔符(默认 `"\n\n"`):+`--dataset 'my-org/my-dataset:fields=q+a:separator=\n'`++**超时**:对于流式数据集,可设置 `timeout`(秒)限制数据收集时间。若超时前无法收集到足够样本,会记录警告并使用已收集的样本。+`--dataset my-org/my-dataset:timeout=600`++**CalibDataset(API)**:编程式使用时,`CalibDataset` 数据类提供了类型化的接口:++ ~~~python+ from auto_round.utils.dataset_utils import CalibDataset+ from auto_round import AutoRound++ # 单字段+ spec = CalibDataset("my-org/my-dataset", split="train", num=1000, fields="text")++ # 多字段拼接+ spec = CalibDataset("my-org/my-dataset", fields=["question", "answer"], separator="\n")++ # 模板+ spec = CalibDataset("my-org/my-dataset", template="{question} {answer}")++ # 直接传入 AutoRound+ ar = AutoRound(model="Qwen/Qwen3-8B", dataset=spec, scheme="W4A16")+ ar.quantize_and_save()+ ~~~+注意:如果没有开启拼接选项,长度小于 args.seqlen 的样本会被舍弃。数据集之间请用英文逗号`,`分隔;单个数据集的参数请用英文冒号`:`分隔;同一参数的多个取值请用英文加号`+`连接。diff --git a/test/unit/common/calibration/test_generic_dataset_loader.py b/test/unit/common/calibration/test_generic_dataset_loader.pynew file mode 100644index 000000000..9aac8b29d--- /dev/null+++ b/test/unit/common/calibration/test_generic_dataset_loader.py@@ -0,0 +1,485 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Mocked-dataset tests for the generic calibration loader.++These tests exercise the main feature path of ``_get_generic_dataset`` and its+integration through ``get_dataset`` without any network access. They cover:++* explicit and default splits forwarded to ``load_dataset``+* automatic text-field detection (non-streaming and streaming)+* message-record extraction via templates / fields+* short / empty-sample filtering+* streaming timeout + partial collection (and the empty-result error path)++All HuggingFace I/O is mocked, so the tests are deterministic and offline.+"""++import time++import datasets+import pytest+from datasets import Dataset++from auto_round import calib_dataset as cd++# Writing map/filter cache artifacts to disk is pointless for these tiny,+# fully in-memory fixtures and would risk cross-test cache collisions.+datasets.disable_caching()+++# ---------------------------------------------------------------------------+# Test doubles+# ---------------------------------------------------------------------------+class FakeTokenizer:+ """Minimal tokenizer mimicking the HF ``__call__`` batch interface.++ Each character maps to its ordinal, so ``len(input_ids) == len(text)``+ (capped at ``max_length``). This makes the downstream ``seqlen`` length+ filter predictable from the raw string length.+ """++ name_or_path = "fake-tokenizer"+ bos_token_id = 1+ eos_token_id = 2++ def __call__(self, texts, truncation=True, max_length=None, **kwargs):+ if isinstance(texts, str):+ texts = [texts]+ input_ids = []+ attention_mask = []+ for t in texts:+ ids = [ord(c) for c in t]+ if max_length is not None:+ ids = ids[:max_length]+ input_ids.append(ids)+ attention_mask.append([1] * len(ids))+ return {"input_ids": input_ids, "attention_mask": attention_mask}+++class FakeIterableDataset:+ """Stand-in for a streaming ``IterableDataset``.++ Deliberately lacks ``column_names`` so the loader falls back to inspecting+ the first sample for field auto-detection (the streaming code path).+ """++ def __init__(self, samples):+ self._samples = list(samples)++ def shuffle(self, seed=None): # order-preserving for deterministic asserts+ return self++ def __iter__(self):+ return iter(self._samples)+++class SlowIterableDataset:+ """Streaming dataset that stalls partway through to trigger a timeout."""++ def __init__(self, quick, slow, delay):+ self._quick = list(quick)+ self._slow = list(slow)+ self._delay = delay++ def shuffle(self, seed=None):+ return self++ def __iter__(self):+ yield from self._quick+ time.sleep(self._delay)+ yield from self._slow+++def tokenizer():+ return FakeTokenizer()+++def _make_load_dataset(return_value, recorded_calls=None):+ """Return a ``load_dataset`` replacement capturing its call kwargs."""++ def _fake_load_dataset(*args, **kwargs):+ if recorded_calls is not None:+ recorded_calls.append({"args": args, "kwargs": kwargs})+ return return_value++ return _fake_load_dataset+++# ---------------------------------------------------------------------------+# Splits forwarded to load_dataset+# ---------------------------------------------------------------------------+class TestSplits:+ def test_explicit_split_forwarded(self, tokenizer, monkeypatch):+ calls = []+ table = Dataset.from_dict({"text": ["hello world foo", "bar baz qux text"]})+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(table, calls))++ cd._get_generic_dataset(+ tokenizer,+ seqlen=4,+ dataset_name="my/ds",+ split=["validation"],+ streaming=False,+ )++ assert calls, "load_dataset was never called"+ # A list split is normalized to a single HF split string before loading.+ assert calls[0]["kwargs"]["split"] == "validation"++ def test_combined_splits_forwarded(self, tokenizer, monkeypatch):+ calls = []+ table = Dataset.from_dict({"text": ["hello world foo", "bar baz qux text"]})+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(table, calls))++ cd._get_generic_dataset(+ tokenizer,+ seqlen=4,+ dataset_name="my/ds",+ split=["train", "validation"],+ streaming=False,+ )++ # Multiple requested splits are combined into HF's "train+validation" form.+ assert calls[0]["kwargs"]["split"] == "train+validation"++ def test_default_split_selects_train(self, tokenizer, monkeypatch):+ calls = []+ # No split requested: load_dataset receives split=None and returns a+ # split dictionary, from which the loader selects the "train" split.+ split_dict = datasets.DatasetDict(+ {+ "train": Dataset.from_dict({"text": ["hello world foo", "bar baz qux text"]}),+ "validation": Dataset.from_dict({"text": ["ignored sample here"]}),+ }+ )+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(split_dict, calls))++ cd._get_generic_dataset(+ tokenizer,+ seqlen=4,+ dataset_name="my/ds",+ split=None,+ streaming=False,+ )++ assert calls[0]["kwargs"]["split"] is None+++# ---------------------------------------------------------------------------+# Split normalization / reduction helpers+# ---------------------------------------------------------------------------+class TestSplitHelpers:+ def test_normalize_split_none(self):+ assert cd._normalize_split(None) is None++ def test_normalize_split_string_passthrough(self):+ assert cd._normalize_split("train") == "train"++ def test_normalize_split_list_combined(self):+ assert cd._normalize_split(["train", "test"]) == "train+test"++ def test_normalize_split_empty_list(self):+ assert cd._normalize_split([]) is None++ def test_reduce_single_dataset_passthrough(self):+ table = Dataset.from_dict({"text": ["a"]})+ assert cd._reduce_to_single_dataset(table) is table++ def test_reduce_dict_prefers_train(self):+ train = Dataset.from_dict({"text": ["train row"]})+ split_dict = datasets.DatasetDict({"validation": Dataset.from_dict({"text": ["v"]}), "train": train})+ assert cd._reduce_to_single_dataset(split_dict) is train++ def test_reduce_dict_first_when_no_train(self):+ val = Dataset.from_dict({"text": ["v"]})+ split_dict = datasets.DatasetDict({"validation": val, "test": Dataset.from_dict({"text": ["t"]})})+ assert cd._reduce_to_single_dataset(split_dict) is val++ def test_reduce_list_combines(self):+ a = Dataset.from_dict({"text": ["a"]})+ b = Dataset.from_dict({"text": ["b"]})+ combined = cd._reduce_to_single_dataset([a, b])+ assert len(combined) == 2+++# ---------------------------------------------------------------------------+# trust_remote_code is passed through to load_dataset (default False)+# ---------------------------------------------------------------------------+class TestTrustRemoteCode:+ def test_defaults_to_false(self, tokenizer, monkeypatch):+ calls = []+ table = Dataset.from_dict({"text": ["hello world foo", "bar baz qux text"]})+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(table, calls))++ cd._get_generic_dataset(tokenizer, seqlen=4, dataset_name="my/ds", split="train", streaming=False)++ assert calls[0]["kwargs"]["trust_remote_code"] is False++ def test_forwarded_when_enabled(self, tokenizer, monkeypatch):+ calls = []+ table = Dataset.from_dict({"text": ["hello world foo", "bar baz qux text"]})+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(table, calls))++ cd._get_generic_dataset(+ tokenizer,+ seqlen=4,+ dataset_name="my/ds",+ split="train",+ streaming=False,+ trust_remote_code=True,+ )++ assert calls[0]["kwargs"]["trust_remote_code"] is True+++# ---------------------------------------------------------------------------+# Auto-detection of the text field+# ---------------------------------------------------------------------------+class TestAutoDetection:+ def test_non_streaming_detects_priority_field(self, tokenizer, monkeypatch):+ # "content" wins over "title" per TEXT_FIELD_PRIORITY and is renamed+ # to "text" internally.+ table = Dataset.from_dict(+ {+ "title": ["t1", "t2"],+ "content": ["the quick brown fox", "lorem ipsum dolor sit"],+ }+ )+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(table))++ ds = cd._get_generic_dataset(+ tokenizer,+ seqlen=4,+ dataset_name="my/ds",+ streaming=False,+ )++ assert "text" in ds.column_names+ assert set(ds["text"]) == {"the quick brown fox", "lorem ipsum dolor sit"}++ def test_streaming_detects_and_collects(self, tokenizer, monkeypatch):+ # Keep the candidate pool == nsamples so collection stops promptly.+ monkeypatch.setenv("AR_CALIB_DATA_MULTIPLIER", "1")+ samples = [{"text": f"streaming sample number {i}"} for i in range(5)]+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(FakeIterableDataset(samples)))++ ds = cd._get_generic_dataset(+ tokenizer,+ seqlen=4,+ dataset_name="my/ds",+ streaming=True,+ nsamples=3,+ )++ assert len(ds) == 3+ for text in ds["text"]:+ assert text.startswith("streaming sample number")+++# ---------------------------------------------------------------------------+# Message / chat records+# ---------------------------------------------------------------------------+class TestMessageRecords:+ def test_template_extracts_message_contents(self, tokenizer, monkeypatch):+ table = Dataset.from_dict(+ {+ "messages": [+ [+ {"role": "user", "content": "hello there"},+ {"role": "assistant", "content": "general kenobi"},+ ],+ [+ {"role": "user", "content": "what is two plus two"},+ {"role": "assistant", "content": "the answer is four"},+ ],+ ]+ }+ )+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(table))++ ds = cd._get_generic_dataset(+ tokenizer,+ seqlen=4,+ dataset_name="my/chat-ds",+ template="{messages}",+ separator=" | ",+ streaming=False,+ )++ assert set(ds["text"]) == {+ "hello there | general kenobi",+ "what is two plus two | the answer is four",+ }++ def test_streaming_message_records(self, tokenizer, monkeypatch):+ monkeypatch.setenv("AR_CALIB_DATA_MULTIPLIER", "1")+ samples = [+ {+ "messages": [+ {"role": "user", "content": f"question {i} goes here"},+ {"role": "assistant", "content": f"response {i} goes here"},+ ]+ }+ for i in range(3)+ ]+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(FakeIterableDataset(samples)))++ ds = cd._get_generic_dataset(+ tokenizer,+ seqlen=4,+ dataset_name="my/chat-ds",+ template="{messages}",+ separator=" ",+ streaming=True,+ nsamples=3,+ )++ assert len(ds) == 3+ for text in ds["text"]:+ assert "goes here" in text+++# ---------------------------------------------------------------------------+# Empty / short sample filtering+# ---------------------------------------------------------------------------+class TestSampleFiltering:+ def test_empty_samples_filtered_non_streaming(self, tokenizer, monkeypatch):+ table = Dataset.from_dict({"text": ["a valid sentence here", " ", "", "another valid line"]})+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(table))++ ds = cd._get_generic_dataset(+ tokenizer,+ seqlen=4,+ dataset_name="my/ds",+ streaming=False,+ )++ assert len(ds) == 2+ assert set(ds["text"]) == {"a valid sentence here", "another valid line"}++ def test_empty_samples_skipped_streaming(self, tokenizer, monkeypatch):+ monkeypatch.setenv("AR_CALIB_DATA_MULTIPLIER", "1")+ samples = [+ {"text": "keep this one"},+ {"text": " "},+ {"text": ""},+ {"text": "keep this too"},+ ]+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(FakeIterableDataset(samples)))++ ds = cd._get_generic_dataset(+ tokenizer,+ seqlen=4,+ dataset_name="my/ds",+ streaming=True,+ nsamples=10,+ )++ assert set(ds["text"]) == {"keep this one", "keep this too"}+++# ---------------------------------------------------------------------------+# Streaming timeout / partial collection+# ---------------------------------------------------------------------------+class TestStreamingTimeout:+ def test_timeout_yields_partial_collection(self, tokenizer, monkeypatch):+ # Two samples arrive instantly, then the stream stalls past the+ # timeout, so only the first two should be collected.+ quick = [{"text": "first good sample"}, {"text": "second good sample"}]+ slow = [{"text": "late sample one"}, {"text": "late sample two"}]+ ds_stream = SlowIterableDataset(quick, slow, delay=1.5)+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(ds_stream))++ ds = cd._get_generic_dataset(+ tokenizer,+ seqlen=4,+ dataset_name="my/ds",+ streaming=True,+ nsamples=100,+ timeout=0.5,+ )++ assert len(ds) == 2+ assert set(ds["text"]) == {"first good sample", "second good sample"}++ def test_no_valid_samples_raises(self, tokenizer, monkeypatch):+ samples = [{"text": ""}, {"text": " "}, {"text": "\n\t"}]+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(FakeIterableDataset(samples)))++ with pytest.raises(ValueError, match="no valid samples"):+ cd._get_generic_dataset(+ tokenizer,+ seqlen=4,+ dataset_name="my/ds",+ streaming=True,+ nsamples=10,+ timeout=30,+ )+++# ---------------------------------------------------------------------------+# Integration through get_dataset (in-process, generic-loader fallback)+# ---------------------------------------------------------------------------+class TestGetDatasetIntegration:+ @pytest.fixture(autouse=True)+ def _in_process(self, monkeypatch):+ # Run preprocessing in-process (no fork/pickle of the mock) and keep+ # the streaming candidate pool small.+ monkeypatch.setenv("AR_DISABLE_DATASET_SUBPROCESS", "1")+ monkeypatch.setenv("AR_CALIB_DATA_MULTIPLIER", "1")+ # Surface any error instead of silently falling back to a network fetch.+ monkeypatch.setattr(cd, "_fallback_to_fineweb_edu", lambda error, *a, **k: (_ for _ in ()).throw(error))++ def test_seqlen_short_samples_dropped(self, tokenizer, monkeypatch):+ # input_ids length == len(text) (truncated to seqlen); samples shorter+ # than seqlen are dropped by the integration-level length filter.+ samples = [+ {"text": "abcdefghij"}, # len 10 -> kept+ {"text": "klmnopqrst"}, # len 10 -> kept+ {"text": "ab"}, # len 2 -> dropped (< seqlen)+ {"text": "cd"}, # len 2 -> dropped (< seqlen)+ ]+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(FakeIterableDataset(samples)))++ ds = cd.get_dataset(+ tokenizer,+ seqlen=5,+ dataset_name="pytest/generic-ds",+ nsamples=10,+ )++ assert len(ds) == 2++ def test_template_spec_routes_to_generic_loader(self, tokenizer, monkeypatch):+ samples = [+ {+ "messages": [+ {"role": "user", "content": "abcdefghij"},+ {"role": "assistant", "content": "klmnopqrst"},+ ]+ }+ for _ in range(3)+ ]+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(FakeIterableDataset(samples)))++ ds = cd.get_dataset(+ tokenizer,+ seqlen=5,+ dataset_name="pytest/chat-ds:template={messages}",+ nsamples=10,+ )++ assert len(ds) >= 1+ assert "input_ids" in ds.column_namesdiff --git a/test/unit/common/core/test_calib_dataset_subprocess.py b/test/unit/common/core/test_calib_dataset_subprocess.pyindex 5b6363242..bbac034b3 100644--- a/test/unit/common/core/test_calib_dataset_subprocess.py+++ b/test/unit/common/core/test_calib_dataset_subprocess.py@@ -122,8 +122,8 @@ def test_subprocess_network_error_falls_back_without_retrying_source(monkeypatchfallback_dataset = Dataset.from_dict({"input_ids": [[1]], "attention_mask": [[1]]})fallback = []- def fallback_to_fineweb(error, tokenizer, seqlen, dataset_name, seed, nsamples):- fallback.append((error, dataset_name))+ def fallback_to_fineweb(error, tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code=False):+ fallback.append((error, dataset_name, trust_remote_code))return fallback_datasetmonkeypatch.setattr(cd.multiprocessing, "get_context", lambda method: _NetworkErrorContext())@@ -138,6 +138,7 @@ def test_subprocess_network_error_falls_back_without_retrying_source(monkeypatchassert result is fallback_datasetassert isinstance(fallback[0][0], ConnectionError)assert fallback[0][1] == "source"+ assert fallback[0][2] is False@pytest.mark.skipif(os.name == "nt", reason="fork is unavailable on Windows")diff --git a/test/unit/common/utils/test_calib_dataset_helpers.py b/test/unit/common/utils/test_calib_dataset_helpers.pyindex 8b1abfebc..f02d6a27b 100644--- a/test/unit/common/utils/test_calib_dataset_helpers.py+++ b/test/unit/common/utils/test_calib_dataset_helpers.py@@ -96,7 +96,7 @@ class TestDatasetNetworkErrors:)assert result is fallback_dataset- load_dataset.assert_called_once_with(tokenizer, 128, "AI-ModelScope/fineweb-edu", 42, 2)+ load_dataset.assert_called_once_with(tokenizer, 128, "AI-ModelScope/fineweb-edu", 42, 2, False)assert "Automatically switching" in warning.call_args.args[0]def test_closed_http_client_is_treated_as_network_error(self):@@ -497,3 +497,71 @@ class TestGetDatasetImpl:)assert dataset.column_names == ["source", "input_ids", "attention_mask"]+++# ---------------------------------------------------------------------------+# _concat_samples_to_seqlen (sample-level packing of short traces)+# ---------------------------------------------------------------------------+class TestConcatSamplesToSeqlen:+ def test_packs_short_samples_into_full_length(self):+ import auto_round.calib_dataset as calib_dataset++ # Four samples of length 3 packed into seqlen=6 -> two full-length rows.+ ds = Dataset.from_dict({"input_ids": [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]]})+ packed = calib_dataset._concat_samples_to_seqlen(ds, seqlen=6)+ assert len(packed) == 2+ for row in packed:+ assert len(row["input_ids"]) == 6+ assert len(row["attention_mask"]) == 6++ def test_oversized_sample_emits_single_sequence(self):+ import auto_round.calib_dataset as calib_dataset++ ds = Dataset.from_dict({"input_ids": [list(range(10))]})+ packed = calib_dataset._concat_samples_to_seqlen(ds, seqlen=4)+ assert len(packed) == 1+ assert len(packed[0]["input_ids"]) == 4++ def test_all_short_returns_original_unpacked(self):+ import auto_round.calib_dataset as calib_dataset++ # Single short sample can never reach seqlen -> original returned.+ ds = Dataset.from_dict({"input_ids": [[1, 2]]})+ packed = calib_dataset._concat_samples_to_seqlen(ds, seqlen=8)+ assert packed is ds+++# ---------------------------------------------------------------------------+# competition_math: chat template rendered once (not twice)+# ---------------------------------------------------------------------------+class TestCompetitionMathChatTemplate:+ class _ChatTokenizer:+ name_or_path = "fake-chat"+ bos_token_id = 1+ eos_token_id = 2++ def __init__(self):+ self.chat_calls = 0++ def apply_chat_template(self, messages, tokenize=False, add_generation_prompt=False, **kwargs):+ self.chat_calls += 1+ return " ".join(f"{m['role']}:{m['content']}" for m in messages)++ def __call__(self, texts, truncation=True, max_length=None, **kwargs):+ if isinstance(texts, str):+ texts = [texts]+ input_ids = [[ord(c) for c in t][:max_length] for t in texts]+ return {"input_ids": input_ids, "attention_mask": [[1] * len(x) for x in input_ids]}++ def test_chat_template_applied_once_per_sample(self, monkeypatch):+ import auto_round.calib_dataset as calib_dataset++ rows = Dataset.from_dict({"problem": ["p1", "p2", "p3"], "solution": ["s1", "s2", "s3"]})+ monkeypatch.setattr(calib_dataset, "load_dataset", lambda *a, **k: rows)++ tok = self._ChatTokenizer()+ calib_dataset.get_competition_math_dataset(tok, seqlen=4, split="train", apply_chat_template=True, nsamples=3)++ # Exactly one render per sample in the loop; the .map() tokenization must+ # NOT re-apply the template (regression guard for double application).+ assert tok.chat_calls == 3diff --git a/test/unit/common/utils/test_dataset_utils.py b/test/unit/common/utils/test_dataset_utils.pynew file mode 100644index 000000000..3b7b761e0--- /dev/null+++ b/test/unit/common/utils/test_dataset_utils.py@@ -0,0 +1,705 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Tests for auto_round.utils.dataset_utils."""++import pytest++from auto_round.utils.dataset_utils import (+ CalibDataset,+ auto_detect_text_field,+ auto_detect_text_field_from_sample,+ build_dataset_spec,+ escape_spec_value,+ extract_conversation_text,+ extract_text_from_sample,+ is_conversation_value,+ normalize_dataset_spec,+ parse_dataset_spec,+ split_unescaped,+ strip_dataset_spec_whitespace,+ unescape_spec_value,+)+++# ---------------------------------------------------------------------------+# auto_detect_text_field+# ---------------------------------------------------------------------------+class TestAutoDetectTextField:+ def test_single_column(self):+ """Single-column dataset should return that column."""++ class FakeDS:+ column_names = ["text"]++ assert auto_detect_text_field(FakeDS()) == "text"++ def test_priority_order(self):+ """'text' should be preferred over 'content'."""++ class FakeDS:+ column_names = ["content", "text", "id"]++ assert auto_detect_text_field(FakeDS()) == "text"++ def test_content_when_no_text(self):+ """'content' should be picked when 'text' is absent."""++ class FakeDS:+ column_names = ["content", "id"]++ assert auto_detect_text_field(FakeDS()) == "content"++ def test_prompt_field(self):+ """'prompt' should be detected when no higher-priority field exists."""++ class FakeDS:+ column_names = ["prompt", "id"]++ assert auto_detect_text_field(FakeDS()) == "prompt"++ def test_fallback_to_longest_string(self):+ """When no known field exists, pick the longest string column."""++ class FakeDS:+ column_names = ["short_col", "long_col", "id"]++ def __iter__(self):+ yield {"short_col": "hi", "long_col": "a much longer text here", "id": 1}+ yield {"short_col": "yo", "long_col": "another long text", "id": 2}++ assert auto_detect_text_field(FakeDS()) == "long_col"++ def test_no_text_field_raises(self):+ """Should raise ValueError when no text field is found."""++ class FakeDS:+ column_names = ["id", "score"]++ def __iter__(self):+ yield {"id": 1, "score": 0.5}++ with pytest.raises(ValueError, match="Cannot auto-detect"):+ auto_detect_text_field(FakeDS())++ def test_features_attribute(self):+ """Should work with features attribute when column_names is absent."""++ class FakeDS:+ features = {"text": "string", "id": "int"}++ def __iter__(self):+ yield {"text": "hello", "id": 1}++ assert auto_detect_text_field(FakeDS()) == "text"+++# ---------------------------------------------------------------------------+# auto_detect_text_field_from_sample+# ---------------------------------------------------------------------------+class TestAutoDetectTextFieldFromSample:+ def test_single_key(self):+ sample = {"text": "hello world"}+ assert auto_detect_text_field_from_sample(sample) == "text"++ def test_priority(self):+ sample = {"content": "hello", "text": "world", "id": 1}+ assert auto_detect_text_field_from_sample(sample) == "text"++ def test_longest_string_fallback(self):+ sample = {"short": "hi", "long": "a much longer text here", "id": 1}+ assert auto_detect_text_field_from_sample(sample) == "long"++ def test_no_text_raises(self):+ sample = {"id": 1, "score": 0.5}+ with pytest.raises(ValueError, match="Cannot auto-detect"):+ auto_detect_text_field_from_sample(sample)+++# ---------------------------------------------------------------------------+# extract_text_from_sample+# ---------------------------------------------------------------------------+class TestExtractTextFromSample:+ def test_single_field_string(self):+ """fields as a single string."""+ sample = {"text": "hello world", "id": 1}+ assert extract_text_from_sample(sample, fields="text") == "hello world"++ def test_single_field_list(self):+ """fields as a single-element list."""+ sample = {"text": "hello world", "id": 1}+ assert extract_text_from_sample(sample, fields=["text"]) == "hello world"++ def test_multiple_fields(self):+ sample = {"question": "What is 2+2?", "answer": "4"}+ result = extract_text_from_sample(sample, fields=["question", "answer"], separator=" ")+ assert result == "What is 2+2? 4"++ def test_template(self):+ sample = {"question": "What is 2+2?", "answer": "4"}+ result = extract_text_from_sample(sample, template="{question} {answer}")+ assert result == "What is 2+2? 4"++ def test_template_with_missing_field(self):+ sample = {"question": "What is 2+2?"}+ with pytest.raises(KeyError, match="answer"):+ extract_text_from_sample(sample, template="{question} {answer}")++ def test_no_field_no_template(self):+ """Should fall back to first string column."""+ sample = {"id": 1, "text": "hello", "score": 0.5}+ assert extract_text_from_sample(sample) == "hello"++ def test_no_text_raises(self):+ sample = {"id": 1, "score": 0.5}+ with pytest.raises(ValueError, match="Cannot extract text"):+ extract_text_from_sample(sample)++ def test_list_field(self):+ sample = {"messages": ["hello", "world"]}+ result = extract_text_from_sample(sample, fields="messages", separator=" ")+ assert result == "hello world"++ def test_list_of_dicts(self):+ """List of message dicts is extracted as a conversation (roles preserved).++ Previously this flattened to an empty string (every row was skipped);+ structured conversations are now extracted with ``role: content`` markers.+ """+ sample = {"messages": [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]}+ result = extract_text_from_sample(sample, fields="messages", separator=" ")+ assert result == "user: hi assistant: hello"++ def test_list_of_dicts_with_template(self):+ """Template mode extracts 'content' from list of dicts."""+ sample = {"messages": [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]}+ result = extract_text_from_sample(sample, template="{messages}", separator=" ")+ assert result == "hi hello"++ def test_template_with_list_field(self):+ sample = {"messages": ["hello", "world"]}+ result = extract_text_from_sample(sample, template="{messages}", separator=" | ")+ assert result == "hello | world"++ def test_template_takes_precedence_over_fields(self):+ """Template should take precedence over fields."""+ sample = {"question": "Q", "answer": "A", "other": "X"}+ result = extract_text_from_sample(sample, fields=["question", "answer"], template="{question}")+ assert result == "Q"+++# ---------------------------------------------------------------------------+# CalibDataset dataclass+# ---------------------------------------------------------------------------+class TestCalibDataset:+ def test_minimal(self):+ spec = CalibDataset("my/ds")+ assert spec.name == "my/ds"+ assert spec.split is None+ assert spec.num is None+ assert spec.concat is False+ assert spec.fields is None+ assert spec.template is None+ assert spec.separator == "\n\n"+ assert spec.apply_chat_template is False+ assert spec.system_prompt is None+ assert spec.timeout == 300++ def test_single_field_string(self):+ spec = CalibDataset("my/ds", fields="text")+ assert spec.fields == "text"++ def test_multiple_fields_list(self):+ spec = CalibDataset("my/ds", fields=["question", "answer"])+ assert spec.fields == ["question", "answer"]++ def test_to_spec_string_minimal(self):+ spec = CalibDataset("my/ds")+ assert spec.to_spec_string() == "my/ds"++ def test_to_spec_string_with_split(self):+ spec = CalibDataset("my/ds", split="train")+ assert spec.to_spec_string() == "my/ds:split=train"++ def test_to_spec_string_with_num(self):+ spec = CalibDataset("my/ds", num=100)+ assert spec.to_spec_string() == "my/ds:num=100"++ def test_to_spec_string_with_fields(self):+ spec = CalibDataset("my/ds", fields=["q", "a"])+ assert spec.to_spec_string() == "my/ds:fields=q+a"++ def test_to_spec_string_single_field(self):+ spec = CalibDataset("my/ds", fields="text")+ assert spec.to_spec_string() == "my/ds:fields=text"++ def test_to_spec_string_with_template(self):+ spec = CalibDataset("my/ds", template="{q} {a}")+ assert spec.to_spec_string() == "my/ds:template={q} {a}"++ def test_to_spec_string_with_separator(self):+ spec = CalibDataset("my/ds", fields=["q", "a"], separator=" ")+ assert spec.to_spec_string() == "my/ds:fields=q+a:separator= "++ def test_to_spec_string_with_timeout(self):+ spec = CalibDataset("my/ds", timeout=600)+ assert spec.to_spec_string() == "my/ds:timeout=600"++ def test_to_spec_string_default_timeout_omitted(self):+ """Default timeout (300) should not appear in spec string."""+ spec = CalibDataset("my/ds", timeout=300)+ assert spec.to_spec_string() == "my/ds"++ def test_to_spec_string_full(self):+ spec = CalibDataset(+ "my/ds",+ split="train",+ num=500,+ fields=["q", "a"],+ separator=" ",+ apply_chat_template=True,+ system_prompt="You are helpful.",+ timeout=600,+ )+ assert (+ spec.to_spec_string()+ == "my/ds:split=train:num=500:fields=q+a:separator= :apply_chat_template=true:system_prompt=You are helpful.:timeout=600"+ )++ def test_from_spec_string(self):+ spec = CalibDataset.from_spec_string("my/ds:split=train:num=100:fields=q+a")+ assert spec.name == "my/ds"+ assert spec.split == ["train"]+ assert spec.num == 100+ assert spec.fields == ["q", "a"]++ def test_from_spec_string_single_field(self):+ spec = CalibDataset.from_spec_string("my/ds:fields=text")+ assert spec.fields == "text"++ def test_from_spec_string_with_timeout(self):+ spec = CalibDataset.from_spec_string("my/ds:timeout=600")+ assert spec.timeout == 600++ def test_from_spec_string_bare_apply_chat_template(self):+ """Bare ``apply_chat_template`` flag survives parsing and round-trips."""+ spec = CalibDataset.from_spec_string("my/ds:apply_chat_template")+ assert spec.apply_chat_template is True+ assert "apply_chat_template=true" in spec.to_spec_string()++ def test_from_spec_string_bare_concat(self):+ """Bare ``concat`` flag survives parsing."""+ spec = CalibDataset.from_spec_string("my/ds:concat")+ assert spec.concat is True++ def test_roundtrip(self):+ """build -> parse -> build should be identity."""+ spec = CalibDataset("my/ds", split="train", num=100, fields=["q", "a"], separator=" ")+ spec_str = spec.to_spec_string()+ parsed = CalibDataset.from_spec_string(spec_str)+ assert parsed.to_spec_string() == spec_str++ def test_roundtrip_template(self):+ spec = CalibDataset("my/ds", template="{question} {answer}", separator=" ")+ spec_str = spec.to_spec_string()+ parsed = CalibDataset.from_spec_string(spec_str)+ assert parsed.template == "{question} {answer}"+ assert parsed.separator == " "++ def test_roundtrip_template_with_colons(self):+ """A template containing ':' must survive serialization (escaped)."""+ tmpl = "Question: {question}\nAnswer: {answer}"+ spec = CalibDataset("my/ds", template=tmpl)+ parsed = CalibDataset.from_spec_string(spec.to_spec_string())+ assert parsed.template == tmpl++ def test_roundtrip_system_prompt_with_comma_and_colon(self):+ """A system prompt with ',' and ':' must not create extra datasets/options."""+ prompt = "Note: be concise, clear, and kind"+ spec = CalibDataset("my/ds", system_prompt=prompt)+ parsed = CalibDataset.from_spec_string(spec.to_spec_string())+ assert parsed.system_prompt == prompt++ def test_comma_in_value_does_not_split_datasets(self):+ """Commas inside option values must stay within a single dataset spec."""+ from auto_round.utils.dataset_utils import split_unescaped++ spec = CalibDataset("my/ds", system_prompt="a, b, c").to_spec_string()+ multi = spec + "," + "pile-10k"+ assert split_unescaped(multi, ",") == [spec, "pile-10k"]++ def test_roundtrip_with_timeout(self):+ spec = CalibDataset("my/ds", num=100, timeout=600)+ spec_str = spec.to_spec_string()+ parsed = CalibDataset.from_spec_string(spec_str)+ assert parsed.timeout == 600+ assert parsed.to_spec_string() == spec_str+++# ---------------------------------------------------------------------------+# build_dataset_spec+# ---------------------------------------------------------------------------+class TestBuildDatasetSpec:+ def test_minimal(self):+ assert build_dataset_spec("my/ds") == "my/ds"++ def test_with_split(self):+ assert build_dataset_spec("my/ds", split="train") == "my/ds:split=train"++ def test_with_num(self):+ assert build_dataset_spec("my/ds", num=100) == "my/ds:num=100"++ def test_with_split_and_num(self):+ assert build_dataset_spec("my/ds", split="train", num=100) == "my/ds:split=train:num=100"++ def test_with_fields_string(self):+ assert build_dataset_spec("my/ds", fields="text") == "my/ds:fields=text"++ def test_with_fields_list(self):+ assert build_dataset_spec("my/ds", fields=["q", "a"]) == "my/ds:fields=q+a"++ def test_with_template(self):+ assert build_dataset_spec("my/ds", template="{q} {a}") == "my/ds:template={q} {a}"++ def test_with_separator(self):+ assert build_dataset_spec("my/ds", fields=["q", "a"], separator=" ") == "my/ds:fields=q+a:separator= "++ def test_with_chat_template(self):+ assert build_dataset_spec("my/ds", apply_chat_template=True) == "my/ds:apply_chat_template=true"++ def test_with_timeout(self):+ assert build_dataset_spec("my/ds", timeout=600) == "my/ds:timeout=600"++ def test_with_default_timeout_omitted(self):+ """Default timeout (300) should not appear in spec string."""+ assert build_dataset_spec("my/ds", timeout=300) == "my/ds"++ def test_full_spec(self):+ assert (+ build_dataset_spec(+ "my/ds", split="train", num=100, fields=["q", "a"], separator=" ", apply_chat_template=True+ )+ == "my/ds:split=train:num=100:fields=q+a:separator= :apply_chat_template=true"+ )++ def test_full_spec_with_timeout(self):+ assert (+ build_dataset_spec("my/ds", split="train", num=100, fields=["q", "a"], separator=" ", timeout=600)+ == "my/ds:split=train:num=100:fields=q+a:separator= :timeout=600"+ )+++# ---------------------------------------------------------------------------+# parse_dataset_spec+# ---------------------------------------------------------------------------+class TestParseDatasetSpec:+ def test_name_only(self):+ result = parse_dataset_spec("my/ds")+ assert result["name"] == "my/ds"+ assert result["split"] is None+ assert result["num"] is None+ assert result["concat"] is False+ assert result["fields"] is None+ assert result["template"] is None+ assert result["separator"] == "\n\n"+ assert result["apply_chat_template"] is False+ assert result["system_prompt"] is None++ def test_split(self):+ result = parse_dataset_spec("my/ds:split=train")+ assert result["name"] == "my/ds"+ assert result["split"] == ["train"]++ def test_multi_split(self):+ result = parse_dataset_spec("my/ds:split=train+test")+ assert result["split"] == ["train", "test"]++ def test_num(self):+ result = parse_dataset_spec("my/ds:num=100")+ assert result["num"] == 100++ def test_concat(self):+ result = parse_dataset_spec("my/ds:concat=true")+ assert result["concat"] is True++ def test_concat_false(self):+ result = parse_dataset_spec("my/ds:concat=false")+ assert result["concat"] is False++ def test_fields_single(self):+ result = parse_dataset_spec("my/ds:fields=text")+ assert result["fields"] == "text"++ def test_fields_multiple(self):+ result = parse_dataset_spec("my/ds:fields=q+a")+ assert result["fields"] == ["q", "a"]++ def test_template(self):+ result = parse_dataset_spec("my/ds:template={q} {a}")+ assert result["template"] == "{q} {a}"++ def test_separator(self):+ result = parse_dataset_spec("my/ds:fields=q+a:separator= ")+ assert result["separator"] == " "++ def test_apply_chat_template(self):+ result = parse_dataset_spec("my/ds:apply_chat_template=true")+ assert result["apply_chat_template"] is True++ def test_bare_apply_chat_template(self):+ """Bare ``apply_chat_template`` flag (no ``=value``) is treated as true."""+ result = parse_dataset_spec("my/ds:apply_chat_template")+ assert result["apply_chat_template"] is True++ def test_bare_concat(self):+ """Bare ``concat`` flag (no ``=value``) is treated as true."""+ result = parse_dataset_spec("my/ds:concat")+ assert result["concat"] is True++ def test_bare_flag_combined_with_kv_options(self):+ """Bare boolean flags coexist with regular key=value options."""+ result = parse_dataset_spec("my/ds:num=100:apply_chat_template:concat")+ assert result["num"] == 100+ assert result["apply_chat_template"] is True+ assert result["concat"] is True++ def test_incomplete_non_boolean_option_ignored(self):+ """An option without ``=`` that is not a known bare flag is ignored."""+ result = parse_dataset_spec("my/ds:split")+ assert result["name"] == "my/ds"+ assert result["split"] is None++ def test_system_prompt(self):+ result = parse_dataset_spec("my/ds:system_prompt=You are helpful.")+ assert result["system_prompt"] == "You are helpful."++ def test_combined(self):+ result = parse_dataset_spec("my/ds:split=train+test:num=100:fields=q+a")+ assert result["name"] == "my/ds"+ assert result["split"] == ["train", "test"]+ assert result["num"] == 100+ assert result["fields"] == ["q", "a"]++ def test_timeout(self):+ result = parse_dataset_spec("my/ds:timeout=600")+ assert result["timeout"] == 600++ def test_timeout_default(self):+ result = parse_dataset_spec("my/ds")+ assert result["timeout"] == 300++ def test_roundtrip(self):+ """build -> parse should be identity."""+ spec = build_dataset_spec("my/ds", split="train", num=100, fields=["q", "a"], separator=" ")+ parsed = parse_dataset_spec(spec)+ assert parsed["name"] == "my/ds"+ assert parsed["split"] == ["train"]+ assert parsed["num"] == 100+ assert parsed["fields"] == ["q", "a"]+ assert parsed["separator"] == " "++ def test_roundtrip_with_timeout(self):+ spec = build_dataset_spec("my/ds", num=100, timeout=600)+ parsed = parse_dataset_spec(spec)+ assert parsed["timeout"] == 600+++# ---------------------------------------------------------------------------+# normalize_dataset_spec+# ---------------------------------------------------------------------------+class TestNormalizeDatasetSpec:+ def test_none_raises(self):+ with pytest.raises(TypeError, match="must be a str"):+ normalize_dataset_spec(None)++ def test_empty_string(self):+ assert normalize_dataset_spec("") == ""++ def test_single_string(self):+ assert normalize_dataset_spec("my/ds") == "my/ds"++ def test_single_calib_dataset(self):+ spec = CalibDataset("my/ds", num=100)+ assert normalize_dataset_spec(spec) == "my/ds:num=100"++ def test_list_of_strings_is_raw_data(self):+ """A list of plain strings is raw calibration text, passed through."""+ result = normalize_dataset_spec(["my/ds", "other/ds"])+ assert result == ["my/ds", "other/ds"]++ def test_list_of_calib_datasets(self):+ specs = [CalibDataset("my/ds", num=100), CalibDataset("other/ds", num=200)]+ result = normalize_dataset_spec(specs)+ assert result == "my/ds:num=100,other/ds:num=200"++ def test_mixed_list_is_raw_data(self):+ """A list mixing strings and CalibDataset objects is raw data."""+ specs = ["my/ds", CalibDataset("other/ds", num=200, timeout=60)]+ result = normalize_dataset_spec(specs)+ assert result == specs++ def test_tuple_of_calib_datasets(self):+ specs = (CalibDataset("my/ds", num=100), CalibDataset("other/ds", num=200))+ result = normalize_dataset_spec(specs)+ assert result == "my/ds:num=100,other/ds:num=200"++ def test_list_with_non_spec_items_is_raw_data(self):+ """A list containing non-str/non-CalibDataset items is raw data."""+ sentinel = object()+ result = normalize_dataset_spec([sentinel])+ assert result == [sentinel]++ def test_empty_list(self):+ assert normalize_dataset_spec([]) == ""+++# ---------------------------------------------------------------------------+# Structured-conversation extraction (messages / conversations fields)+# ---------------------------------------------------------------------------+class TestConversationExtraction:+ def test_is_conversation_value_hf_schema(self):+ value = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]+ assert is_conversation_value(value) is True++ def test_is_conversation_value_sharegpt_schema(self):+ value = [{"from": "human", "value": "hi"}, {"from": "gpt", "value": "hello"}]+ assert is_conversation_value(value) is True++ def test_is_conversation_value_plain_string_list(self):+ assert is_conversation_value(["hi", "hello"]) is False++ def test_is_conversation_value_non_list(self):+ assert is_conversation_value("hi") is False+ assert is_conversation_value([]) is False+ assert is_conversation_value([{"foo": "bar"}]) is False++ def test_extract_conversation_text_preserves_roles(self):+ value = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]+ assert extract_conversation_text(value, separator=" ") == "user: hi assistant: hello"++ def test_extract_conversation_text_sharegpt(self):+ value = [{"from": "human", "value": "hi"}, {"from": "gpt", "value": "hello"}]+ assert extract_conversation_text(value, separator="\n") == "human: hi\ngpt: hello"++ def test_extract_conversation_text_multimodal_content_parts(self):+ value = [{"role": "user", "content": [{"type": "text", "text": "hi"}, {"type": "text", "text": "there"}]}]+ assert extract_conversation_text(value, separator=" ") == "user: hi there"++ def test_single_field_messages_not_empty(self):+ """Regression: a 'messages' field of dicts must not flatten to ''."""+ sample = {"messages": [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "yo"}]}+ result = extract_text_from_sample(sample, fields="messages", separator=" ")+ assert result == "user: hi assistant: yo"+ assert result.strip() != ""++ def test_fallback_detects_conversation(self):+ """Auto fallback (no fields) extracts a conversation instead of ''."""+ sample = {"conversations": [{"from": "human", "value": "hi"}, {"from": "gpt", "value": "yo"}]}+ result = extract_text_from_sample(sample)+ assert "hi" in result and "yo" in result+++# ---------------------------------------------------------------------------+# auto_detect_text_field with unresolved streaming metadata (column_names=None)+# ---------------------------------------------------------------------------+class TestAutoDetectUnresolvedMetadata:+ class _FakeIterable:+ """Mimics a streaming IterableDataset whose schema is not yet resolved."""++ column_names = None+ features = None++ def __init__(self, rows):+ self._rows = rows++ def __iter__(self):+ return iter(self._rows)++ def test_none_column_names_falls_back_to_sampling(self):+ ds = self._FakeIterable([{"text": "the quick brown fox", "id": 1}])+ # Must not raise TypeError from list(None); detects from a sample instead.+ assert auto_detect_text_field(ds) == "text"++ def test_none_metadata_single_column(self):+ ds = self._FakeIterable([{"body": "hello world"}])+ assert auto_detect_text_field(ds) == "body"+++# ---------------------------------------------------------------------------+# Spec value escaping (preserve ':' and ',' in option values)+# ---------------------------------------------------------------------------+class TestSpecEscaping:+ def test_escape_unescape_roundtrip(self):+ for raw in ["plain", "a: b", "a, b", "x\\y", "line1\nline2", "tab\there"]:+ assert unescape_spec_value(escape_spec_value(raw)) == raw++ def test_escape_protects_delimiters(self):+ escaped = escape_spec_value("a:b,c")+ assert ":" not in escaped.replace("\\:", "")+ assert "," not in escaped.replace("\\,", "")++ def test_split_unescaped_ignores_escaped_delims(self):+ assert split_unescaped("a\\:b:c", ":") == ["a\\:b", "c"]+ assert split_unescaped("a\\,b,c", ",") == ["a\\,b", "c"]++ def test_split_unescaped_maxsplit(self):+ assert split_unescaped("a:b:c", ":", maxsplit=1) == ["a", "b:c"]++ def test_template_with_colon_roundtrip(self):+ tmpl = "Question: {q}\nAnswer: {a}"+ parsed = parse_dataset_spec(CalibDataset("my/ds", template=tmpl).to_spec_string())+ assert parsed["template"] == tmpl++ def test_system_prompt_with_comma_roundtrip(self):+ prompt = "Be concise, clear: and kind"+ parsed = parse_dataset_spec(CalibDataset("my/ds", system_prompt=prompt).to_spec_string())+ assert parsed["system_prompt"] == prompt++ def test_comma_in_value_not_split_as_dataset(self):+ spec = CalibDataset("my/ds", system_prompt="a, b, c").to_spec_string()+ multi = spec + ",pile-10k"+ assert split_unescaped(multi, ",") == [spec, "pile-10k"]++ def test_legacy_unescaped_spec_still_parses(self):+ """Specs without any escapes keep their original parsing behavior."""+ result = parse_dataset_spec("my/ds:split=train+test:num=100:fields=q+a")+ assert result["name"] == "my/ds"+ assert result["split"] == ["train", "test"]+ assert result["fields"] == ["q", "a"]++ def test_legacy_separator_escape_newline(self):+ assert parse_dataset_spec("my/ds:separator=\\n:fields=text")["separator"] == "\n"+++# ---------------------------------------------------------------------------+# strip_dataset_spec_whitespace (preserve spaces inside option values)+# ---------------------------------------------------------------------------+class TestStripSpecWhitespace:+ def test_preserves_separator_space(self):+ assert strip_dataset_spec_whitespace("my/ds:separator= ") == "my/ds:separator= "++ def test_preserves_template_spaces(self):+ spec = "my/ds:template={question} {answer}"+ assert strip_dataset_spec_whitespace(spec) == spec++ def test_trims_cosmetic_whitespace_around_commas(self):+ assert strip_dataset_spec_whitespace("pile-10k , ultrachat") == "pile-10k,ultrachat"++ def test_trims_name_and_keys(self):+ assert strip_dataset_spec_whitespace(" my/ds : fields=text") == "my/ds:fields=text"++ def test_preserves_escaped_value_spaces(self):+ spec = CalibDataset("my/ds", system_prompt="a, b: c").to_spec_string()+ # Round-trips through whitespace stripping without corrupting the value.+ stripped = strip_dataset_spec_whitespace(spec)+ assert parse_dataset_spec(stripped)["system_prompt"] == "a, b: c"