Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[Experimental]Add generic HF dataset support and two more datasets (#2363)

Wenhua Chenge31b77284e8b
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 604a28ae3..472706173 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -16,7 +16,7 @@ from __future__ import annotations
import functools
import inspect
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Optional, Union
import torch
@@ -30,13 +30,14 @@ from auto_round.scheme_entry import (
resolve_entry_scheme,
)
from auto_round.schemes import QuantizationScheme
+from auto_round.utils.dataset_utils import CalibDataset, normalize_dataset_spec
from auto_round.utils.device_manager import normalize_default_device_map
if TYPE_CHECKING:
from auto_round.algorithms.quantization.config import QuantizationConfig
from auto_round.algorithms.quantization.rtn.config import RTNConfig
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
- from auto_round.compressors.base import BaseOrchestrator as BaseCompressor
+ from auto_round.compressors.base import BaseCompressor
# ---------------------------------------------------------------------------
@@ -541,7 +542,7 @@ class _CompressorBuilder:
tokenizer=None,
platform="hf",
format=None,
- dataset="NeelNanda/pile-10k",
+ dataset: str | CalibDataset | list | tuple | torch.utils.data.DataLoader | None = None,
low_gpu_mem_usage: bool = False,
device_map: str | torch.device | int | dict = 0,
iters: int | None = None,
@@ -558,8 +559,15 @@ class _CompressorBuilder:
from auto_round.algorithms.registry import normalize_algorithm_config
from auto_round.compressors.orchestrator import CompressionOrchestrator as Compressor
from auto_round.compressors.utils import check_need_act_calibration
+ from auto_round.utils.dataset_utils import CalibDataset
from auto_round.utils.model import is_model_free_route
+ # Normalize CalibDataset objects to spec strings
+ if isinstance(dataset, CalibDataset):
+ dataset = dataset.to_spec_string()
+ elif isinstance(dataset, (list, tuple)):
+ dataset = normalize_dataset_spec(dataset)
+
if alg_configs is None:
alg_configs = "signround"
# TODO wenhuach if key in kwargs could override scheme and alg_config, we should pop and override,
@@ -714,7 +722,8 @@ class AutoRound:
Use config instances to provide algorithm-specific options, such
as ``SignRoundConfig(iters=50)`` or ``AWQConfig(apply_clip=True)``.
layer_config: Optional per-layer quantization overrides.
- dataset: Calibration dataset name, samples, or dataloader.
+ dataset: Calibration dataset name, samples, dataloader, or a
+ :class:`~auto_round.utils.dataset_utils.CalibDataset` object.
seqlen: Calibration sequence length.
nsamples: Number of calibration samples.
batch_size: Calibration batch size.
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index 499e630eb..09df68453 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -29,8 +29,10 @@ logging.getLogger("datasets").setLevel(logging.WARNING)
import torch
from datasets import (
Dataset,
+ DatasetDict,
Features,
IterableDataset,
+ IterableDatasetDict,
Sequence,
Value,
concatenate_datasets,
@@ -42,6 +44,15 @@ from torch.utils.data import DataLoader
from . import envs
from .utils import is_local_path, logger
+from .utils.dataset_utils import (
+ CalibDataset,
+ auto_detect_text_field,
+ auto_detect_text_field_from_sample,
+ extract_text_from_sample,
+ normalize_dataset_spec,
+ split_unescaped,
+ unescape_spec_value,
+)
CALIB_DATASETS = {}
_GITHUB_CODE_CLEAN_MAX_DATASETS_VERSION = Version("3.6.0")
@@ -84,7 +95,7 @@ def _get_dataset_network_error(error):
return None
-def _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples):
+def _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code=False):
"""Switch to ModelScope FineWeb-Edu after a calibration dataset network failure."""
network_error = _get_dataset_network_error(error)
if network_error is None:
@@ -108,15 +119,17 @@ def _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamp
network_error,
_FINEWEB_EDU_MODELSCOPE_DATASET,
)
- return _get_dataset_impl(tokenizer, seqlen, _FINEWEB_EDU_MODELSCOPE_DATASET, seed, nsamples)
+ return _get_dataset_impl(tokenizer, seqlen, _FINEWEB_EDU_MODELSCOPE_DATASET, seed, nsamples, trust_remote_code)
-def _preprocess_dataset_in_subprocess(result_queue, tokenizer, seqlen, dataset_name, seed, nsamples, output_path):
+def _preprocess_dataset_in_subprocess(
+ result_queue, tokenizer, seqlen, dataset_name, seed, nsamples, output_path, trust_remote_code=False
+):
"""Run dataset preprocessing and report network failures to the parent process."""
# The parent's OpenMP thread pool does not survive fork; keep torch single-threaded here.
torch.set_num_threads(1)
try:
- dataset = _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)
+ dataset = _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code)
dataset.save_to_disk(output_path)
except Exception as error:
network_error = _get_dataset_network_error(error)
@@ -175,35 +188,83 @@ def register_dataset(name):
return register
-def apply_chat_template_to_samples(samples, tokenizer, seqlen, system_prompt=None):
- rendered_messages = []
- # if system_prompt is None: ## remove system prompt as models like deepseek don't recommend using it
- # system_prompt = "You are a helpful assistant."
- for text in samples:
- message = []
- if system_prompt is not None and system_prompt != "":
- message.append({"role": "system", "content": system_prompt})
+def render_chat_sample(tokenizer, text, system_prompt=None, tools=None):
+ """Render a single conversation to a string with the chat template.
- if isinstance(text, list) and isinstance(text[0], dict):
- message += text
- else:
- message.append({"role": "user", "content": text})
+ Args:
+ tokenizer: The tokenizer providing ``apply_chat_template``.
+ text: Either a plain user string or a list of chat messages (dicts with
+ ``role``/``content``).
+ system_prompt: Optional global system prompt. It is only prepended when
+ the conversation does not already start with its own system turn.
+ tools: Optional tool definitions for this conversation, passed to
+ ``apply_chat_template(..., tools=...)``.
+
+ Returns:
+ The rendered chat string.
+ """
+ message = []
+ is_full_conversation = isinstance(text, list) and len(text) > 0 and isinstance(text[0], dict)
+
+ # Respect a per-sample system prompt: if the conversation already starts
+ # with its own system message, do not prepend the global system_prompt
+ # (otherwise the rendered text would contain two system turns).
+ has_own_system = is_full_conversation and text[0].get("role") == "system"
+ if system_prompt is not None and system_prompt != "" and not has_own_system:
+ message.append({"role": "system", "content": system_prompt})
+
+ if is_full_conversation:
+ message += text
+ else:
+ message.append({"role": "user", "content": text})
+
+ sample_tools = tools or None
+
+ # For full multi-turn conversations ending in an assistant reply we want
+ # the whole trace as calibration text (including tool_calls / tool
+ # responses), so we do not append a generation prompt.
+ add_generation_prompt = not (is_full_conversation and message[-1].get("role") == "assistant")
+
+ template_kwargs = {"tokenize": False, "add_generation_prompt": add_generation_prompt}
+ if sample_tools is not None:
+ template_kwargs["tools"] = sample_tools
+
+ try:
+ return tokenizer.apply_chat_template(message, **template_kwargs)
+ except Exception:
+ logger.warning("Failed to apply chat template. removing the system role in chat history.")
+ message_modified = [msg for msg in message if msg["role"] != "system"]
try:
- chat_templated = tokenizer.apply_chat_template(
- message,
- tokenize=False,
- add_generation_prompt=True,
- )
+ return tokenizer.apply_chat_template(message_modified, **template_kwargs)
except Exception:
- logger.warning("Failed to apply chat template. removing the system role in chat history.")
- message_modified = [msg for msg in message if msg["role"] != "system"]
- chat_templated = tokenizer.apply_chat_template(
- message_modified,
- tokenize=False,
- add_generation_prompt=True,
- )
+ # Last resort: drop tools in case the template does not support them.
+ template_kwargs.pop("tools", None)
+ return tokenizer.apply_chat_template(message_modified, **template_kwargs)
+
+
+def apply_chat_template_to_samples(samples, tokenizer, seqlen, system_prompt=None, tools=None):
+ """Render samples with the tokenizer chat template.
- rendered_messages.append(chat_templated)
+ Args:
+ samples: A list where each item is either a plain user string or a list
+ of chat messages (dicts with ``role``/``content``).
+ tokenizer: The tokenizer providing ``apply_chat_template``.
+ seqlen: Maximum sequence length for tokenization.
+ system_prompt: Optional system prompt prepended to each conversation.
+ tools: Optional per-sample tool definitions aligned with ``samples``
+ (a list the same length as ``samples``; each entry is the ``tools``
+ list for that conversation, or ``None``). Passed to
+ ``apply_chat_template(..., tools=...)`` so agent/tool-calling data
+ keeps its tool schema. When the conversation already contains the
+ final assistant turn (full multi-turn agent traces), the generation
+ prompt is not appended.
+ """
+ rendered_messages = []
+ # if system_prompt is None: ## remove system prompt as models like deepseek don't recommend using it
+ # system_prompt = "You are a helpful assistant."
+ for idx, text in enumerate(samples):
+ sample_tools = tools[idx] if (tools is not None and idx < len(tools)) else None
+ rendered_messages.append(render_chat_sample(tokenizer, text, system_prompt, sample_tools))
example = tokenizer(rendered_messages, truncation=True, max_length=seqlen)
return example
@@ -370,7 +431,14 @@ def get_fineweb_edu_dataset(
@register_dataset(["BAAI/CCI3-HQ", "CCI3-HQ"])
def get_cci3_hq_dataset(
- tokenizer, seqlen, dataset_name="BAAI/CCI3-HQ", split=None, seed=42, apply_chat_template=False, system_prompt=None
+ tokenizer,
+ seqlen,
+ dataset_name="BAAI/CCI3-HQ",
+ split=None,
+ seed=42,
+ apply_chat_template=False,
+ system_prompt=None,
+ nsamples=512,
):
"""Returns a dataloader for the specified dataset and split.
@@ -392,7 +460,7 @@ def get_cci3_hq_dataset(
)
calib_dataset = load_dataset("BAAI/CCI3-HQ", split="train", streaming=True)
- calib_dataset = calib_dataset.shuffle(seed=seed).take(10000)
+ calib_dataset = calib_dataset.shuffle(seed=seed).take(nsamples * envs.AR_CALIB_DATA_MULTIPLIER)
calib_dataset = calib_dataset.map(tokenizer_function, batched=True)
return calib_dataset
@@ -407,6 +475,7 @@ def get_github_code_clean_dataset(
seed=42,
apply_chat_template=False,
system_prompt=None,
+ nsamples=512,
):
"""Returns a dataloader for the specified dataset and split.
@@ -462,7 +531,9 @@ def get_github_code_clean_dataset(
else:
raise
calib_dataset = concatenate_datasets([dataset_mit, dataset_apache])
- calib_dataset = calib_dataset.shuffle(seed=seed).take(10000) ##TODO concat data'shuffle may have bugs
+ calib_dataset = calib_dataset.shuffle(seed=seed).take(
+ nsamples * envs.AR_CALIB_DATA_MULTIPLIER
+ ) ##TODO concat data'shuffle may have bugs
calib_dataset = calib_dataset.map(tokenizer_function, batched=True)
return calib_dataset
@@ -477,6 +548,7 @@ def get_opencode_instruct_dataset(
seed=42,
apply_chat_template=False,
system_prompt=None,
+ nsamples=512,
):
"""Return tokenized coding instructions and responses from OpenCodeInstruct."""
split = "train" if split is None else split
@@ -492,7 +564,7 @@ def get_opencode_instruct_dataset(
)
dataset = load_dataset("nvidia/OpenCodeInstruct", split=split, streaming=True)
- dataset = dataset.shuffle(seed=seed).take(10000)
+ dataset = dataset.shuffle(seed=seed).take(nsamples * envs.AR_CALIB_DATA_MULTIPLIER)
samples = []
for data in dataset:
if apply_chat_template:
@@ -570,6 +642,103 @@ def get_opencode_instruct_dataset(
return calib_dataset
+@register_dataset(["qwedsacf/competition_math", "competition_math"])
+def get_competition_math_dataset(
+ tokenizer,
+ seqlen,
+ dataset_name="qwedsacf/competition_math",
+ split=None,
+ seed=42,
+ apply_chat_template=False,
+ system_prompt=None,
+ nsamples=512,
+):
+ """Return a calibration dataset built from the competition_math corpus.
+
+ Each sample concatenates the ``problem`` and ``solution`` fields. Because the
+ individual problem/solution pairs are much shorter than typical calibration
+ sequence lengths (e.g. 2048), the tokenized samples are packed together into
+ full ``seqlen``-length sequences instead of being dropped by the downstream
+ length filter.
+
+ Args:
+ tokenizer: The tokenizer to be used for tokenization.
+ seqlen: The maximum/target sequence length.
+ dataset_name: Dataset identifier (defaults to ``qwedsacf/competition_math``).
+ split: The data split to use. Defaults to ``train``.
+ seed: The random seed for shuffling.
+ apply_chat_template: Whether to apply the tokenizer's chat template.
+ system_prompt: Optional system prompt for the chat template.
+ nsamples: Number of calibration samples requested. The candidate pool is
+ ``nsamples * AR_CALIB_DATA_MULTIPLIER`` before packing.
+
+ Returns:
+ A HuggingFace ``Dataset`` of packed, tokenized calibration sequences.
+ """
+ split = "train" if split is None else split
+ if isinstance(split, list):
+ if len(split) != 1:
+ raise ValueError("competition_math supports only one split at a time.")
+ split = split[0]
+
+ # The chat template (when requested) is already rendered per-sample in the
+ # loop below, so tokenize the resulting "text" plainly here to avoid applying
+ # the template a second time (which would duplicate role markers/prompts).
+ tokenizer_function = get_tokenizer_function(tokenizer, seqlen)
+
+ # competition_math is small; load it fully instead of streaming.
+ # Always use the canonical Hub repo id (the short "competition_math" alias
+ # is only a registry key, not a valid Hub dataset name).
+ dataset = load_dataset("qwedsacf/competition_math", split=split, trust_remote_code=False)
+ dataset = dataset.shuffle(seed=seed)
+ max_candidates = nsamples * envs.AR_CALIB_DATA_MULTIPLIER
+ if len(dataset) > max_candidates:
+ dataset = dataset.select(range(max_candidates))
+
+ samples = []
+ for data in dataset:
+ problem = (data.get("problem") or "").strip()
+ solution = (data.get("solution") or "").strip()
+ if not problem and not solution:
+ continue
+ if apply_chat_template:
+ messages = []
+ if system_prompt is not None and system_prompt != "":
+ messages.append({"role": "system", "content": system_prompt})
+ messages.extend(
+ [
+ {"role": "user", "content": problem},
+ {"role": "assistant", "content": solution},
+ ]
+ )
+ try:
+ text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
+ except Exception:
+ logger.warning("Failed to apply chat template. removing the system role in chat history.")
+ messages = [message for message in messages if message["role"] != "system"]
+ text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
+ samples.append({"text": text})
+ else:
+ samples.append({"text": f"{problem}\n\n{solution}"})
+ random.Random(seed).shuffle(samples)
+
+ calib_dataset = Dataset.from_list(samples)
+ calib_dataset = calib_dataset.map(
+ tokenizer_function,
+ batched=True,
+ new_fingerprint=_make_map_fingerprint(
+ calib_dataset, tokenizer, seqlen, apply_chat_template, system_prompt, "text"
+ ),
+ )
+
+ # Sample-level concat by default: pack whole samples until each reaches
+ # seqlen, then reset, so short problem/solution pairs are not dropped by the
+ # downstream length filter.
+ calib_dataset = _concat_samples_to_seqlen(calib_dataset, seqlen)
+
+ return calib_dataset
+
+
@register_dataset(["HuggingFaceH4/ultrachat_200k", "ultrachat_200k"])
def get_ultrachat_dataset(
tokenizer,
@@ -579,6 +748,7 @@ def get_ultrachat_dataset(
seed=42,
apply_chat_template=True,
system_prompt=None,
+ nsamples=512,
):
if split is None:
split = "train_sft"
@@ -587,7 +757,7 @@ def get_ultrachat_dataset(
raise ValueError(f"split must be one of {all_splits} for ultrachat_200k ")
dataset = load_dataset("HuggingFaceH4/ultrachat_200k", split=split, streaming=True, trust_remote_code=True)
- dataset = dataset.shuffle(seed=seed).take(20000)
+ dataset = dataset.shuffle(seed=seed).take(nsamples * envs.AR_CALIB_DATA_MULTIPLIER)
def is_instruct_tokenizer(tokenizer):
try:
@@ -598,12 +768,9 @@ def get_ultrachat_dataset(
is_instruct = is_instruct_tokenizer(tokenizer)
- if is_instruct and not apply_chat_template:
- logger.info("Tokenizer looks like an instruct/chat model, but apply_chat_template=False. Setting to True.")
- apply_chat_template = True
- elif not is_instruct and apply_chat_template:
+ if not is_instruct and apply_chat_template:
logger.info("Tokenizer is not an instruct/chat model, but apply_chat_template=True. Setting to False.")
- apply_chat_template = False
+ apply_chat_template = False
def tokenize_example_batch(examples):
if not apply_chat_template:
@@ -619,6 +786,250 @@ def get_ultrachat_dataset(
return dataset
+@register_dataset(["openbmb/UltraData-SFT-Agent-2609", "UltraData-SFT-Agent-2609"])
+def get_ultradata_agent_dataset(
+ tokenizer,
+ seqlen,
+ dataset_name="openbmb/UltraData-SFT-Agent-2609",
+ split=None,
+ seed=42,
+ apply_chat_template=True,
+ system_prompt=None,
+ nsamples=512,
+):
+ """Agent/tool-calling calibration dataset.
+
+ Each sample carries a ``messages`` conversation (system/user/assistant with
+ ``tool_calls``/tool turns) and a ``tools`` schema list. The chat template is
+ rendered with ``tools=`` so the quantized model sees full tool-calling
+ traces. ``apply_chat_template`` is forced on because the raw ``messages``
+ structure is only meaningful once templated.
+
+ The dataset ships multiple configs (``Code-Agent``, ``Search-Agent``,
+ ``General-Agent``, ``Tool-Use``); all of them are loaded and interleaved so
+ calibration covers the full range of agent behaviors.
+
+ Length-aware sampling: conversations in this dataset can be very long. A
+ sample whose *messages alone* already exceed ``seqlen`` would have its actual
+ conversation content truncated, so such samples are deprioritized. We first
+ collect samples whose rendered messages fit within ``seqlen`` (even if
+ ``messages + tools`` exceeds it, since only the tool schema overflows); only
+ if there are not enough of those do we fall back to the oversized ones.
+ """
+ if split is None:
+ split = "train"
+
+ # Always load from the canonical Hub repo id, regardless of whether the user
+ # passed the full name or the short alias ("UltraData-SFT-Agent-2609").
+ hub_repo_id = "openbmb/UltraData-SFT-Agent-2609"
+
+ # This dataset is split into multiple configs; mix them all so the quantized
+ # model sees the full range of agent behaviors (code / search / general /
+ # tool-use). We avoid ``interleave_datasets`` on purpose: the configs have
+ # slightly different ``tools`` feature schemas (e.g. an extra ``title`` key),
+ # which makes HF feature alignment fail. Since we only read ``messages`` and
+ # ``tools`` and render them ourselves, we scan each config independently and
+ # then balance the final selection across configs.
+ configs = ["Code-Agent", "Search-Agent", "General-Agent", "Tool-Use"]
+ sub_datasets = []
+ for cfg in configs:
+ try:
+ d = load_dataset(hub_repo_id, cfg, split=split, streaming=True)
+ d = d.shuffle(seed=seed, buffer_size=1000)
+ sub_datasets.append(d)
+ except Exception as e: # noqa: BLE001
+ logger.warning("Failed to load config '%s' of %s: %s", cfg, hub_repo_id, e)
+
+ if not sub_datasets:
+ raise ValueError(f"Could not load any config of {hub_repo_id} (tried {configs}).")
+
+ if not apply_chat_template:
+ logger.info(
+ "apply_chat_template=False is ignored for %s; agent/tool data requires chat templating.", hub_repo_id
+ )
+
+ # target = nsamples * envs.AR_CALIB_DATA_MULTIPLIER
+ target = nsamples * 2
+ n_cfg = len(sub_datasets)
+ # Per-config quota so every config contributes a balanced share. Collect a
+ # bit extra per config so that, if one config is short, others can cover the
+ # shortfall during the balanced round-robin selection below.
+ per_cfg_quota = -(-target // n_cfg) # ceil(target / n_cfg)
+ # Safety bound per config: don't scan a whole (huge) config looking for
+ # short-enough samples when they are scarce.
+ max_scan_per_cfg = target
+
+ # The per-sample system prompt in this dataset is very long and pushes most
+ # samples past ``seqlen``. Drop it with high probability so the bulk of the
+ # calibration data reflects the actual conversation/tool content, while a
+ # small fraction still keeps the system prompt so the model sees that format.
+ drop_system_prob = 0.99
+ rng = random.Random(seed)
+
+ def _maybe_drop_system(messages):
+ """Drop a leading system message with probability ``drop_system_prob``."""
+ if (
+ messages
+ and isinstance(messages[0], dict)
+ and messages[0].get("role") == "system"
+ and rng.random() < drop_system_prob
+ ):
+ return messages[1:]
+ return messages
+
+ def _token_len(text):
+ return len(tokenizer(text, add_special_tokens=False)["input_ids"])
+
+ def _approx_word_count(messages):
+ """Cheap whitespace-split length estimate of the raw conversation.
+
+ Token count is almost always >= whitespace word count (subword / code /
+ JSON only increase it), and the chat template only adds more tokens. So
+ if this estimate already exceeds ``seqlen`` the templated sample is
+ definitely longer than ``seqlen`` and we can skip the expensive
+ ``apply_chat_template`` + tokenization entirely.
+ """
+ count = 0
+ for msg in messages:
+ content = msg.get("content")
+ if content:
+ count += len(content.split())
+ tool_calls = msg.get("tool_calls")
+ if tool_calls:
+ count += len(str(tool_calls).split())
+ return count
+
+ def _build_length_conversation(messages):
+ """Build the message list used for the messages-only length check.
+
+ Mirrors the system-prompt handling of ``render_chat_sample`` (tools are
+ excluded here on purpose: we filter on the messages-only length).
+ """
+ conv = []
+ has_own_system = len(messages) > 0 and isinstance(messages[0], dict) and messages[0].get("role") == "system"
+ if system_prompt is not None and system_prompt != "" and not has_own_system:
+ conv.append({"role": "system", "content": system_prompt})
+ conv += messages
+ return conv
+
+ def _batch_msg_lengths(conversations):
+ """Batch-render conversations and return their token lengths.
+
+ We render to strings first (``tokenize=False`` on a batch reliably
+ returns a list[str]) and then tokenize the strings in one batched call.
+ This avoids the version-dependent return type of
+ ``apply_chat_template(batch, tokenize=True)`` (which can be a dict-like
+ ``BatchEncoding`` and make per-sample length extraction wrong).
+ ``add_generation_prompt=False`` because these are full traces and the few
+ generation-prompt tokens are negligible for a length filter.
+ """
+ if not conversations:
+ return []
+ try:
+ rendered = tokenizer.apply_chat_template(conversations, tokenize=False, add_generation_prompt=False)
+ except Exception:
+ # Fallback: per-sample, dropping the system role if the template fails.
+ rendered = []
+ for conv in conversations:
+ try:
+ rendered.append(tokenizer.apply_chat_template(conv, tokenize=False, add_generation_prompt=False))
+ except Exception:
+ conv2 = [m for m in conv if m.get("role") != "system"]
+ rendered.append(tokenizer.apply_chat_template(conv2, tokenize=False, add_generation_prompt=False))
+ enc = tokenizer(rendered, add_special_tokens=False)
+ return [len(ids) for ids in enc["input_ids"]]
+
+ # Scan each config separately. Store raw (messages, tools) and only render
+ # the finally selected samples with tools, so we don't pay the cost of fully
+ # rendering the (large, mostly unused) fallback pool. The accurate length
+ # check is done in a single batched template call per config.
+ preferred_by_cfg = [[] for _ in sub_datasets] # messages fit within seqlen
+ fallback_by_cfg = [[] for _ in sub_datasets] # messages exceed seqlen (gets truncated)
+
+ for i, d in enumerate(sub_datasets):
+ scanned = 0
+ candidates = [] # (messages, tools) that passed the cheap word-count filter
+ for sample in d:
+ if len(candidates) >= per_cfg_quota or scanned >= max_scan_per_cfg:
+ break
+ scanned += 1
+ messages = sample["messages"]
+ tools = sample.get("tools") if isinstance(sample, dict) else sample["tools"]
+
+ # Decide once (at read time) whether to drop the long system prompt,
+ # so word-count, length check and final render all stay consistent.
+ messages = _maybe_drop_system(messages)
+
+ # Fast path: if even the raw whitespace word count exceeds seqlen, the
+ # sample is definitely too long -> skip rendering/tokenizing.
+ if _approx_word_count(messages) > seqlen:
+ if len(fallback_by_cfg[i]) < per_cfg_quota:
+ fallback_by_cfg[i].append((messages, tools))
+ continue
+ candidates.append((messages, tools))
+
+ # Batched accurate length check for this config's candidates.
+ lengths = _batch_msg_lengths([_build_length_conversation(m) for m, _ in candidates])
+ for (messages, tools), length in zip(candidates, lengths):
+ if length <= seqlen:
+ preferred_by_cfg[i].append((messages, tools))
+ elif len(fallback_by_cfg[i]) < per_cfg_quota:
+ fallback_by_cfg[i].append((messages, tools))
+
+ def _round_robin_take(pools, limit, already):
+ """Pull items round-robin across per-config pools until ``limit``."""
+ out = []
+ idx = [0] * len(pools)
+ while already + len(out) < limit and any(idx[i] < len(pools[i]) for i in range(len(pools))):
+ for i in range(len(pools)):
+ if already + len(out) >= limit:
+ break
+ if idx[i] < len(pools[i]):
+ out.append(pools[i][idx[i]])
+ idx[i] += 1
+ return out
+
+ # Balanced selection: round-robin across configs' preferred pools first,
+ # then top up (still round-robin) from the fallback pools if short.
+ selected = _round_robin_take(preferred_by_cfg, target, 0)
+ n_preferred = len(selected)
+ if len(selected) < target:
+ selected += _round_robin_take(fallback_by_cfg, target, len(selected))
+ if len(selected) > n_preferred:
+ logger.info(
+ "%s: only %d samples fit messages<=seqlen; topping up with %d oversized samples.",
+ hub_repo_id,
+ n_preferred,
+ len(selected) - n_preferred,
+ )
+
+ # Global shuffle so configs/tiers are mixed across the calibration set.
+ random.Random(seed).shuffle(selected)
+
+ # Render the final selection with tools included.
+ texts = [render_chat_sample(tokenizer, messages, system_prompt, tools=tools) for messages, tools in selected]
+
+ import datasets
+
+ calib_dataset = datasets.Dataset.from_list([{"text": t} for t in texts])
+ calib_dataset = calib_dataset.map(
+ lambda examples: tokenizer(examples["text"], truncation=True, max_length=seqlen),
+ batched=True,
+ remove_columns=["text"],
+ )
+
+ # Pack whole traces up to seqlen before returning. "Preferred" traces are
+ # chosen on their messages-only length (<= seqlen) and can remain shorter
+ # than seqlen even after rendering tools, so returning them unpacked would
+ # let _get_dataset_impl.filter_func drop them -- leaving calibration
+ # undersized despite available oversized fallback data. Sample-level packing
+ # concatenates short traces into full-length sequences, while oversized
+ # traces (already truncated to seqlen) each emit a single full-length
+ # sequence, so no usable data is lost.
+ calib_dataset = _concat_samples_to_seqlen(calib_dataset, seqlen)
+ return calib_dataset
+
+
@register_dataset("openbmb/Ultra-FineWeb")
def get_ultrafinweb_dataset(
tokenizer,
@@ -628,19 +1039,20 @@ def get_ultrafinweb_dataset(
seed=42,
apply_chat_template=True,
system_prompt=None,
+ nsamples=512,
):
if split is not None:
if split not in ["en", "zh"]:
raise ValueError("split must be one of ['en', 'zh'] for Ultra-FineWeb dataset")
- calib_dataset = load_dataset("openbmb/Ultra-FineWeb", split=split, streaming=True, trust_remote_code=True)
+ calib_dataset = load_dataset("openbmb/Ultra-FineWeb", split=split, streaming=True)
else:
- calib_dataset = load_dataset("openbmb/Ultra-FineWeb", split="en", streaming=True, trust_remote_code=True)
+ calib_dataset = load_dataset("openbmb/Ultra-FineWeb", split="en", streaming=True)
# dataset_ch = load_dataset("openbmb/Ultra-FineWeb", split='zh',
# streaming=True, trust_remote_code=True).shuffle(seed=seed).take(2000)
# calib_dataset = concatenate_datasets([dataset_en, dataset_ch]) ##concat dasetset could not shuffle
- calib_dataset = calib_dataset.shuffle(seed=seed).take(20000)
+ calib_dataset = calib_dataset.shuffle(seed=seed).take(nsamples * envs.AR_CALIB_DATA_MULTIPLIER)
def get_default_tokenizer_function():
def default_tokenizer_function(examples):
@@ -923,6 +1335,330 @@ def get_local_dataset(
return calib_dataset
+def _concat_samples_to_seqlen(calib_dataset, seqlen):
+ """Pack whole samples together until each reaches ``seqlen`` tokens.
+
+ Unlike the continuous packing in :func:`concat_dataset_element` (where a
+ sample's leftover tokens carry over into the next sequence, splitting
+ samples across boundaries), this packs *whole* samples into a buffer and,
+ once the buffer reaches ``seqlen``, emits a single ``seqlen``-length
+ sequence and resets the buffer. The next sample therefore starts a fresh
+ concatenation; any trailing buffer shorter than ``seqlen`` is dropped.
+
+ Args:
+ calib_dataset: A tokenized ``Dataset`` with an ``input_ids`` column.
+ seqlen: Target sequence length for each packed sample.
+
+ Returns:
+ A new ``Dataset`` with packed ``input_ids`` / ``attention_mask`` columns.
+ """
+ packed = []
+ buffer_ids = []
+ for example in calib_dataset:
+ ids = example["input_ids"]
+ if isinstance(ids, torch.Tensor):
+ ids = ids.tolist()
+ buffer_ids.extend(ids)
+ if len(buffer_ids) >= seqlen:
+ trimmed = buffer_ids[:seqlen]
+ packed.append({"input_ids": trimmed, "attention_mask": [1] * seqlen})
+ # Reset: the next sample starts a fresh concatenation.
+ buffer_ids = []
+
+ if not packed:
+ logger.warning(
+ "Sample-level concat produced no full-length sequences (all samples "
+ "shorter than seqlen=%d combined did not reach it). Returning the "
+ "original dataset unpacked.",
+ seqlen,
+ )
+ return calib_dataset
+
+ return Dataset.from_list(packed)
+
+
+def _normalize_split(split):
+ """Reduce a requested split spec to a single ``load_dataset`` split string.
+
+ The dispatcher passes splits as lists (e.g. ``["train", "test"]``); combine
+ every requested split into HuggingFace's ``"train+test"`` form so they are
+ loaded together instead of only the first one being used. Returns ``None``
+ when nothing was requested, so the caller can pick an available default from
+ the resulting split dictionary.
+ """
+ if split is None:
+ return None
+ if isinstance(split, (list, tuple)):
+ parts = [str(part) for part in split if part]
+ return "+".join(parts) if parts else None
+ return split
+
+
+def _reduce_to_single_dataset(loaded):
+ """Reduce a ``load_dataset`` result to a single (Iterable)Dataset.
+
+ When a concrete split string is passed, ``load_dataset`` already returns a
+ single dataset and this is a no-op. When no split is requested it returns a
+ split dictionary; in that case pick a sensible default (prefer ``train``,
+ otherwise the first available split) so downstream ``.map()`` / column
+ access receive a single dataset rather than a dict or list.
+ """
+ if isinstance(loaded, (DatasetDict, IterableDatasetDict)):
+ keys = list(loaded.keys())
+ if not keys:
+ raise ValueError("Loaded dataset contains no splits to select from.")
+ key = "train" if "train" in keys else keys[0]
+ return loaded[key]
+ # Defensive: a list is returned if a list split ever reaches load_dataset;
+ # combine the splits so callers always receive a single dataset.
+ if isinstance(loaded, list):
+ if not loaded:
+ raise ValueError("Loaded dataset contains no splits to combine.")
+ if len(loaded) == 1:
+ return loaded[0]
+ if any(isinstance(part, IterableDataset) for part in loaded):
+ from datasets import interleave_datasets
+
+ return interleave_datasets(loaded)
+ return concatenate_datasets(loaded)
+ return loaded
+
+
+def _get_generic_dataset(
+ tokenizer,
+ seqlen,
+ dataset_name,
+ split=None,
+ seed=42,
+ apply_chat_template=False,
+ system_prompt=None,
+ fields=None,
+ template=None,
+ separator="\n\n",
+ streaming=True,
+ nsamples=None,
+ timeout=300,
+ concat=False,
+ trust_remote_code=False,
+):
+ """Generic dataset loader that works with any HuggingFace dataset.
+
+ Automatically detects the text field if not specified, or uses the
+ user-provided fields/template to extract text.
+
+ By default uses streaming mode to avoid downloading the entire dataset.
+ Samples are collected until ``nsamples`` valid samples (meeting ``seqlen``)
+ are gathered, or the ``timeout`` is reached.
+
+ Args:
+ tokenizer: The tokenizer to use for tokenization.
+ seqlen: The maximum sequence length.
+ dataset_name: HuggingFace dataset name or local path.
+ split: The data split to use.
+ seed: Random seed for shuffling.
+ apply_chat_template: Whether to apply chat template.
+ system_prompt: Optional system prompt.
+ fields: Text field(s) to extract. A single string for one column,
+ or a list of strings for multiple columns (concatenated with
+ *separator*). If ``None``, the text field is auto-detected.
+ template: Template string with {field} placeholders.
+ separator: Separator for multi-field concatenation.
+ streaming: Whether to use streaming mode (default True for remote
+ datasets to avoid downloading the full dataset).
+ nsamples: Maximum number of samples to collect (for streaming mode).
+ If None, collects up to 10000 samples.
+ timeout: Timeout in seconds for streaming collection. If the dataset
+ cannot provide enough samples within this time, a warning is
+ logged and the collected samples are used.
+ concat: When True, pack whole samples together until each reaches
+ ``seqlen`` tokens (resetting after every full sequence), so that
+ datasets whose individual samples are shorter than ``seqlen`` are
+ not dropped by the downstream length filter.
+ trust_remote_code: Passed through to ``datasets.load_dataset``. Controlled
+ by the AutoRound interface so the user decides whether to execute
+ remote dataset-loading scripts. Defaults to ``False``.
+
+ Returns:
+ A tokenized HuggingFace Dataset.
+ """
+ import time
+
+ # Determine if we should use streaming
+ # Local files are always loaded fully (they're already on disk)
+ is_local = is_local_path(dataset_name)
+ use_streaming = streaming and not is_local
+
+ # Normalize the requested split(s) once so local, streaming and
+ # non-streaming branches behave consistently. The dispatcher passes splits
+ # as lists; ``_normalize_split`` combines them into HF's "train+test" form
+ # (or ``None`` when nothing was requested).
+ split_arg = _normalize_split(split)
+
+ # Load the dataset
+ if is_local:
+ # Local file: use json or text loader (always full load)
+ if dataset_name.endswith((".json", ".jsonl")):
+ loaded = load_dataset("json", data_files=dataset_name, split=split_arg)
+ elif dataset_name.endswith(".txt"):
+ loaded = load_dataset("text", data_files=dataset_name, split=split_arg)
+ else:
+ loaded = load_dataset(dataset_name, split=split_arg)
+ else:
+ # HuggingFace dataset. ``trust_remote_code`` is passed through from the
+ # AutoRound interface so the user controls whether remote code runs.
+ loaded = load_dataset(
+ dataset_name,
+ split=split_arg,
+ trust_remote_code=trust_remote_code,
+ streaming=use_streaming,
+ )
+
+ # When no split was requested, ``load_dataset`` returns a split dictionary;
+ # reduce it (and any list result) to a single dataset for extraction below.
+ calib_dataset = _reduce_to_single_dataset(loaded)
+
+ # Determine the text field
+ if fields is None and template is None:
+ # Auto-detect. Streaming IterableDatasets may expose ``column_names`` as
+ # None before the schema is resolved, so check for non-None metadata and
+ # otherwise detect from a single sample.
+ if getattr(calib_dataset, "column_names", None) is not None:
+ fields = auto_detect_text_field(calib_dataset)
+ else:
+ # IterableDataset / unresolved schema – grab first sample to detect
+ first_sample = next(iter(calib_dataset))
+ fields = auto_detect_text_field_from_sample(first_sample)
+ logger.info(f"Auto-detected text field '{fields}' for dataset '{dataset_name}'")
+
+ # Build the text extraction function
+ def _extract_text_from_sample(sample):
+ """Extract text from a single sample dict."""
+ try:
+ return extract_text_from_sample(
+ sample,
+ fields=fields,
+ template=template,
+ separator=separator,
+ )
+ except (KeyError, ValueError) as e:
+ logger.warning(f"Skipping sample due to field extraction error: {e}")
+ return None
+
+ # Tokenizer function
+ tokenizer_function = get_tokenizer_function(
+ tokenizer, seqlen, apply_chat_template=apply_chat_template, system_prompt=system_prompt
+ )
+
+ if use_streaming:
+ # --- Streaming mode: collect samples until nsamples or timeout ---
+ # Over-collect by AR_CALIB_DATA_MULTIPLIER (same as registered datasets)
+ # so that enough samples survive the downstream seqlen filter in
+ # ``_get_dataset_impl`` (short samples < seqlen are dropped there).
+ target = nsamples * envs.AR_CALIB_DATA_MULTIPLIER if nsamples is not None else 10000
+ start_time = time.time()
+ collected_texts = []
+ total_scanned = 0
+ skipped = 0
+
+ logger.info(
+ f"Streaming dataset '{dataset_name}': collecting up to {target} samples " f"(timeout={timeout}s)..."
+ )
+
+ # Shuffle the iterable dataset for random sampling
+ iter_ds = calib_dataset.shuffle(seed=seed)
+
+ for sample in iter_ds:
+ elapsed = time.time() - start_time
+ if elapsed > timeout:
+ logger.warning(
+ f"Dataset '{dataset_name}' streaming timeout reached ({timeout}s). "
+ f"Collected {len(collected_texts)}/{target} samples after scanning "
+ f"{total_scanned} samples ({skipped} skipped). "
+ f"Using available samples for calibration."
+ )
+ break
+
+ total_scanned += 1
+ text = _extract_text_from_sample(sample)
+ if text is None or not text.strip():
+ skipped += 1
+ continue
+
+ collected_texts.append(text)
+
+ if len(collected_texts) >= target:
+ break
+
+ # Progress logging every 1000 samples
+ if total_scanned % 1000 == 0:
+ logger.info(
+ f" Streaming '{dataset_name}': {len(collected_texts)}/{target} "
+ f"valid samples collected ({total_scanned} scanned, {skipped} skipped, "
+ f"{elapsed:.1f}s elapsed)"
+ )
+
+ if len(collected_texts) == 0:
+ raise ValueError(
+ f"Dataset '{dataset_name}' yielded no valid samples within {timeout}s. "
+ f"Scanned {total_scanned} samples. Check the dataset format or "
+ f"specify 'fields' explicitly."
+ )
+
+ if len(collected_texts) < target:
+ logger.warning(
+ f"Dataset '{dataset_name}': only {len(collected_texts)}/{target} samples "
+ f"collected (scanned {total_scanned} total, {skipped} skipped). "
+ f"Calibration will use {len(collected_texts)} samples."
+ )
+
+ # Build a Dataset from collected texts
+ calib_dataset = Dataset.from_dict({"text": collected_texts})
+
+ else:
+ # --- Non-streaming mode: full dataset ---
+ def _extract_text_batched(examples):
+ """Extract text from batched examples."""
+ texts = []
+ for i in range(len(next(iter(examples.values())))):
+ sample = {k: v[i] for k, v in examples.items()}
+ text = _extract_text_from_sample(sample)
+ if text is not None:
+ texts.append(text)
+ else:
+ texts.append("") # placeholder, will be filtered later
+ return {"text": texts}
+
+ if fields is not None or template is not None:
+ calib_dataset = calib_dataset.map(
+ _extract_text_batched,
+ batched=True,
+ remove_columns=calib_dataset.column_names if hasattr(calib_dataset, "column_names") else None,
+ )
+ else:
+ # Auto-detected field: rename to "text" if needed
+ if fields != "text":
+ calib_dataset = calib_dataset.rename_column(fields, "text")
+
+ calib_dataset = calib_dataset.shuffle(seed=seed)
+
+ # Filter out empty/short texts and tokenize
+ calib_dataset = calib_dataset.filter(lambda x: len(x["text"].strip()) > 0)
+
+ calib_dataset = calib_dataset.map(
+ tokenizer_function,
+ batched=True,
+ new_fingerprint=_make_map_fingerprint(
+ calib_dataset, tokenizer, seqlen, apply_chat_template, system_prompt, "text"
+ ),
+ )
+
+ # Pack short samples into full seqlen-length sequences when requested.
+ if concat:
+ calib_dataset = _concat_samples_to_seqlen(calib_dataset, seqlen)
+
+ return calib_dataset
+
+
def get_dataset_len(dataset):
"""Calculates the length of a dataset.
@@ -989,7 +1725,9 @@ def select_dataset(dataset, indices):
return dataset
-def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, nsamples=512):
+def _get_dataset_impl(
+ tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, nsamples=512, trust_remote_code=False
+):
"""Internal implementation: generate a dataset for calibration.
Args:
@@ -1006,7 +1744,15 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed
Returns:
Dataset: The processed dataset ready for calibration.
"""
- dataset_names = dataset_name.split(",")
+ # Normalize CalibDataset objects to spec strings
+ if isinstance(dataset_name, CalibDataset):
+ dataset_name = dataset_name.to_spec_string()
+ elif isinstance(dataset_name, (list, tuple)):
+ dataset_name = normalize_dataset_spec(dataset_name)
+
+ # Split on unescaped commas so that commas inside option values (e.g. a
+ # system_prompt or template) are not mistaken for dataset separators.
+ dataset_names = split_unescaped(dataset_name, ",")
def cast_dataset_columns(dataset):
features = dict(dataset.features)
@@ -1073,24 +1819,88 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed
split = None
do_concat = False
apply_chat_template = False
+ fields = None
+ template = None
+ separator = "\n\n"
+ timeout = 300
if ":" in name:
- name, split_list = name.split(":")[0], name.split(":")[1:]
- for ele in split_list:
- key, values = ele.split("=")[0], ele.split("=")[1:]
+ # Split on unescaped ":" so values containing ":" (e.g. templates)
+ # are not truncated; the first segment is the dataset name.
+ segments = split_unescaped(name, ":")
+ name = segments[0]
+ for ele in segments[1:]:
+ if "=" in ele:
+ key, raw_value = ele.split("=", 1)
+ elif ele.strip() in ("concat", "apply_chat_template"):
+ # Preserve the bare boolean syntax (e.g. "pile-10k:apply_chat_template").
+ key, raw_value = ele, "true"
+ else:
+ # Ignore incomplete non-boolean options.
+ continue
+ key = key.strip()
+ # Restore escaped delimiters / whitespace; keep separator spaces.
+ value = unescape_spec_value(raw_value if key == "separator" else raw_value.strip())
if key == "split":
- split = values[0].split("+")
- if key == "num":
- data_lens[name] = int(values[0])
- if key == "concat":
- do_concat = not (len(values) > 0 and values[0].lower() == "false")
- if key == "apply_chat_template":
- apply_chat_template = not (len(values) > 0 and values[0].lower() == "false")
- if key == "system_prompt":
- system_prompt = values[0]
+ split = value.split("+")
+ elif key == "num":
+ data_lens[name] = int(value)
+ elif key == "concat":
+ do_concat = value.lower() != "false"
+ elif key == "apply_chat_template":
+ apply_chat_template = value.lower() != "false"
+ elif key == "system_prompt":
+ system_prompt = value
apply_chat_template = True
- if is_local_path(name):
- get_dataset = CALIB_DATASETS.get("local")
+ elif key == "fields":
+ # Single field: "fields=text" -> "text"
+ # Multiple fields: "fields=q+a" -> ["q", "a"]
+ if "+" in value:
+ fields = value.split("+")
+ else:
+ fields = value
+ elif key == "template":
+ template = value
+ elif key == "separator":
+ separator = value
+ elif key == "timeout":
+ timeout = int(value)
+ # If user explicitly specified fields/template, use the generic
+ # loader regardless of whether the dataset is registered.
+ use_generic = fields is not None or template is not None
+
+ if use_generic or is_local_path(name):
+ if is_local_path(name) and not use_generic:
+ # Local file without explicit field spec: use the local handler
+ dataset = CALIB_DATASETS["local"](
+ tokenizer,
+ seqlen,
+ seed=seed,
+ split=split,
+ dataset_name=name,
+ apply_chat_template=apply_chat_template,
+ system_prompt=system_prompt,
+ )
+ else:
+ # Generic loader (for local files with field spec, or any HF dataset)
+ dataset = _get_generic_dataset(
+ tokenizer,
+ seqlen,
+ dataset_name=name,
+ split=split,
+ seed=seed,
+ apply_chat_template=apply_chat_template,
+ system_prompt=system_prompt,
+ fields=fields,
+ template=template,
+ separator=separator,
+ nsamples=nsamples,
+ timeout=timeout,
+ concat=do_concat,
+ trust_remote_code=trust_remote_code,
+ )
+ # Packing already handled inside the generic loader.
+ do_concat = False
else:
calib_name = name
if name not in CALIB_DATASETS:
@@ -1100,26 +1910,59 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed
calib_name = key
break
get_dataset = CALIB_DATASETS.get(calib_name)
- if get_dataset is None:
- filtered_keys = [k for k in CALIB_DATASETS if "/" not in k]
- raise ValueError(
- f"Dataset '{name}' is not found. Please choose from the supported datasets: {filtered_keys}."
- )
- dataset_kwargs = dict(
- tokenizer=tokenizer,
- seqlen=seqlen,
- seed=seed,
- split=split,
- dataset_name=name,
- apply_chat_template=apply_chat_template,
- system_prompt=system_prompt,
- )
- if get_dataset is get_fineweb_edu_dataset:
- dataset_kwargs["max_samples"] = max(
- _FINEWEB_EDU_MIN_CANDIDATES,
- data_lens.get(name, nsamples),
- )
- dataset = get_dataset(**dataset_kwargs)
+ if get_dataset is None:
+ # Fallback: use generic loader for any HuggingFace dataset
+ logger.info(f"Dataset '{name}' not in registry, using generic loader with auto field detection.")
+ dataset = _get_generic_dataset(
+ tokenizer,
+ seqlen,
+ dataset_name=name,
+ split=split,
+ seed=seed,
+ apply_chat_template=apply_chat_template,
+ system_prompt=system_prompt,
+ fields=fields,
+ template=template,
+ separator=separator,
+ nsamples=nsamples,
+ timeout=timeout,
+ concat=do_concat,
+ trust_remote_code=trust_remote_code,
+ )
+ # Packing already handled inside the generic loader.
+ do_concat = False
+ else:
+ if get_dataset is None:
+ filtered_keys = [k for k in CALIB_DATASETS.keys() if "/" not in k]
+ raise ValueError(
+ f"Dataset '{name}' is not found. Please choose from the supported datasets: {filtered_keys}."
+ )
+ dataset_kwargs = dict(
+ tokenizer=tokenizer,
+ seqlen=seqlen,
+ seed=seed,
+ split=split,
+ dataset_name=name,
+ apply_chat_template=apply_chat_template,
+ system_prompt=system_prompt,
+ )
+ if get_dataset is get_fineweb_edu_dataset:
+ dataset_kwargs["max_samples"] = max(
+ _FINEWEB_EDU_MIN_CANDIDATES,
+ data_lens.get(name, nsamples),
+ )
+ elif get_dataset in (
+ get_cci3_hq_dataset,
+ get_github_code_clean_dataset,
+ get_opencode_instruct_dataset,
+ get_competition_math_dataset,
+ get_ultrachat_dataset,
+ get_ultrafinweb_dataset,
+ get_ultradata_agent_dataset,
+ ):
+ dataset_kwargs["nsamples"] = nsamples
+ dataset = get_dataset(**dataset_kwargs)
+
if do_concat:
dataset = concat_dataset_element(dataset)
@@ -1194,7 +2037,7 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed
return dataset_final
-def get_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, nsamples=512):
+def get_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, nsamples=512, trust_remote_code=False):
"""Generate a dataset for calibration.
Uses a subprocess for preprocessing to ensure all temporary memory is fully
@@ -1209,19 +2052,26 @@ def get_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, n
Args:
tokenizer: The tokenizer to use for tokenization.
seqlen (int): The exact sequence length.
- dataset_name (str, optional): Dataset name(s) separated by commas.
+ dataset_name (str, optional): Dataset name(s) separated by commas, or a
+ ``CalibDataset`` object. Defaults to "NeelNanda/pile-10k".
seed (int, optional): Random seed for reproducibility. Defaults to 42.
nsamples (int, optional): Total number of samples to include. Defaults to 512.
Returns:
Dataset: The processed dataset ready for calibration.
"""
+ # Normalize CalibDataset objects to spec strings
+ if isinstance(dataset_name, CalibDataset):
+ dataset_name = dataset_name.to_spec_string()
+ elif isinstance(dataset_name, (list, tuple)):
+ dataset_name = normalize_dataset_spec(dataset_name)
+
# Allow disabling subprocess mode via environment variable
if envs.AR_DISABLE_DATASET_SUBPROCESS:
try:
- return _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)
+ return _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code)
except Exception as error:
- return _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples)
+ return _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code)
# Run preprocessing in a subprocess so all temporary memory is freed on exit.
logger.info("Preprocessing calibration dataset in a subprocess to avoid memory leaks...")
@@ -1240,7 +2090,16 @@ def get_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, n
result_queue = ctx.Queue()
p = ctx.Process(
target=_preprocess_dataset_in_subprocess,
- args=(result_queue, tokenizer, seqlen, dataset_name, seed, nsamples, os.path.join(temp_dir, "dataset")),
+ args=(
+ result_queue,
+ tokenizer,
+ seqlen,
+ dataset_name,
+ seed,
+ nsamples,
+ os.path.join(temp_dir, "dataset"),
+ trust_remote_code,
+ ),
)
p.start()
p.join()
@@ -1265,15 +2124,19 @@ def get_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, n
logger.warning(f"Subprocess dataset preprocessing failed ({e}), falling back to in-process mode.")
if subprocess_network_error is not None:
- return _fallback_to_fineweb_edu(subprocess_network_error, tokenizer, seqlen, dataset_name, seed, nsamples)
+ return _fallback_to_fineweb_edu(
+ subprocess_network_error, tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code
+ )
try:
- return _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples)
+ return _get_dataset_impl(tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code)
except Exception as error:
- return _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples)
+ return _fallback_to_fineweb_edu(error, tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code)
-def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, bs=8, nsamples=512):
+def get_dataloader(
+ tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42, bs=8, nsamples=512, trust_remote_code=False
+):
"""Generate a DataLoader for calibration using specified parameters.
Args:
@@ -1316,6 +2179,6 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed=42
res = {"input_ids": input_ids_new, "attention_mask": attention_mask_new}
return res
- dataset_final = get_dataset(tokenizer, seqlen, dataset_name, seed, nsamples)
+ dataset_final = get_dataset(tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code)
calib_dataloader = DataLoader(dataset_final, batch_size=bs, shuffle=False, collate_fn=collate_batch)
return calib_dataloader
diff --git a/auto_round/calibration/base.py b/auto_round/calibration/base.py
index 539d9f418..90afb9040 100644
--- a/auto_round/calibration/base.py
+++ b/auto_round/calibration/base.py
@@ -41,6 +41,7 @@ class Calibrator(ABC):
def __init__(self, compressor: "BaseCompressor", **kwargs) -> None:
self.model = compressor.model_context.model
self.tokenizer = compressor.model_context.tokenizer
+ self.trust_remote_code = getattr(compressor.model_context, "trust_remote_code", False)
self.dataset = compressor.dataset
self.seed = compressor.seed
self.low_gpu_mem_usage = compressor.low_gpu_mem_usage
diff --git a/auto_round/calibration/diffusion.py b/auto_round/calibration/diffusion.py
index c278872bc..0b2270bfe 100644
--- a/auto_round/calibration/diffusion.py
+++ b/auto_round/calibration/diffusion.py
@@ -114,7 +114,11 @@ class DiffusionCalibrator(LLMCalibrator):
)
requires_image = self._requires_calibration_image()
if isinstance(self.dataset, str):
- dataset = self.dataset.replace(" ", "")
+ from auto_round.utils.dataset_utils import strip_dataset_spec_whitespace
+
+ # Strip only cosmetic whitespace; spaces inside option values
+ # (e.g. separator / template / system_prompt) are preserved.
+ dataset = strip_dataset_spec_whitespace(self.dataset)
self.dataloader, self.batch_size = get_diffusion_dataloader(
dataset=dataset, bs=bs, seed=self.seed, nsamples=nsamples, image_required=requires_image
)
diff --git a/auto_round/calibration/llm.py b/auto_round/calibration/llm.py
index 287d8c769..41ed11322 100644
--- a/auto_round/calibration/llm.py
+++ b/auto_round/calibration/llm.py
@@ -288,11 +288,14 @@ class LLMCalibrator(Calibrator):
Verbatim port of the legacy ``Compressor.calib`` (LLM path only).
"""
from auto_round.calib_dataset import get_dataloader
+ from auto_round.utils.dataset_utils import strip_dataset_spec_whitespace
need_attention_mask = True
if isinstance(self.dataset, str):
need_attention_mask = False # all supported datasets do not use pad
- dataset = self.dataset.replace(" ", "") # remove all whitespaces
+ # Strip only cosmetic whitespace; spaces inside option values
+ # (e.g. separator / template / system_prompt) are preserved.
+ dataset = strip_dataset_spec_whitespace(self.dataset)
# slow here
self.dataloader = get_dataloader(
@@ -302,6 +305,7 @@ class LLMCalibrator(Calibrator):
self.seed,
bs,
nsamples,
+ trust_remote_code=self.trust_remote_code,
)
else:
self.dataloader = self.dataset
diff --git a/auto_round/calibration/mllm.py b/auto_round/calibration/mllm.py
index 24493aa20..e2e8d2ab2 100644
--- a/auto_round/calibration/mllm.py
+++ b/auto_round/calibration/mllm.py
@@ -87,12 +87,16 @@ class MLLMCalibrator(LLMCalibrator):
logger.info(f"Using MLLM template: {template_name}")
- dataset = self.dataset.replace(" ", "") if isinstance(self.dataset, str) else self.dataset
+ from auto_round.utils.dataset_utils import strip_dataset_spec_whitespace
+
+ # Strip only cosmetic whitespace; spaces inside option values
+ # (e.g. separator / template / system_prompt) are preserved.
+ dataset = strip_dataset_spec_whitespace(self.dataset) if isinstance(self.dataset, str) else self.dataset
if dataset is None:
dataset = template_obj.default_dataset
if isinstance(dataset, str):
- dataset = dataset.replace(" ", "")
+ dataset = strip_dataset_spec_whitespace(dataset)
# Switch text-only dataset to MLLM dataset when quant_nontext_module=True,
# as text datasets cannot calibrate vision modules.
from auto_round.calib_dataset import CALIB_DATASETS
@@ -130,7 +134,9 @@ class MLLMCalibrator(LLMCalibrator):
"(vision/audio towers are not being quantized, so text-only "
"calibration through the full-model forward is sufficient)."
)
- self.dataloader = get_dataloader(tokenizer, self.seqlen, dataset, self.seed, bs, nsamples)
+ self.dataloader = get_dataloader(
+ tokenizer, self.seqlen, dataset, self.seed, bs, nsamples, trust_remote_code=self.trust_remote_code
+ )
else:
orig_bs = self.batch_size
(
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 5c3d3de4d..bf973f581 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -75,6 +75,7 @@ from auto_round.utils import (
revert_checkpoint_conversion_mapping,
revert_name_with_weight_transforms,
)
+from auto_round.utils.dataset_utils import CalibDataset
from auto_round.utils.device import (
_force_trim_malloc,
patch_xpu_sdpa_drop_causal_mask,
@@ -247,7 +248,7 @@ class BaseOrchestrator:
ignore_layers: str = "",
quant_lm_head: bool = False,
to_quant_block_names: str | list[str] | None = None,
- dataset: str | list | tuple | torch.utils.data.DataLoader | None = None,
+ dataset: str | list | tuple | torch.utils.data.DataLoader | CalibDataset | None = None,
**kwargs,
) -> None:
# ``CalibrationContext`` is the single source of truth for calibration
@@ -256,6 +257,13 @@ class BaseOrchestrator:
# via property forwarders. ``_resolve_scheme`` later wires this same
# instance onto the quantizer so the two share state.
from auto_round.calibration.state import CalibrationContext
+ from auto_round.utils.dataset_utils import CalibDataset, normalize_dataset_spec
+
+ # Normalize CalibDataset objects to spec strings
+ if isinstance(dataset, CalibDataset):
+ dataset = dataset.to_spec_string()
+ elif isinstance(dataset, (list, tuple)):
+ dataset = normalize_dataset_spec(dataset)
dataset_was_explicitly_set = dataset is not None
self.dataset = dataset if dataset_was_explicitly_set else "NeelNanda/pile-10k"
diff --git a/auto_round/envs.py b/auto_round/envs.py
index c0337fb75..f0140b89e 100644
--- a/auto_round/envs.py
+++ b/auto_round/envs.py
@@ -39,6 +39,7 @@ if TYPE_CHECKING:
AR_QUANTIZE_BAGEL_MOE_GEN: bool = False
AR_NVFP4_FUSED_LAYER_GLOBAL_SCALE: bool = True
AR_ALLOW_W8_ASYM: bool = False
+ AR_CALIB_DATA_MULTIPLIER: int = 100
def _get_optional_positive_int_env(name: str) -> int | None:
@@ -66,6 +67,9 @@ environment_variables: dict[str, Callable[[], Any]] = {
"AR_OMP_NUM_THREADS": lambda: os.getenv("AR_OMP_NUM_THREADS", None),
"AR_DISABLE_OFFLOAD": lambda: os.getenv("AR_DISABLE_OFFLOAD", "0").lower() in ("1", "true", "yes"),
"AR_DISABLE_DATASET_SUBPROCESS": lambda: os.getenv("AR_DISABLE_DATASET_SUBPROCESS", "0").lower() in ("1", "true"),
+ # Multiplier applied to nsamples to determine the candidate pool size for
+ # streaming calibration datasets. The pool is nsamples * AR_CALIB_DATA_MULTIPLIER.
+ "AR_CALIB_DATA_MULTIPLIER": lambda: int(os.getenv("AR_CALIB_DATA_MULTIPLIER", "100")),
"AR_DISABLE_COPY_MTP_WEIGHTS": lambda: os.getenv("AR_DISABLE_COPY_MTP_WEIGHTS", "0").lower()
in ("1", "true", "yes"),
# Device for the disk-streamed calibration forward pass in
diff --git a/auto_round/utils/__init__.py b/auto_round/utils/__init__.py
index 62fcc1506..1775c9a85 100644
--- a/auto_round/utils/__init__.py
+++ b/auto_round/utils/__init__.py
@@ -21,6 +21,15 @@ from auto_round.utils.weight_handler import (
detect_weight_type,
is_quantized_input_module,
)
+from auto_round.utils.dataset_utils import (
+ CalibDataset,
+ auto_detect_text_field,
+ auto_detect_text_field_from_sample,
+ build_dataset_spec,
+ extract_text_from_sample,
+ normalize_dataset_spec,
+ parse_dataset_spec,
+)
def copy_missing_tensors_from_source(*args, **kwargs):
diff --git a/auto_round/utils/dataset_utils.py b/auto_round/utils/dataset_utils.py
new file mode 100644
index 000000000..83892e617
--- /dev/null
+++ b/auto_round/utils/dataset_utils.py
@@ -0,0 +1,906 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utilities for flexible calibration dataset loading and field extraction.
+
+This module provides:
+
+- **CalibDataset** dataclass for a clean, typed API to describe calibration datasets.
+- **Auto-detection** of the text field in a HuggingFace ``Dataset`` by inspecting
+ column names and sampling values.
+- **Field extraction** with support for single fields, multiple fields, and
+ user-defined templates (``{field}`` placeholders).
+- **Spec-string building and parsing** for CLI / string-based usage.
+
+Example::
+
+ from auto_round.utils.dataset_utils import CalibDataset
+
+ # Single field
+ spec = CalibDataset("my-org/my-dataset", split="train", num=1000, fields="text")
+
+ # Multiple fields concatenated
+ spec = CalibDataset("my-org/my-dataset", fields=["question", "answer"], separator="\\n")
+
+ # Template
+ spec = CalibDataset("my-org/my-dataset", template="{question} {answer}")
+
+ # Pass directly to AutoRound
+ from auto_round import AutoRound
+ model = AutoRound(model, dataset=spec)
+"""
+
+import re
+from dataclasses import dataclass
+from typing import Any, Union
+
+# ---------------------------------------------------------------------------
+# CalibDataset dataclass
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class CalibDataset:
+ """Structured specification for a calibration dataset.
+
+ This is the preferred way to specify calibration data programmatically.
+ It can be passed directly to ``AutoRound(dataset=...)`` or converted to a
+ spec string via :meth:`to_spec_string`.
+
+ Args:
+ name: Dataset name (HuggingFace ID) or local file path.
+ split: Split name(s). A single string or a list of strings.
+ num: Number of samples to take from this dataset.
+ concat: Whether to concatenate samples into full-length sequences.
+ fields: Text field(s) to extract. A single string for one column,
+ or a list of strings for multiple columns (concatenated with
+ *separator*). If ``None``, the text field is auto-detected.
+ template: A format string with ``{field_name}`` placeholders.
+ Takes precedence over *fields* when both are set.
+ separator: Separator used when concatenating multiple fields.
+ Defaults to ``"\\n\\n"``.
+ apply_chat_template: Whether to apply the tokenizer's chat template.
+ system_prompt: Optional system prompt for chat template.
+ timeout: Seconds to wait for data collection before falling back to
+ whatever has been collected so far (default: 300).
+
+ Example::
+
+ >>> spec = CalibDataset("my-org/my-ds", split="train", num=500, fields="text")
+ >>> spec = CalibDataset("my-org/my-ds", fields=["question", "answer"], separator="\\n")
+ >>> spec = CalibDataset("my-org/my-ds", template="{question} {answer}")
+ """
+
+ name: str
+ split: str | list[str] | None = None
+ num: int | None = None
+ concat: bool = False
+ fields: str | list[str] | None = None
+ template: str | None = None
+ separator: str = "\n\n"
+ apply_chat_template: bool = False
+ system_prompt: str | None = None
+ timeout: int = 300
+
+ def to_spec_string(self) -> str:
+ """Convert to a spec string (the format used in CLI / ``dataset`` param).
+
+ Returns:
+ A string like ``"my/ds:split=train:num=100:fields=q+a"``.
+ """
+ parts = [self.name]
+
+ if self.split is not None:
+ if isinstance(self.split, list):
+ parts.append(f"split={'+'.join(escape_spec_value(s) for s in self.split)}")
+ else:
+ parts.append(f"split={escape_spec_value(self.split)}")
+
+ if self.num is not None:
+ parts.append(f"num={self.num}")
+
+ if self.concat:
+ parts.append("concat=true")
+
+ if self.template is not None:
+ parts.append(f"template={escape_spec_value(self.template)}")
+ elif self.fields is not None:
+ if isinstance(self.fields, str):
+ parts.append(f"fields={escape_spec_value(self.fields)}")
+ else:
+ parts.append(f"fields={'+'.join(escape_spec_value(f) for f in self.fields)}")
+
+ # Only include separator if it differs from the default and fields/template is used
+ if self.separator != "\n\n" and (self.template is not None or self.fields is not None):
+ parts.append(f"separator={escape_spec_value(self.separator)}")
+
+ if self.apply_chat_template:
+ parts.append("apply_chat_template=true")
+
+ if self.system_prompt is not None:
+ parts.append(f"system_prompt={escape_spec_value(self.system_prompt)}")
+
+ if self.timeout != 300:
+ parts.append(f"timeout={self.timeout}")
+
+ return ":".join(parts)
+
+ @classmethod
+ def from_spec_string(cls, spec: str) -> "CalibDataset":
+ """Parse a spec string into a ``CalibDataset`` instance.
+
+ Args:
+ spec: A spec string like ``"my/ds:split=train:num=100:fields=q+a"``.
+
+ Returns:
+ A ``CalibDataset`` instance.
+ """
+ return cls(**parse_dataset_spec(spec))
+
+
+# ---------------------------------------------------------------------------
+# Auto-detection of text fields
+# ---------------------------------------------------------------------------
+
+#: Priority-ordered list of column names commonly used for text content in
+#: HuggingFace datasets. The first match (in this order) wins when multiple
+#: candidates are present.
+TEXT_FIELD_PRIORITY: list[str] = [
+ "text",
+ "content",
+ "prompt",
+ "question",
+ "instruction",
+ "input",
+ "code",
+ "conversations",
+ "messages",
+ "dialog",
+ "utterance",
+ "utterances",
+ "sentence",
+ "paragraph",
+ "article",
+ "body",
+ "abstract",
+ "title",
+ "caption",
+ "description",
+ "label",
+ "answer",
+ "response",
+ "output",
+ "completion",
+ "translation",
+ "summary",
+ "context",
+ "passage",
+ "document",
+ "doc",
+ "raw_text",
+ "raw",
+]
+
+#: Regex that matches ``{field_name}`` placeholders in a template string.
+_TEMPLATE_PLACEHOLDER_RE = re.compile(r"\{(\w+)\}")
+
+
+def _is_text_value(value: Any) -> bool:
+ """Return ``True`` if *value* looks like a text string (non-trivial length)."""
+ if isinstance(value, str):
+ return len(value.strip()) > 0
+ return False
+
+
+def _is_text_list(value: Any) -> bool:
+ """Return ``True`` if *value* is a list of non-empty strings (e.g. messages)."""
+ if not isinstance(value, (list, tuple)):
+ return False
+ if len(value) == 0:
+ return False
+ # At least half of the elements should be non-empty strings
+ text_count = sum(1 for v in value if isinstance(v, str) and len(v.strip()) > 0)
+ return text_count >= len(value) // 2
+
+
+#: Keys commonly used for the role of a chat message across dataset schemas
+#: (HF ``role``, ShareGPT ``from``, etc.).
+_MESSAGE_ROLE_KEYS: tuple[str, ...] = ("role", "from", "speaker")
+#: Keys commonly used for the textual content of a chat message.
+_MESSAGE_CONTENT_KEYS: tuple[str, ...] = ("content", "value", "text", "message")
+
+
+def _message_content(message: dict[str, Any]) -> str:
+ """Extract the textual content from a single chat-message dict.
+
+ Handles both plain-string content and structured (e.g. multimodal) content
+ that is itself a list of parts such as ``[{"type": "text", "text": ...}]``.
+ """
+ for key in _MESSAGE_CONTENT_KEYS:
+ if key in message and message[key] is not None:
+ content = message[key]
+ if isinstance(content, (list, tuple)):
+ parts = []
+ for part in content:
+ if isinstance(part, dict):
+ parts.append(str(part.get("text", part.get("content", ""))))
+ elif isinstance(part, str):
+ parts.append(part)
+ return " ".join(p for p in parts if p)
+ return str(content)
+ return ""
+
+
+def _message_role(message: dict[str, Any]) -> str | None:
+ """Extract the role/speaker of a single chat-message dict, if present."""
+ for key in _MESSAGE_ROLE_KEYS:
+ if message.get(key):
+ return str(message[key])
+ return None
+
+
+def is_conversation_value(value: Any) -> bool:
+ """Return ``True`` if *value* is a list of chat-message dicts.
+
+ Recognizes structured conversations stored under fields like ``messages`` or
+ ``conversations`` where each element is a dict carrying a content field
+ (``content``/``value``/...). Such values would otherwise flatten to an empty
+ string in :func:`extract_text_from_sample`, causing every row to be skipped.
+ """
+ if not isinstance(value, (list, tuple)) or len(value) == 0:
+ return False
+ dict_messages = [item for item in value if isinstance(item, dict)]
+ # Require the list to be predominantly message dicts.
+ if len(dict_messages) < max(1, len(value) // 2):
+ return False
+ # At least one message must expose a recognizable content field.
+ return any(any(k in msg for k in _MESSAGE_CONTENT_KEYS) for msg in dict_messages)
+
+
+def extract_conversation_text(value: Any, separator: str = "\n") -> str:
+ """Flatten a list of chat-message dicts into plain text for calibration.
+
+ Role markers (``role: content``) are preserved so text-only calibration
+ retains the conversational structure. Plain-string elements are kept as-is.
+ """
+ parts: list[str] = []
+ for message in value:
+ if isinstance(message, dict):
+ content = _message_content(message)
+ if not content:
+ continue
+ role = _message_role(message)
+ parts.append(f"{role}: {content}" if role else content)
+ elif isinstance(message, str) and message.strip():
+ parts.append(message)
+ return separator.join(parts)
+
+
+def auto_detect_text_field(dataset, sample_size: int = 10) -> str:
+ """Auto-detect the most likely text field in a HuggingFace ``Dataset``.
+
+ Strategy (in order of priority):
+
+ 1. If the dataset has exactly **one** column, use it.
+ 2. Check for well-known text field names (``text``, ``content``, ``prompt``,
+ ``question``, …) in :data:`TEXT_FIELD_PRIORITY` order.
+ 3. Among all remaining string columns, pick the one with the longest average
+ text length across the first *sample_size* rows.
+
+ Args:
+ dataset: A HuggingFace ``Dataset`` or ``IterableDataset``.
+ sample_size: Number of rows to sample for heuristic scoring. Defaults
+ to 10.
+
+ Returns:
+ The name of the detected text field.
+
+ Raises:
+ ValueError: If no suitable text field can be found.
+ """
+ # --- 1. Single-column shortcut ------------------------------------------
+ # Note: HF ``IterableDataset`` can expose ``column_names``/``features`` as
+ # ``None`` before the schema is resolved (streaming is the generic loader's
+ # default), so check for non-None metadata rather than mere attribute
+ # presence; otherwise ``list(None)`` would raise ``TypeError``.
+ if getattr(dataset, "column_names", None) is not None:
+ columns = list(dataset.column_names)
+ elif getattr(dataset, "features", None) is not None:
+ columns = list(dataset.features.keys())
+ else:
+ # Schema not resolved (e.g. streaming IterableDataset) – grab one sample.
+ columns = list(next(iter(dataset)).keys())
+
+ if len(columns) == 1:
+ return columns[0]
+
+ # --- 2. Well-known names ------------------------------------------------
+ for candidate in TEXT_FIELD_PRIORITY:
+ if candidate in columns:
+ return candidate
+
+ # --- 3. Heuristic: longest average string length ------------------------
+ # Sample a few rows to estimate average text length per column.
+ samples = []
+ for i, row in enumerate(dataset):
+ if i >= sample_size:
+ break
+ samples.append(row)
+
+ if not samples:
+ raise ValueError(f"Cannot auto-detect text field: dataset is empty. " f"Available columns: {columns}")
+
+ best_field = None
+ best_avg_len = 0.0
+ for col in columns:
+ total_len = 0
+ count = 0
+ for row in samples:
+ val = row.get(col)
+ if isinstance(val, str):
+ total_len += len(val)
+ count += 1
+ elif isinstance(val, (list, tuple)):
+ # Sum lengths of string elements (e.g. messages / conversations)
+ for item in val:
+ if isinstance(item, str):
+ total_len += len(item)
+ elif isinstance(item, dict) and "content" in item:
+ total_len += len(str(item["content"]))
+ count += 1
+ if count > 0:
+ avg_len = total_len / count
+ if avg_len > best_avg_len:
+ best_avg_len = avg_len
+ best_field = col
+
+ if best_field is None:
+ raise ValueError(
+ f"Cannot auto-detect a text field in dataset. "
+ f"Available columns: {columns}. "
+ f"Please specify the field explicitly via the 'field' or 'fields' parameter."
+ )
+ return best_field
+
+
+def auto_detect_text_field_from_sample(sample: dict[str, Any]) -> str:
+ """Auto-detect the most likely text field from a single sample dict.
+
+ Uses the same priority logic as :func:`auto_detect_text_field` but operates
+ on a single row instead of a full dataset. Useful for streaming /
+ ``IterableDataset`` where column names may not be available upfront.
+
+ Args:
+ sample: A single row (dict) from the dataset.
+
+ Returns:
+ The name of the detected text field.
+
+ Raises:
+ ValueError: If no suitable text field can be found.
+ """
+ if len(sample) == 1:
+ return next(iter(sample.keys()))
+
+ for candidate in TEXT_FIELD_PRIORITY:
+ if candidate in sample:
+ return candidate
+
+ best_field = None
+ best_len = 0
+ for key, val in sample.items():
+ if isinstance(val, str):
+ if len(val) > best_len:
+ best_len = len(val)
+ best_field = key
+ elif isinstance(val, (list, tuple)):
+ total_len = sum(len(str(v)) for v in val if isinstance(v, str))
+ if total_len > best_len:
+ best_len = total_len
+ best_field = key
+
+ if best_field is None:
+ raise ValueError(
+ f"Cannot auto-detect a text field from sample. "
+ f"Available fields: {list(sample.keys())}. "
+ f"Please specify the field explicitly via the 'field' or 'fields' parameter."
+ )
+ return best_field
+
+
+# ---------------------------------------------------------------------------
+# Text extraction from a single sample
+# ---------------------------------------------------------------------------
+
+
+def extract_text_from_sample(
+ sample: dict[str, Any],
+ fields: str | list[str] | None = None,
+ template: str | None = None,
+ separator: str = "\n\n",
+) -> str:
+ """Extract a text string from a single dataset sample.
+
+ Three modes are supported (checked in order):
+
+ 1. **Template** – if *template* is given, ``{field_name}`` placeholders are
+ replaced with the corresponding values from *sample*.
+ 2. **Multiple fields** – if *fields* is a list with more than one entry,
+ the values are concatenated with *separator*.
+ 3. **Single field** – if *fields* is a string (or a single-element list),
+ the value of that column is returned as-is.
+
+ If none of the above are provided, the first string-valued column is used.
+
+ Args:
+ sample: A single row (dict) from the dataset.
+ fields: A single field name (str) or a list of field names to
+ concatenate. If ``None``, auto-detection is used.
+ template: A format string with ``{field_name}`` placeholders.
+ Takes precedence over *fields*.
+ separator: Separator used when concatenating multiple fields.
+ Defaults to ``"\\n\\n"``.
+
+ Returns:
+ The extracted text string.
+
+ Raises:
+ KeyError: If a referenced field is not present in *sample*.
+ ValueError: If no field can be determined.
+ """
+ # --- Template mode ------------------------------------------------------
+ if template is not None:
+
+ def _replace(match: re.Match) -> str:
+ key = match.group(1)
+ if key not in sample:
+ raise KeyError(
+ f"Template references field '{key}' which is not in the sample. "
+ f"Available fields: {list(sample.keys())}"
+ )
+ val = sample[key]
+ if isinstance(val, (list, tuple)):
+ # Flatten list-of-dicts (e.g. messages) or list-of-strings
+ parts = []
+ for item in val:
+ if isinstance(item, dict):
+ parts.append(str(item.get("content", item)))
+ else:
+ parts.append(str(item))
+ return separator.join(parts)
+ return str(val)
+
+ return _TEMPLATE_PLACEHOLDER_RE.sub(_replace, template)
+
+ # --- Normalize fields to a list ------------------------------------------
+ if fields is None:
+ field_list: list[str] | None = None
+ elif isinstance(fields, str):
+ field_list = [fields]
+ else:
+ field_list = list(fields)
+
+ # --- Multi-field mode ---------------------------------------------------
+ if field_list is not None and len(field_list) > 1:
+ parts = []
+ for f in field_list:
+ if f not in sample:
+ raise KeyError(f"Field '{f}' not found in sample. Available fields: {list(sample.keys())}")
+ val = sample[f]
+ if is_conversation_value(val):
+ parts.append(extract_conversation_text(val, separator))
+ elif isinstance(val, (list, tuple)):
+ parts.append(separator.join(str(v) for v in val if isinstance(v, str)))
+ else:
+ parts.append(str(val))
+ return separator.join(parts)
+
+ # --- Single-field mode --------------------------------------------------
+ if field_list is not None and len(field_list) == 1:
+ f = field_list[0]
+ if f not in sample:
+ raise KeyError(f"Field '{f}' not found in sample. Available fields: {list(sample.keys())}")
+ val = sample[f]
+ # Structured conversation (list of role/content dicts): preserve roles +
+ # content instead of flattening to an empty string (which would make the
+ # downstream loader skip every row, even with apply_chat_template=True).
+ if is_conversation_value(val):
+ return extract_conversation_text(val, separator)
+ if isinstance(val, (list, tuple)):
+ return separator.join(str(v) for v in val if isinstance(v, str))
+ return str(val)
+
+ # --- Fallback: first string column ---------------------------------------
+ for val in sample.values():
+ if isinstance(val, str) and len(val.strip()) > 0:
+ return val
+ if is_conversation_value(val):
+ return extract_conversation_text(val, separator)
+ if isinstance(val, (list, tuple)) and _is_text_list(val):
+ return separator.join(str(v) for v in val if isinstance(v, str))
+
+ raise ValueError(
+ f"Cannot extract text from sample. Available fields: {list(sample.keys())}. "
+ f"Please specify 'fields' or 'template' explicitly."
+ )
+
+
+# ---------------------------------------------------------------------------# Dataset normalization
+# ---------------------------------------------------------------------------
+
+
+def normalize_dataset_spec(dataset: Union[str, "CalibDataset", list, tuple]) -> str | list | tuple:
+ """Normalize a dataset specification to a spec string or pass through raw data.
+
+ Accepts a ``CalibDataset`` instance, a plain string, or a list/tuple.
+ ``CalibDataset`` objects are converted to their spec-string representation
+ via :meth:`CalibDataset.to_spec_string`.
+
+ A list/tuple is treated as **raw calibration data** (returned unchanged)
+ unless it consists solely of ``CalibDataset`` objects, in which case the
+ specs are joined into a comma-separated spec string. A list of plain
+ strings is therefore raw calibration text (each sample is tokenized on the
+ fly), not a list of dataset names. To combine multiple *named* datasets,
+ pass a single comma-separated string (e.g. ``"ds1,ds2"``) or a list of
+ ``CalibDataset`` objects.
+
+ Args:
+ dataset: A dataset specification — a string, a ``CalibDataset``, or a
+ list/tuple of raw samples / ``CalibDataset`` objects.
+
+ Returns:
+ A comma-separated spec string suitable for ``_get_dataset_impl``, or
+ the original list/tuple when it holds raw calibration data.
+
+ Raises:
+ TypeError: If *dataset* is ``None``.
+ """
+ if dataset is None:
+ raise TypeError(
+ f"dataset must be a str, CalibDataset, or list of str/CalibDataset, got {type(dataset).__name__}"
+ )
+ if isinstance(dataset, CalibDataset):
+ return dataset.to_spec_string()
+ if isinstance(dataset, str):
+ return dataset
+ if isinstance(dataset, (list, tuple)):
+ # A list/tuple is raw calibration data (returned unchanged) unless it
+ # is a list of CalibDataset specs. A list of plain strings is raw
+ # calibration text (each sample tokenized on the fly), not a list of
+ # dataset names. Multiple named datasets use a comma-separated string
+ # or a list of CalibDataset objects.
+ if not dataset:
+ return ""
+ if all(isinstance(item, CalibDataset) for item in dataset):
+ return ",".join(item.to_spec_string() for item in dataset)
+ return dataset
+ # Non-spec types (DataLoader, BatchEncoding, etc.) are passed through.
+ return dataset
+
+
+# ---------------------------------------------------------------------------# Spec-string building and parsing
+# ---------------------------------------------------------------------------
+
+#: Backslash-escape sequences used to protect structural delimiters (and
+#: whitespace control chars) inside option values of a dataset spec string.
+#: ``:`` separates key=value options, ``,`` separates datasets, and ``\`` is the
+#: escape introducer; ``\n`` / ``\t`` keep newline / tab round-trippable.
+_SPEC_ESCAPE_MAP = {
+ "\\": "\\\\",
+ ":": "\\:",
+ ",": "\\,",
+ "\n": "\\n",
+ "\t": "\\t",
+}
+#: Reverse mapping (escape character -> literal) used when unescaping.
+_SPEC_UNESCAPE_MAP = {"\\": "\\", ":": ":", ",": ",", "n": "\n", "t": "\t"}
+
+
+def escape_spec_value(value: str) -> str:
+ """Escape structural delimiters inside a spec option value.
+
+ Protects ``:``, ``,`` and ``\\`` (plus newline/tab) so that values such as
+ ``template='Question: {q}\\nAnswer: {a}'`` or ``system_prompt='a, b'`` survive
+ a serialize/parse round-trip instead of being split into extra options or
+ extra datasets.
+ """
+ return "".join(_SPEC_ESCAPE_MAP.get(ch, ch) for ch in value)
+
+
+def unescape_spec_value(value: str) -> str:
+ """Reverse :func:`escape_spec_value` (also accepts legacy ``\\n`` / ``\\t``)."""
+ out: list[str] = []
+ i = 0
+ n = len(value)
+ while i < n:
+ ch = value[i]
+ if ch == "\\" and i + 1 < n and value[i + 1] in _SPEC_UNESCAPE_MAP:
+ out.append(_SPEC_UNESCAPE_MAP[value[i + 1]])
+ i += 2
+ continue
+ out.append(ch)
+ i += 1
+ return "".join(out)
+
+
+def split_unescaped(spec: str, delim: str, maxsplit: int = -1) -> list[str]:
+ """Split *spec* on *delim* occurrences that are not backslash-escaped.
+
+ Escape sequences (``\\:`` / ``\\,`` / ``\\\\`` / ...) are preserved verbatim in
+ the returned parts so the caller can :func:`unescape_spec_value` them.
+ """
+ parts: list[str] = []
+ current: list[str] = []
+ i = 0
+ n = len(spec)
+ splits = 0
+ while i < n:
+ ch = spec[i]
+ if ch == "\\" and i + 1 < n:
+ # Keep the escape sequence intact (two chars) for later unescaping.
+ current.append(ch)
+ current.append(spec[i + 1])
+ i += 2
+ continue
+ if ch == delim and (maxsplit < 0 or splits < maxsplit):
+ parts.append("".join(current))
+ current = []
+ splits += 1
+ i += 1
+ continue
+ current.append(ch)
+ i += 1
+ parts.append("".join(current))
+ return parts
+
+
+def strip_dataset_spec_whitespace(spec: str) -> str:
+ """Remove cosmetic whitespace from a calibration dataset spec string.
+
+ A spec has the form ``name[:key=value[:key=value]][,name2:...]``. Users may
+ add cosmetic spaces around the comma separators, dataset names, and option
+ keys; those are stripped. Spaces *inside* option values are meaningful and
+ kept intact — e.g. ``separator= `` or ``template={question} {answer}`` —
+ so serialized :class:`CalibDataset` options are not silently altered the way
+ a blanket ``spec.replace(" ", "")`` would (turning ``separator=' '`` into an
+ empty separator and ``{question} {answer}`` into ``{question}{answer}``).
+
+ Args:
+ spec: The dataset spec string.
+
+ Returns:
+ The spec with cosmetic whitespace removed and value whitespace preserved.
+ """
+ entries = []
+ for entry in split_unescaped(spec, ","):
+ segments = split_unescaped(entry, ":")
+ if len(segments) == 1:
+ # Plain dataset name (no options): safe to trim surrounding spaces.
+ entries.append(segments[0].strip())
+ continue
+ normalized = [segments[0].strip()]
+ for option in segments[1:]:
+ if "=" in option:
+ key, value = option.split("=", 1)
+ # Trim the key only; preserve all whitespace inside the value.
+ normalized.append(f"{key.strip()}={value}")
+ else:
+ normalized.append(option.strip())
+ entries.append(":".join(normalized))
+ return ",".join(entries)
+
+
+#: Keys that are recognised in a dataset spec string.
+_SPEC_KEYS = {
+ "split",
+ "num",
+ "concat",
+ "apply_chat_template",
+ "system_prompt",
+ "fields",
+ "template",
+ "separator",
+ "timeout",
+}
+
+
+def build_dataset_spec(
+ name: str,
+ split: str | list[str] | None = None,
+ num: int | None = None,
+ concat: bool = False,
+ fields: str | list[str] | None = None,
+ template: str | None = None,
+ separator: str = "\n\n",
+ apply_chat_template: bool = False,
+ system_prompt: str | None = None,
+ timeout: int = 300,
+) -> str:
+ """Build a dataset spec string from individual parameters.
+
+ This is the inverse of :func:`parse_dataset_spec` and is useful for
+ programmatically constructing dataset specifications.
+
+ Args:
+ name: Dataset name (HuggingFace ID or local path).
+ split: Split name(s). A single string or a list of strings.
+ num: Number of samples to take from this dataset.
+ concat: Whether to concatenate samples into full-length sequences.
+ fields: Text field(s) to extract. A single string for one column,
+ or a list of strings for multiple columns (concatenated with
+ *separator*). If ``None``, the text field is auto-detected.
+ template: A format string with ``{field_name}`` placeholders.
+ Takes precedence over *fields* when both are set.
+ separator: Separator used when concatenating multiple fields.
+ Defaults to ``"\\n\\n"``.
+ apply_chat_template: Whether to apply the tokenizer's chat template.
+ system_prompt: Optional system prompt for chat template.
+ timeout: Timeout in seconds for streaming data collection (default: 300).
+
+ Returns:
+ A spec string like ``"dataset:split=train:num=100:fields=q+a"``.
+
+ Example::
+
+ >>> build_dataset_spec("my/ds", split="train", num=500, fields=["q", "a"])
+ 'my/ds:split=train:num=500:fields=q+a'
+
+ >>> build_dataset_spec("my/ds", fields="text")
+ 'my/ds:fields=text'
+
+ >>> build_dataset_spec("my/ds", template="{question} {answer}", separator=" ")
+ 'my/ds:template={question} {answer}:separator= '
+ """
+ parts = [name]
+
+ if split is not None:
+ if isinstance(split, list):
+ parts.append(f"split={'+'.join(escape_spec_value(s) for s in split)}")
+ else:
+ parts.append(f"split={escape_spec_value(split)}")
+
+ if num is not None:
+ parts.append(f"num={num}")
+
+ if concat:
+ parts.append("concat=true")
+
+ if template is not None:
+ parts.append(f"template={escape_spec_value(template)}")
+ elif fields is not None:
+ if isinstance(fields, str):
+ parts.append(f"fields={escape_spec_value(fields)}")
+ else:
+ parts.append(f"fields={'+'.join(escape_spec_value(f) for f in fields)}")
+
+ # Only include separator if it differs from the default and fields/template is used
+ if separator != "\n\n" and (template is not None or fields is not None):
+ parts.append(f"separator={escape_spec_value(separator)}")
+
+ if apply_chat_template:
+ parts.append("apply_chat_template=true")
+
+ if system_prompt is not None:
+ parts.append(f"system_prompt={escape_spec_value(system_prompt)}")
+
+ if timeout != 300:
+ parts.append(f"timeout={timeout}")
+
+ return ":".join(parts)
+
+
+def parse_dataset_spec(spec: str) -> dict[str, Any]:
+ """Parse a dataset spec string into a structured dict.
+
+ The spec string format is::
+
+ dataset_name[:key=value[:key=value[...]]]
+
+ Recognised keys: ``split``, ``num``, ``concat``, ``apply_chat_template``,
+ ``system_prompt``, ``fields``, ``template``, ``separator``, ``timeout``.
+
+ Multi-value keys (``split``, ``fields``) use ``+`` as the intra-value
+ separator.
+
+ Args:
+ spec: The spec string to parse.
+
+ Returns:
+ A dict with the following keys (all optional except ``name``):
+
+ - ``name`` (str): Dataset name / path.
+ - ``split`` (list[str] | None): Split name(s).
+ - ``num`` (int | None): Number of samples.
+ - ``concat`` (bool): Whether to concatenate.
+ - ``fields`` (str | list[str] | None): Text field(s) to extract.
+ - ``template`` (str | None): Template string with ``{field}`` placeholders.
+ - ``separator`` (str): Separator for multi-field concatenation.
+ - ``apply_chat_template`` (bool): Whether to apply chat template.
+ - ``system_prompt`` (str | None): System prompt.
+
+ Example::
+
+ >>> parse_dataset_spec("my/ds:split=train+test:num=100:fields=q+a")
+ {'name': 'my/ds', 'split': ['train', 'test'], 'num': 100,
+ 'concat': False, 'fields': ['q', 'a'],
+ 'template': None, 'separator': '\\n\\n',
+ 'apply_chat_template': False, 'system_prompt': None}
+ """
+ result: dict[str, Any] = {
+ "name": spec,
+ "split": None,
+ "num": None,
+ "concat": False,
+ "fields": None,
+ "template": None,
+ "separator": "\n\n",
+ "apply_chat_template": False,
+ "system_prompt": None,
+ "timeout": 300,
+ }
+
+ if ":" not in spec:
+ return result
+
+ # Split on unescaped ":" so that ":" inside option values (e.g. a template
+ # like "Question: {q}") does not truncate the value. The first segment is
+ # the dataset name; the rest are key=value options.
+ segments = split_unescaped(spec, ":")
+ result["name"] = unescape_spec_value(segments[0]).strip()
+
+ for part in segments[1:]:
+ if not part:
+ continue
+ if "=" in part:
+ key, value = part.split("=", 1)
+ elif part.strip() in ("concat", "apply_chat_template"):
+ # Preserve the bare boolean syntax (e.g. "pile-10k:apply_chat_template").
+ key, value = part, "true"
+ else:
+ # Ignore incomplete non-boolean options.
+ continue
+ key = key.strip()
+ # Don't strip value for separator (whitespace is meaningful)
+ if key != "separator":
+ value = value.strip()
+ # Restore escaped delimiters / whitespace control chars in the value.
+ value = unescape_spec_value(value)
+
+ if key == "split":
+ result["split"] = value.split("+") if value else None
+ elif key == "num":
+ result["num"] = int(value)
+ elif key == "concat":
+ result["concat"] = value.lower() != "false"
+ elif key == "apply_chat_template":
+ result["apply_chat_template"] = value.lower() != "false"
+ elif key == "system_prompt":
+ result["system_prompt"] = value
+ elif key == "fields":
+ # Single field: "fields=text" -> "text"
+ # Multiple fields: "fields=q+a" -> ["q", "a"]
+ if "+" in value:
+ result["fields"] = value.split("+")
+ else:
+ result["fields"] = value
+ elif key == "template":
+ result["template"] = value
+ elif key == "separator":
+ result["separator"] = value
+ elif key == "timeout":
+ result["timeout"] = int(value)
+ else:
+ # Unknown key – store it for forward compatibility
+ result[key] = value
+
+ return result
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 31e0baedf..86349fda5 100755
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -73,6 +73,12 @@ calibration data and will be downloaded automatically from the datasets Hub. Oth
- `mbpp` for code
- `nvidia/OpenCodeInstruct` for code instruction data
- `openbmb/Ultra-FineWeb`
+- `openbmb/UltraData-SFT-Agent-2609` (experimental) for agent/tool-calling data. Each sample carries a `messages` conversation
+ (with `tool_calls`/tool turns) and a `tools` schema that is rendered through the chat template (`apply_chat_template`
+ is always enabled), so the quantized model sees full tool-calling traces. All configs (`Code-Agent`, `Search-Agent`,
+ `General-Agent`, `Tool-Use`) are loaded and interleaved
+- `qwedsacf/competition_math` (experimental) for math (the `problem` and `solution` fields are concatenated, and short
+ samples are packed into full `seqlen` sequences by default)
### Customized Dataset
@@ -107,12 +113,59 @@ and huggingface dataset are supported. You could specify splits of a dataset by
**Samples concatenation**: An optional setting allows users to concatenate calibration samples
using `--dataset NeelNanda/pile-10k:concat=True`.
All samples will be concatenated first, then split into chunks of seqlen length.
+For the generic loader (any HuggingFace dataset, or one specified with `fields`/`template`),
+`concat=True` performs sample-level packing instead: whole samples are accumulated until the buffer
+reaches `seqlen`, a full-length sequence is emitted, and the buffer is reset so the next sample starts
+a fresh concatenation. This prevents datasets whose individual samples are shorter than `seqlen`
+(e.g. `qwedsacf/competition_math`) from being dropped by the length filter.
**Apply chat template**: Using `--dataset NeelNanda/pile-10k:apply_chat_template` enables application of a chat template
to the calibration data before tokenization. This is commonly used for instruct-style models during generation. To
customize the system prompt,
use:`--dataset 'NeelNanda/pile-10k:apply_chat_template:system_prompt="You are a helpful assistant."'`
+**Generic dataset loading** (experimental): Any HuggingFace dataset can be used directly without prior registration. The text field
+is auto-detected by inspecting column names and sampling values. For example:
+`--dataset my-org/my-dataset:split=train:num=512`
+
+> **Note**: Generic dataset loading (including `fields`, `template`, `separator`, `timeout`, auto field detection, and the
+> `CalibDataset` API below) is **experimental** and its behavior/interface may change in future releases.
+
+**Field selection**: Use `fields` to explicitly specify the text column(s). A single field:
+`--dataset my-org/my-dataset:fields=text`. Multiple fields are concatenated with a separator:
+`--dataset my-org/my-dataset:fields=question+answer`
+
+**Template**: Use `template` with `{field}` placeholders to build custom text from multiple columns.
+Takes precedence over `fields` when both are set.
+`--dataset 'my-org/my-dataset:template={question} {answer}'`
+
+**Separator**: When concatenating multiple fields, customize the separator (default `"\n\n"`):
+`--dataset 'my-org/my-dataset:fields=q+a:separator=\n'`
+
+**Timeout**: For streaming datasets, set `timeout` (seconds) to bound data collection. If the dataset
+cannot provide enough samples within the timeout, a warning is logged and the collected samples are used.
+`--dataset my-org/my-dataset:timeout=600`
+
+**CalibDataset (API)**: For programmatic usage, the `CalibDataset` dataclass provides a typed interface:
+
+ ~~~python
+ from auto_round.utils.dataset_utils import CalibDataset
+ from auto_round import AutoRound
+
+ # Single field
+ spec = CalibDataset("my-org/my-dataset", split="train", num=1000, fields="text")
+
+ # Multiple fields concatenated
+ spec = CalibDataset("my-org/my-dataset", fields=["question", "answer"], separator="\n")
+
+ # Template
+ spec = CalibDataset("my-org/my-dataset", template="{question} {answer}")
+
+ # Pass directly to AutoRound
+ ar = AutoRound(model="Qwen/Qwen3-8B", dataset=spec, scheme="W4A16")
+ ar.quantize_and_save()
+ ~~~
+
Note: If the concatenation option is not enabled, samples shorter than args.seqlen will be dropped.
Please use ',' to split datasets, ':' to split parameters of a dataset and '+' to add values for one targeted parameter.
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index b40b40ccb..2d78a0df5 100755
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -73,6 +73,10 @@ pip install auto-round
- `mbpp`:用于代码场景
- `nvidia/OpenCodeInstruct`:用于代码指令数据
- `openbmb/Ultra-FineWeb`
+- `openbmb/UltraData-SFT-Agent-2609`(实验性):用于智能体/工具调用(tool-calling)数据。每个样本都包含一个 `messages` 对话
+ (含 `tool_calls`/tool 轮次)以及 `tools` 工具 schema,并通过聊天模板渲染(始终启用 `apply_chat_template`),
+ 因此量化模型可以看到完整的工具调用轨迹。该数据集的所有配置(`Code-Agent`、`Search-Agent`、`General-Agent`、`Tool-Use`)都会被加载并交错混合
+- `qwedsacf/competition_math`(实验性):用于数学场景(会拼接 `problem` 与 `solution` 字段,并默认将较短的样本打包成完整的 `seqlen` 序列)
### 自定义数据集
**建议用户还是尽量不使用 padding 的数据**。虽然对于 padding 过的数据我们有做特殊处理,但是目前验证的比较少。
@@ -102,10 +106,48 @@ pip install auto-round
**数据集组合**:可使用 `--dataset` 参数组合不同数据集并设置其他的参数。示例代码 `--dataset ./tmp.json,NeelNanda/pile-10k:num=256,mbpp:num=128`。此参数同时支持本地标定文件和 Hugging Face 数据集,同时还可用 `split = split1+split2` 指定某个数据集的多个拆分子集。
-**样本拼接**:可使用 `--dataset NeelNanda/pile-10k:concat=True` 拼接标定样本。在该模式下,会先将所有样本拼接成完整文本,再按 seqlen 长度切分出来。
+**样本拼接**:可使用 `--dataset NeelNanda/pile-10k:concat=True` 拼接标定样本。在该模式下,会先将所有样本拼接成完整文本,再按 seqlen 长度切分出来。对于通用加载器(任意 HuggingFace 数据集,或通过 `fields`/`template` 指定的数据集),`concat=True` 会改为执行“样本级拼接”:不断累积整条样本直到缓冲区达到 `seqlen`,输出一条完整长度的序列后清空缓冲区,下一条样本重新开始拼接。这样可以避免单条样本短于 `seqlen` 的数据集(例如 `qwedsacf/competition_math`)被长度过滤丢弃。
**启用对话模板**:可使用 `--dataset NeelNanda/pile-10k:apply_chat_template` 在分词前为标定数据应用对话模板,这在指令式模型的生成任务中比较常用。若需自定义系统提示词,可使用 `--dataset 'NeelNanda/pile-10k:apply_chat_template:system_prompt="你是一个乐于助人的智能助手。"'`
+**通用数据集加载**(实验性):任何 HuggingFace 数据集都可以直接使用,无需预先注册。文本字段会通过检查列名和采样值自动检测。例如:
+`--dataset my-org/my-dataset:split=train:num=512`
+
+> **注意**:通用数据集加载(包括 `fields`、`template`、`separator`、`timeout`、自动字段检测,以及下文的 `CalibDataset` API)为**实验性**功能,其行为和接口在未来版本中可能发生变化。
+
+**字段选择**:使用 `fields` 参数显式指定文本列。单个字段:
+`--dataset my-org/my-dataset:fields=text`。多个字段会用分隔符拼接:
+`--dataset my-org/my-dataset:fields=question+answer`
+
+**模板**:使用 `template` 配合 `{field}` 占位符从多列构建自定义文本。同时设置时优先于 `fields`。
+`--dataset 'my-org/my-dataset:template={question} {answer}'`
+
+**分隔符**:拼接多个字段时可自定义分隔符(默认 `"\n\n"`):
+`--dataset 'my-org/my-dataset:fields=q+a:separator=\n'`
+
+**超时**:对于流式数据集,可设置 `timeout`(秒)限制数据收集时间。若超时前无法收集到足够样本,会记录警告并使用已收集的样本。
+`--dataset my-org/my-dataset:timeout=600`
+
+**CalibDataset(API)**:编程式使用时,`CalibDataset` 数据类提供了类型化的接口:
+
+ ~~~python
+ from auto_round.utils.dataset_utils import CalibDataset
+ from auto_round import AutoRound
+
+ # 单字段
+ spec = CalibDataset("my-org/my-dataset", split="train", num=1000, fields="text")
+
+ # 多字段拼接
+ spec = CalibDataset("my-org/my-dataset", fields=["question", "answer"], separator="\n")
+
+ # 模板
+ spec = CalibDataset("my-org/my-dataset", template="{question} {answer}")
+
+ # 直接传入 AutoRound
+ ar = AutoRound(model="Qwen/Qwen3-8B", dataset=spec, scheme="W4A16")
+ ar.quantize_and_save()
+ ~~~
+
注意:如果没有开启拼接选项,长度小于 args.seqlen 的样本会被舍弃。
数据集之间请用英文逗号`,`分隔;单个数据集的参数请用英文冒号`:`分隔;同一参数的多个取值请用英文加号`+`连接。
diff --git a/test/unit/common/calibration/test_generic_dataset_loader.py b/test/unit/common/calibration/test_generic_dataset_loader.py
new file mode 100644
index 000000000..9aac8b29d
--- /dev/null
+++ b/test/unit/common/calibration/test_generic_dataset_loader.py
@@ -0,0 +1,485 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Mocked-dataset tests for the generic calibration loader.
+
+These tests exercise the main feature path of ``_get_generic_dataset`` and its
+integration through ``get_dataset`` without any network access. They cover:
+
+* explicit and default splits forwarded to ``load_dataset``
+* automatic text-field detection (non-streaming and streaming)
+* message-record extraction via templates / fields
+* short / empty-sample filtering
+* streaming timeout + partial collection (and the empty-result error path)
+
+All HuggingFace I/O is mocked, so the tests are deterministic and offline.
+"""
+
+import time
+
+import datasets
+import pytest
+from datasets import Dataset
+
+from auto_round import calib_dataset as cd
+
+# Writing map/filter cache artifacts to disk is pointless for these tiny,
+# fully in-memory fixtures and would risk cross-test cache collisions.
+datasets.disable_caching()
+
+
+# ---------------------------------------------------------------------------
+# Test doubles
+# ---------------------------------------------------------------------------
+class FakeTokenizer:
+ """Minimal tokenizer mimicking the HF ``__call__`` batch interface.
+
+ Each character maps to its ordinal, so ``len(input_ids) == len(text)``
+ (capped at ``max_length``). This makes the downstream ``seqlen`` length
+ filter predictable from the raw string length.
+ """
+
+ name_or_path = "fake-tokenizer"
+ bos_token_id = 1
+ eos_token_id = 2
+
+ def __call__(self, texts, truncation=True, max_length=None, **kwargs):
+ if isinstance(texts, str):
+ texts = [texts]
+ input_ids = []
+ attention_mask = []
+ for t in texts:
+ ids = [ord(c) for c in t]
+ if max_length is not None:
+ ids = ids[:max_length]
+ input_ids.append(ids)
+ attention_mask.append([1] * len(ids))
+ return {"input_ids": input_ids, "attention_mask": attention_mask}
+
+
+class FakeIterableDataset:
+ """Stand-in for a streaming ``IterableDataset``.
+
+ Deliberately lacks ``column_names`` so the loader falls back to inspecting
+ the first sample for field auto-detection (the streaming code path).
+ """
+
+ def __init__(self, samples):
+ self._samples = list(samples)
+
+ def shuffle(self, seed=None): # order-preserving for deterministic asserts
+ return self
+
+ def __iter__(self):
+ return iter(self._samples)
+
+
+class SlowIterableDataset:
+ """Streaming dataset that stalls partway through to trigger a timeout."""
+
+ def __init__(self, quick, slow, delay):
+ self._quick = list(quick)
+ self._slow = list(slow)
+ self._delay = delay
+
+ def shuffle(self, seed=None):
+ return self
+
+ def __iter__(self):
+ yield from self._quick
+ time.sleep(self._delay)
+ yield from self._slow
+
+
+def tokenizer():
+ return FakeTokenizer()
+
+
+def _make_load_dataset(return_value, recorded_calls=None):
+ """Return a ``load_dataset`` replacement capturing its call kwargs."""
+
+ def _fake_load_dataset(*args, **kwargs):
+ if recorded_calls is not None:
+ recorded_calls.append({"args": args, "kwargs": kwargs})
+ return return_value
+
+ return _fake_load_dataset
+
+
+# ---------------------------------------------------------------------------
+# Splits forwarded to load_dataset
+# ---------------------------------------------------------------------------
+class TestSplits:
+ def test_explicit_split_forwarded(self, tokenizer, monkeypatch):
+ calls = []
+ table = Dataset.from_dict({"text": ["hello world foo", "bar baz qux text"]})
+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(table, calls))
+
+ cd._get_generic_dataset(
+ tokenizer,
+ seqlen=4,
+ dataset_name="my/ds",
+ split=["validation"],
+ streaming=False,
+ )
+
+ assert calls, "load_dataset was never called"
+ # A list split is normalized to a single HF split string before loading.
+ assert calls[0]["kwargs"]["split"] == "validation"
+
+ def test_combined_splits_forwarded(self, tokenizer, monkeypatch):
+ calls = []
+ table = Dataset.from_dict({"text": ["hello world foo", "bar baz qux text"]})
+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(table, calls))
+
+ cd._get_generic_dataset(
+ tokenizer,
+ seqlen=4,
+ dataset_name="my/ds",
+ split=["train", "validation"],
+ streaming=False,
+ )
+
+ # Multiple requested splits are combined into HF's "train+validation" form.
+ assert calls[0]["kwargs"]["split"] == "train+validation"
+
+ def test_default_split_selects_train(self, tokenizer, monkeypatch):
+ calls = []
+ # No split requested: load_dataset receives split=None and returns a
+ # split dictionary, from which the loader selects the "train" split.
+ split_dict = datasets.DatasetDict(
+ {
+ "train": Dataset.from_dict({"text": ["hello world foo", "bar baz qux text"]}),
+ "validation": Dataset.from_dict({"text": ["ignored sample here"]}),
+ }
+ )
+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(split_dict, calls))
+
+ cd._get_generic_dataset(
+ tokenizer,
+ seqlen=4,
+ dataset_name="my/ds",
+ split=None,
+ streaming=False,
+ )
+
+ assert calls[0]["kwargs"]["split"] is None
+
+
+# ---------------------------------------------------------------------------
+# Split normalization / reduction helpers
+# ---------------------------------------------------------------------------
+class TestSplitHelpers:
+ def test_normalize_split_none(self):
+ assert cd._normalize_split(None) is None
+
+ def test_normalize_split_string_passthrough(self):
+ assert cd._normalize_split("train") == "train"
+
+ def test_normalize_split_list_combined(self):
+ assert cd._normalize_split(["train", "test"]) == "train+test"
+
+ def test_normalize_split_empty_list(self):
+ assert cd._normalize_split([]) is None
+
+ def test_reduce_single_dataset_passthrough(self):
+ table = Dataset.from_dict({"text": ["a"]})
+ assert cd._reduce_to_single_dataset(table) is table
+
+ def test_reduce_dict_prefers_train(self):
+ train = Dataset.from_dict({"text": ["train row"]})
+ split_dict = datasets.DatasetDict({"validation": Dataset.from_dict({"text": ["v"]}), "train": train})
+ assert cd._reduce_to_single_dataset(split_dict) is train
+
+ def test_reduce_dict_first_when_no_train(self):
+ val = Dataset.from_dict({"text": ["v"]})
+ split_dict = datasets.DatasetDict({"validation": val, "test": Dataset.from_dict({"text": ["t"]})})
+ assert cd._reduce_to_single_dataset(split_dict) is val
+
+ def test_reduce_list_combines(self):
+ a = Dataset.from_dict({"text": ["a"]})
+ b = Dataset.from_dict({"text": ["b"]})
+ combined = cd._reduce_to_single_dataset([a, b])
+ assert len(combined) == 2
+
+
+# ---------------------------------------------------------------------------
+# trust_remote_code is passed through to load_dataset (default False)
+# ---------------------------------------------------------------------------
+class TestTrustRemoteCode:
+ def test_defaults_to_false(self, tokenizer, monkeypatch):
+ calls = []
+ table = Dataset.from_dict({"text": ["hello world foo", "bar baz qux text"]})
+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(table, calls))
+
+ cd._get_generic_dataset(tokenizer, seqlen=4, dataset_name="my/ds", split="train", streaming=False)
+
+ assert calls[0]["kwargs"]["trust_remote_code"] is False
+
+ def test_forwarded_when_enabled(self, tokenizer, monkeypatch):
+ calls = []
+ table = Dataset.from_dict({"text": ["hello world foo", "bar baz qux text"]})
+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(table, calls))
+
+ cd._get_generic_dataset(
+ tokenizer,
+ seqlen=4,
+ dataset_name="my/ds",
+ split="train",
+ streaming=False,
+ trust_remote_code=True,
+ )
+
+ assert calls[0]["kwargs"]["trust_remote_code"] is True
+
+
+# ---------------------------------------------------------------------------
+# Auto-detection of the text field
+# ---------------------------------------------------------------------------
+class TestAutoDetection:
+ def test_non_streaming_detects_priority_field(self, tokenizer, monkeypatch):
+ # "content" wins over "title" per TEXT_FIELD_PRIORITY and is renamed
+ # to "text" internally.
+ table = Dataset.from_dict(
+ {
+ "title": ["t1", "t2"],
+ "content": ["the quick brown fox", "lorem ipsum dolor sit"],
+ }
+ )
+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(table))
+
+ ds = cd._get_generic_dataset(
+ tokenizer,
+ seqlen=4,
+ dataset_name="my/ds",
+ streaming=False,
+ )
+
+ assert "text" in ds.column_names
+ assert set(ds["text"]) == {"the quick brown fox", "lorem ipsum dolor sit"}
+
+ def test_streaming_detects_and_collects(self, tokenizer, monkeypatch):
+ # Keep the candidate pool == nsamples so collection stops promptly.
+ monkeypatch.setenv("AR_CALIB_DATA_MULTIPLIER", "1")
+ samples = [{"text": f"streaming sample number {i}"} for i in range(5)]
+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(FakeIterableDataset(samples)))
+
+ ds = cd._get_generic_dataset(
+ tokenizer,
+ seqlen=4,
+ dataset_name="my/ds",
+ streaming=True,
+ nsamples=3,
+ )
+
+ assert len(ds) == 3
+ for text in ds["text"]:
+ assert text.startswith("streaming sample number")
+
+
+# ---------------------------------------------------------------------------
+# Message / chat records
+# ---------------------------------------------------------------------------
+class TestMessageRecords:
+ def test_template_extracts_message_contents(self, tokenizer, monkeypatch):
+ table = Dataset.from_dict(
+ {
+ "messages": [
+ [
+ {"role": "user", "content": "hello there"},
+ {"role": "assistant", "content": "general kenobi"},
+ ],
+ [
+ {"role": "user", "content": "what is two plus two"},
+ {"role": "assistant", "content": "the answer is four"},
+ ],
+ ]
+ }
+ )
+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(table))
+
+ ds = cd._get_generic_dataset(
+ tokenizer,
+ seqlen=4,
+ dataset_name="my/chat-ds",
+ template="{messages}",
+ separator=" | ",
+ streaming=False,
+ )
+
+ assert set(ds["text"]) == {
+ "hello there | general kenobi",
+ "what is two plus two | the answer is four",
+ }
+
+ def test_streaming_message_records(self, tokenizer, monkeypatch):
+ monkeypatch.setenv("AR_CALIB_DATA_MULTIPLIER", "1")
+ samples = [
+ {
+ "messages": [
+ {"role": "user", "content": f"question {i} goes here"},
+ {"role": "assistant", "content": f"response {i} goes here"},
+ ]
+ }
+ for i in range(3)
+ ]
+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(FakeIterableDataset(samples)))
+
+ ds = cd._get_generic_dataset(
+ tokenizer,
+ seqlen=4,
+ dataset_name="my/chat-ds",
+ template="{messages}",
+ separator=" ",
+ streaming=True,
+ nsamples=3,
+ )
+
+ assert len(ds) == 3
+ for text in ds["text"]:
+ assert "goes here" in text
+
+
+# ---------------------------------------------------------------------------
+# Empty / short sample filtering
+# ---------------------------------------------------------------------------
+class TestSampleFiltering:
+ def test_empty_samples_filtered_non_streaming(self, tokenizer, monkeypatch):
+ table = Dataset.from_dict({"text": ["a valid sentence here", " ", "", "another valid line"]})
+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(table))
+
+ ds = cd._get_generic_dataset(
+ tokenizer,
+ seqlen=4,
+ dataset_name="my/ds",
+ streaming=False,
+ )
+
+ assert len(ds) == 2
+ assert set(ds["text"]) == {"a valid sentence here", "another valid line"}
+
+ def test_empty_samples_skipped_streaming(self, tokenizer, monkeypatch):
+ monkeypatch.setenv("AR_CALIB_DATA_MULTIPLIER", "1")
+ samples = [
+ {"text": "keep this one"},
+ {"text": " "},
+ {"text": ""},
+ {"text": "keep this too"},
+ ]
+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(FakeIterableDataset(samples)))
+
+ ds = cd._get_generic_dataset(
+ tokenizer,
+ seqlen=4,
+ dataset_name="my/ds",
+ streaming=True,
+ nsamples=10,
+ )
+
+ assert set(ds["text"]) == {"keep this one", "keep this too"}
+
+
+# ---------------------------------------------------------------------------
+# Streaming timeout / partial collection
+# ---------------------------------------------------------------------------
+class TestStreamingTimeout:
+ def test_timeout_yields_partial_collection(self, tokenizer, monkeypatch):
+ # Two samples arrive instantly, then the stream stalls past the
+ # timeout, so only the first two should be collected.
+ quick = [{"text": "first good sample"}, {"text": "second good sample"}]
+ slow = [{"text": "late sample one"}, {"text": "late sample two"}]
+ ds_stream = SlowIterableDataset(quick, slow, delay=1.5)
+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(ds_stream))
+
+ ds = cd._get_generic_dataset(
+ tokenizer,
+ seqlen=4,
+ dataset_name="my/ds",
+ streaming=True,
+ nsamples=100,
+ timeout=0.5,
+ )
+
+ assert len(ds) == 2
+ assert set(ds["text"]) == {"first good sample", "second good sample"}
+
+ def test_no_valid_samples_raises(self, tokenizer, monkeypatch):
+ samples = [{"text": ""}, {"text": " "}, {"text": "\n\t"}]
+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(FakeIterableDataset(samples)))
+
+ with pytest.raises(ValueError, match="no valid samples"):
+ cd._get_generic_dataset(
+ tokenizer,
+ seqlen=4,
+ dataset_name="my/ds",
+ streaming=True,
+ nsamples=10,
+ timeout=30,
+ )
+
+
+# ---------------------------------------------------------------------------
+# Integration through get_dataset (in-process, generic-loader fallback)
+# ---------------------------------------------------------------------------
+class TestGetDatasetIntegration:
+ @pytest.fixture(autouse=True)
+ def _in_process(self, monkeypatch):
+ # Run preprocessing in-process (no fork/pickle of the mock) and keep
+ # the streaming candidate pool small.
+ monkeypatch.setenv("AR_DISABLE_DATASET_SUBPROCESS", "1")
+ monkeypatch.setenv("AR_CALIB_DATA_MULTIPLIER", "1")
+ # Surface any error instead of silently falling back to a network fetch.
+ monkeypatch.setattr(cd, "_fallback_to_fineweb_edu", lambda error, *a, **k: (_ for _ in ()).throw(error))
+
+ def test_seqlen_short_samples_dropped(self, tokenizer, monkeypatch):
+ # input_ids length == len(text) (truncated to seqlen); samples shorter
+ # than seqlen are dropped by the integration-level length filter.
+ samples = [
+ {"text": "abcdefghij"}, # len 10 -> kept
+ {"text": "klmnopqrst"}, # len 10 -> kept
+ {"text": "ab"}, # len 2 -> dropped (< seqlen)
+ {"text": "cd"}, # len 2 -> dropped (< seqlen)
+ ]
+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(FakeIterableDataset(samples)))
+
+ ds = cd.get_dataset(
+ tokenizer,
+ seqlen=5,
+ dataset_name="pytest/generic-ds",
+ nsamples=10,
+ )
+
+ assert len(ds) == 2
+
+ def test_template_spec_routes_to_generic_loader(self, tokenizer, monkeypatch):
+ samples = [
+ {
+ "messages": [
+ {"role": "user", "content": "abcdefghij"},
+ {"role": "assistant", "content": "klmnopqrst"},
+ ]
+ }
+ for _ in range(3)
+ ]
+ monkeypatch.setattr(cd, "load_dataset", _make_load_dataset(FakeIterableDataset(samples)))
+
+ ds = cd.get_dataset(
+ tokenizer,
+ seqlen=5,
+ dataset_name="pytest/chat-ds:template={messages}",
+ nsamples=10,
+ )
+
+ assert len(ds) >= 1
+ assert "input_ids" in ds.column_names
diff --git a/test/unit/common/core/test_calib_dataset_subprocess.py b/test/unit/common/core/test_calib_dataset_subprocess.py
index 5b6363242..bbac034b3 100644
--- a/test/unit/common/core/test_calib_dataset_subprocess.py
+++ b/test/unit/common/core/test_calib_dataset_subprocess.py
@@ -122,8 +122,8 @@ def test_subprocess_network_error_falls_back_without_retrying_source(monkeypatch
fallback_dataset = Dataset.from_dict({"input_ids": [[1]], "attention_mask": [[1]]})
fallback = []
- def fallback_to_fineweb(error, tokenizer, seqlen, dataset_name, seed, nsamples):
- fallback.append((error, dataset_name))
+ def fallback_to_fineweb(error, tokenizer, seqlen, dataset_name, seed, nsamples, trust_remote_code=False):
+ fallback.append((error, dataset_name, trust_remote_code))
return fallback_dataset
monkeypatch.setattr(cd.multiprocessing, "get_context", lambda method: _NetworkErrorContext())
@@ -138,6 +138,7 @@ def test_subprocess_network_error_falls_back_without_retrying_source(monkeypatch
assert result is fallback_dataset
assert isinstance(fallback[0][0], ConnectionError)
assert fallback[0][1] == "source"
+ assert fallback[0][2] is False
@pytest.mark.skipif(os.name == "nt", reason="fork is unavailable on Windows")
diff --git a/test/unit/common/utils/test_calib_dataset_helpers.py b/test/unit/common/utils/test_calib_dataset_helpers.py
index 8b1abfebc..f02d6a27b 100644
--- a/test/unit/common/utils/test_calib_dataset_helpers.py
+++ b/test/unit/common/utils/test_calib_dataset_helpers.py
@@ -96,7 +96,7 @@ class TestDatasetNetworkErrors:
)
assert result is fallback_dataset
- load_dataset.assert_called_once_with(tokenizer, 128, "AI-ModelScope/fineweb-edu", 42, 2)
+ load_dataset.assert_called_once_with(tokenizer, 128, "AI-ModelScope/fineweb-edu", 42, 2, False)
assert "Automatically switching" in warning.call_args.args[0]
def test_closed_http_client_is_treated_as_network_error(self):
@@ -497,3 +497,71 @@ class TestGetDatasetImpl:
)
assert dataset.column_names == ["source", "input_ids", "attention_mask"]
+
+
+# ---------------------------------------------------------------------------
+# _concat_samples_to_seqlen (sample-level packing of short traces)
+# ---------------------------------------------------------------------------
+class TestConcatSamplesToSeqlen:
+ def test_packs_short_samples_into_full_length(self):
+ import auto_round.calib_dataset as calib_dataset
+
+ # Four samples of length 3 packed into seqlen=6 -> two full-length rows.
+ ds = Dataset.from_dict({"input_ids": [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]]})
+ packed = calib_dataset._concat_samples_to_seqlen(ds, seqlen=6)
+ assert len(packed) == 2
+ for row in packed:
+ assert len(row["input_ids"]) == 6
+ assert len(row["attention_mask"]) == 6
+
+ def test_oversized_sample_emits_single_sequence(self):
+ import auto_round.calib_dataset as calib_dataset
+
+ ds = Dataset.from_dict({"input_ids": [list(range(10))]})
+ packed = calib_dataset._concat_samples_to_seqlen(ds, seqlen=4)
+ assert len(packed) == 1
+ assert len(packed[0]["input_ids"]) == 4
+
+ def test_all_short_returns_original_unpacked(self):
+ import auto_round.calib_dataset as calib_dataset
+
+ # Single short sample can never reach seqlen -> original returned.
+ ds = Dataset.from_dict({"input_ids": [[1, 2]]})
+ packed = calib_dataset._concat_samples_to_seqlen(ds, seqlen=8)
+ assert packed is ds
+
+
+# ---------------------------------------------------------------------------
+# competition_math: chat template rendered once (not twice)
+# ---------------------------------------------------------------------------
+class TestCompetitionMathChatTemplate:
+ class _ChatTokenizer:
+ name_or_path = "fake-chat"
+ bos_token_id = 1
+ eos_token_id = 2
+
+ def __init__(self):
+ self.chat_calls = 0
+
+ def apply_chat_template(self, messages, tokenize=False, add_generation_prompt=False, **kwargs):
+ self.chat_calls += 1
+ return " ".join(f"{m['role']}:{m['content']}" for m in messages)
+
+ def __call__(self, texts, truncation=True, max_length=None, **kwargs):
+ if isinstance(texts, str):
+ texts = [texts]
+ input_ids = [[ord(c) for c in t][:max_length] for t in texts]
+ return {"input_ids": input_ids, "attention_mask": [[1] * len(x) for x in input_ids]}
+
+ def test_chat_template_applied_once_per_sample(self, monkeypatch):
+ import auto_round.calib_dataset as calib_dataset
+
+ rows = Dataset.from_dict({"problem": ["p1", "p2", "p3"], "solution": ["s1", "s2", "s3"]})
+ monkeypatch.setattr(calib_dataset, "load_dataset", lambda *a, **k: rows)
+
+ tok = self._ChatTokenizer()
+ calib_dataset.get_competition_math_dataset(tok, seqlen=4, split="train", apply_chat_template=True, nsamples=3)
+
+ # Exactly one render per sample in the loop; the .map() tokenization must
+ # NOT re-apply the template (regression guard for double application).
+ assert tok.chat_calls == 3
diff --git a/test/unit/common/utils/test_dataset_utils.py b/test/unit/common/utils/test_dataset_utils.py
new file mode 100644
index 000000000..3b7b761e0
--- /dev/null
+++ b/test/unit/common/utils/test_dataset_utils.py
@@ -0,0 +1,705 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for auto_round.utils.dataset_utils."""
+
+import pytest
+
+from auto_round.utils.dataset_utils import (
+ CalibDataset,
+ auto_detect_text_field,
+ auto_detect_text_field_from_sample,
+ build_dataset_spec,
+ escape_spec_value,
+ extract_conversation_text,
+ extract_text_from_sample,
+ is_conversation_value,
+ normalize_dataset_spec,
+ parse_dataset_spec,
+ split_unescaped,
+ strip_dataset_spec_whitespace,
+ unescape_spec_value,
+)
+
+
+# ---------------------------------------------------------------------------
+# auto_detect_text_field
+# ---------------------------------------------------------------------------
+class TestAutoDetectTextField:
+ def test_single_column(self):
+ """Single-column dataset should return that column."""
+
+ class FakeDS:
+ column_names = ["text"]
+
+ assert auto_detect_text_field(FakeDS()) == "text"
+
+ def test_priority_order(self):
+ """'text' should be preferred over 'content'."""
+
+ class FakeDS:
+ column_names = ["content", "text", "id"]
+
+ assert auto_detect_text_field(FakeDS()) == "text"
+
+ def test_content_when_no_text(self):
+ """'content' should be picked when 'text' is absent."""
+
+ class FakeDS:
+ column_names = ["content", "id"]
+
+ assert auto_detect_text_field(FakeDS()) == "content"
+
+ def test_prompt_field(self):
+ """'prompt' should be detected when no higher-priority field exists."""
+
+ class FakeDS:
+ column_names = ["prompt", "id"]
+
+ assert auto_detect_text_field(FakeDS()) == "prompt"
+
+ def test_fallback_to_longest_string(self):
+ """When no known field exists, pick the longest string column."""
+
+ class FakeDS:
+ column_names = ["short_col", "long_col", "id"]
+
+ def __iter__(self):
+ yield {"short_col": "hi", "long_col": "a much longer text here", "id": 1}
+ yield {"short_col": "yo", "long_col": "another long text", "id": 2}
+
+ assert auto_detect_text_field(FakeDS()) == "long_col"
+
+ def test_no_text_field_raises(self):
+ """Should raise ValueError when no text field is found."""
+
+ class FakeDS:
+ column_names = ["id", "score"]
+
+ def __iter__(self):
+ yield {"id": 1, "score": 0.5}
+
+ with pytest.raises(ValueError, match="Cannot auto-detect"):
+ auto_detect_text_field(FakeDS())
+
+ def test_features_attribute(self):
+ """Should work with features attribute when column_names is absent."""
+
+ class FakeDS:
+ features = {"text": "string", "id": "int"}
+
+ def __iter__(self):
+ yield {"text": "hello", "id": 1}
+
+ assert auto_detect_text_field(FakeDS()) == "text"
+
+
+# ---------------------------------------------------------------------------
+# auto_detect_text_field_from_sample
+# ---------------------------------------------------------------------------
+class TestAutoDetectTextFieldFromSample:
+ def test_single_key(self):
+ sample = {"text": "hello world"}
+ assert auto_detect_text_field_from_sample(sample) == "text"
+
+ def test_priority(self):
+ sample = {"content": "hello", "text": "world", "id": 1}
+ assert auto_detect_text_field_from_sample(sample) == "text"
+
+ def test_longest_string_fallback(self):
+ sample = {"short": "hi", "long": "a much longer text here", "id": 1}
+ assert auto_detect_text_field_from_sample(sample) == "long"
+
+ def test_no_text_raises(self):
+ sample = {"id": 1, "score": 0.5}
+ with pytest.raises(ValueError, match="Cannot auto-detect"):
+ auto_detect_text_field_from_sample(sample)
+
+
+# ---------------------------------------------------------------------------
+# extract_text_from_sample
+# ---------------------------------------------------------------------------
+class TestExtractTextFromSample:
+ def test_single_field_string(self):
+ """fields as a single string."""
+ sample = {"text": "hello world", "id": 1}
+ assert extract_text_from_sample(sample, fields="text") == "hello world"
+
+ def test_single_field_list(self):
+ """fields as a single-element list."""
+ sample = {"text": "hello world", "id": 1}
+ assert extract_text_from_sample(sample, fields=["text"]) == "hello world"
+
+ def test_multiple_fields(self):
+ sample = {"question": "What is 2+2?", "answer": "4"}
+ result = extract_text_from_sample(sample, fields=["question", "answer"], separator=" ")
+ assert result == "What is 2+2? 4"
+
+ def test_template(self):
+ sample = {"question": "What is 2+2?", "answer": "4"}
+ result = extract_text_from_sample(sample, template="{question} {answer}")
+ assert result == "What is 2+2? 4"
+
+ def test_template_with_missing_field(self):
+ sample = {"question": "What is 2+2?"}
+ with pytest.raises(KeyError, match="answer"):
+ extract_text_from_sample(sample, template="{question} {answer}")
+
+ def test_no_field_no_template(self):
+ """Should fall back to first string column."""
+ sample = {"id": 1, "text": "hello", "score": 0.5}
+ assert extract_text_from_sample(sample) == "hello"
+
+ def test_no_text_raises(self):
+ sample = {"id": 1, "score": 0.5}
+ with pytest.raises(ValueError, match="Cannot extract text"):
+ extract_text_from_sample(sample)
+
+ def test_list_field(self):
+ sample = {"messages": ["hello", "world"]}
+ result = extract_text_from_sample(sample, fields="messages", separator=" ")
+ assert result == "hello world"
+
+ def test_list_of_dicts(self):
+ """List of message dicts is extracted as a conversation (roles preserved).
+
+ Previously this flattened to an empty string (every row was skipped);
+ structured conversations are now extracted with ``role: content`` markers.
+ """
+ sample = {"messages": [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]}
+ result = extract_text_from_sample(sample, fields="messages", separator=" ")
+ assert result == "user: hi assistant: hello"
+
+ def test_list_of_dicts_with_template(self):
+ """Template mode extracts 'content' from list of dicts."""
+ sample = {"messages": [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]}
+ result = extract_text_from_sample(sample, template="{messages}", separator=" ")
+ assert result == "hi hello"
+
+ def test_template_with_list_field(self):
+ sample = {"messages": ["hello", "world"]}
+ result = extract_text_from_sample(sample, template="{messages}", separator=" | ")
+ assert result == "hello | world"
+
+ def test_template_takes_precedence_over_fields(self):
+ """Template should take precedence over fields."""
+ sample = {"question": "Q", "answer": "A", "other": "X"}
+ result = extract_text_from_sample(sample, fields=["question", "answer"], template="{question}")
+ assert result == "Q"
+
+
+# ---------------------------------------------------------------------------
+# CalibDataset dataclass
+# ---------------------------------------------------------------------------
+class TestCalibDataset:
+ def test_minimal(self):
+ spec = CalibDataset("my/ds")
+ assert spec.name == "my/ds"
+ assert spec.split is None
+ assert spec.num is None
+ assert spec.concat is False
+ assert spec.fields is None
+ assert spec.template is None
+ assert spec.separator == "\n\n"
+ assert spec.apply_chat_template is False
+ assert spec.system_prompt is None
+ assert spec.timeout == 300
+
+ def test_single_field_string(self):
+ spec = CalibDataset("my/ds", fields="text")
+ assert spec.fields == "text"
+
+ def test_multiple_fields_list(self):
+ spec = CalibDataset("my/ds", fields=["question", "answer"])
+ assert spec.fields == ["question", "answer"]
+
+ def test_to_spec_string_minimal(self):
+ spec = CalibDataset("my/ds")
+ assert spec.to_spec_string() == "my/ds"
+
+ def test_to_spec_string_with_split(self):
+ spec = CalibDataset("my/ds", split="train")
+ assert spec.to_spec_string() == "my/ds:split=train"
+
+ def test_to_spec_string_with_num(self):
+ spec = CalibDataset("my/ds", num=100)
+ assert spec.to_spec_string() == "my/ds:num=100"
+
+ def test_to_spec_string_with_fields(self):
+ spec = CalibDataset("my/ds", fields=["q", "a"])
+ assert spec.to_spec_string() == "my/ds:fields=q+a"
+
+ def test_to_spec_string_single_field(self):
+ spec = CalibDataset("my/ds", fields="text")
+ assert spec.to_spec_string() == "my/ds:fields=text"
+
+ def test_to_spec_string_with_template(self):
+ spec = CalibDataset("my/ds", template="{q} {a}")
+ assert spec.to_spec_string() == "my/ds:template={q} {a}"
+
+ def test_to_spec_string_with_separator(self):
+ spec = CalibDataset("my/ds", fields=["q", "a"], separator=" ")
+ assert spec.to_spec_string() == "my/ds:fields=q+a:separator= "
+
+ def test_to_spec_string_with_timeout(self):
+ spec = CalibDataset("my/ds", timeout=600)
+ assert spec.to_spec_string() == "my/ds:timeout=600"
+
+ def test_to_spec_string_default_timeout_omitted(self):
+ """Default timeout (300) should not appear in spec string."""
+ spec = CalibDataset("my/ds", timeout=300)
+ assert spec.to_spec_string() == "my/ds"
+
+ def test_to_spec_string_full(self):
+ spec = CalibDataset(
+ "my/ds",
+ split="train",
+ num=500,
+ fields=["q", "a"],
+ separator=" ",
+ apply_chat_template=True,
+ system_prompt="You are helpful.",
+ timeout=600,
+ )
+ assert (
+ spec.to_spec_string()
+ == "my/ds:split=train:num=500:fields=q+a:separator= :apply_chat_template=true:system_prompt=You are helpful.:timeout=600"
+ )
+
+ def test_from_spec_string(self):
+ spec = CalibDataset.from_spec_string("my/ds:split=train:num=100:fields=q+a")
+ assert spec.name == "my/ds"
+ assert spec.split == ["train"]
+ assert spec.num == 100
+ assert spec.fields == ["q", "a"]
+
+ def test_from_spec_string_single_field(self):
+ spec = CalibDataset.from_spec_string("my/ds:fields=text")
+ assert spec.fields == "text"
+
+ def test_from_spec_string_with_timeout(self):
+ spec = CalibDataset.from_spec_string("my/ds:timeout=600")
+ assert spec.timeout == 600
+
+ def test_from_spec_string_bare_apply_chat_template(self):
+ """Bare ``apply_chat_template`` flag survives parsing and round-trips."""
+ spec = CalibDataset.from_spec_string("my/ds:apply_chat_template")
+ assert spec.apply_chat_template is True
+ assert "apply_chat_template=true" in spec.to_spec_string()
+
+ def test_from_spec_string_bare_concat(self):
+ """Bare ``concat`` flag survives parsing."""
+ spec = CalibDataset.from_spec_string("my/ds:concat")
+ assert spec.concat is True
+
+ def test_roundtrip(self):
+ """build -> parse -> build should be identity."""
+ spec = CalibDataset("my/ds", split="train", num=100, fields=["q", "a"], separator=" ")
+ spec_str = spec.to_spec_string()
+ parsed = CalibDataset.from_spec_string(spec_str)
+ assert parsed.to_spec_string() == spec_str
+
+ def test_roundtrip_template(self):
+ spec = CalibDataset("my/ds", template="{question} {answer}", separator=" ")
+ spec_str = spec.to_spec_string()
+ parsed = CalibDataset.from_spec_string(spec_str)
+ assert parsed.template == "{question} {answer}"
+ assert parsed.separator == " "
+
+ def test_roundtrip_template_with_colons(self):
+ """A template containing ':' must survive serialization (escaped)."""
+ tmpl = "Question: {question}\nAnswer: {answer}"
+ spec = CalibDataset("my/ds", template=tmpl)
+ parsed = CalibDataset.from_spec_string(spec.to_spec_string())
+ assert parsed.template == tmpl
+
+ def test_roundtrip_system_prompt_with_comma_and_colon(self):
+ """A system prompt with ',' and ':' must not create extra datasets/options."""
+ prompt = "Note: be concise, clear, and kind"
+ spec = CalibDataset("my/ds", system_prompt=prompt)
+ parsed = CalibDataset.from_spec_string(spec.to_spec_string())
+ assert parsed.system_prompt == prompt
+
+ def test_comma_in_value_does_not_split_datasets(self):
+ """Commas inside option values must stay within a single dataset spec."""
+ from auto_round.utils.dataset_utils import split_unescaped
+
+ spec = CalibDataset("my/ds", system_prompt="a, b, c").to_spec_string()
+ multi = spec + "," + "pile-10k"
+ assert split_unescaped(multi, ",") == [spec, "pile-10k"]
+
+ def test_roundtrip_with_timeout(self):
+ spec = CalibDataset("my/ds", num=100, timeout=600)
+ spec_str = spec.to_spec_string()
+ parsed = CalibDataset.from_spec_string(spec_str)
+ assert parsed.timeout == 600
+ assert parsed.to_spec_string() == spec_str
+
+
+# ---------------------------------------------------------------------------
+# build_dataset_spec
+# ---------------------------------------------------------------------------
+class TestBuildDatasetSpec:
+ def test_minimal(self):
+ assert build_dataset_spec("my/ds") == "my/ds"
+
+ def test_with_split(self):
+ assert build_dataset_spec("my/ds", split="train") == "my/ds:split=train"
+
+ def test_with_num(self):
+ assert build_dataset_spec("my/ds", num=100) == "my/ds:num=100"
+
+ def test_with_split_and_num(self):
+ assert build_dataset_spec("my/ds", split="train", num=100) == "my/ds:split=train:num=100"
+
+ def test_with_fields_string(self):
+ assert build_dataset_spec("my/ds", fields="text") == "my/ds:fields=text"
+
+ def test_with_fields_list(self):
+ assert build_dataset_spec("my/ds", fields=["q", "a"]) == "my/ds:fields=q+a"
+
+ def test_with_template(self):
+ assert build_dataset_spec("my/ds", template="{q} {a}") == "my/ds:template={q} {a}"
+
+ def test_with_separator(self):
+ assert build_dataset_spec("my/ds", fields=["q", "a"], separator=" ") == "my/ds:fields=q+a:separator= "
+
+ def test_with_chat_template(self):
+ assert build_dataset_spec("my/ds", apply_chat_template=True) == "my/ds:apply_chat_template=true"
+
+ def test_with_timeout(self):
+ assert build_dataset_spec("my/ds", timeout=600) == "my/ds:timeout=600"
+
+ def test_with_default_timeout_omitted(self):
+ """Default timeout (300) should not appear in spec string."""
+ assert build_dataset_spec("my/ds", timeout=300) == "my/ds"
+
+ def test_full_spec(self):
+ assert (
+ build_dataset_spec(
+ "my/ds", split="train", num=100, fields=["q", "a"], separator=" ", apply_chat_template=True
+ )
+ == "my/ds:split=train:num=100:fields=q+a:separator= :apply_chat_template=true"
+ )
+
+ def test_full_spec_with_timeout(self):
+ assert (
+ build_dataset_spec("my/ds", split="train", num=100, fields=["q", "a"], separator=" ", timeout=600)
+ == "my/ds:split=train:num=100:fields=q+a:separator= :timeout=600"
+ )
+
+
+# ---------------------------------------------------------------------------
+# parse_dataset_spec
+# ---------------------------------------------------------------------------
+class TestParseDatasetSpec:
+ def test_name_only(self):
+ result = parse_dataset_spec("my/ds")
+ assert result["name"] == "my/ds"
+ assert result["split"] is None
+ assert result["num"] is None
+ assert result["concat"] is False
+ assert result["fields"] is None
+ assert result["template"] is None
+ assert result["separator"] == "\n\n"
+ assert result["apply_chat_template"] is False
+ assert result["system_prompt"] is None
+
+ def test_split(self):
+ result = parse_dataset_spec("my/ds:split=train")
+ assert result["name"] == "my/ds"
+ assert result["split"] == ["train"]
+
+ def test_multi_split(self):
+ result = parse_dataset_spec("my/ds:split=train+test")
+ assert result["split"] == ["train", "test"]
+
+ def test_num(self):
+ result = parse_dataset_spec("my/ds:num=100")
+ assert result["num"] == 100
+
+ def test_concat(self):
+ result = parse_dataset_spec("my/ds:concat=true")
+ assert result["concat"] is True
+
+ def test_concat_false(self):
+ result = parse_dataset_spec("my/ds:concat=false")
+ assert result["concat"] is False
+
+ def test_fields_single(self):
+ result = parse_dataset_spec("my/ds:fields=text")
+ assert result["fields"] == "text"
+
+ def test_fields_multiple(self):
+ result = parse_dataset_spec("my/ds:fields=q+a")
+ assert result["fields"] == ["q", "a"]
+
+ def test_template(self):
+ result = parse_dataset_spec("my/ds:template={q} {a}")
+ assert result["template"] == "{q} {a}"
+
+ def test_separator(self):
+ result = parse_dataset_spec("my/ds:fields=q+a:separator= ")
+ assert result["separator"] == " "
+
+ def test_apply_chat_template(self):
+ result = parse_dataset_spec("my/ds:apply_chat_template=true")
+ assert result["apply_chat_template"] is True
+
+ def test_bare_apply_chat_template(self):
+ """Bare ``apply_chat_template`` flag (no ``=value``) is treated as true."""
+ result = parse_dataset_spec("my/ds:apply_chat_template")
+ assert result["apply_chat_template"] is True
+
+ def test_bare_concat(self):
+ """Bare ``concat`` flag (no ``=value``) is treated as true."""
+ result = parse_dataset_spec("my/ds:concat")
+ assert result["concat"] is True
+
+ def test_bare_flag_combined_with_kv_options(self):
+ """Bare boolean flags coexist with regular key=value options."""
+ result = parse_dataset_spec("my/ds:num=100:apply_chat_template:concat")
+ assert result["num"] == 100
+ assert result["apply_chat_template"] is True
+ assert result["concat"] is True
+
+ def test_incomplete_non_boolean_option_ignored(self):
+ """An option without ``=`` that is not a known bare flag is ignored."""
+ result = parse_dataset_spec("my/ds:split")
+ assert result["name"] == "my/ds"
+ assert result["split"] is None
+
+ def test_system_prompt(self):
+ result = parse_dataset_spec("my/ds:system_prompt=You are helpful.")
+ assert result["system_prompt"] == "You are helpful."
+
+ def test_combined(self):
+ result = parse_dataset_spec("my/ds:split=train+test:num=100:fields=q+a")
+ assert result["name"] == "my/ds"
+ assert result["split"] == ["train", "test"]
+ assert result["num"] == 100
+ assert result["fields"] == ["q", "a"]
+
+ def test_timeout(self):
+ result = parse_dataset_spec("my/ds:timeout=600")
+ assert result["timeout"] == 600
+
+ def test_timeout_default(self):
+ result = parse_dataset_spec("my/ds")
+ assert result["timeout"] == 300
+
+ def test_roundtrip(self):
+ """build -> parse should be identity."""
+ spec = build_dataset_spec("my/ds", split="train", num=100, fields=["q", "a"], separator=" ")
+ parsed = parse_dataset_spec(spec)
+ assert parsed["name"] == "my/ds"
+ assert parsed["split"] == ["train"]
+ assert parsed["num"] == 100
+ assert parsed["fields"] == ["q", "a"]
+ assert parsed["separator"] == " "
+
+ def test_roundtrip_with_timeout(self):
+ spec = build_dataset_spec("my/ds", num=100, timeout=600)
+ parsed = parse_dataset_spec(spec)
+ assert parsed["timeout"] == 600
+
+
+# ---------------------------------------------------------------------------
+# normalize_dataset_spec
+# ---------------------------------------------------------------------------
+class TestNormalizeDatasetSpec:
+ def test_none_raises(self):
+ with pytest.raises(TypeError, match="must be a str"):
+ normalize_dataset_spec(None)
+
+ def test_empty_string(self):
+ assert normalize_dataset_spec("") == ""
+
+ def test_single_string(self):
+ assert normalize_dataset_spec("my/ds") == "my/ds"
+
+ def test_single_calib_dataset(self):
+ spec = CalibDataset("my/ds", num=100)
+ assert normalize_dataset_spec(spec) == "my/ds:num=100"
+
+ def test_list_of_strings_is_raw_data(self):
+ """A list of plain strings is raw calibration text, passed through."""
+ result = normalize_dataset_spec(["my/ds", "other/ds"])
+ assert result == ["my/ds", "other/ds"]
+
+ def test_list_of_calib_datasets(self):
+ specs = [CalibDataset("my/ds", num=100), CalibDataset("other/ds", num=200)]
+ result = normalize_dataset_spec(specs)
+ assert result == "my/ds:num=100,other/ds:num=200"
+
+ def test_mixed_list_is_raw_data(self):
+ """A list mixing strings and CalibDataset objects is raw data."""
+ specs = ["my/ds", CalibDataset("other/ds", num=200, timeout=60)]
+ result = normalize_dataset_spec(specs)
+ assert result == specs
+
+ def test_tuple_of_calib_datasets(self):
+ specs = (CalibDataset("my/ds", num=100), CalibDataset("other/ds", num=200))
+ result = normalize_dataset_spec(specs)
+ assert result == "my/ds:num=100,other/ds:num=200"
+
+ def test_list_with_non_spec_items_is_raw_data(self):
+ """A list containing non-str/non-CalibDataset items is raw data."""
+ sentinel = object()
+ result = normalize_dataset_spec([sentinel])
+ assert result == [sentinel]
+
+ def test_empty_list(self):
+ assert normalize_dataset_spec([]) == ""
+
+
+# ---------------------------------------------------------------------------
+# Structured-conversation extraction (messages / conversations fields)
+# ---------------------------------------------------------------------------
+class TestConversationExtraction:
+ def test_is_conversation_value_hf_schema(self):
+ value = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]
+ assert is_conversation_value(value) is True
+
+ def test_is_conversation_value_sharegpt_schema(self):
+ value = [{"from": "human", "value": "hi"}, {"from": "gpt", "value": "hello"}]
+ assert is_conversation_value(value) is True
+
+ def test_is_conversation_value_plain_string_list(self):
+ assert is_conversation_value(["hi", "hello"]) is False
+
+ def test_is_conversation_value_non_list(self):
+ assert is_conversation_value("hi") is False
+ assert is_conversation_value([]) is False
+ assert is_conversation_value([{"foo": "bar"}]) is False
+
+ def test_extract_conversation_text_preserves_roles(self):
+ value = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}]
+ assert extract_conversation_text(value, separator=" ") == "user: hi assistant: hello"
+
+ def test_extract_conversation_text_sharegpt(self):
+ value = [{"from": "human", "value": "hi"}, {"from": "gpt", "value": "hello"}]
+ assert extract_conversation_text(value, separator="\n") == "human: hi\ngpt: hello"
+
+ def test_extract_conversation_text_multimodal_content_parts(self):
+ value = [{"role": "user", "content": [{"type": "text", "text": "hi"}, {"type": "text", "text": "there"}]}]
+ assert extract_conversation_text(value, separator=" ") == "user: hi there"
+
+ def test_single_field_messages_not_empty(self):
+ """Regression: a 'messages' field of dicts must not flatten to ''."""
+ sample = {"messages": [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "yo"}]}
+ result = extract_text_from_sample(sample, fields="messages", separator=" ")
+ assert result == "user: hi assistant: yo"
+ assert result.strip() != ""
+
+ def test_fallback_detects_conversation(self):
+ """Auto fallback (no fields) extracts a conversation instead of ''."""
+ sample = {"conversations": [{"from": "human", "value": "hi"}, {"from": "gpt", "value": "yo"}]}
+ result = extract_text_from_sample(sample)
+ assert "hi" in result and "yo" in result
+
+
+# ---------------------------------------------------------------------------
+# auto_detect_text_field with unresolved streaming metadata (column_names=None)
+# ---------------------------------------------------------------------------
+class TestAutoDetectUnresolvedMetadata:
+ class _FakeIterable:
+ """Mimics a streaming IterableDataset whose schema is not yet resolved."""
+
+ column_names = None
+ features = None
+
+ def __init__(self, rows):
+ self._rows = rows
+
+ def __iter__(self):
+ return iter(self._rows)
+
+ def test_none_column_names_falls_back_to_sampling(self):
+ ds = self._FakeIterable([{"text": "the quick brown fox", "id": 1}])
+ # Must not raise TypeError from list(None); detects from a sample instead.
+ assert auto_detect_text_field(ds) == "text"
+
+ def test_none_metadata_single_column(self):
+ ds = self._FakeIterable([{"body": "hello world"}])
+ assert auto_detect_text_field(ds) == "body"
+
+
+# ---------------------------------------------------------------------------
+# Spec value escaping (preserve ':' and ',' in option values)
+# ---------------------------------------------------------------------------
+class TestSpecEscaping:
+ def test_escape_unescape_roundtrip(self):
+ for raw in ["plain", "a: b", "a, b", "x\\y", "line1\nline2", "tab\there"]:
+ assert unescape_spec_value(escape_spec_value(raw)) == raw
+
+ def test_escape_protects_delimiters(self):
+ escaped = escape_spec_value("a:b,c")
+ assert ":" not in escaped.replace("\\:", "")
+ assert "," not in escaped.replace("\\,", "")
+
+ def test_split_unescaped_ignores_escaped_delims(self):
+ assert split_unescaped("a\\:b:c", ":") == ["a\\:b", "c"]
+ assert split_unescaped("a\\,b,c", ",") == ["a\\,b", "c"]
+
+ def test_split_unescaped_maxsplit(self):
+ assert split_unescaped("a:b:c", ":", maxsplit=1) == ["a", "b:c"]
+
+ def test_template_with_colon_roundtrip(self):
+ tmpl = "Question: {q}\nAnswer: {a}"
+ parsed = parse_dataset_spec(CalibDataset("my/ds", template=tmpl).to_spec_string())
+ assert parsed["template"] == tmpl
+
+ def test_system_prompt_with_comma_roundtrip(self):
+ prompt = "Be concise, clear: and kind"
+ parsed = parse_dataset_spec(CalibDataset("my/ds", system_prompt=prompt).to_spec_string())
+ assert parsed["system_prompt"] == prompt
+
+ def test_comma_in_value_not_split_as_dataset(self):
+ spec = CalibDataset("my/ds", system_prompt="a, b, c").to_spec_string()
+ multi = spec + ",pile-10k"
+ assert split_unescaped(multi, ",") == [spec, "pile-10k"]
+
+ def test_legacy_unescaped_spec_still_parses(self):
+ """Specs without any escapes keep their original parsing behavior."""
+ result = parse_dataset_spec("my/ds:split=train+test:num=100:fields=q+a")
+ assert result["name"] == "my/ds"
+ assert result["split"] == ["train", "test"]
+ assert result["fields"] == ["q", "a"]
+
+ def test_legacy_separator_escape_newline(self):
+ assert parse_dataset_spec("my/ds:separator=\\n:fields=text")["separator"] == "\n"
+
+
+# ---------------------------------------------------------------------------
+# strip_dataset_spec_whitespace (preserve spaces inside option values)
+# ---------------------------------------------------------------------------
+class TestStripSpecWhitespace:
+ def test_preserves_separator_space(self):
+ assert strip_dataset_spec_whitespace("my/ds:separator= ") == "my/ds:separator= "
+
+ def test_preserves_template_spaces(self):
+ spec = "my/ds:template={question} {answer}"
+ assert strip_dataset_spec_whitespace(spec) == spec
+
+ def test_trims_cosmetic_whitespace_around_commas(self):
+ assert strip_dataset_spec_whitespace("pile-10k , ultrachat") == "pile-10k,ultrachat"
+
+ def test_trims_name_and_keys(self):
+ assert strip_dataset_spec_whitespace(" my/ds : fields=text") == "my/ds:fields=text"
+
+ def test_preserves_escaped_value_spaces(self):
+ spec = CalibDataset("my/ds", system_prompt="a, b: c").to_spec_string()
+ # Round-trips through whitespace stripping without corrupting the value.
+ stripped = strip_dataset_spec_whitespace(spec)
+ assert parse_dataset_spec(stripped)["system_prompt"] == "a, b: c"