PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[Feature]: speedup autoscheme with multi-process and cache files (#2083)
Xin He288f848b89cf
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex e1824b92a..7dbf1a6c3 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -14,7 +14,11 @@import copyimport gc+import hashlib+import jsonimport math+import os+import timefrom dataclasses import asdictfrom functools import wrapsfrom typing import Iterable, Optional, Union@@ -48,6 +52,7 @@ from auto_round.data_type.gguf import ()from auto_round.data_type.utils import get_quant_func, reshape_pad_tensor_by_group_size, revert_tensor_by_padfrom auto_round.logger import logger+from auto_round.modeling.fused_moe.replace_modules import safe_to_cpu_from auto_round.schemes import QuantizationScheme, preset_name_to_schemefrom auto_round.utils import (SUPPORTED_LAYER_TYPES,@@ -61,6 +66,7 @@ from auto_round.utils import (get_module,is_mllm_model,llm_load_model,+ load_model,mllm_load_model,parse_available_devices,set_avg_auto_device_map,@@ -152,7 +158,8 @@ class AutoSchemeWrapperLinear(WrapperLinear):self.x_diff = Nonereturn None- qdq_x.register_hook(save_grad)+ if qdq_x.requires_grad:+ qdq_x.register_hook(save_grad)return qdq_x, scale, zpdef _qdq_weight(self, value, min_scale, max_scale):@@ -185,7 +192,8 @@ class AutoSchemeWrapperLinear(WrapperLinear):self.mix_score = self.weight_score + self.act_scorereturn None- qdq_w.register_hook(save_grad)+ if qdq_w.requires_grad:+ qdq_w.register_hook(save_grad)return qdq_w, 1.0, None@@ -304,7 +312,8 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):self.x_diff = Nonereturn None- qdq_x.register_hook(save_grad)+ if qdq_x.requires_grad:+ qdq_x.register_hook(save_grad)return qdq_x, scale, zpdef _qdq_weight(self, value, min_scale, max_scale):@@ -607,11 +616,16 @@ def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_dev# Enable gradients for the output of the last blockif module.tmp_name == block_names[-1]:if isinstance(result, torch.Tensor):- result = result.requires_grad_(True)+ if result.is_floating_point():+ result = result.requires_grad_(True)elif isinstance(result, tuple):- result = tuple(r.requires_grad_(True) if isinstance(r, torch.Tensor) else r for r in result)+ result = tuple(+ r.requires_grad_(True) if isinstance(r, torch.Tensor) and r.is_floating_point() else r+ for r in result+ )- pbar.update(1)+ if pbar is not None:+ pbar.update(1)return resultreturn new_forward@@ -775,9 +789,12 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, schif isinstance(block_output, tuple):# For tuple outputs, we usually care about the first element (hidden states)main_output = block_output[0]- main_output = main_output.requires_grad_(True)- else:+ if isinstance(main_output, torch.Tensor) and main_output.is_floating_point():+ main_output = main_output.requires_grad_(True)+ elif isinstance(block_output, torch.Tensor) and block_output.is_floating_point():main_output = block_output.requires_grad_(True)+ else:+ main_output = block_output# Backward pass for the current blocktorch.autograd.backward(@@ -789,7 +806,11 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, sch# Extract gradients w.r.t. the block inputif block_input_args and isinstance(block_input_args[0], torch.Tensor):- current_grad = block_input_args[0].grad.detach().clone()+ if block_input_args[0].grad is not None:+ current_grad = block_input_args[0].grad.detach().clone()+ else:+ logger.warning(f"No gradient found for input of {block_name}, stopping backward replay")+ breakelse:logger.warning(f"No suitable input gradient found for {block_name}")break@@ -800,7 +821,8 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, sch# clear_memory(device_list=major_device) # this one is very slow and seems does not affect max ram usagememory_monitor.update()- pbar.update(1)+ if pbar is not None:+ pbar.update(1)_log_batch_avg_loss(model,@@ -867,6 +889,8 @@ def get_score_for_scheme(if name in fixed_layer_scheme.keys():continuem = get_module(model, name)+ if m is None:+ raise RuntimeError(f"AutoScheme scoring layer {name!r} is missing after model preprocessing")if not check_to_quantized(m):layer_bits, _ = compute_layer_bits(m, ignore_scale_zp_bits)scores_dict[name] = [layer_bits, 0.0]@@ -895,7 +919,9 @@ def get_score_for_scheme(device = major_deviceelse:device = m.weight.device- m.tuning_device = m.weight.device+ # Replacement materialization may create fresh expert Linear modules+ # without the metadata assigned on the pre-materialized tree.+ m.tuning_device = devicenew_m = WrapperLayer(m,@@ -1226,9 +1252,10 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = None(layer_names, scheme) for each layer, or (None, None) if no feasiblesolution exists."""- # dp: total_params -> (accumulated_loss, chosen_path)- # The path is stored as a tuple to avoid the high overhead of repeatedly- # copying Python lists for every DP transition.+ # dp: total_params -> (accumulated_loss, path_node)+ # Each path node points to its parent. Tuple/list concatenation still copies+ # the entire path on every transition, which becomes quadratic for large+ # models; linked nodes keep each transition O(1) and are expanded only once.dp: dict[int, tuple[float, tuple]] = {0: (0.0, ())}for layer_name, opts in layers.items():new_dp: dict[int, tuple[float, tuple]] = {}@@ -1240,7 +1267,7 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = Nonecontinuenew_loss = cur_loss + loss_cost- new_path = cur_path + ((layer_names, scheme),)+ new_path = (cur_path, layer_names, scheme)# Keep the path with smaller loss for the same parameter budgetif np_total not in new_dp or new_loss < new_dp[np_total][0]:@@ -1285,7 +1312,12 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = None# Select the solution with the minimum lossbest_params = min(dp.keys(), key=lambda k: dp[k][0])best_loss, best_path = dp[best_params]- return best_loss, list(best_path)+ path = []+ while best_path:+ best_path, layer_names, scheme = best_path+ path.append((layer_names, scheme))+ path.reverse()+ return best_loss, pathdef move_module_to_tuning_device(module, major_device="cpu"):@@ -1321,7 +1353,7 @@ def move_module_to_tuning_device(module, major_device="cpu"):for n, m in module.named_modules():if hasattr(m, "orig_layer"):- target = m.orig_layer.tuning_device+ target = getattr(m.orig_layer, "tuning_device", getattr(m, "tuning_device", major_device))m.to(target)_move_own_tensors(m, target)elif hasattr(m, "tuning_device"):@@ -1432,6 +1464,388 @@ def _apply_head_trick(head_name, schemes, sorted_indices, target_bits, target_patotal_scores[head_name] = filtered+# ---------------------------------------------------------------------------+# AutoScheme scoring cache helpers+# ---------------------------------------------------------------------------+++def _scheme_repr(s):+ """Normalize a scheme (str/QuantizationScheme/dict) to a hashable repr."""+ if isinstance(s, str):+ return s+ if isinstance(s, QuantizationScheme):+ return json.dumps(asdict(s), sort_keys=True, default=str)+ if isinstance(s, dict):+ return json.dumps(s, sort_keys=True, default=str)+ return str(s)+++def _autoscheme_cache_key(+ model_name,+ dataset,+ nsamples,+ seqlen,+ batch_size,+ quant_layer_names,+ fixed_layer_scheme,+ scheme,+ force_mllm,+ low_gpu_mem_usage,+):+ """Return a 16-char hex digest that uniquely identifies a **single-scheme** scoring run.++ The key covers every parameter that directly affects per-layer loss values+ **except** ``avg_bits`` / ``target_bits`` (only drive the DP step).+ Unlike the old version, this key is generated **per-scheme**, not per-run,+ so caching is granular: adding/removing schemes doesn't invalidate cached+ scores for unchanged schemes.+ """+ key_data = {+ "model_name": model_name,+ "dataset": dataset,+ "nsamples": nsamples,+ "seqlen": seqlen,+ "batch_size": batch_size,+ "quant_layer_names": sorted(quant_layer_names),+ "fixed_layer_scheme": {k: v for k, v in sorted(fixed_layer_scheme.items())},+ "scheme": _scheme_repr(scheme),+ "force_mllm": force_mllm,+ "low_gpu_mem_usage": low_gpu_mem_usage,+ }+ key_str = json.dumps(key_data, sort_keys=True, default=str)+ return hashlib.sha256(key_str.encode()).hexdigest()[:16]+++def _autoscheme_cache_path(cache_key, scheme_index):+ """Return the full path to the JSON cache file for a **single scheme**.++ Each scheme gets its own cache file under ``{AR_WORK_SPACE}/auto_scheme_cache/``+ to enable granular reuse: adding/removing schemes or changing non-scoring+ parameters (e.g., target_bits) doesn't invalidate caches for unmodified schemes.++ Args:+ cache_key: Per-scheme cache key (includes model, dataset, scheme, etc.)+ scheme_index: Index of the scheme (for human readability in filenames)+ """+ from auto_round import envs as _envs++ cache_dir = os.path.join(_envs.AR_WORK_SPACE, "auto_scheme_cache")+ os.makedirs(cache_dir, exist_ok=True)+ return os.path.join(cache_dir, f"scheme_{scheme_index:02d}_{cache_key}.json")+++def _save_autoscheme_scores(+ cache_path,+ cache_key,+ scheme_index,+ scheme_dict,+ layer_scores,+ total_loss_for_scheme,+ total_params,+):+ """Persist scoring results for **a single scheme** to *cache_path* as JSON.++ Each scheme's scores are stored independently so that adding/removing schemes+ or changing unrelated parameters (e.g., target_bits) doesn't invalidate cached+ scores for unchanged schemes.++ Schema version 3 (per-scheme, per-op cache)::++ {+ "version": 3,+ "score_granularity": "per_op",+ "cache_key": "<hex>",+ "scheme_index": 0,+ "scheme": { ... scheme dict ... },+ "created_at": "<ISO datetime>",+ "layer_scores": { layer_key: [bits, loss], ... },+ "total_loss_for_scheme": 1.234,+ "total_params": 12345+ }+ """+ if cache_path is None or cache_key is None:+ return+ # Persist only per-layer independent scores. Grouping (e.g. shared_layers+ # or MoE expert groups) is intentionally NOT stored here — callers should+ # re-apply grouping when loading a cache so the on-disk format stays+ # per-op and backward/forward compatible.+ data = {+ "version": 3,+ "score_granularity": "per_op",+ "cache_key": cache_key,+ "scheme_index": scheme_index,+ "scheme": scheme_dict,+ "created_at": time.strftime("%Y-%m-%dT%H:%M:%S"),+ "layer_scores": layer_scores,+ "total_loss_for_scheme": total_loss_for_scheme,+ "total_params": total_params,+ }+ try:+ with open(cache_path, "w", encoding="utf-8") as _f:+ json.dump(data, _f, indent=2, default=str)+ logger.info("AutoScheme: per-scheme cache saved → %s", cache_path)+ except Exception as _exc: # noqa: BLE001+ logger.warning("AutoScheme: failed to save per-scheme cache: %s", _exc)+++def _load_autoscheme_scores(cache_path):+ """Load and validate a **single-scheme** per-op scoring cache file (version 3).++ Returns the parsed dict with keys ``layer_scores``, ``total_loss_for_scheme``,+ and ``total_params`` on success, or ``None`` if the file is missing, malformed,+ or fails the version sanity check.+ """+ _required = ("layer_scores", "total_loss_for_scheme", "total_params")+ try:+ with open(cache_path, encoding="utf-8") as _f:+ data = json.load(_f)+ if data.get("version") != 3 or data.get("score_granularity") != "per_op":+ logger.warning(+ "AutoScheme: per-scheme cache schema mismatch "+ "(expected version=3, score_granularity=per_op; got version=%s, score_granularity=%s)",+ data.get("version"),+ data.get("score_granularity"),+ )+ return None+ for _k in _required:+ if _k not in data:+ logger.warning("AutoScheme: per-scheme cache missing required field %s", _k)+ return None+ return data+ except Exception as _exc: # noqa: BLE001+ logger.warning("AutoScheme: failed to read per-scheme cache %s: %s", cache_path, _exc)+ return None+++def _is_per_op_cache_compatible(cached_data, quant_layer_names, fixed_layer_scheme):+ """Return whether a cache contains exactly one score for every non-fixed quant layer."""+ expected_layers = set(quant_layer_names) - set(fixed_layer_scheme)+ return set(cached_data["layer_scores"]) == expected_layers+++def _refresh_cached_layer_bits(+ model,+ quant_layer_names,+ fixed_layer_scheme,+ scheme,+ cached_layer_scores,+ ignore_scale_zp_bits,+):+ """Recompute bit costs for the current accounting mode while preserving cached losses."""+ apply_quant_scheme(+ model,+ quant_layer_names=quant_layer_names,+ fixed_layer_scheme=fixed_layer_scheme,+ scheme=scheme,+ )+ refreshed_scores = {}+ for name, (_, loss) in cached_layer_scores.items():+ bits, _ = compute_layer_bits(get_module(model, name), ignore_scale_zp_bits)+ refreshed_scores[name] = [bits, loss]+ return refreshed_scores+++def _assign_scheme_worker_devices(worker_count, available_devices):+ """Assign workers round-robin within the devices selected by the caller."""+ if not available_devices:+ raise ValueError("available_devices must contain at least one device")+ return [available_devices[worker_index % len(available_devices)] for worker_index in range(worker_count)]+++class _ProgressQueueProxy:+ """Forward worker progress events to the parent process that owns tqdm."""++ def __init__(self, progress_queue):+ self.progress_queue = progress_queue++ def update(self, steps=1):+ self.progress_queue.put(("update", steps))++ def write(self, message):+ self.progress_queue.put(("write", message))+++def _drain_progress_queue(progress_queue, pbar):+ """Apply all currently queued worker progress events to the parent tqdm instance."""+ import queue++ while True:+ try:+ event, payload = progress_queue.get_nowait()+ except queue.Empty:+ break+ if event == "update":+ pbar.update(payload)+ elif event == "write":+ pbar.write(payload)+++def _get_worker_memory_report(worker_device):+ """Return this worker's peak RAM and VRAM for its assigned logical device."""+ memory_monitor.update(device_list=worker_device)+ device_key = str(worker_device).split(":")[-1]+ return {+ "device": device_key,+ "peak_ram": memory_monitor.peak_ram,+ "peak_vram": memory_monitor.peak_vram.get(device_key, 0.0),+ }+++def _merge_worker_memory_reports(monitor, reports):+ """Merge child-process RAM/VRAM peaks into the parent monitor."""+ # Worker reports are process-local. Sum them to estimate the concurrent+ # worker peak, then add the parent RSS after workers have exited. The+ # parent's sampled peak may already contain the full live process tree,+ # so retain whichever aggregate is larger rather than adding both.+ worker_peak_ram = sum(report.get("peak_ram", 0.0) for report in reports)+ parent_ram = monitor._process_tree_rss() if hasattr(monitor, "_process_tree_rss") else monitor.peak_ram+ monitor.peak_ram = max(monitor.peak_ram, parent_ram + worker_peak_ram)++ worker_peaks = {}+ for report in reports:+ device = str(report["device"])+ worker_peaks[device] = worker_peaks.get(device, 0.0) + report["peak_vram"]+ for device, peak_vram in worker_peaks.items():+ monitor.peak_vram[device] = max(monitor.peak_vram.get(device, 0.0), peak_vram)+++def _get_scheme_worker_count(num_schemes, num_gpus):+ """Use one worker per scoring scheme; workers may share a visible GPU."""+ if num_gpus < 1:+ raise ValueError("AutoScheme multiprocessing requires at least one GPU")+ return num_schemes+++def _load_scheme_worker_model(model_name, use_model_replacements, low_cpu_mem_usage):+ """Load an isolated worker model without an extra full-size CPU initialization copy."""+ return load_model(+ model_name,+ device="cpu",+ use_auto_mapping=False,+ use_model_replacements=use_model_replacements,+ low_cpu_mem_usage=low_cpu_mem_usage,+ )+++def _score_scheme_worker(args):+ """Score one scheme and return its index, scores, and worker VRAM peak."""+ import traceback as _tb++ (+ index,+ scheme,+ model_name,+ is_vlm,+ quant_layer_names,+ fixed_layer_scheme,+ dataset,+ nsamples,+ seqlen,+ batch_size,+ need_weight_grad,+ enable_torch_compile,+ low_cpu_mem_usage,+ low_gpu_mem_usage,+ ignore_scale_zp_bits,+ force_mllm,+ use_model_replacements,+ worker_device,+ total_schemes,+ progress_queue,+ ) = args++ from auto_round.auto_scheme.utils import _scheme_short_name as _short_name+ from auto_round.auto_scheme.utils import apply_quant_scheme as _apply_quant_scheme+ from auto_round.auto_scheme.utils import compute_layer_bits as _compute_layer_bits+ from auto_round.utils import get_block_names as _get_block_names+ from auto_round.utils import get_module as _get_module++ try:+ model, tokenizer, processor, _, _, is_vlm, _ = _load_scheme_worker_model(+ model_name,+ use_model_replacements,+ low_cpu_mem_usage,+ )+ except Exception as exc:+ raise RuntimeError(+ f"_score_scheme_worker[{index}]: failed to load model {model_name!r}\n{_tb.format_exc()}"+ ) from exc++ safe_to_cpu_(model)+ block_names = _get_block_names(model, quant_vision=force_mllm)[0]+ for block_name in block_names:+ block = _get_module(model, block_name)+ block.in_block = True+ for _, module in block.named_modules():+ module.in_block = True+ for _, module in model.named_modules():+ if len(list(module.children())) == 0:+ if not hasattr(module, "in_block"):+ module.in_block = False+ if not module.in_block and low_gpu_mem_usage:+ module.to(worker_device)++ from auto_round.modeling.fused_moe.replace_modules import materialize_model_++ materialize_model_(model)+ # MoE materialization can replace fused expert modules with newly-created+ # Linear layers. Assign tuning metadata only after the final module tree exists.+ for layer_name in quant_layer_names:+ layer = _get_module(model, layer_name)+ if layer is None:+ raise RuntimeError(+ f"_score_scheme_worker[{index}]: layer {layer_name!r} is missing after model preprocessing"+ )+ if not hasattr(layer, "tuning_device"):+ layer.tuning_device = worker_device+ layer.tmp_name = layer_name++ _apply_quant_scheme(+ model,+ quant_layer_names=quant_layer_names,+ fixed_layer_scheme=fixed_layer_scheme,+ scheme=scheme,+ )++ is_bf16 = isinstance(scheme, str) and scheme.upper() == "BF16"+ if isinstance(scheme, dict):+ is_bf16 = scheme.get("bits", 16) >= 16 and scheme.get("act_bits", 16) >= 16+ if is_bf16:+ scores = {}+ for layer_name in quant_layer_names:+ if layer_name in fixed_layer_scheme:+ continue+ layer_bits, _ = _compute_layer_bits(_get_module(model, layer_name), ignore_scale_zp_bits)+ scores[layer_name] = [layer_bits, 0.0]+ return index, scores, _get_worker_memory_report(worker_device)++ from auto_round.auto_scheme.delta_loss import get_score_for_scheme++ scores = get_score_for_scheme(+ model,+ tokenizer,+ quant_layer_names,+ fixed_layer_scheme,+ dataset,+ ignore_scale_zp_bits=ignore_scale_zp_bits,+ pbar=_ProgressQueueProxy(progress_queue),+ nsamples=nsamples,+ seqlen=seqlen,+ need_weight_grad=need_weight_grad,+ enable_torch_compile=enable_torch_compile,+ low_gpu_mem_usage=low_gpu_mem_usage,+ major_device=worker_device,+ batch_size=batch_size,+ offload_context=None,+ processor=processor,+ is_vlm=is_vlm,+ force_mllm=force_mllm,+ model_name=model_name,+ scheme_tag=f"{index + 1}/{total_schemes} {_short_name(scheme)}",+ )+ return index, scores, _get_worker_memory_report(worker_device)++def _gen_layer_config(auto_scheme: AutoScheme,model: Union[str, torch.nn.Module],@@ -1539,8 +1953,8 @@ def _gen_layer_config(return Trueif isinstance(scheme, QuantizationScheme):scheme = asdict(scheme)- if scheme["bits"] >= 16 and scheme["act_bits"] >= 16:- return True+ if isinstance(scheme, dict):+ return scheme.get("bits", 16) >= 16 and scheme.get("act_bits", 16) >= 16return Falsefrom auto_round import envs as _envs@@ -1670,108 +2084,387 @@ def _gen_layer_config(layer_numel[_n] = _npoptions_scores = []+ pbar = None++ # ---- Scoring cache (per-scheme) -------------------------------------------------- #+ # Each scheme gets its own cache file so that adding/removing schemes or changing+ # unrelated parameters (e.g., target_bits) doesn't invalidate cached scores for+ # unchanged schemes.+ _model_id_for_cache = model_name or getattr(getattr(model, "config", None), "_name_or_path", None)++ # In per-scheme caching, each scheme is checked independently for cache hits.+ # We always prepare the common resources (imatrix, pbar, etc.) since they're+ # reusable across schemes and their setup cost is negligible compared to scoring.+ if True:+ if need_imatrix:+ dataloader = get_dataloader(+ tokenizer,+ seqlen=max(seqlen * 2, 2048),+ dataset_name=dataset,+ seed=42,+ bs=batch_size,+ nsamples=min(nsamples, 128),+ )+ logger.info("start to compute imatrix in AutoScheme")+ cal_imatrix(model, dataloader, major_device, low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage)+ memory_monitor.update()+ memory_monitor.log_summary()+ logger.info("finish calculating imatrix")- if need_imatrix:- dataloader = get_dataloader(- tokenizer,- seqlen=max(seqlen * 2, 2048),- dataset_name=dataset,- seed=42,- bs=batch_size,- nsamples=min(nsamples, 128),- )- logger.info("start to compute imatrix in AutoScheme")- cal_imatrix(model, dataloader, major_device, low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage)- memory_monitor.update()- memory_monitor.log_summary()- logger.info("finish calculating imatrix")-- # Register hooks and clear all block weights before the scheme loop.- # Hooks will transparently reload weights on demand during forward passes.- if offload_context is not None:- offload_context.add_offload_hooks(model, block_name)+ # Register hooks and clear all block weights before the scheme loop.+ # Hooks will transparently reload weights on demand during forward passes.+ if offload_context is not None:+ offload_context.add_offload_hooks(model, block_name)++ pbar = tqdm(total=pbar_cnt, desc="Generating AutoScheme")+ scored_layer_names = set(quant_layer_names + embedding_layers_names)++ def _group_per_op_scores(index, per_op_scores):+ """Apply the current shared-layer grouping without mutating cached per-op scores."""+ grouped_scores = {}+ remaining = {name: list(score) for name, score in per_op_scores.items()}+ for share_layer in shared_layers:+ param_bits = 0+ tmp_loss = 0+ name_list = []+ for name in share_layer:+ if name in remaining:+ bits, loss = remaining.pop(name)+ param_bits += bits+ tmp_loss += loss+ name_list.append(name)+ if name_list:+ grouped_scores[name_list[0]] = [index, param_bits, tmp_loss, name_list]+ for name, (bits, loss) in remaining.items():+ grouped_scores[name] = [index, bits, loss, [name]]+ return grouped_scores++ def _record_scheme_scores(index, per_op_scores):+ grouped_scores = _group_per_op_scores(index, per_op_scores)+ total_loss = sum(item[2] for item in grouped_scores.values())+ for key, item in grouped_scores.items():+ total_scores.setdefault(key, []).append(item)+ options_scores.append(total_loss)+ return total_loss++ def _save_per_op_scores(index, scheme, cache_key, cache_path, per_op_scores):+ if cache_key is None or cache_path is None:+ return+ if isinstance(scheme, str):+ scheme_dict = asdict(preset_name_to_scheme(scheme))+ elif isinstance(scheme, QuantizationScheme):+ scheme_dict = asdict(scheme)+ else:+ scheme_dict = scheme if isinstance(scheme, dict) else {"preset": str(scheme)}+ _save_autoscheme_scores(+ cache_path=cache_path,+ cache_key=cache_key,+ scheme_index=index,+ scheme_dict=scheme_dict,+ layer_scores={name: list(score) for name, score in per_op_scores.items()},+ total_loss_for_scheme=sum(score[1] for score in per_op_scores.values()),+ total_params=sum(+ (+ module.weight.numel()+ if hasattr(module, "weight") and module.weight is not None+ else getattr(module, "_cached_weight_numel", 0)+ )+ for name, module in model.named_modules()+ if name in scored_layer_names+ ),+ )- pbar = tqdm(total=pbar_cnt, desc="Generating AutoScheme")- for index, scheme in enumerate(schemes):- scheme_tag = f"{index + 1}/{len(schemes)} {_scheme_short_name(scheme)}"- logger.info(f"AutoScheme transition: switch to scheme {index + 1}/{len(schemes)} ({scheme})")- apply_quant_scheme(- model, quant_layer_names=quant_layer_names, fixed_layer_scheme=fixed_layer_scheme, scheme=scheme- )- scores = {} # name: bits, loss- if check_bf16_scheme(scheme):- for n in quant_layer_names:- if n in fixed_layer_scheme.keys():- continue- m = get_module(model, n)- bits, _ = compute_layer_bits(m, auto_scheme.ignore_scale_zp_bits)- scores[n] = [bits, 0.0]- else:- scores = get_score_for_scheme(- model,- tokenizer,- quant_layer_names,- fixed_layer_scheme,- dataset,- ignore_scale_zp_bits=auto_scheme.ignore_scale_zp_bits,- pbar=pbar,+ scheme_cache_meta = []+ for index, scheme in enumerate(schemes):+ if check_bf16_scheme(scheme) or _model_id_for_cache is None:+ scheme_cache_meta.append((None, None, None))+ if check_bf16_scheme(scheme):+ logger.info(+ "AutoScheme: scheme %d/%d (%s) is a BF16 baseline; skipping scoring and cache lookup.",+ index + 1,+ len(schemes),+ _scheme_short_name(scheme),+ )+ continue+ cache_key = _autoscheme_cache_key(+ model_name=_model_id_for_cache,+ dataset=dataset,nsamples=nsamples,seqlen=seqlen,- need_weight_grad=need_weight_grad,- enable_torch_compile=enable_torch_compile,- low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage,- major_device=major_device,batch_size=batch_size,- offload_context=offload_context,- processor=processor,- is_vlm=is_vlm,+ quant_layer_names=quant_layer_names,+ fixed_layer_scheme=fixed_layer_scheme,+ scheme=scheme,force_mllm=force_mllm,- model_name=model_name,- scheme_tag=scheme_tag,+ low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage,)- # Track peak RAM after each scheme scoring- memory_monitor.update()- memory_monitor.log_summary()-- new_scores = {}- for share_layer in shared_layers:- param_bits = 0- tmp_loss = 0- name_list = []- for name in share_layer:- if name in scores.keys():- param_bits += scores[name][0]- tmp_loss += scores[name][1]- name_list.append(name)- scores.pop(name)- new_scores[name_list[0]] = [index, param_bits, tmp_loss, name_list]- for name, item in scores.items():- new_scores[name] = [index, item[0], item[1], [name]]- options_total_loss = 0.0- for key, item in new_scores.items():- options_total_loss += item[2]- if key in total_scores:- total_scores[key].append(item)- else:- total_scores[key] = [item]- options_scores.append(options_total_loss)- logger.info(- f"AutoScheme transition: scheme {index + 1}/{len(schemes)} "- f"scoring finished (total_loss={options_total_loss:.6f})"+ cache_path = _autoscheme_cache_path(cache_key, index)+ cached_data = _load_autoscheme_scores(cache_path) if os.path.exists(cache_path) else None+ if cached_data is not None and not _is_per_op_cache_compatible(+ cached_data, quant_layer_names, fixed_layer_scheme+ ):+ logger.warning(+ "AutoScheme: cache %s does not contain complete per-op scores; rescoring scheme %d",+ cache_path,+ index,+ )+ cached_data = None+ scheme_cache_meta.append((cache_key, cache_path, cached_data))++ uncached_indices = [+ index+ for index, (_, _, cached) in enumerate(scheme_cache_meta)+ if not check_bf16_scheme(schemes[index]) and cached is None+ ]+ worker_device_pool = [device for device in device_list if str(device).startswith("cuda:")]+ num_gpus = len(worker_device_pool)+ parallel_enabled = _envs.AR_ENABLE_AUTO_SCHEME_PARALLEL+ can_parallel = (+ parallel_enabled+ and _model_id_for_cache is not None+ and num_gpus >= 1+ and len(uncached_indices) >= 2+ and not need_imatrix)- clear_memory(device_list=device_list)+ if not parallel_enabled and len(uncached_indices) >= 2:+ logger.info(+ "AutoScheme: parallel scoring is disabled; set AR_ENABLE_AUTO_SCHEME_PARALLEL=1 to enable it. "+ "scoring %d uncached non-BF16 schemes serially.",+ len(uncached_indices),+ )- # Remove hooks and restore original weights from disk for final bit-budget computations- if offload_context is not None:- offload_context.remove_offload_hooks(model, block_name)+ from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase- total_params = 0- for n, m in model.named_modules():- if n in quant_layer_names + embedding_layers_names:- n_param = m.weight.numel()- if n_param == 0 and hasattr(m, "_cached_weight_numel"):- n_param = m._cached_weight_numel- total_params += n_param+ use_model_replacements = any(isinstance(module, ReplacementModuleBase) for module in model.modules())++ parallel_done = False+ if can_parallel:+ try:+ import torch.multiprocessing as multiprocessing++ def _serialize_scheme(scheme):+ return asdict(scheme) if isinstance(scheme, QuantizationScheme) else scheme++ num_workers = _get_scheme_worker_count(len(uncached_indices), num_gpus)+ worker_devices = _assign_scheme_worker_devices(len(uncached_indices), worker_device_pool)+ logger.info(+ "AutoScheme: starting %d parallel scoring workers for %d uncached non-BF16 schemes "+ "(devices=%s; workers may share devices)",+ num_workers,+ len(uncached_indices),+ worker_device_pool,+ )+ logger.info(+ "AutoScheme: if parallel scoring runs out of RAM/VRAM or automatic serial fallback cannot "+ "recover, set AR_ENABLE_AUTO_SCHEME_PARALLEL=0 and rerun."+ )+ spawn_context = multiprocessing.get_context("spawn")+ with spawn_context.Manager() as manager:+ progress_queue = manager.Queue()+ worker_args = [+ (+ index,+ _serialize_scheme(schemes[index]),+ _model_id_for_cache,+ is_vlm,+ list(quant_layer_names),+ dict(fixed_layer_scheme),+ dataset,+ nsamples,+ seqlen,+ batch_size,+ need_weight_grad,+ enable_torch_compile,+ auto_scheme.low_cpu_mem_usage,+ auto_scheme.low_gpu_mem_usage,+ auto_scheme.ignore_scale_zp_bits,+ force_mllm,+ use_model_replacements,+ worker_devices[slot],+ len(schemes),+ progress_queue,+ )+ for slot, index in enumerate(uncached_indices)+ ]+ with spawn_context.Pool(processes=num_workers) as pool:+ async_results = pool.map_async(_score_scheme_worker, worker_args)+ while not async_results.ready():+ _drain_progress_queue(progress_queue, pbar)+ memory_monitor.update_cpu()+ async_results.wait(timeout=0.1)+ _drain_progress_queue(progress_queue, pbar)+ worker_results = async_results.get()++ parallel_results = {index: scores for index, scores, _ in worker_results}+ _merge_worker_memory_reports(memory_monitor, [report for _, _, report in worker_results])+ logger.info(+ "AutoScheme parallel aggregate memory (parent + %d workers): %s",+ len(worker_results),+ memory_monitor.get_summary(),+ )++ for index, scheme in enumerate(schemes):+ cache_key, cache_path, cached_data = scheme_cache_meta[index]+ if check_bf16_scheme(scheme):+ apply_quant_scheme(+ model,+ quant_layer_names=quant_layer_names,+ fixed_layer_scheme=fixed_layer_scheme,+ scheme=scheme,+ )+ per_op_scores = {}+ for name in quant_layer_names:+ if name in fixed_layer_scheme:+ continue+ bits, _ = compute_layer_bits(get_module(model, name), auto_scheme.ignore_scale_zp_bits)+ per_op_scores[name] = [bits, 0.0]+ elif cached_data is not None:+ logger.info(+ "AutoScheme: loading per-scheme cache for scheme %d from %s."+ " Delete this file to disable reuse and rescore.",+ index,+ cache_path,+ )+ per_op_scores = _refresh_cached_layer_bits(+ model,+ quant_layer_names,+ fixed_layer_scheme,+ scheme,+ cached_data["layer_scores"],+ auto_scheme.ignore_scale_zp_bits,+ )+ if not check_bf16_scheme(scheme):+ pbar.update(pbar_cnt // effective_scheme_num if effective_scheme_num > 0 else 1)+ else:+ per_op_scores = parallel_results[index]+ _save_per_op_scores(index, scheme, cache_key, cache_path, per_op_scores)+ total_loss = _record_scheme_scores(index, per_op_scores)+ logger.info(+ "AutoScheme transition: scheme %d/%d scoring finished (total_loss=%.6f)",+ index + 1,+ len(schemes),+ total_loss,+ )+ parallel_done = True+ logger.info("AutoScheme: parallel scoring completed.")+ post_scoring_started = time.perf_counter()+ except Exception as parallel_error: # noqa: BLE001+ logger.warning(+ "AutoScheme: parallel scoring failed, falling back to serial: %s. "+ "If fallback cannot recover (for example after RAM/VRAM exhaustion), rerun with "+ "AR_ENABLE_AUTO_SCHEME_PARALLEL=0.",+ parallel_error,+ )+ total_scores.clear()+ options_scores.clear()+ pbar.reset(total=pbar_cnt)++ if not parallel_done:+ if uncached_indices:+ from auto_round.modeling.fused_moe.replace_modules import materialize_model_++ materialize_model_(model)+ for index, scheme in enumerate(schemes):+ scheme_tag = f"{index + 1}/{len(schemes)} {_scheme_short_name(scheme)}"+ logger.info(f"AutoScheme transition: switch to scheme {index + 1}/{len(schemes)} ({scheme})")+ cache_key, cache_path, cached_data = scheme_cache_meta[index]++ if cached_data is not None:+ logger.info(+ "AutoScheme: loading per-scheme cache for scheme %d from %s."+ " Delete this file to disable reuse and rescore.",+ index,+ cache_path,+ )+ per_op_scores = _refresh_cached_layer_bits(+ model,+ quant_layer_names,+ fixed_layer_scheme,+ scheme,+ cached_data["layer_scores"],+ auto_scheme.ignore_scale_zp_bits,+ )+ if not check_bf16_scheme(scheme):+ pbar.update(pbar_cnt // effective_scheme_num if effective_scheme_num > 0 else 1)+ else:+ apply_quant_scheme(+ model,+ quant_layer_names=quant_layer_names,+ fixed_layer_scheme=fixed_layer_scheme,+ scheme=scheme,+ )+ if check_bf16_scheme(scheme):+ per_op_scores = {}+ for name in quant_layer_names:+ if name in fixed_layer_scheme:+ continue+ bits, _ = compute_layer_bits(get_module(model, name), auto_scheme.ignore_scale_zp_bits)+ per_op_scores[name] = [bits, 0.0]+ else:+ per_op_scores = get_score_for_scheme(+ model,+ tokenizer,+ quant_layer_names,+ fixed_layer_scheme,+ dataset,+ ignore_scale_zp_bits=auto_scheme.ignore_scale_zp_bits,+ pbar=pbar,+ nsamples=nsamples,+ seqlen=seqlen,+ need_weight_grad=need_weight_grad,+ enable_torch_compile=enable_torch_compile,+ low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage,+ major_device=major_device,+ batch_size=batch_size,+ offload_context=offload_context,+ processor=processor,+ is_vlm=is_vlm,+ force_mllm=force_mllm,+ model_name=model_name,+ scheme_tag=scheme_tag,+ )+ memory_monitor.update()+ memory_monitor.log_summary()+ if not check_bf16_scheme(scheme):+ _save_per_op_scores(index, scheme, cache_key, cache_path, per_op_scores)++ total_loss = _record_scheme_scores(index, per_op_scores)+ logger.info(+ "AutoScheme transition: scheme %d/%d scoring finished (total_loss=%.6f)",+ index + 1,+ len(schemes),+ total_loss,+ )+ clear_memory(device_list=device_list)++ # Serial scoring leaves the main model configured with the final scheme and+ # applies fixed_layer_scheme as a side effect. Parallel workers and cache hits+ # do not touch the main model, so restore that state before bit-budget math.+ if schemes:+ apply_quant_scheme(+ model,+ quant_layer_names=quant_layer_names,+ fixed_layer_scheme=fixed_layer_scheme,+ scheme=schemes[-1],+ )++ # Remove hooks and restore original weights from disk for final bit-budget computations+ if offload_context is not None:+ offload_context.remove_offload_hooks(model, block_name)++ total_params = 0+ for n, m in model.named_modules():+ if n in quant_layer_names + embedding_layers_names:+ n_param = m.weight.numel()+ if n_param == 0 and hasattr(m, "_cached_weight_numel"):+ n_param = m._cached_weight_numel+ total_params += n_param++ if parallel_done:+ logger.info(+ "AutoScheme post-scoring: model restore and parameter accounting took %.2fs",+ time.perf_counter() - post_scoring_started,+ )target_params_cnt = int(total_params * target_bits)sorted_indices = sorted(range(len(options_scores)), key=lambda i: options_scores[i])@@ -1903,19 +2596,25 @@ def _gen_layer_config(if target_params_cnt <= 0:raise ValueError("Avg bits is too small")+ cleanup_started = time.perf_counter()remove_quant_scheme(model) # Must place after minus fixed_layermemory_monitor.update()memory_monitor.log_summary()+ logger.info(+ "AutoScheme post-scoring: scheme cleanup and memory accounting took %.2fs",+ time.perf_counter() - cleanup_started,+ )+ dp_started = time.perf_counter()best_loss, best_path = choose_bits_per_layer_with_path(total_scores, target_params_cnt)+ logger.info(+ "AutoScheme post-scoring: DP selection took %.2fs (layers=%d)",+ time.perf_counter() - dp_started,+ len(total_scores),+ )+if best_path is None:- min_dp_avg = min_dp_bits / max(sum(layer_numel.get(n, 0) for n in total_scores), 1)- raise ValueError(- f"target_bits is infeasible: after subtracting fixed/embedding layers the remaining "- f"budget ({target_params_cnt} bits) is below the minimum the given options can reach "- f"({min_dp_bits} bits, avg {min_dp_avg:.3f}). Raise target_bits, add a lower-bit option, "- f"or lower the bits of fixed layers."- )+ raise ValueError("Avg bits is too small")# print(best_loss, best_path) # TODO better loglayer_config = copy.deepcopy(fixed_layer_scheme)@@ -1929,13 +2628,15 @@ def _gen_layer_config(layer_scheme = options[item[1]]for layer_name in layer_names:layer_config[layer_name] = asdict(layer_scheme)+ reporting_started = time.perf_counter()_log_scheme_loss_matrix(total_scores, options, block_name, model=model, layer_numel=layer_numel)_describe_layer_config(layer_config, total_scores, options, block_name, model=model)+ logger.info("AutoScheme post-scoring: result reporting took %.2fs", time.perf_counter() - reporting_started)if model_name is not None:model = Nonedel modelelse:- model = model.to("cpu") # TODO this requires large ram+ safe_to_cpu_(model)if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:import accelerate@@ -1960,7 +2661,8 @@ def _gen_layer_config(low_cpu_str = "enabled" if auto_scheme.low_cpu_mem_usage else "disabled"memory_monitor.log_summary(f"AutoScheme complete (low_cpu_mem_usage={low_cpu_str})")- pbar.close()+ if pbar is not None:+ pbar.close()return layer_config@@ -1989,30 +2691,11 @@ def gen_layer_config(is_vlm = Falseif isinstance(model, str):model_name = model- # Detect VLM (Qwen-VL / Qwen3-VL / LLaVA / etc.) and load via the MLLM- # path so we get a usable ``processor`` for the multimodal calibration- # dataloader. The block walker auto-detects VLMs too and skips the- # vision tower when scoring.- is_vlm = is_mllm_model(model_name)- if is_vlm:- model, processor, tokenizer, _ = mllm_load_model(- model_name,- device="cpu",- use_auto_mapping=False,- )- else:- # Load model on CPU only; do not apply automatic device map or tuning-aware placement at load time.- model, tokenizer, _ = llm_load_model(model_name, device_map="cpu")+ model, tokenizer, processor, _, _, is_vlm, _ = load_model(model_name, device="cpu", use_auto_mapping=False)else:# Object passed in: still try to detect VLM so we can pick the right dataloader later.- try:- is_vlm = is_mllm_model(model)- except Exception: # noqa: BLE001- is_vlm = False-- from auto_round.modeling.fused_moe.replace_modules import materialize_model_+ _, _, _, _, _, is_vlm, _ = load_model(model)- materialize_model_(model)# ---- Vision-tower scoring requires a full backward ---- ## ``model_forward_low_gpu`` only walks the language tower (it uses# ``get_block_names(model)[0]``, which excludes vision blocks by default)@@ -2041,7 +2724,7 @@ def gen_layer_config(else:model = dispatch_model_by_all_available_devices(model, device_map)else:- model.to("cpu")+ safe_to_cpu_(model)if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:import accelerate@@ -2112,7 +2795,7 @@ def gen_layer_config("Fallback to CPU for automatic scheme generation."" Using multiple devices is strongly recommended (e.g., --device_map 0,1,2,3).")- model.to("cpu")+ safe_to_cpu_(model)for n, m in model.named_modules():if hasattr(m, "orig_layer"):set_module(model, n, m.orig_layer)diff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.pyindex 7068b7fbc..a65fc05ba 100644--- a/auto_round/auto_scheme/gen_auto_scheme.py+++ b/auto_round/auto_scheme/gen_auto_scheme.py@@ -27,7 +27,7 @@ from auto_round.utils import get_layer_features, get_module@dataclassclass AutoScheme:- avg_bits: float+ avg_bits: Union[float, list[float]]options: Union[str, list[Union[QuantizationScheme, str]], tuple[Union[QuantizationScheme, str], ...]]shared_layers: Optional[Iterable[Iterable[str]]] = Nonemethod: str = "default"@@ -126,31 +126,41 @@ class GenScheme:if not isinstance(self.dataset, str):raise TypeError("`dataset` must be a string, got {type(self.dataset).__name__}.")- target = self.auto_scheme.avg_bits+ targets = (+ self.auto_scheme.avg_bits if isinstance(self.auto_scheme.avg_bits, list) else [self.auto_scheme.avg_bits]+ )+ if not targets:+ raise ValueError("`avg_bits` must not be an empty list.")+min_avg_bit = self.min_avg_bitmax_avg_bit = self.max_avg_bit- logger.info("Average bits range: [%.3f, %.3f], target = %.3f", min_avg_bit, max_avg_bit, target)- if abs(target - min_avg_bit) < 1e-3 or abs(target - max_avg_bit) < 1e-3:+ normalized_targets = []+ for target in targets:+ logger.info("Average bits range: [%.3f, %.3f], target = %.3f", min_avg_bit, max_avg_bit, target)if abs(target - min_avg_bit) < 1e-3:target = min_avg_bit- else:+ elif abs(target - max_avg_bit) < 1e-3:target = max_avg_bit- self.auto_scheme.avg_bits = target-- if not (min_avg_bit <= target <= max_avg_bit):- details = ", ".join(f"{opt}={bits:.3f}" for opt, bits in getattr(self, "_option_avg_bits", []))- hint = (- "Provide at least one option whose avg_bits is <= target and one whose avg_bits is >= target. "- "Note that layers fixed to high-precision (e.g. lm_head/embed kept at bf16) inflate the lower bound; "- "use `quant_lm_head=True`/adjust `layer_config`/`ignore_layers` if needed."- )- raise ValueError(- f"Target avg_bits={target:.3f} is outside the achievable range "- f"[{min_avg_bit:.3f}, {max_avg_bit:.3f}] for options={list(self.auto_scheme.options)} "- f"(per-option avg_bits: {details}). {hint}"- )-- def get_layer_config(self) -> dict[str, dict]:+ normalized_targets.append(target)++ if not (min_avg_bit <= target <= max_avg_bit):+ details = ", ".join(f"{opt}={bits:.3f}" for opt, bits in getattr(self, "_option_avg_bits", []))+ hint = (+ "Provide at least one option whose avg_bits is <= target and one whose avg_bits is >= target. "+ "Note that layers fixed to high-precision (e.g. lm_head/embed kept at bf16) "+ "inflate the lower bound; use `quant_lm_head=True`/adjust `layer_config`/`ignore_layers` if needed."+ )+ raise ValueError(+ f"Target avg_bits={target:.3f} is outside the achievable range "+ f"[{min_avg_bit:.3f}, {max_avg_bit:.3f}] for options={list(self.auto_scheme.options)} "+ f"(per-option avg_bits: {details}). {hint}"+ )++ self.auto_scheme.avg_bits = (+ normalized_targets if isinstance(self.auto_scheme.avg_bits, list) else normalized_targets[0]+ )++ def get_layer_config(self) -> dict[str, dict] | dict[float, dict[str, dict]]:method_name = self.auto_scheme.methodfrom auto_round import auto_scheme@@ -158,6 +168,20 @@ class GenScheme:if self.auto_scheme.low_gpu_mem_usage:self.enable_torch_compile = False+ target_bits = self.auto_scheme.avg_bits+ if not isinstance(target_bits, list):+ return self._get_layer_config(method_func)++ layer_configs = {}+ try:+ for target_bit in target_bits:+ self.auto_scheme.avg_bits = target_bit+ layer_configs[target_bit] = self._get_layer_config(method_func)+ finally:+ self.auto_scheme.avg_bits = target_bits+ return layer_configs++ def _get_layer_config(self, method_func) -> dict[str, dict]:layer_config = method_func(self.auto_scheme,self.model,diff --git a/auto_round/auto_scheme/utils.py b/auto_round/auto_scheme/utils.pyindex caa7c2151..7b5d869a1 100644--- a/auto_round/auto_scheme/utils.py+++ b/auto_round/auto_scheme/utils.py@@ -636,16 +636,13 @@ def _log_batch_avg_loss(model, batch_idx: int, pbar=None, block_names=None, tota)-def _build_layer_config_header_rows(columns: list[str]) -> list[list[str]]:+def _build_layer_config_header_rows(columns: list[str], has_expert_layers: bool = False) -> list[list[str]]:"""Build a compact two-row header for the layer-config matrix.The first row keeps a shared prefix for each grouped set of columns (forexample ``mlp`` or ``self_attn``), while the second row keeps the leafsuffix (for example ``down_proj``)."""- if not columns:- return [["block"], [""]]-leaves = []first_row = []prev_prefix = None@@ -664,6 +661,10 @@ def _build_layer_config_header_rows(columns: list[str]) -> list[list[str]]:first_row.append("")prev_prefix = None+ if has_expert_layers:+ first_row.append("mlp")+ leaves.append("experts")+header_rows = [["block"] + first_row]header_rows.append([""] + leaves)return header_rows@@ -711,7 +712,7 @@ def _log_scheme_loss_matrix(total_scores, options, block_names, model=None, layehas_expert_layers = any(block_expert_keys.get(b) for b in block_names)block_display_names = [_short_summary_name(b) for b in block_names]- header_rows = _build_layer_config_header_rows(columns)+ header_rows = _build_layer_config_header_rows(columns, has_expert_layers)# Fixed cell width for scientific-notation values: "1.234E-05" = 9 chars._CELL_W = 9@@ -727,11 +728,8 @@ def _log_scheme_loss_matrix(total_scores, options, block_names, model=None, layewidths[col] = max(widths[col], len(prefix))header_keys = ["block"] + columns + (["experts"] if has_expert_layers else [])- header_vals = [header_rows[0][0]] + header_rows[0][1:]- header_vals_row2 = ([""]) + (header_rows[1][1:] if len(header_rows) > 1 else [""] * len(columns))- if has_expert_layers:- header_vals.append("experts")- header_vals_row2.append("")+ header_vals = header_rows[0]+ header_vals_row2 = header_rows[1]def _fmt_row(values: list[str], keys: list[str]) -> str:return "|".join(v.ljust(widths[k]) for v, k in zip(values, keys))@@ -862,7 +860,7 @@ def _describe_layer_config(layer_config, total_scores, options, block_names, modhas_expert_layers = any(block_expert_layers.get(block_name) for block_name in block_names)block_display_names = [_short_summary_name(block_name) for block_name in block_names]- header_rows = _build_layer_config_header_rows(columns)+ header_rows = _build_layer_config_header_rows(columns, has_expert_layers)# Width is driven by the leaf name and cell content — NOT the full dotted column name,# so columns stay tight (e.g. "down_proj" width, not "mlp.down_proj" width).@@ -872,7 +870,9 @@ def _describe_layer_config(layer_config, total_scores, options, block_names, modmax_cell_len = max((len(block_cells.get(b, {}).get(col, "-")) for b in block_names), default=1)widths[col] = max(len(leaf), max_cell_len)if has_expert_layers:- widths["experts"] = max(len("experts"), max((len(v) for v in expert_text_by_block.values()), default=1))+ widths["experts"] = max(+ len("mlp"), len("experts"), max((len(v) for v in expert_text_by_block.values()), default=1)+ )# Ensure the first column of each prefix group is wide enough for the prefix text in header row 1.for col, prefix in zip(columns, header_rows[0][1:]):if prefix:@@ -882,11 +882,8 @@ def _describe_layer_config(layer_config, total_scores, options, block_names, modreturn "|".join(v.ljust(widths[k]) for v, k in zip(values, keys))header_keys = ["block"] + columns + (["experts"] if has_expert_layers else [])- header_vals = [header_rows[0][0]] + header_rows[0][1:]- if len(header_rows) > 1:- header_vals_row2 = [header_rows[1][0]] + header_rows[1][1:]- else:- header_vals_row2 = [""] + [""] * len(columns)+ header_vals = header_rows[0]+ header_vals_row2 = header_rows[1]logger.info("AutoScheme final layer_config matrix:")logger.info("AutoScheme note: cell=`scheme`.")diff --git a/auto_round/cli/main.py b/auto_round/cli/main.pyindex a09467e3e..67d38c7ad 100644--- a/auto_round/cli/main.py+++ b/auto_round/cli/main.py@@ -332,6 +332,38 @@ def tune(args):from auto_round.auto_scheme import AutoScheme+ # Normalize --options: accepts both space-separated (nargs="+" list) and comma-separated string.+ # Examples: --options W4A16 W8A16 OR --options W4A16,W8A16+ if args.options is not None:+ flat = ",".join(args.options) # handles list; each element may itself contain commas+ args.options = ",".join(p.strip() for p in flat.split(",") if p.strip())++ # Normalize --shared_layers: supports three forms per invocation:+ # - all bare tokens (no commas): treated as one group+ # e.g. --shared_layers l1 l2 → [['l1', 'l2']]+ # - comma-containing tokens: each token is its own group+ # e.g. --shared_layers l1,l2 l3,l4 → [['l1','l2'], ['l3','l4']]+ # - single comma token: one group+ # e.g. --shared_layers l1,l2 → [['l1', 'l2']]+ # Multiple --shared_layers flags always produce multiple groups (one per flag when each flag+ # yields exactly one group, or more if a flag contains comma-tokens).+ if args.shared_layers is not None:+ normalized_groups = []+ for invocation in args.shared_layers:+ # invocation is a list of tokens from one --shared_layers flag (nargs="+")+ if any("," in token for token in invocation):+ # at least one comma token → each token becomes its own group+ for token in invocation:+ group = [p.strip() for p in token.split(",") if p.strip()]+ if group:+ normalized_groups.append(group)+ else:+ # all bare names → bundle into a single group+ group = [p.strip() for p in invocation if p.strip()]+ if group:+ normalized_groups.append(group)+ args.shared_layers = normalized_groups or None+if args.avg_bits is not None:if args.options is None:raise ValueError("please set --options for auto scheme")diff --git a/auto_round/cli/parser.py b/auto_round/cli/parser.pyindex 85f2bfdcc..ce68bf283 100644--- a/auto_round/cli/parser.py+++ b/auto_round/cli/parser.py@@ -134,7 +134,13 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Argu)rt.add_argument("--output_dir", default="./tmp_autoround", type=str, help="Directory to save quantized artifacts.")rt.add_argument("--avg_bits", "--target_bits", default=None, type=float, help="Average target bits for AutoScheme.")- rt.add_argument("--options", default=None, type=str, help="AutoScheme options, for example 'W4A16,W8A16'.")+ rt.add_argument(+ "--options",+ default=None,+ type=str,+ nargs="+",+ help="AutoScheme options. Accepts comma-separated ('W4A16,W8A16') or space-separated (W4A16 W8A16).",+ )rt.add_argument("--low_gpu_mem_usage", action="store_true", help="Enable memory-efficient mode by offloading features to CPU.")@@ -157,7 +163,8 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Argunargs="+",action="append",default=None,- help="Ensure listed layers share the same quantization data type.",+ help="Ensure listed layers share the same quantization data type."+ " Accepts space-separated (--shared_layers l1 l2) or comma-separated ('l1,l2') per group.",)rt.add_argument("--static_kv_dtype",diff --git a/auto_round/compressors/entry.py b/auto_round/compressors/entry.pyindex ce6b826e8..9cdff97af 100644--- a/auto_round/compressors/entry.py+++ b/auto_round/compressors/entry.py@@ -369,7 +369,12 @@ class AutoRound(object):# Model-free routing is now supported directly by the new entry path.model_free_iters = 0 if isinstance(quant_config, RTNConfig) else getattr(quant_config, "iters", None)model_free_disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", None)- route_decision_kwargs = dict(route_kwargs, format=format)+ route_decision_kwargs = dict(+ route_kwargs,+ format=format,+ static_kv_dtype=base_kwargs.get("static_kv_dtype"),+ static_attention_dtype=base_kwargs.get("static_attention_dtype"),+ )if is_model_free_route(model, scheme, model_free_iters, model_free_disable_opt_rtn, route_decision_kwargs):from auto_round.compressors.model_free import ModelFreeCompressordiff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyindex 8f07b5138..8c171a82a 100644--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -120,6 +120,8 @@ _BLOCK_NAME_TO_IGNORE = ["shared_expert_gate.", ".gate.", "embed", "conv"]# Preset schemes that model-free mode can produce.# INT presets use ``auto_round:auto_gptq`` packing; MXFP presets use# ``mxfp4-pack-quantized`` or ``mxfp8-quantized`` (compressed-tensors) packing.+# BF16 acts as a full-precision default — all layers stay in BF16 unless+# overridden by layer_config.## Note: ``W3A16`` (3-bit) is intentionally excluded. 3-bit packing requires# in_features to be padded to a multiple of pack_factor=10, which the current@@ -133,6 +135,7 @@ SUPPORTED_PRESET_SCHEMES: tuple[str, ...] = ("W8A16","MXFP4","MXFP8",+ "BF16",)# Allowed ``bits`` values for integer WOQ.@@ -1048,14 +1051,13 @@ def _build_mxfp_quantization_config(:mod:`auto_round.export.export_to_llmcompressor.export_to_fp`."""from auto_round.export.export_to_llmcompressor.config import (- check_compressed_tensors_supported,initialize_quantization,)- check_compressed_tensors_supported(raise_error=True)+ bits = default_scheme.get("bits", 4)+ is_fp_default = (bits or 0) >= 16 # BF16/FP16 full-precision default- bits = default_scheme["bits"]- if bits not in _SUPPORTED_MXFP_BITS:+ if not is_fp_default and bits not in _SUPPORTED_MXFP_BITS:raise ValueError(f"Unsupported MXFP bits={bits} for model-free output.")# Default ignore list: any layer present in ignored_layers (deduped) that@@ -1077,13 +1079,24 @@ def _build_mxfp_quantization_config(layer_bits = scheme.get("bits", bits) if scheme is not None else bitsscheme_groups.setdefault(layer_bits, []).append(layer)else:- scheme_groups[bits] = list(quantized_layers)+ if not is_fp_default:+ scheme_groups[bits] = list(quantized_layers)+ # else: BF16 default with no layer_config → no MXFP layers; scheme_groups stays {}if len(scheme_groups) <= 1:- # Single scheme — existing behavior.- scheme_name = "MXFP4" if bits == 4 else "MXFP8"- fmt = "mxfp4-pack-quantized" if bits == 4 else "mxfp8-quantized"+ # Single scheme — use the actual MXFP bits from the group, not the+ # default bits (which may be 16 for a BF16 default scheme).+ actual_bits = next(iter(scheme_groups.keys())) if scheme_groups else bits+ if actual_bits not in _SUPPORTED_MXFP_BITS:+ raise ValueError(f"Unsupported MXFP bits={actual_bits} for model-free output.")+ scheme_name = "MXFP4" if actual_bits == 4 else "MXFP8"+ fmt = "mxfp4-pack-quantized" if actual_bits == 4 else "mxfp8-quantized"qconfig = initialize_quantization(scheme=scheme_name, ignore=ignore)+ if is_fp_default and scheme_groups:+ targets = list(quantized_layers)+ if any(".experts." in name for name in quantized_layers):+ targets.insert(0, "RoutedExperts")+ qconfig.config_groups["group_0"].targets = targetsqconfig = qconfig.to_dict()qconfig["format"] = fmtqconfig["provider"] = "auto-round"@@ -1128,6 +1141,217 @@ def _build_mxfp_quantization_config(return full_dict+def _build_mxfp_autoround_quantization_config(+ default_scheme: dict,+ quantized_layers: list[str],+ ignored_layers: list[str],+ layer_config: dict | None = None,+) -> dict:+ """Build an auto-round style quantization_config for MXFP4 / MXFP8.++ Unlike :func:`_build_mxfp_quantization_config` which produces a+ compressed-tensors / llm-compressor style config, this function produces+ an ``auto-round`` style config (``quant_method="auto-round"``,+ ``packing_format="auto_round:llm_compressor"``). The on-disk weight+ tensors are identical; only the ``quantization_config`` metadata differs.++ The generated config mirrors the layout of the regular AutoRound MXFP+ export, including activation quantization fields (``act_bits`` /+ ``act_data_type`` / …) from the scheme, ``enable_quanted_input``, and+ ``lm_head`` in ``extra_config`` when it was quantized.+ """+ from collections import Counter++ from auto_round.version import __version__++ bits = default_scheme.get("bits", 4)+ group_size = default_scheme.get("group_size", 32)+ is_fp_default = (bits or 0) >= 16++ # For BF16 default + MXFP layer_config overrides, derive the dominant+ # MXFP bit-width from the layers that were actually quantized.+ if is_fp_default and layer_config and quantized_layers:+ temp_matcher = _PatternMatcher(+ ignore_patterns=[],+ layer_config=layer_config,+ default_scheme=default_scheme,+ )+ mxfp_counter: Counter = Counter()+ for layer in quantized_layers:+ scheme = temp_matcher.resolve_scheme(f"{layer}.weight")+ if scheme is None:+ continue+ lb = scheme.get("bits")+ ldt = (scheme.get("data_type") or "").lower()+ if lb and lb < 16 and is_mx_fp(ldt):+ mxfp_counter[lb] += 1+ if mxfp_counter:+ bits, _ = mxfp_counter.most_common(1)[0]+ group_size = 32++ if (bits or 0) < 1 or bits not in _SUPPORTED_MXFP_BITS:+ bits = 4 # safe fallback++ qconfig: dict = {+ "quant_method": "auto-round",+ "packing_format": "auto_round:llm_compressor",+ "bits": bits,+ "group_size": group_size or 32,+ "sym": True, # MXFP is always symmetric+ "data_type": "mx_fp",+ "iters": 0,+ "model_free": True,+ "autoround_version": __version__,+ "enable_quanted_input": False,+ }++ # Carry activation quantization fields from the scheme (e.g. MXFP4 has+ # act_bits=4, act_data_type="mx_fp", act_dynamic=True, act_group_size=32,+ # act_sym=True). Including these keeps the config consistent with the+ # output produced by the regular AutoRound MXFP export flow.+ for act_key in ("act_bits", "act_data_type", "act_dynamic", "act_group_size", "act_sym"):+ val = default_scheme.get(act_key)+ if val is not None:+ qconfig[act_key] = val++ scheme_keys = [f.name for f in fields(QuantizationScheme)]+ extra_config: dict = {}+ non_linear_ops = ["embed", "conv"]+ non_linear_re = re.compile("|".join(re.escape(op) for op in non_linear_ops))++ if layer_config:+ for layer_name, cfg in layer_config.items():+ if not isinstance(cfg, dict):+ continue+ cfg_bits = cfg.get("bits", bits)+ if cfg_bits >= 16:+ extra_config[layer_name] = {k: cfg[k] for k in scheme_keys if cfg.get(k) is not None}+ continue+ differs = any(+ cfg.get(key) is not None and cfg[key] != default_scheme.get(key)+ for key in ("bits", "group_size", "sym", "data_type")+ )+ if differs:+ extra_config[layer_name] = {k: cfg[k] for k in scheme_keys if cfg.get(k) is not None}++ quantized_set = set(quantized_layers)+ unique_ignored = list(dict.fromkeys(ignored_layers))+ for layer_name in unique_ignored:+ if layer_name in quantized_set or layer_name in extra_config:+ continue+ if non_linear_re.search(layer_name):+ continue+ extra_config[layer_name] = {"bits": 16, "data_type": "float"}++ # lm_head: when explicitly quantized, record its full scheme in extra_config+ # (mirrors the regular AutoRound MXFP export behavior).+ if "lm_head" in quantized_set and "lm_head" not in extra_config:+ lm_head_cfg = (layer_config or {}).get("lm_head", default_scheme)+ extra_config["lm_head"] = {k: lm_head_cfg.get(k) for k in scheme_keys if lm_head_cfg.get(k) is not None}++ if extra_config:+ qconfig["extra_config"] = extra_config++ return qconfig+++def _layer_config_has_mxfp(layer_config: dict | None) -> bool:+ """Return True if any layer_config entry requests MXFP quantization.++ Handles values that are plain strings (preset names), dicts (possibly+ with a ``"scheme"`` key or a ``"data_type"`` key), or+ :class:`QuantizationScheme` instances.+ """+ if not layer_config:+ return False+ for val in layer_config.values():+ if isinstance(val, str):+ try:+ s = _normalize_scheme(val.upper())+ if is_mx_fp((s.data_type or "").lower()):+ return True+ except Exception:+ pass+ elif isinstance(val, dict):+ # Direct data_type field+ if is_mx_fp((val.get("data_type") or "").lower()):+ return True+ # Nested 'scheme' key (e.g. {"scheme": "MXFP4"})+ scheme_val = val.get("scheme")+ if isinstance(scheme_val, str):+ try:+ s = _normalize_scheme(scheme_val.upper())+ if is_mx_fp((s.data_type or "").lower()):+ return True+ except Exception:+ pass+ elif isinstance(val, QuantizationScheme):+ if is_mx_fp((val.data_type or "").lower()):+ return True+ return False+++def _is_full_precision_default(scheme_input: Any) -> bool:+ """Return True when *scheme_input* represents a full-precision default (bits >= 16).++ Used to detect BF16/FP16 schemes where no weights are quantized by+ default but layer_config overrides are still applied.+ """+ try:+ s = _normalize_scheme(scheme_input)+ return (s.bits or 0) >= 16 and (s.act_bits or 16) >= 16+ except Exception:+ return False+++def _derive_dominant_int_scheme(+ quantized_layers: list[str],+ layer_config: dict,+ fallback: dict,+) -> dict | None:+ """Infer the dominant INT scheme from layers that were actually quantized.++ Used when the default scheme is full-precision (BF16/FP16) to produce a+ quantization_config where the most common INT scheme becomes the top-level+ default and the BF16 layers appear as ``bits=16`` exceptions in+ ``extra_config``. This matches the layout expected by AutoRound loaders+ (dominant quantized bits at the top, full-precision overrides below).++ Returns ``None`` when no INT layers were found in *quantized_layers*.+ """+ from collections import Counter++ temp_matcher = _PatternMatcher(+ ignore_patterns=[],+ layer_config=layer_config,+ default_scheme=fallback,+ )+ counter: "Counter[tuple]" = Counter()+ for layer in quantized_layers:+ scheme = temp_matcher.resolve_scheme(f"{layer}.weight")+ if scheme is None:+ continue+ bits = scheme.get("bits")+ if bits is None or bits >= 16:+ continue+ data_type = (scheme.get("data_type") or "int").lower()+ if is_mx_fp(data_type):+ continue # MXFP is handled by the dedicated path+ key = (bits, scheme.get("group_size") or fallback.get("group_size"), bool(scheme.get("sym", True)), data_type)+ counter[key] += 1++ if not counter:+ return None++ (bits, group_size, sym, data_type), _ = counter.most_common(1)[0]+ return {+ "bits": bits,+ "group_size": group_size,+ "sym": sym,+ "data_type": data_type,+ }++def _build_quantization_config(default_scheme: dict,layer_config: dict,@@ -1135,10 +1359,27 @@ def _build_quantization_config(quantized_layers: list[str],ignored_layers: list[str],block_name_to_quantize: Optional[list[str]] = None,+ format: str = "auto_round",) -> dict:"""Build a quantization_config dict compatible with auto-round format."""- # MXFP (mx_fp) uses the llm-compressor / compressed-tensors style config.- if is_mx_fp((default_scheme.get("data_type") or "int").lower()):+ # MXFP (mx_fp) supports two output styles depending on *format*:+ # - "llm_compressor" → compressed-tensors / llm-compressor style config+ # (produced by _build_mxfp_quantization_config).+ # - "auto_round" / "auto_round:auto_gptq" → auto-round style config+ # (produced by _build_mxfp_autoround_quantization_config).+ # Also route to MXFP config when the default is full-precision (BF16/FP16)+ # but layer_config contains MXFP overrides.+ data_type = (default_scheme.get("data_type") or "int").lower()+ default_bits = default_scheme.get("bits", 4)+ is_fp_default = (default_bits or 0) >= 16 and not is_mx_fp(data_type)+ if is_mx_fp(data_type) or (is_fp_default and _layer_config_has_mxfp(layer_config)):+ if format in ("auto_round", "auto_round:auto_gptq"):+ return _build_mxfp_autoround_quantization_config(+ default_scheme=default_scheme,+ quantized_layers=quantized_layers,+ ignored_layers=ignored_layers,+ layer_config=layer_config,+ )return _build_mxfp_quantization_config(default_scheme=default_scheme,quantized_layers=quantized_layers,@@ -1146,6 +1387,15 @@ def _build_quantization_config(layer_config=layer_config,)+ # When the default is full-precision (BF16/FP16) but INT layers were+ # quantized via layer_config, derive the dominant INT scheme and use it as+ # the top-level defaults so the output config reads as+ # "INT-quantized with BF16 exceptions" rather than the inverse.+ if is_fp_default and quantized_layers:+ dominant = _derive_dominant_int_scheme(quantized_layers, layer_config, default_scheme)+ if dominant is not None:+ default_scheme = dominant+from auto_round.version import __version__scheme_keys = [f.name for f in fields(QuantizationScheme)]@@ -1421,6 +1671,11 @@ def _validate_supported_scheme(bits = scheme_obj.bitsact_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16+ # Full-precision (BF16/FP16) default: all layers stay in full precision+ # unless layer_config provides lower-bit overrides.+ if (bits or 0) >= 16 and act_bits >= 16:+ return+# MXFP weight-only path: accept mx_fp data type with bits in {4, 8}.# Activation quantization for MXFP is dynamic at inference time, so the# weight-only RTN path here is independent of act_bits.@@ -1429,7 +1684,7 @@ def _validate_supported_scheme(# name is provided. Variants such as MXFP4_RCEIL / MXFP8_RCEIL use a# different activation format; silently mapping them to "MXFP4" /# "MXFP8" in the output config would misrepresent the requested scheme.- if isinstance(scheme_input, str) and scheme_input not in ("MXFP4", "MXFP8"):+ if isinstance(scheme_input, str) and scheme_input.upper() not in ("MXFP4", "MXFP8"):raise ValueError(f"Model-free mode only supports MXFP preset names 'MXFP4' and 'MXFP8', "f"but got '{scheme_input}'. "@@ -1504,12 +1759,16 @@ def _looks_like_auto_scheme(scheme: Any) -> bool:def _validate_auto_scheme_options(auto_scheme: Any) -> str:- """Validate that every AutoScheme option is model-free-packable.+ """Validate AutoScheme options for model-free packing.++ Returns the single quantized data-type family shared by all options+ (``"int"`` or ``"mx_fp"``). ``BF16`` (or any >=16-bit no-op scheme) is+ allowed and treated as "keep layer in full precision" during conversion,+ so it does not contribute to the returned family.- Returns the single data-type family shared by all options- (``"int"`` or ``"mx_fp"``). Raises ``ValueError`` when any option is- unsupported or when INT and MXFP options are mixed (they use different- packing formats and cannot be produced in one model-free run).+ Raises ``ValueError`` when any non-BF16 option is unsupported or when INT+ and MXFP options are mixed (they use different packing formats and cannot+ be produced in one model-free run)."""options = list(getattr(auto_scheme, "options", []) or [])if not options:@@ -1530,6 +1789,14 @@ def _validate_auto_scheme_options(auto_scheme: Any) -> str:else:scheme_obj = None+ # AutoScheme may include BF16/no-op options. In model-free flow these+ # layers become ignore_layers entries (full precision), so they are+ # allowed and excluded from family checks.+ if scheme_obj is not None:+ act_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16+ if (scheme_obj.bits or 0) >= 16 and act_bits >= 16:+ continue+# GGUF k-quants carry super_bits and are not packable by the model-free# RTN kernel even though their data_type is nominally "int".if scheme_obj is None or getattr(scheme_obj, "super_bits", None) is not None:@@ -1584,9 +1851,33 @@ def _convert_auto_scheme_layer_config(if not isinstance(cfg, dict):continuebits = cfg.get("bits")+ data_type_raw = (cfg.get("data_type") or "").strip()++ # Infer bits from compact dtype aliases that embed bit-width in the name,+ # e.g. "mxfp8" → 8, "MXFP4" → 4, "mx_fp4" → 4.+ if bits is None and data_type_raw:+ dt_lower = data_type_raw.lower()+ for prefix in ("mxfp", "mx_fp", "nvfp", "nv_fp"):+ tail = dt_lower[len(prefix) :]+ if dt_lower.startswith(prefix) and tail.isdigit():+ bits = int(tail)+ break+if bits is None:continue+clean = {k: cfg[k] for k in scheme_keys if cfg.get(k) is not None}+ clean["bits"] = bits # honour inferred value when not explicit++ # Normalise compact dtype aliases to canonical forms understood by+ # the quantization kernels ("mxfp8" / "MXFP4" → "mx_fp").+ if data_type_raw:+ dt_lower = data_type_raw.lower()+ if dt_lower.startswith("mxfp") or dt_lower.startswith("mx_fp"):+ clean["data_type"] = "mx_fp"+ elif dt_lower.startswith("nvfp") or dt_lower.startswith("nv_fp"):+ clean["data_type"] = "nv_fp"+if bits >= 16:fp16_layers.append(name)continue@@ -2029,6 +2320,7 @@ class _ModelFreeCompressorCore:ignore_patterns=self.ignore_patterns,quantized_layers=self.all_quantized_layers,ignored_layers=self.all_ignored_layers,+ format=self.format,)self.config["quantization_config"] = quantization_config@@ -2470,7 +2762,7 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):family = _validate_auto_scheme_options(self.scheme_input)accepted_formats = {"auto_round", "auto_round:auto_gptq"}if family == "mx_fp":- accepted_formats = {"llm_compressor"}+ accepted_formats = {"llm_compressor", "auto_round", "auto_round:auto_gptq"}if format not in accepted_formats:logger.warning(@@ -2509,10 +2801,18 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):"auto_round","auto_round:auto_gptq",}- # MXFP only supports the llm_compressor format (INT string preset,- # or an AutoScheme run whose options resolved to the MXFP family).- if self.scheme_input in ["MXFP4", "MXFP8"] or self._auto_scheme_family == "mx_fp":- _accepted_formats = ["llm_compressor"]+ # MXFP supports both llm_compressor (compressed-tensors) and auto_round formats.+ # The only difference is the quantization_config metadata; on-disk weights are identical.+ normalized_scheme = (+ _normalize_scheme(self.scheme_input) if not _looks_like_auto_scheme(self.scheme_input) else None+ )+ if (+ normalized_scheme is not None and is_mx_fp((normalized_scheme.data_type or "").lower())+ ) or self._auto_scheme_family == "mx_fp":+ _accepted_formats = {"llm_compressor", "auto_round", "auto_round:auto_gptq"}+ elif _is_full_precision_default(self.scheme_input) and _layer_config_has_mxfp(self.layer_config_input):+ # BF16 default with MXFP layer_config overrides.+ _accepted_formats = {"llm_compressor", "auto_round", "auto_round:auto_gptq"}if format not in _accepted_formats:logger.warning(f"Format '{format}' is not supported by model-free mode for scheme '{self.scheme_input}'; "@@ -2524,11 +2824,16 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):if self.user_scheme_overrides:self.scheme_input = _apply_scheme_overrides(self.scheme_input, self.user_scheme_overrides)- # Temporarily point output_dir at what the caller requested- orig = self.output_dir+ # Temporarily point output_dir and format at what the caller requested+ orig_dir = self.output_dir+ orig_fmt = self.formatself.output_dir = output_dir- out_path = self.run()- self.output_dir = orig+ self.format = format+ try:+ out_path = self.run()+ finally:+ self.output_dir = orig_dir+ self.format = orig_fmtself.quantized = Truereturn None, out_pathdiff --git a/auto_round/compressors/orchestrator.py b/auto_round/compressors/orchestrator.pyindex c355cb566..6e0e5d12b 100644--- a/auto_round/compressors/orchestrator.py+++ b/auto_round/compressors/orchestrator.py@@ -283,6 +283,7 @@ class CompressionOrchestrator(BaseOrchestrator):if len(device_manager.device_list) > 1 and not self.model_context.is_diffusion:accelerate.hooks.remove_hook_from_submodules(m)mv_module_from_gpu(m)+ clear_memory(device_list=device_manager.device_list)memory_monitor.log_summary()# ── Infrastructure: immediate_pack / shard write ──────────────────diff --git a/auto_round/envs.py b/auto_round/envs.pyindex 714eaacc6..0769754cf 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -26,6 +26,7 @@ if TYPE_CHECKING:LLAMA_CPP_ROOT: Optional[str] = NoneAR_AUTO_SCHEME_NSAMPLES: Optional[int] = NoneAR_AUTO_SCHEME_BATCH_SIZE: Optional[int] = None+ AR_ENABLE_AUTO_SCHEME_PARALLEL: bool = Falsedef _get_optional_positive_int_env(name: str) -> Optional[int]:@@ -85,6 +86,10 @@ environment_variables: dict[str, Callable[[], Any]] = {# when ``AutoScheme.batch_size`` is not explicitly set.# When unset, AutoScheme uses its built-in heuristic (8 for low GPU memory mode, 1 for normal mode)."AR_AUTO_SCHEME_BATCH_SIZE": lambda: _get_optional_positive_int_env("AR_AUTO_SCHEME_BATCH_SIZE"),+ # Enables AutoScheme to score schemes in parallel. Disabled by default to+ # avoid multiple model-loading workers exhausting host RAM or device memory.+ "AR_ENABLE_AUTO_SCHEME_PARALLEL": lambda: os.getenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "0").lower()+ in ("1", "true", "yes"),}diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.pyindex a276a5bff..af05f2f13 100644--- a/auto_round/modeling/fused_moe/replace_modules.py+++ b/auto_round/modeling/fused_moe/replace_modules.py@@ -155,14 +155,12 @@ def release_original_module_(model: torch.nn.Module) -> None:def safe_to_cpu_(model: torch.nn.Module) -> None:- # If no replacement happened, move model to CPU directly- if global_state.replaced_module_count == 0:+ """Move a model to CPU unless it still contains unmaterialized meta tensors."""+ has_meta_tensor = any(param.device.type == "meta" for param in model.parameters()) or any(+ buffer.device.type == "meta" for buffer in model.buffers()+ )+ if not has_meta_tensor:model.to("cpu")- return- else:- # TODO: (yiliu30) there might be some edge cases where some modules are replaced- # and we need to move them to CPU safely.- passclass ReplacementModuleBase(ABC, torch.nn.Module):diff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex 5364a8280..c0b809388 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -21,7 +21,6 @@ import sysimport tempfilefrom contextlib import ContextDecorator, contextmanagerfrom functools import lru_cache-from itertools import combinationsfrom threading import Lockfrom typing import Any, Callable, Optional, Union@@ -1026,66 +1025,36 @@ def partition_dict_numbers(number_dict, n):"""Partition a dictionary of numbers into N groups with approximately equal sums"""- # Edge cases- if n > len(number_dict):- groups = []- for key, value in number_dict.items():- groups.append({key: value})- for _ in range(n - len(number_dict)):- groups.append({})- return groups-- if n == len(number_dict):- return [{key: value} for key, value in number_dict.items()]-- total_sum = sum(number_dict.values())- # target = total_sum / n # Use float for better precision+ # Quick handling of edge cases+ if n <= 0:+ return []items = list(number_dict.items())- result = []- remaining = items.copy()-- def find_optimal_subset(arr, target):- """Find subset with sum closest to target"""- best_subset = []- best_diff = float("inf")-- # Try all possible subset sizes- for r in range(1, len(arr) + 1):- for combo in combinations(arr, r):- current_sum = sum(value for _, value in combo)- current_diff = abs(current_sum - target)-- # If we found a perfect match, return immediately- if current_diff == 0:- return list(combo)-- # Update the best subset if this is better- if current_diff < best_diff and current_sum <= total_sum:- best_diff = current_diff- best_subset = list(combo)-- return best_subset-- # Distribute items into n-1 groups- for i in range(n - 1):- if not remaining:- break+ m = len(items)- # Calculate dynamic target based on remaining items- remaining_target = sum(value for _, value in remaining) / (n - i)- subset = find_optimal_subset(remaining, remaining_target)+ if n >= m:+ # Put each item in its own bucket, then pad empty buckets+ groups = [{k: v} for k, v in items]+ groups.extend({} for _ in range(n - m))+ return groups- result.append(dict(subset))+ # Greedy largest-first balancing (Longest Processing Time / LPT-like):+ # - Sort items by value descending+ # - Assign each item to the group with the current smallest sum+ # Complexity: O(m log n) which scales well for large m (layers)+ groups_sums = [0.0] * n+ groups = [dict() for _ in range(n)]- # Remove allocated items- for item in subset:- remaining.remove(item)+ # Sort items descending by size+ items_sorted = sorted(items, key=lambda kv: kv[1], reverse=True)- # Last group gets all remaining items- result.append(dict(remaining))+ for key, val in items_sorted:+ # choose the group with minimum current sum+ idx = min(range(n), key=lambda i: groups_sums[i])+ groups[idx][key] = val+ groups_sums[idx] += val- return result+ return groupsdef dispatch_model_block_wise(model: torch.nn.Module, device_map: str, max_mem_ratio=0.9):@@ -1338,13 +1307,24 @@ class MemoryMonitor:self.peak_vram = {} # {device_id: peak_mb}self.enabled = True+ @staticmethod+ def _process_tree_rss() -> float:+ """Return the combined RSS of this process and all live descendants in GB."""+ process = psutil.Process()+ rss = process.memory_info().rss+ for child in process.children(recursive=True):+ try:+ rss += child.memory_info().rss+ except (psutil.NoSuchProcess, psutil.AccessDenied):+ continue+ return rss / 1024**3+def update(self, device_list=None):"""Update current memory usage and track peaks."""if not self.enabled:return# Track RAM- process = psutil.Process()- current_ram = process.memory_info().rss / 1024**3 # GB+ current_ram = self._process_tree_rss()self.peak_ram = max(self.peak_ram, current_ram)if device_list is None: # TODO this has issue, wait for clean_memory all pass device_listdevice_list = [0]@@ -1386,8 +1366,7 @@ class MemoryMonitor:def update_cpu(self):if not self.enabled:return- process = psutil.Process()- current_ram = process.memory_info().rss / 1024**3 # GB+ current_ram = self._process_tree_rss()self.peak_ram = max(self.peak_ram, current_ram)def update_hpu(self, device_list=None):diff --git a/auto_round/utils/device_manager.py b/auto_round/utils/device_manager.pyindex 1e99a93cf..b3aa5be8f 100644--- a/auto_round/utils/device_manager.py+++ b/auto_round/utils/device_manager.py@@ -410,9 +410,35 @@ class ARDevice:return _DeviceIndexContext(self, index)def total_memory(self, index: int = 0) -> int:- fn = getattr(self._module, "get_memory_info", None)+ return self.mem_get_info(index)[1]- return fn(index)[1] if callable(fn) else None # pylint: disable=E1102+ def mem_get_info(self, index: int = 0) -> tuple[int, int]:+ """Return ``(free_bytes, total_bytes)`` for ``index``."""+ modules = (self._module, self.get_device_module(self.type))+ for module in dict.fromkeys(modules):+ if module is None:+ continue+ for name in ("get_memory_info", "mem_get_info"):+ fn = getattr(module, name, None)+ if not callable(fn):+ continue+ try:+ free, total = fn(index) # pylint: disable=E1102+ if int(total) > 0:+ return int(free), int(total)+ except Exception:+ continue++ module = self.get_device_module(self.type)+ get_properties = getattr(module, "get_device_properties", None)+ if callable(get_properties):+ try:+ total = int(get_properties(index).total_memory)+ if total > 0:+ return max(0, total - self.memory_reserved(index)), total+ except Exception:+ pass+ return 0, 0def memory_reserved(self, index: int = 0) -> int:if self._module is None:@@ -432,17 +458,6 @@ class ARDevice:except Exception:return 0- # def mem_get_info(self, index: int = 0) -> tuple[int, int]:- # """Return ``(free_bytes, total_bytes)`` for ``index``.- #- # Falls back to ``total - reserved`` when the backend lacks a native- # ``mem_get_info`` implementation.- # """- # module = self.get_device_module(self.type) if self._module is _accelerator_api() else self._module- # fn = getattr(module, "get_memory_info", None)- #- # return fn(index) if callable(fn) else (0, 0) # pylint: disable=E1102-# -- numeric format / mixed-precision policy ---------------------------def supports_bf16(self) -> bool:"""Whether this backend can execute the ``bfloat16`` data type."""@@ -1055,7 +1070,15 @@ def get_device_memory(i: int = 0) -> int:dev_mgr = get_current_device_manager()if not dev_mgr.is_available() or dev_mgr.type == "cpu":raise RuntimeError("No supported device found (CUDA/XPU/HPU/...).")- return dev_mgr.total_memory(i) / 1024 / 1024 / 1024++ total_memory = dev_mgr.total_memory(i)+ if total_memory is None:+ raise RuntimeError(f"Failed to query total memory for device index {i} on backend {dev_mgr.type}.")+ if total_memory <= 0:+ raise RuntimeError(+ f"Detected non-positive total memory ({total_memory}) for device index {i} on backend {dev_mgr.type}."+ )+ return total_memory / 1024**3def _clear_memory_for_cpu_and_cuda(diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex cc2c761e1..d84b18b64 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -356,11 +356,22 @@ def llm_load_model(if device_str is not None and "hpu" in device_str:torch_dtype = torch.bfloat16+ if "dtype" in kwargs:+ torch_dtype = kwargs.pop("dtype")+ kwargs.pop("torch_dtype", None)+ elif "torch_dtype" in kwargs:+ torch_dtype = kwargs.pop("torch_dtype")+ dtype_key = (+ "dtype"+ if platform == "hf" and version.parse(transformers.__version__) >= version.parse("4.56.0")+ else "torch_dtype"+ )load_kwargs = {- "torch_dtype": torch_dtype,+ dtype_key: torch_dtype,"trust_remote_code": trust_remote_code,"device_map": "auto" if use_auto_mapping else None,}+ load_kwargs.update(kwargs)if version.parse(transformers.__version__) >= version.parse("5.0.0"):is_mxfp4 = _is_mxfp4_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)@@ -925,6 +936,119 @@ def diffusion_load_model(return pipe, model.to(device)+def load_model(+ pretrained_model_name_or_path: Union[str, torch.nn.Module],+ platform: str = "hf",+ model_dtype: str = None,+ trust_remote_code: bool = True,+ device: str = "cpu",+ use_auto_mapping: bool = None,+ use_model_replacements: bool = False,+ **kwargs,+) -> tuple:+ """Unified model loader supporting LLM, MLLM (VLM), and diffusion models.++ Automatically detects model type (text-only LLM / multimodal MLLM / diffusion+ pipeline) via ``is_mllm_model`` / ``is_diffusion_model`` and dispatches to the+ corresponding loader. When a pre-loaded ``torch.nn.Module`` is passed the+ function skips loading and only performs type detection.++ Args:+ pretrained_model_name_or_path: HuggingFace model ID, local path, or an+ already-loaded ``torch.nn.Module``.+ platform: ``"hf"`` (default) or ``"model_scope"``.+ model_dtype: Cast the model to this dtype after loading (e.g. ``"bf16"``).+ trust_remote_code: Passed through to every HF ``from_pretrained`` call.+ device: Device to load the model onto (default ``"cpu"``).+ use_auto_mapping: Override ``device_map="auto"`` for MLLM/diffusion loaders.+ ``None`` means each sub-loader decides its own default.+ use_model_replacements: Apply AutoRound replacement modules and special-model+ forward patches after loading. Intended for isolated quantization workers.+ **kwargs: Extra keyword arguments forwarded verbatim to the active sub-loader.++ Returns:+ A 7-tuple ``(model, tokenizer, processor, image_processor, pipe, is_mllm,+ is_diffusion)`` where unused fields are ``None`` / ``False``.+ """++ def _prepare_model(model):+ if not use_model_replacements:+ return model+ from auto_round.special_model_handler import _handle_special_model, update_module++ model = update_module(model, formats=None, cleanup_original=False)+ return _handle_special_model(model)++ model = None+ tokenizer = None+ processor = None+ image_processor = None+ pipe = None+ _is_mllm = False+ _is_diffusion = False++ if isinstance(pretrained_model_name_or_path, str):+ if is_mllm_model(pretrained_model_name_or_path, platform=platform):+ _kwargs = dict(+ platform=platform,+ device=device,+ model_dtype=model_dtype,+ trust_remote_code=trust_remote_code,+ )+ if use_auto_mapping is not None:+ _kwargs["use_auto_mapping"] = use_auto_mapping+ _kwargs.update(kwargs)+ model, processor, tokenizer, image_processor = mllm_load_model(pretrained_model_name_or_path, **_kwargs)+ model = _prepare_model(model)+ _is_mllm = True++ elif is_diffusion_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code):+ _kwargs = dict(+ platform=platform,+ device=device,+ model_dtype=model_dtype,+ trust_remote_code=trust_remote_code,+ )+ if use_auto_mapping is not None:+ _kwargs["use_auto_mapping"] = use_auto_mapping+ _kwargs.update(kwargs)+ pipe, model = diffusion_load_model(pretrained_model_name_or_path, **_kwargs)+ model = _prepare_model(model)+ _is_diffusion = True++ else:+ # Plain text LLM+ # Apply class-level block patches (e.g. unfused-MoE replacements) before+ # ``from_pretrained`` so the model is loaded with the correct architecture.+ from auto_round.modeling.unfused_moe import apply_model_monkey_patches++ apply_model_monkey_patches(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)++ _kwargs = dict(+ platform=platform,+ device=device,+ model_dtype=model_dtype,+ trust_remote_code=trust_remote_code,+ )+ _kwargs.update(kwargs)+ model, tokenizer = llm_load_model(pretrained_model_name_or_path, **_kwargs)+ model = _prepare_model(model)+ else:+ # Already a loaded model object – detect type without triggering any file I/O.+ try:+ _is_mllm = is_mllm_model(pretrained_model_name_or_path)+ except Exception:+ pass+ if not _is_mllm:+ try:+ _is_diffusion = is_diffusion_model(pretrained_model_name_or_path)+ except Exception:+ pass+ model = _prepare_model(pretrained_model_name_or_path)++ return model, tokenizer, processor, image_processor, pipe, _is_mllm, _is_diffusion++def is_pure_text_model(model):"""verify on: phi-3.5, Mistral-Small-3.1, gemma-3, qwen2-vl,"""if hasattr(model, "config") and hasattr(model.config, "vision_config"):@@ -2310,6 +2434,11 @@ def is_model_free_route(explicit = bool(kwargs.get("model_free", False))disabled = bool(kwargs.get("disable_model_free", False))+ has_static_attention_quantization = any(+ kwargs.get(key) is not None for key in ("static_kv_dtype", "static_attention_dtype")+ )+ if has_static_attention_quantization:+ return Falseif explicit:return True# Only auto-route when format is auto_round (or not specified).diff --git a/docs/environments.md b/docs/environments.mdindex 296bf6c28..3ccd400c0 100644--- a/docs/environments.md+++ b/docs/environments.md@@ -141,6 +141,16 @@ export AR_AUTO_SCHEME_NSAMPLES=1 # set 1 for quick executionexport AR_AUTO_SCHEME_BATCH_SIZE=1```+### AR_ENABLE_AUTO_SCHEME_PARALLEL+- **Description**: Enables multiprocessing across AutoScheme candidates. Keep it disabled when parallel model-loading workers could exhaust host RAM or device memory.+- **Default**: `False` (schemes are scored serially)+- **Valid Values**: `"1"`, `"true"`, `"yes"` (case-insensitive) enable parallel scoring; any other value keeps parallel scoring disabled+- **Usage**: Set this before running AutoScheme to enable parallel candidate scoring++```bash+export AR_ENABLE_AUTO_SCHEME_PARALLEL=1+```+## Usage Examples### Setting Environment Variablesdiff --git a/docs/environments_CN.md b/docs/environments_CN.mdindex c04c45568..e2100d60a 100644--- a/docs/environments_CN.md+++ b/docs/environments_CN.md@@ -141,6 +141,16 @@ export AR_AUTO_SCHEME_NSAMPLES=1export AR_AUTO_SCHEME_BATCH_SIZE=1```+### AR_ENABLE_AUTO_SCHEME_PARALLEL+- **描述**:启用 AutoScheme 候选方案之间的多进程并行。当并行加载多个模型可能耗尽主机内存或显存时,请保持关闭。+- **默认值**:`False`(串行评分各方案)+- **有效值**:`"1"`、`"true"`、`"yes"`(不区分大小写)表示启用并行评分;其他值保持关闭+- **用途**:运行 AutoScheme 前设置,以启用候选方案并行评分++```bash+export AR_ENABLE_AUTO_SCHEME_PARALLEL=1+```+## 使用示例### 设置环境变量diff --git a/test/fixtures.py b/test/fixtures.pyindex ffc628401..258b05456 100644--- a/test/fixtures.py+++ b/test/fixtures.py@@ -401,7 +401,8 @@ def mock_fp8_capable_device():def clean_tmp_model_folder():yieldshutil.rmtree("./tmp", ignore_errors=True) # unittest default workspace- shutil.rmtree("./tmp_autoround", ignore_errors=True) # autoround default workspace+ shutil.rmtree("./ar_work_space", ignore_errors=True) # autoround default workspace+ shutil.rmtree("./tmp_autoround", ignore_errors=True) # autoround default model output path# Create objective fixtures for testingdiff --git a/test/test_cpu/export/test_llmc_format.py b/test/test_cpu/export/test_llmc_format.pyindex 7982bf8bd..8a6e6d95f 100644--- a/test/test_cpu/export/test_llmc_format.py+++ b/test/test_cpu/export/test_llmc_format.py@@ -105,6 +105,7 @@ class TestLLMC:iters=0,disable_opt_rtn=True,scheme=scheme,+ disable_model_free=True,)compressed_model, tmp_path = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_projdiff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.pyindex 1d25ba719..cb21213f7 100644--- a/test/test_cpu/models/test_moe_model.py+++ b/test/test_cpu/models/test_moe_model.py@@ -22,6 +22,7 @@ def quantize_model(model, output_dir, scheme, iters=0, ignore_layers="self_attn,seqlen=32,ignore_layers=ignore_layers,disable_opt_rtn=disable_opt_rtn,+ disable_model_free=True,)quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)return quantized_model, save_folder@@ -98,6 +99,7 @@ def test_qwen3_vl_moe_mxfp(tiny_qwen3_vl_moe_model_path, tmp_path):seqlen=32,iters=0,disable_opt_rtn=True,+ disable_model_free=True,ignore_layers="self_attn,lm_head, mlp.gate",)quantized_model, quantized_model_path = autoround.quantize_and_save(format="auto_round", output_dir=tmp_path)diff --git a/test/test_cpu/quantization/test_model_free.py b/test/test_cpu/quantization/test_model_free.pyindex bceede445..8849281b9 100644--- a/test/test_cpu/quantization/test_model_free.py+++ b/test/test_cpu/quantization/test_model_free.py@@ -18,12 +18,40 @@ import jsonimport osimport pytest+++def test_quantize_and_save_restores_temporary_state_on_failure(monkeypatch):+ from auto_round.compressors.model_free import ModelFreeCompressor++ compressor = ModelFreeCompressor.__new__(ModelFreeCompressor)+ compressor.scheme_input = "W4A16"+ compressor.layer_config_input = None+ compressor.user_scheme_overrides = None+ compressor._auto_scheme_family = None+ compressor.output_dir = "original-output"+ compressor.format = "auto_round"+ compressor.quantized = False++ def fail_run():+ raise RuntimeError("save failed")++ monkeypatch.setattr(compressor, "run", fail_run)++ with pytest.raises(RuntimeError, match="save failed"):+ compressor.quantize_and_save("temporary-output", format="auto_round:auto_gptq")++ assert compressor.output_dir == "original-output"+ assert compressor.format == "auto_round"+ assert compressor.quantized is False++import torchfrom safetensors import safe_openfrom safetensors.torch import save_filefrom auto_round import AutoRoundfrom auto_round.compressors.model_free import (+ _build_mxfp_autoround_quantization_config,_build_mxfp_quantization_config,_build_quantization_config,_convert_auto_scheme_layer_config,@@ -43,6 +71,7 @@ from auto_round.compressors.model_free import (is_model_free_supported_scheme,)from auto_round.schemes import QuantizationScheme+from auto_round.utils.model import is_model_free_routefrom ...envs import require_compressed_tensors@@ -50,6 +79,19 @@ from ...envs import require_compressed_tensors# Helpers# ---------------------------------------------------------------------------+[email protected]("scheme", ["MXFP8", "mxfp8"])+def test_model_free_route_accepts_mxfp_scheme_case_insensitively(scheme):+ assert is_model_free_route("model", scheme, 0, True, {})++[email protected]("dtype_key", ["static_kv_dtype", "static_attention_dtype"])[email protected]("explicit", [False, True])+def test_model_free_route_rejects_static_attention_quantization(dtype_key, explicit):+ kwargs = {dtype_key: "fp8", "model_free": explicit}+ assert not is_model_free_route("model", "W4A16", 0, True, kwargs)++_LLAMA_CFG = {"architectures": ["LlamaForCausalLM"], "model_type": "llama"}_DEFAULT_SCHEME = {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"}@@ -503,7 +545,9 @@ class TestModelFreeMXFP:}model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)output_dir = str(tmp_path / "output")- _ModelFreeCompressorCore(model_name_or_path=model_dir, output_dir=output_dir, scheme=scheme).run()+ _ModelFreeCompressorCore(+ model_name_or_path=model_dir, output_dir=output_dir, scheme=scheme, format="llm_compressor"+ ).run()qc = _read_qconfig(output_dir)assert qc["format"] == fmtassert qc["quant_method"] == "compressed-tensors"@@ -551,6 +595,20 @@ class TestModelFreeMXFP:assert "lm_head" in cfg["ignore"]assert len(cfg["config_groups"]) == 1+ @require_compressed_tensors+ def test_build_mxfp_config_bf16_default_targets_only_overrides(self):+ """A single MXFP override must not describe all BF16 Linear layers as quantized."""+ default = {"bits": 16, "group_size": -1, "sym": True, "data_type": "bf16"}+ layer_config = {+ "model.layers.0.self_attn.q_proj": {"bits": 4, "group_size": 32, "data_type": "mx_fp"},+ }+ quantized = ["model.layers.0.self_attn.q_proj"]++ cfg = _build_mxfp_quantization_config(default, quantized, [], layer_config=layer_config)++ assert cfg["format"] == "mxfp4-pack-quantized"+ assert cfg["config_groups"]["group_0"]["targets"] == quantized+@require_compressed_tensorsdef test_build_mxfp_mixed_config_two_groups(self):"""Mixed MXFP4+MXFP8: override layers get explicit targets; default gets ["Linear"]."""@@ -643,6 +701,7 @@ class TestModelFreeMXFP:output_dir=output_dir,scheme="MXFP4",layer_config=layer_config,+ format="llm_compressor",).run()qc = _read_qconfig(output_dir)@@ -853,7 +912,9 @@ class TestLLMCompressorMXFPSource:}model_dir = _make_model_dir(tmp_path, _LLMCOMPRESSOR_MXFP_CFG_FP8, tensors)output_dir = str(tmp_path / "output")- _ModelFreeCompressorCore(model_name_or_path=model_dir, output_dir=output_dir, scheme="MXFP8").run()+ _ModelFreeCompressorCore(+ model_name_or_path=model_dir, output_dir=output_dir, scheme="MXFP8", format="llm_compressor"+ ).run()qc = _read_qconfig(output_dir)assert qc["format"] == "mxfp8-quantized"@@ -934,7 +995,9 @@ class TestLLMCompressorMXFPSource:}model_dir = _make_model_dir(tmp_path, _DEEPSEEK_V4_CFG, tensors)output_dir = str(tmp_path / "output")- _ModelFreeCompressorCore(model_name_or_path=model_dir, output_dir=output_dir, scheme="MXFP8").run()+ _ModelFreeCompressorCore(+ model_name_or_path=model_dir, output_dir=output_dir, scheme="MXFP8", format="llm_compressor"+ ).run()qc = _read_qconfig(output_dir)assert qc["format"] == "mxfp8-quantized"@@ -979,11 +1042,10 @@ class TestLLMCompressorMXFPSource:# ===========================================================================-_SUPPORTED = ["W2A16", "W2A16G32", "W2A16G64", "W4A16", "W4A16_MIXED", "W8A16", "MXFP4", "MXFP8"]+_SUPPORTED = ["W2A16", "W2A16G32", "W2A16G64", "W4A16", "W4A16_MIXED", "W8A16", "MXFP4", "MXFP8", "BF16"]_UNSUPPORTED = ["W3A16","FPW8A16",- "BF16","MXINT4","NVFP4","FP8_BLOCK",@@ -1022,11 +1084,16 @@ class TestSchemeValidation:layer_config=core.layer_config,ignore_patterns=core.ignore_patterns,)- assert "model.layers.0.mlp.fc1" in quantized- if name.startswith("MXFP"):- assert "model.layers.0.mlp.fc1.weight_scale" in output+ if name == "BF16":+ # BF16 default: no weights quantized; the layer is kept in full precision.+ assert len(quantized) == 0+ assert "model.layers.0.mlp.fc1.weight" in outputelse:- assert "model.layers.0.mlp.fc1.qweight" in output+ assert "model.layers.0.mlp.fc1" in quantized+ if name.startswith("MXFP"):+ assert "model.layers.0.mlp.fc1.weight_scale" in output+ else:+ assert "model.layers.0.mlp.fc1.qweight" in output@pytest.mark.parametrize("name", _UNSUPPORTED)def test_unsupported_raises(self, tmp_path, name):@@ -1400,6 +1467,11 @@ class TestModelFreeAutoScheme:assert _validate_auto_scheme_options(AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "W8A16"))) == "int"+ def test_validate_options_int_family_with_bf16(self):+ from auto_round import AutoScheme++ assert _validate_auto_scheme_options(AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16"))) == "int"+def test_validate_options_mxfp_family(self):from auto_round import AutoScheme@@ -1440,6 +1512,22 @@ class TestModelFreeAutoScheme:assert "model.embed_tokens" not in per_layerassert fp16_layers == ["model.embed_tokens"]+ def test_convert_layer_config_infers_mxfp_bits_from_dtype_alias(self):+ generated = {+ "model.layers.0.q_proj": {"group_size": 32, "sym": True, "data_type": "mxfp8"},+ "model.layers.0.k_proj": {"group_size": 32, "sym": True, "data_type": "MXFP4"},+ }++ base_scheme, per_layer, fp16_layers = _convert_auto_scheme_layer_config(generated)++ assert fp16_layers == []+ assert per_layer["model.layers.0.q_proj"]["bits"] == 8+ assert per_layer["model.layers.0.q_proj"]["data_type"] == "mx_fp"+ assert per_layer["model.layers.0.k_proj"]["bits"] == 4+ assert per_layer["model.layers.0.k_proj"]["data_type"] == "mx_fp"+ assert base_scheme.data_type == "mx_fp"+ assert base_scheme.bits in (4, 8)+def test_e2e_int_auto_scheme(self, tmp_path, tiny_opt_model_path):from auto_round import AutoScheme@@ -1468,3 +1556,269 @@ class TestModelFreeAutoScheme:qc = _read_qconfig(output_dir)assert qc["quant_method"] == "compressed-tensors"assert qc["provider"] == "auto-round"+++# ===========================================================================+# MXFP auto-round format (packing_format=auto_round:llm_compressor)+# ===========================================================================+++class TestMXFPAutoRoundFormat:+ """Tests for MXFP4/MXFP8 model-free quantization with format='auto_round'.++ The weight tensors on disk are identical to the llm_compressor path;+ only the ``quantization_config`` metadata differs.+ """++ # ------------------------------------------------------------------+ # Unit tests: _build_mxfp_autoround_quantization_config+ # ------------------------------------------------------------------++ @require_compressed_tensors+ def test_mxfp4_top_level_fields(self):+ """MXFP4 scheme → all mandatory top-level fields are correct."""+ from dataclasses import asdict++ from auto_round.schemes import PRESET_SCHEMES++ default = {k: v for k, v in asdict(PRESET_SCHEMES["MXFP4"]).items() if v is not None}+ cfg = _build_mxfp_autoround_quantization_config(default, quantized_layers=["layer.fc1"], ignored_layers=[])+ assert cfg["quant_method"] == "auto-round"+ assert cfg["packing_format"] == "auto_round:llm_compressor"+ assert cfg["bits"] == 4+ assert cfg["group_size"] == 32+ assert cfg["data_type"] == "mx_fp"+ assert cfg["sym"] is True+ assert cfg["enable_quanted_input"] is False+ assert cfg["model_free"] is True+ assert "autoround_version" in cfg++ @require_compressed_tensors+ def test_mxfp4_activation_fields(self):+ """MXFP4 scheme carries act_* fields matching the real AutoRound output."""+ from dataclasses import asdict++ from auto_round.schemes import PRESET_SCHEMES++ default = {k: v for k, v in asdict(PRESET_SCHEMES["MXFP4"]).items() if v is not None}+ cfg = _build_mxfp_autoround_quantization_config(default, quantized_layers=["layer.fc1"], ignored_layers=[])+ assert cfg["act_bits"] == 4+ assert cfg["act_data_type"] == "mx_fp"+ assert cfg["act_dynamic"] is True+ assert cfg["act_group_size"] == 32+ assert cfg["act_sym"] is True++ @require_compressed_tensors+ def test_mxfp8_bits(self):+ """MXFP8 scheme → bits=8, act_bits=8."""+ from dataclasses import asdict++ from auto_round.schemes import PRESET_SCHEMES++ default = {k: v for k, v in asdict(PRESET_SCHEMES["MXFP8"]).items() if v is not None}+ cfg = _build_mxfp_autoround_quantization_config(default, quantized_layers=["layer.fc1"], ignored_layers=[])+ assert cfg["bits"] == 8+ assert cfg["act_bits"] == 8+ assert cfg["packing_format"] == "auto_round:llm_compressor"++ @require_compressed_tensors+ def test_ignored_layers_in_extra_config(self):+ """Ignored layers (non-embed/conv) go into extra_config as bits=16."""+ default = {+ "bits": 4,+ "group_size": 32,+ "sym": True,+ "data_type": "mx_fp",+ "act_bits": 4,+ "act_data_type": "mx_fp",+ "act_dynamic": True,+ "act_group_size": 32,+ "act_sym": True,+ }+ ignored = ["lm_head", "model.embed_tokens", "model.conv1", "model.layers.0.mlp.gate"]+ cfg = _build_mxfp_autoround_quantization_config(+ default,+ quantized_layers=["model.layers.0.fc1"],+ ignored_layers=ignored,+ )+ extra = cfg.get("extra_config", {})+ assert extra.get("lm_head") == {"bits": 16, "data_type": "float"}+ assert extra.get("model.layers.0.mlp.gate") == {"bits": 16, "data_type": "float"}+ # embed / conv are non-Linear — filtered out+ assert "model.embed_tokens" not in extra+ assert "model.conv1" not in extra++ @require_compressed_tensors+ def test_quantized_lm_head_in_extra_config(self):+ """When lm_head is quantized it must appear in extra_config with its scheme."""+ default = {+ "bits": 4,+ "group_size": 32,+ "sym": True,+ "data_type": "mx_fp",+ "act_bits": 4,+ "act_data_type": "mx_fp",+ "act_dynamic": True,+ "act_group_size": 32,+ "act_sym": True,+ }+ cfg = _build_mxfp_autoround_quantization_config(+ default,+ quantized_layers=["model.layers.0.fc1", "lm_head"],+ ignored_layers=[],+ )+ extra = cfg.get("extra_config", {})+ assert "lm_head" in extra+ assert extra["lm_head"]["bits"] == 4+ assert extra["lm_head"]["data_type"] == "mx_fp"++ @require_compressed_tensors+ def test_build_quantization_config_routes_mxfp_to_autoround(self):+ """_build_quantization_config with format='auto_round' routes MXFP to auto-round style."""+ default = {+ "bits": 4,+ "group_size": 32,+ "sym": True,+ "data_type": "mx_fp",+ "act_bits": 4,+ "act_data_type": "mx_fp",+ "act_dynamic": True,+ "act_group_size": 32,+ "act_sym": True,+ }+ cfg = _build_quantization_config(+ default_scheme=default,+ layer_config={},+ ignore_patterns=[],+ quantized_layers=["layer.fc1"],+ ignored_layers=[],+ format="auto_round",+ )+ assert cfg["quant_method"] == "auto-round"+ assert cfg["packing_format"] == "auto_round:llm_compressor"++ @require_compressed_tensors+ def test_build_quantization_config_mxfp_llmcompressor_path_unchanged(self):+ """_build_quantization_config with format='llm_compressor' still produces compressed-tensors."""+ default = {"bits": 4, "group_size": 32, "sym": True, "data_type": "mx_fp"}+ cfg = _build_quantization_config(+ default_scheme=default,+ layer_config={},+ ignore_patterns=[],+ quantized_layers=["layer.fc1"],+ ignored_layers=["lm_head"],+ format="llm_compressor",+ )+ assert cfg["quant_method"] == "compressed-tensors"+ assert cfg["format"] == "mxfp4-pack-quantized"++ # ------------------------------------------------------------------+ # End-to-end tests via AutoRound.quantize_and_save+ # ------------------------------------------------------------------++ @require_compressed_tensors+ def test_e2e_mxfp4_config_fields(self, tmp_path):+ """MXFP4 + format='auto_round': quantization_config matches real AutoRound output."""+ tensors = {+ "model.layers.0.self_attn.q_proj.weight": torch.randn(128, 128),+ "model.layers.0.fc1.weight": torch.randn(512, 128),+ "lm_head.weight": torch.randn(1000, 128),+ }+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)+ output_dir = str(tmp_path / "output")+ AutoRound(model=model_dir, scheme="MXFP4", model_free=True).quantize_and_save(output_dir, format="auto_round")+ qc = _read_qconfig(output_dir)+ assert qc["quant_method"] == "auto-round"+ assert qc["packing_format"] == "auto_round:llm_compressor"+ assert qc["bits"] == 4+ assert qc["data_type"] == "mx_fp"+ assert qc["act_bits"] == 4+ assert qc["act_data_type"] == "mx_fp"+ assert qc["enable_quanted_input"] is False+ assert qc["model_free"] is True+ # lm_head kept full-precision → extra_config entry+ extra = qc.get("extra_config", {})+ assert extra.get("lm_head") == {"bits": 16, "data_type": "float"}++ @require_compressed_tensors+ def test_e2e_mxfp4_weight_tensors(self, tmp_path):+ """MXFP4 + format='auto_round': on-disk weight layout is unchanged (weight_packed + weight_scale)."""+ tensors = {+ "model.layers.0.fc1.weight": torch.randn(512, 128),+ "lm_head.weight": torch.randn(1000, 128),+ }+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)+ output_dir = str(tmp_path / "output")+ AutoRound(model=model_dir, scheme="MXFP4", model_free=True).quantize_and_save(output_dir, format="auto_round")+ keys = _read_output_keys(output_dir)+ assert "model.layers.0.fc1.weight_packed" in keys+ assert "model.layers.0.fc1.weight_scale" in keys+ assert "lm_head.weight" in keys+ assert "lm_head.weight_packed" not in keys++ @require_compressed_tensors+ def test_e2e_mxfp8_autoround_format(self, tmp_path):+ """MXFP8 + format='auto_round': bits=8, weight layout uses float8 .weight."""+ tensors = {+ "model.layers.0.fc1.weight": torch.randn(128, 128),+ "lm_head.weight": torch.randn(1000, 128),+ }+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)+ output_dir = str(tmp_path / "output")+ AutoRound(model=model_dir, scheme="MXFP8", model_free=True).quantize_and_save(output_dir, format="auto_round")+ qc = _read_qconfig(output_dir)+ assert qc["quant_method"] == "auto-round"+ assert qc["packing_format"] == "auto_round:llm_compressor"+ assert qc["bits"] == 8+ assert qc["act_bits"] == 8+ keys = _read_output_keys(output_dir)+ assert "model.layers.0.fc1.weight" in keys+ assert "model.layers.0.fc1.weight_scale" in keys++ @require_compressed_tensors+ def test_e2e_mxfp4_quant_lm_head_extra_config(self, tmp_path):+ """When lm_head is quantized, it appears in extra_config with its full scheme."""+ tensors = {+ "model.layers.0.fc1.weight": torch.randn(128, 128),+ "lm_head.weight": torch.randn(1000, 128),+ }+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)+ output_dir = str(tmp_path / "output")+ AutoRound(model=model_dir, scheme="MXFP4", model_free=True, quant_lm_head=True).quantize_and_save(+ output_dir, format="auto_round"+ )+ qc = _read_qconfig(output_dir)+ extra = qc.get("extra_config", {})+ assert "lm_head" in extra+ assert extra["lm_head"]["bits"] == 4+ assert extra["lm_head"]["data_type"] == "mx_fp"+ keys = _read_output_keys(output_dir)+ assert "lm_head.weight_packed" in keys++ @require_compressed_tensors+ def test_e2e_mxfp4_autoround_format_same_weights_as_llmcompressor(self, tmp_path):+ """auto_round and llm_compressor format produce identical weight bytes."""+ tensors = {"model.layers.0.fc1.weight": torch.randn(64, 128)}+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)++ out_ar = str(tmp_path / "out_ar")+ out_llm = str(tmp_path / "out_llm")+ AutoRound(model=model_dir, scheme="MXFP4", model_free=True).quantize_and_save(out_ar, format="auto_round")+ AutoRound(model=model_dir, scheme="MXFP4", model_free=True).quantize_and_save(out_llm, format="llm_compressor")++ # Weight bytes must match; only quantization_config differs.+ def _load_tensor(directory, name):+ for f in os.listdir(directory):+ if f.endswith(".safetensors"):+ with safe_open(os.path.join(directory, f), framework="pt") as sf:+ if name in sf.keys():+ return sf.get_tensor(name)+ return None++ wp_ar = _load_tensor(out_ar, "model.layers.0.fc1.weight_packed")+ ws_ar = _load_tensor(out_ar, "model.layers.0.fc1.weight_scale")+ wp_llm = _load_tensor(out_llm, "model.layers.0.fc1.weight_packed")+ ws_llm = _load_tensor(out_llm, "model.layers.0.fc1.weight_scale")+ assert wp_ar is not None and wp_llm is not None+ assert torch.equal(wp_ar, wp_llm), "weight_packed must be identical"+ assert torch.equal(ws_ar, ws_llm), "weight_scale must be identical"diff --git a/test/test_cpu/schemes/test_auto_scheme.py b/test/test_cpu/schemes/test_auto_scheme.pyindex 304b4d755..d6a2ce168 100644--- a/test/test_cpu/schemes/test_auto_scheme.py+++ b/test/test_cpu/schemes/test_auto_scheme.py@@ -1,9 +1,209 @@+import json+import osimport shutilimport pytest-from transformers import AutoRoundConfigfrom auto_round import AutoRound, AutoScheme+from auto_round.auto_scheme.utils import _build_layer_config_header_rows, _short_summary_name+++ "model_type, expected",+ [+ ("mllm", ("model", "tokenizer", "processor", "image_processor", None, True, False)),+ ("diffusion", ("model", None, None, None, "pipe", False, True)),+ ("llm", ("model", "tokenizer", None, None, None, False, False)),+ ],+)+def test_load_model_returns_named_values_for_each_model_type(monkeypatch, model_type, expected):+ import auto_round.utils.model as model_utils++ monkeypatch.setattr(model_utils, "is_mllm_model", lambda *_args, **_kwargs: model_type == "mllm")+ monkeypatch.setattr(model_utils, "is_diffusion_model", lambda *_args, **_kwargs: model_type == "diffusion")+ monkeypatch.setattr(+ model_utils,+ "mllm_load_model",+ lambda *_args, **_kwargs: ("model", "processor", "tokenizer", "image_processor"),+ )+ monkeypatch.setattr(model_utils, "diffusion_load_model", lambda *_args, **_kwargs: ("pipe", "model"))+ monkeypatch.setattr(model_utils, "llm_load_model", lambda *_args, **_kwargs: ("model", "tokenizer"))+ monkeypatch.setattr("auto_round.modeling.unfused_moe.apply_model_monkey_patches", lambda *_args, **_kwargs: None)++ assert model_utils.load_model("model-id") == expected+++def test_env_ar_auto_scheme_nsamples_overrides_default(monkeypatch):+ """AR_AUTO_SCHEME_NSAMPLES env var should override the built-in nsamples heuristic."""+ import auto_round.envs as envs++ monkeypatch.setenv("AR_AUTO_SCHEME_NSAMPLES", "7")+ assert envs.AR_AUTO_SCHEME_NSAMPLES == 7+++def test_env_ar_auto_scheme_batch_size_overrides_default(monkeypatch):+ """AR_AUTO_SCHEME_BATCH_SIZE env var should override the built-in batch_size default."""+ import auto_round.envs as envs++ monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "4")+ assert envs.AR_AUTO_SCHEME_BATCH_SIZE == 4+++def test_env_ar_auto_scheme_batch_size_zero_raises(monkeypatch):+ """Zero value for AR_AUTO_SCHEME_BATCH_SIZE should raise ValueError."""+ import pytest++ import auto_round.envs as envs++ monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "0")+ with pytest.raises(ValueError):+ _ = envs.AR_AUTO_SCHEME_BATCH_SIZE+++def test_env_ar_enable_auto_scheme_parallel(monkeypatch):+ import auto_round.envs as envs++ monkeypatch.delenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", raising=False)+ assert envs.AR_ENABLE_AUTO_SCHEME_PARALLEL is False+ monkeypatch.setenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "1")+ assert envs.AR_ENABLE_AUTO_SCHEME_PARALLEL is True+++def test_build_layer_config_header_rows_merges_adjacent_prefixes():+ """Adjacent columns with the same prefix should be merged into one compact header cell."""+ columns = ["mlp.down_proj", "mlp.gate_proj", "self_attn.q_proj", "self_attn.v_proj"]+ assert _build_layer_config_header_rows(columns) == [+ ["block", "mlp", "", "self_attn", ""],+ ["", "down_proj", "gate_proj", "q_proj", "v_proj"],+ ]+++def test_build_layer_config_header_rows_includes_experts_under_mlp():+ columns = ["mlp.down_proj", "self_attn.q_proj"]+ assert _build_layer_config_header_rows(columns, has_expert_layers=True) == [+ ["block", "mlp", "self_attn", "mlp"],+ ["", "down_proj", "q_proj", "experts"],+ ]+++def test_short_summary_name_keeps_one_field_before_numeric_suffix():+ """Numeric block suffixes should be shortened to keep the preceding field."""+ assert _short_summary_name("model.layers.0") == "layers.0"+++def test_get_layer_config_supports_avg_bits_list(monkeypatch):+ from auto_round import auto_scheme+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme, GenScheme++ scheme = AutoScheme(avg_bits=[2.5, 3.5], options=("W2A16", "W4A16"))+ generator = GenScheme.__new__(GenScheme)+ generator.auto_scheme = scheme+ generator.model = object()+ generator.quant_layer_names = []+ generator.fixed_layer_scheme = {}+ generator.dataset = "pile-10k"+ generator.tokenizer = object()+ generator.device_map = None+ generator.enable_torch_compile = False+ generator.min_avg_bit_scheme = "W2A16"+ generator.processor = None++ def method_func(auto_scheme, *args, **kwargs):+ return {"layer": {"bits": auto_scheme.avg_bits}}++ monkeypatch.setitem(auto_scheme.AUTO_SCHEME_METHODS, "default", method_func)++ assert generator.get_layer_config() == {+ 2.5: {"layer": {"bits": 2.5}},+ 3.5: {"layer": {"bits": 3.5}},+ }+ assert scheme.avg_bits == [2.5, 3.5]+++def test_choose_bits_per_layer_reconstructs_optimal_path():+ """DP parent pointers should preserve the optimal choices in layer order."""+ from auto_round.auto_scheme.delta_loss import choose_bits_per_layer_with_path++ layers = {+ "layer.0": [(0, 2, 4.0, ["layer.0"]), (1, 4, 1.0, ["layer.0"])],+ "layer.1": [(0, 2, 3.0, ["layer.1"]), (1, 4, 0.5, ["layer.1"])],+ }++ loss, path = choose_bits_per_layer_with_path(layers, P=6)++ assert loss == 4.0+ assert path == [(["layer.0"], 1), (["layer.1"], 0)]+++def test_build_expert_groups_groups_experts_per_block():+ """Expert layers in the same block should be grouped together."""+ import torch+ from torch import nn++ from auto_round.auto_scheme.utils import build_expert_groups++ # Build a minimal MoE-like model with 2 blocks, each with 2 experts having 2 projections+ class FakeModel(nn.Module):+ def __init__(self):+ super().__init__()+ self.model = nn.Module()+ self.model.layers = nn.ModuleList()+ for i in range(2):+ block = nn.Module()+ block.mlp = nn.Module()+ block.mlp.experts = nn.ModuleList()+ for j in range(2):+ expert = nn.Module()+ expert.gate_proj = nn.Linear(8, 8, bias=False)+ expert.up_proj = nn.Linear(8, 8, bias=False)+ expert.down_proj = nn.Linear(8, 8, bias=False)+ block.mlp.experts.append(expert)+ block.self_attn = nn.Module()+ block.self_attn.q_proj = nn.Linear(8, 8, bias=False)+ self.model.layers.append(block)++ model = FakeModel()+ quant_layer_names = [n for n, m in model.named_modules() if isinstance(m, nn.Linear)]+ fixed_layer_scheme = {}++ groups = build_expert_groups(model, quant_layer_names, fixed_layer_scheme)+ # Should have 2 groups (one per block), each containing all 6 expert projections+ assert len(groups) == 2+ for group in groups:+ expert_layers = [n for n in group if "experts" in n]+ assert len(expert_layers) == 6 # 2 experts * 3 projections+ # Non-expert layers (q_proj) should NOT be in the group+ assert all("self_attn" not in n for n in group)+++def test_build_expert_groups_skips_fixed_layers():+ """Expert layers already in fixed_layer_scheme should not be grouped."""+ import torch+ from torch import nn++ from auto_round.auto_scheme.utils import build_expert_groups++ class FakeModel(nn.Module):+ def __init__(self):+ super().__init__()+ self.model = nn.Module()+ self.model.layers = nn.ModuleList()+ block = nn.Module()+ block.mlp = nn.Module()+ block.mlp.experts = nn.ModuleList()+ for j in range(2):+ expert = nn.Module()+ expert.gate_proj = nn.Linear(8, 8, bias=False)+ block.mlp.experts.append(expert)+ self.model.layers.append(block)++ model = FakeModel()+ quant_layer_names = [n for n, m in model.named_modules() if isinstance(m, nn.Linear)]+ # Fix all expert layers+ fixed_layer_scheme = {n: {} for n in quant_layer_names if "experts" in n}++ groups = build_expert_groups(model, quant_layer_names, fixed_layer_scheme)+ assert len(groups) == 0class TestAutoScheme:@@ -19,13 +219,28 @@ class TestAutoScheme:def test_auto_scheme_export(self, tiny_opt_model_path):model_name = tiny_opt_model_path+ int_save_dir = os.path.join(self.save_dir, "int")scheme = AutoScheme(avg_bits=2, options=("W2A16"), nsamples=1, ignore_scale_zp_bits=True)ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)- ar.quantize_and_save(self.save_dir)+ _, int_model_path = ar.quantize_and_save(int_save_dir)++ with open(os.path.join(int_model_path, "config.json")) as f:+ int_config = json.load(f)["quantization_config"]+ assert int_config["quant_method"] == "auto-round"+ assert int_config["bits"] == 2+ assert int_config["data_type"] == "int"+ mxfp_save_dir = os.path.join(self.save_dir, "mxfp")scheme = AutoScheme(avg_bits=4, options=("mxfp4"), nsamples=1, ignore_scale_zp_bits=True)ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)- ar.quantize_and_save(self.save_dir)+ _, mxfp_model_path = ar.quantize_and_save(mxfp_save_dir)++ with open(os.path.join(mxfp_model_path, "config.json")) as f:+ mxfp_config = json.load(f)["quantization_config"]+ assert mxfp_config["quant_method"] == "auto-round"+ assert mxfp_config["bits"] == 4+ assert mxfp_config["data_type"] == "mx_fp"+ assert os.path.exists(os.path.join(int_model_path, "config.json"))def test_gguf_user_fixed_embedding_budget(self, tiny_qwen_model_path):"""Regression test: a user-fixed embedding must be budget-priced at its fixed bits.@@ -127,3 +342,439 @@ class TestAutoScheme:avg_bits, _ = compute_avg_bits_for_model(model)print(avg_bits)assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3++ def test_cache_files_saved_with_correct_format(self, tiny_opt_model_path, tmp_path, monkeypatch):+ """After AutoScheme runs, per-scheme JSON cache files must exist with individual layer scores."""+ import glob+ import json++ from auto_round.auto_scheme.delta_loss import _load_autoscheme_scores++ cache_dir = str(tmp_path / "ar_work_space")+ monkeypatch.setenv("AR_WORK_SPACE", cache_dir)++ scheme = AutoScheme(+ avg_bits=3,+ options=("W2A16", "W4A16"),+ nsamples=1,+ ignore_scale_zp_bits=True,+ )+ ar = AutoRound(model=tiny_opt_model_path, scheme=scheme, iters=0, nsamples=1)+ _, layer_config = ar.quantize()++ # Cache files must exist — one per scheme (2 schemes here)+ cache_files = glob.glob(f"{cache_dir}/auto_scheme_cache/scheme_*.json")+ assert (+ len(cache_files) == 2+ ), f"Expected 2 cache files (one per scheme), found {len(cache_files)}: {cache_files}"++ for path in cache_files:+ data = _load_autoscheme_scores(path)+ assert data is not None, f"Cache file {path} could not be loaded"+ assert data["version"] == 3+ assert data["score_granularity"] == "per_op"+ assert "layer_scores" in data+ assert "total_loss_for_scheme" in data++ # Every layer in layer_scores must have a [bits, loss] pair (individual, not merged)+ for layer_name, score_pair in data["layer_scores"].items():+ assert (+ isinstance(score_pair, list) and len(score_pair) == 2+ ), f"layer_scores[{layer_name!r}] should be [bits, loss], got {score_pair}"+ bits, loss = score_pair+ assert bits > 0, f"bits must be positive for {layer_name}"+ assert loss >= 0, f"loss must be non-negative for {layer_name}"++ # All non-fixed linear layers in layer_config should appear individually in cache+ for layer_name in layer_config:+ assert layer_name in data["layer_scores"], (+ f"Layer {layer_name!r} missing from cache {path} — "+ f"cache may have stored merged group scores instead of individual scores"+ )++ def test_different_avg_bits_produces_different_layer_config(self, tiny_opt_model_path):+ """Changing avg_bits should change the resulting layer_config."""+ scheme_low = AutoScheme(+ avg_bits=2.5,+ options=("W2A16", "W4A16"),+ nsamples=1,+ ignore_scale_zp_bits=True,+ )+ ar_low = AutoRound(model=tiny_opt_model_path, scheme=scheme_low, iters=0, nsamples=1)+ _, config_low = ar_low.quantize()++ scheme_high = AutoScheme(+ avg_bits=3.5,+ options=("W2A16", "W4A16"),+ nsamples=1,+ ignore_scale_zp_bits=True,+ )+ ar_high = AutoRound(model=tiny_opt_model_path, scheme=scheme_high, iters=0, nsamples=1)+ _, config_high = ar_high.quantize()++ low_avg = sum(v["bits"] for v in config_low.values() if "bits" in v) / max(+ len([v for v in config_low.values() if "bits" in v]), 1+ )+ high_avg = sum(v["bits"] for v in config_high.values() if "bits" in v) / max(+ len([v for v in config_high.values() if "bits" in v]), 1+ )+ assert high_avg > low_avg, (+ f"avg_bits=4 should produce higher average bits than avg_bits=2, "+ f"got low={low_avg:.2f} high={high_avg:.2f}"+ )++ def test_shared_layers_assigns_same_bits(self, tiny_opt_model_path):+ """With shared_layers=[q_proj,k_proj,v_proj], all three must get the same bits per block."""+ scheme = AutoScheme(+ avg_bits=5,+ options=("MXFP4", "MXFP8"),+ nsamples=1,+ ignore_scale_zp_bits=True,+ shared_layers=[["fc1", "fc2"]],+ )+ ar = AutoRound(model=tiny_opt_model_path, scheme=scheme, iters=0, nsamples=1)+ _, layer_config = ar.quantize()++ scheme = AutoScheme(+ avg_bits=5,+ options=("MXFP4", "MXFP8"),+ nsamples=1,+ ignore_scale_zp_bits=True,+ shared_layers=[["q_proj", "k_proj", "v_proj"]],+ )+ ar = AutoRound(model=tiny_opt_model_path, scheme=scheme, iters=0, nsamples=1)+ _, layer_config = ar.quantize()++ # Collect per-block q/k/v bits+ block_qkv: dict[str, dict[str, int]] = {}+ for name, cfg in layer_config.items():+ if "bits" not in cfg:+ continue+ for proj in ("q_proj", "k_proj", "v_proj"):+ if name.endswith("." + proj):+ prefix = name[: -len(proj) - 1]+ block_qkv.setdefault(prefix, {})[proj] = cfg["bits"]++ assert block_qkv, "No q/k/v layers found in layer_config"+ for prefix, proj_bits in block_qkv.items():+ present = list(proj_bits.keys())+ bits_values = list(proj_bits.values())+ assert len(set(bits_values)) == 1, (+ f"Block {prefix!r}: q/k/v should all have the same bits with shared_layers, "+ f"got {dict(zip(present, bits_values))}"+ )+++def test_autoscheme_cache_key_different_for_different_schemes():+ """Per-scheme cache: different schemes should produce different cache keys."""+ from auto_round.auto_scheme.delta_loss import _autoscheme_cache_key++ key_w4 = _autoscheme_cache_key(+ model_name="test-model",+ dataset="pile-10k",+ nsamples=16,+ seqlen=256,+ batch_size=8,+ quant_layer_names=["layer.0"],+ fixed_layer_scheme={},+ scheme="W4A16",+ force_mllm=False,+ low_gpu_mem_usage=True,+ )+ key_w8 = _autoscheme_cache_key(+ model_name="test-model",+ dataset="pile-10k",+ nsamples=16,+ seqlen=256,+ batch_size=8,+ quant_layer_names=["layer.0"],+ fixed_layer_scheme={},+ scheme="W8A16",+ force_mllm=False,+ low_gpu_mem_usage=True,+ )+ assert key_w4 != key_w8+ assert len(key_w4) == 16 # sha256 truncated to 16 chars+++def test_autoscheme_cache_key_insensitive_to_layer_order():+ """Per-scheme cache: layer order should not affect the key (internally sorted)."""+ from auto_round.auto_scheme.delta_loss import _autoscheme_cache_key++ key1 = _autoscheme_cache_key(+ model_name="test-model",+ dataset="pile-10k",+ nsamples=16,+ seqlen=256,+ batch_size=8,+ quant_layer_names=["layer.0", "layer.1"],+ fixed_layer_scheme={},+ scheme="W4A16",+ force_mllm=False,+ low_gpu_mem_usage=True,+ )+ key2 = _autoscheme_cache_key(+ model_name="test-model",+ dataset="pile-10k",+ nsamples=16,+ seqlen=256,+ batch_size=8,+ quant_layer_names=["layer.1", "layer.0"], # Different order+ fixed_layer_scheme={},+ scheme="W4A16",+ force_mllm=False,+ low_gpu_mem_usage=True,+ )+ assert key1 == key2 # Should match after internal sorting+++def test_autoscheme_cache_key_changes_only_with_scoring_config():+ """Execution settings that affect scoring invalidate the cache; bit accounting does not."""+ from auto_round.auto_scheme.delta_loss import _autoscheme_cache_key++ kwargs = {+ "model_name": "test-model",+ "dataset": "pile-10k",+ "nsamples": 16,+ "seqlen": 256,+ "batch_size": 8,+ "quant_layer_names": ["layer.0"],+ "fixed_layer_scheme": {},+ "scheme": "W4A16",+ "force_mllm": False,+ "low_gpu_mem_usage": True,+ }+ baseline = _autoscheme_cache_key(**kwargs)++ assert baseline != _autoscheme_cache_key(**{**kwargs, "low_gpu_mem_usage": False})+++def test_refresh_cached_layer_bits_preserves_loss():+ import torch+ from torch import nn++ from auto_round.auto_scheme.delta_loss import _refresh_cached_layer_bits++ model = nn.Module()+ model.layer = nn.Linear(8, 4, bias=False)+ scheme = {"bits": 4, "group_size": 4, "sym": True, "data_type": "int", "act_bits": 16}+ cached_scores = {"layer": [999, 1.25]}++ with_overhead = _refresh_cached_layer_bits(model, ["layer"], {}, scheme, cached_scores, False)+ without_overhead = _refresh_cached_layer_bits(model, ["layer"], {}, scheme, cached_scores, True)++ assert with_overhead["layer"][0] > without_overhead["layer"][0]+ assert without_overhead["layer"] == [model.layer.weight.numel() * 4, 1.25]+ assert cached_scores == {"layer": [999, 1.25]}+++def test_autoscheme_cache_save_and_load(tmp_path):+ """Per-scheme cache: scores can be saved and loaded correctly."""+ from auto_round.auto_scheme.delta_loss import (+ _load_autoscheme_scores,+ _save_autoscheme_scores,+ )++ cache_key = "test_key_123"+ cache_path = os.path.join(str(tmp_path), f"scheme_00_{cache_key}.json")++ scheme_dict = {"bits": 4, "act_bits": 16}+ layer_scores = {+ "layer.0": [4, 1.2],+ "layer.1": [4, 0.9],+ }+ total_loss = 2.1+ total_params = 1000000++ _save_autoscheme_scores(+ cache_path,+ cache_key,+ 0,+ scheme_dict,+ layer_scores,+ total_loss,+ total_params,+ )++ loaded = _load_autoscheme_scores(cache_path)+ assert loaded is not None+ assert loaded["layer_scores"] == layer_scores+ assert loaded["total_loss_for_scheme"] == total_loss+ assert loaded["total_params"] == total_params+++def test_parallel_progress_events_are_applied_in_parent():+ """Worker progress events should update only the parent-owned progress bar."""+ import queue++ from auto_round.auto_scheme.delta_loss import _drain_progress_queue, _ProgressQueueProxy++ class FakeProgressBar:+ def __init__(self):+ self.steps = 0+ self.messages = []++ def update(self, steps):+ self.steps += steps++ def write(self, message):+ self.messages.append(message)++ progress_queue = queue.Queue()+ worker_progress = _ProgressQueueProxy(progress_queue)+ parent_progress = FakeProgressBar()++ worker_progress.update(2)+ worker_progress.write("scheme progress")+ worker_progress.update()+ _drain_progress_queue(progress_queue, parent_progress)++ assert parent_progress.steps == 3+ assert parent_progress.messages == ["scheme progress"]+ assert progress_queue.empty()+++def test_assign_scheme_worker_devices_round_robin():+ from auto_round.auto_scheme.delta_loss import _assign_scheme_worker_devices++ assert _assign_scheme_worker_devices(5, ["cuda:2", "cuda:5"]) == [+ "cuda:2",+ "cuda:5",+ "cuda:2",+ "cuda:5",+ "cuda:2",+ ]+++def test_assign_scheme_worker_devices_shares_single_gpu():+ from auto_round.auto_scheme.delta_loss import _assign_scheme_worker_devices++ assert _assign_scheme_worker_devices(3, ["cuda:3"]) == ["cuda:3", "cuda:3", "cuda:3"]+++def test_assign_scheme_worker_devices_rejects_no_gpu():+ from auto_round.auto_scheme.delta_loss import _assign_scheme_worker_devices++ with pytest.raises(ValueError, match="at least one"):+ _assign_scheme_worker_devices(1, [])+++def test_scheme_worker_count_allows_workers_to_share_gpus():+ from auto_round.auto_scheme.delta_loss import _get_scheme_worker_count++ assert _get_scheme_worker_count(5, 1) == 5+ assert _get_scheme_worker_count(5, 2) == 5+ assert _get_scheme_worker_count(1, 4) == 1+++def test_scheme_worker_count_rejects_no_gpu():+ from auto_round.auto_scheme.delta_loss import _get_scheme_worker_count++ with pytest.raises(ValueError, match="at least one GPU"):+ _get_scheme_worker_count(2, 0)+++def test_opt_scheme_worker_uses_low_cpu_memory_loading(monkeypatch):+ from test.helpers import opt_name_or_path++ from auto_round.auto_scheme import delta_loss++ captured = {}++ def fake_load_model(model_name, **kwargs):+ captured["model_name"] = model_name+ captured.update(kwargs)+ return ("model", "tokenizer", None, None, None, False, False)++ monkeypatch.setattr(delta_loss, "load_model", fake_load_model)++ result = delta_loss._load_scheme_worker_model(opt_name_or_path, True, True)++ assert result[0] == "model"+ assert captured == {+ "model_name": opt_name_or_path,+ "device": "cpu",+ "use_auto_mapping": False,+ "use_model_replacements": True,+ "low_cpu_mem_usage": True,+ }+++def test_per_op_cache_compatibility_rejects_grouped_scores():+ from auto_round.auto_scheme.delta_loss import _is_per_op_cache_compatible++ quant_layers = ["layer.0", "layer.1", "fixed"]+ fixed_layers = {"fixed": {"bits": 8}}++ assert _is_per_op_cache_compatible(+ {"layer_scores": {"layer.0": [4, 1.0], "layer.1": [4, 2.0]}},+ quant_layers,+ fixed_layers,+ )+ assert not _is_per_op_cache_compatible(+ {"layer_scores": {"layer.0": [8, 3.0]}},+ quant_layers,+ fixed_layers,+ )+++def test_version_two_cache_is_rejected(tmp_path):+ import json++ from auto_round.auto_scheme.delta_loss import _load_autoscheme_scores++ cache_path = tmp_path / "scheme_00_old.json"+ cache_path.write_text(+ json.dumps(+ {+ "version": 2,+ "layer_scores": {"layer.0": [4, 1.0]},+ "total_loss_for_scheme": 1.0,+ "total_params": 4,+ }+ ),+ encoding="utf-8",+ )++ assert _load_autoscheme_scores(cache_path) is None+++def test_worker_memory_reports_cover_all_processes_and_devices():+ from auto_round.auto_scheme.delta_loss import _merge_worker_memory_reports++ class FakeMemoryMonitor:+ peak_ram = 1.0+ peak_vram = {"0": 1.0}++ @staticmethod+ def _process_tree_rss():+ return 1.25++ monitor = FakeMemoryMonitor()+ _merge_worker_memory_reports(+ monitor,+ [+ {"device": "0", "peak_ram": 0.5, "peak_vram": 2.0},+ {"device": "0", "peak_ram": 0.75, "peak_vram": 3.0},+ {"device": "1", "peak_ram": 0.25, "peak_vram": 4.0},+ ],+ )++ assert monitor.peak_ram == 2.75+ assert monitor.peak_vram == {"0": 5.0, "1": 4.0}+++def test_replacement_wrapper_without_tuning_device_uses_major_device():+ import torch++ from auto_round.auto_scheme.delta_loss import move_module_to_tuning_device++ class ReplacementWrapper(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.orig_layer = torch.nn.Linear(2, 2)++ wrapper = ReplacementWrapper()+ move_module_to_tuning_device(wrapper, major_device="cpu")++ assert wrapper.orig_layer.weight.device.type == "cpu"diff --git a/test/test_cpu/schemes/test_auto_scheme_internal.py b/test/test_cpu/schemes/test_auto_scheme_internal.pydeleted file mode 100644index d5080fa86..000000000--- a/test/test_cpu/schemes/test_auto_scheme_internal.py+++ /dev/null@@ -1,116 +0,0 @@-import os--from auto_round.auto_scheme.gen_auto_scheme import AutoScheme-from auto_round.auto_scheme.utils import _build_layer_config_header_rows, _short_summary_name---def test_env_ar_auto_scheme_nsamples_overrides_default(monkeypatch):- """AR_AUTO_SCHEME_NSAMPLES env var should override the built-in nsamples heuristic."""- import auto_round.envs as envs-- monkeypatch.setenv("AR_AUTO_SCHEME_NSAMPLES", "7")- assert envs.AR_AUTO_SCHEME_NSAMPLES == 7---def test_env_ar_auto_scheme_batch_size_overrides_default(monkeypatch):- """AR_AUTO_SCHEME_BATCH_SIZE env var should override the built-in batch_size default."""- import auto_round.envs as envs-- monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "4")- assert envs.AR_AUTO_SCHEME_BATCH_SIZE == 4---def test_env_ar_auto_scheme_batch_size_zero_raises(monkeypatch):- """Zero value for AR_AUTO_SCHEME_BATCH_SIZE should raise ValueError."""- import pytest-- import auto_round.envs as envs-- monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "0")- with pytest.raises(ValueError):- _ = envs.AR_AUTO_SCHEME_BATCH_SIZE---def test_build_layer_config_header_rows_merges_adjacent_prefixes():- """Adjacent columns with the same prefix should be merged into one compact header cell."""- columns = ["mlp.down_proj", "mlp.gate_proj", "self_attn.q_proj", "self_attn.v_proj"]- assert _build_layer_config_header_rows(columns) == [- ["block", "mlp", "", "self_attn", ""],- ["", "down_proj", "gate_proj", "q_proj", "v_proj"],- ]---def test_short_summary_name_keeps_one_field_before_numeric_suffix():- """Numeric block suffixes should be shortened to keep the preceding field."""- assert _short_summary_name("model.layers.0") == "layers.0"---def test_build_expert_groups_groups_experts_per_block():- """Expert layers in the same block should be grouped together."""- import torch- from torch import nn-- from auto_round.auto_scheme.utils import build_expert_groups-- # Build a minimal MoE-like model with 2 blocks, each with 2 experts having 2 projections- class FakeModel(nn.Module):- def __init__(self):- super().__init__()- self.model = nn.Module()- self.model.layers = nn.ModuleList()- for i in range(2):- block = nn.Module()- block.mlp = nn.Module()- block.mlp.experts = nn.ModuleList()- for j in range(2):- expert = nn.Module()- expert.gate_proj = nn.Linear(8, 8, bias=False)- expert.up_proj = nn.Linear(8, 8, bias=False)- expert.down_proj = nn.Linear(8, 8, bias=False)- block.mlp.experts.append(expert)- block.self_attn = nn.Module()- block.self_attn.q_proj = nn.Linear(8, 8, bias=False)- self.model.layers.append(block)-- model = FakeModel()- quant_layer_names = [n for n, m in model.named_modules() if isinstance(m, nn.Linear)]- fixed_layer_scheme = {}-- groups = build_expert_groups(model, quant_layer_names, fixed_layer_scheme)- # Should have 2 groups (one per block), each containing all 6 expert projections- assert len(groups) == 2- for group in groups:- expert_layers = [n for n in group if "experts" in n]- assert len(expert_layers) == 6 # 2 experts * 3 projections- # Non-expert layers (q_proj) should NOT be in the group- assert all("self_attn" not in n for n in group)---def test_build_expert_groups_skips_fixed_layers():- """Expert layers already in fixed_layer_scheme should not be grouped."""- import torch- from torch import nn-- from auto_round.auto_scheme.utils import build_expert_groups-- class FakeModel(nn.Module):- def __init__(self):- super().__init__()- self.model = nn.Module()- self.model.layers = nn.ModuleList()- block = nn.Module()- block.mlp = nn.Module()- block.mlp.experts = nn.ModuleList()- for j in range(2):- expert = nn.Module()- expert.gate_proj = nn.Linear(8, 8, bias=False)- block.mlp.experts.append(expert)- self.model.layers.append(block)-- model = FakeModel()- quant_layer_names = [n for n, m in model.named_modules() if isinstance(m, nn.Linear)]- # Fix all expert layers- fixed_layer_scheme = {n: {} for n in quant_layer_names if "experts" in n}-- groups = build_expert_groups(model, quant_layer_names, fixed_layer_scheme)- assert len(groups) == 0diff --git a/test/test_cpu/utils/test_cli_usage.py b/test/test_cpu/utils/test_cli_usage.pyindex 81c490ad7..ac937ec92 100644--- a/test/test_cpu/utils/test_cli_usage.py+++ b/test/test_cpu/utils/test_cli_usage.py@@ -249,3 +249,61 @@ def test_legacy_disable_flags_map_to_enable_bools():assert args.enable_minmax_tuning is Falseassert args.enable_quanted_input is False+++def _normalize_options(raw):+ if raw is None:+ return None+ flat = ",".join(raw)+ return ",".join(p.strip() for p in flat.split(",") if p.strip())+++def _normalize_shared_layers(raw):+ if raw is None:+ return None+ normalized_groups = []+ for invocation in raw:+ if any("," in token for token in invocation):+ for token in invocation:+ group = [p.strip() for p in token.split(",") if p.strip()]+ if group:+ normalized_groups.append(group)+ else:+ group = [p.strip() for p in invocation if p.strip()]+ if group:+ normalized_groups.append(group)+ return normalized_groups or None+++def test_options_comma_space_separated():+ """--options accepts comma-separated and space-separated values."""+ from auto_round.cli.parser import build_quantize_parser++ p = build_quantize_parser()+ assert _normalize_options(p.parse_args(["--avg_bits", "4", "--options", "W4A16,W8A16"]).options) == "W4A16,W8A16"+ assert _normalize_options(p.parse_args(["--avg_bits", "4", "--options", "W4A16", "W8A16"]).options) == "W4A16,W8A16"+ assert p.parse_args(["--model", "dummy"]).options is None+++def test_shared_layers_normalize():+ """--shared_layers: bare tokens → one group; comma tokens → one group each; a,b c,d → two groups."""+ from auto_round.cli.parser import build_quantize_parser++ p = build_quantize_parser()+ # bare tokens → one group+ assert _normalize_shared_layers(+ p.parse_args(["--model", "dummy", "--shared_layers", "l1", "l2"]).shared_layers+ ) == [["l1", "l2"]]+ # comma token → one group+ assert _normalize_shared_layers(p.parse_args(["--model", "dummy", "--shared_layers", "l1,l2"]).shared_layers) == [+ ["l1", "l2"]+ ]+ # a,b c,d → two groups (replaces --shared_layers a,b --shared_layers c,d)+ assert _normalize_shared_layers(+ p.parse_args(["--model", "dummy", "--shared_layers", "l1,l2", "l3,l4"]).shared_layers+ ) == [["l1", "l2"], ["l3", "l4"]]+ # multiple flags → multiple groups+ assert _normalize_shared_layers(+ p.parse_args(["--model", "dummy", "--shared_layers", "l1", "l2", "--shared_layers", "l3,l4"]).shared_layers+ ) == [["l1", "l2"], ["l3", "l4"]]+ assert p.parse_args(["--model", "dummy"]).shared_layers is Nonediff --git a/test/test_cuda/models/test_fp8_model.py b/test/test_cuda/models/test_fp8_model.pyindex 91599e1cf..7fc32a0f0 100644--- a/test/test_cuda/models/test_fp8_model.py+++ b/test/test_cuda/models/test_fp8_model.py@@ -43,6 +43,7 @@ class TestAutoRound:iters=0,disable_opt_rtn=True,low_cpu_mem_usage=False,+ disable_model_free=True,)quantized_model, quantized_model_path = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)assert quantized_model is not None, "Quantized model should not be None."