Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[Feature]: speedup autoscheme with multi-process and cache files (#2083)

Xin He288f848b89cf
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index e1824b92a..7dbf1a6c3 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -14,7 +14,11 @@
import copy
import gc
+import hashlib
+import json
import math
+import os
+import time
from dataclasses import asdict
from functools import wraps
from typing import Iterable, Optional, Union
@@ -48,6 +52,7 @@ from auto_round.data_type.gguf import (
)
from auto_round.data_type.utils import get_quant_func, reshape_pad_tensor_by_group_size, revert_tensor_by_pad
from auto_round.logger import logger
+from auto_round.modeling.fused_moe.replace_modules import safe_to_cpu_
from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
from auto_round.utils import (
SUPPORTED_LAYER_TYPES,
@@ -61,6 +66,7 @@ from auto_round.utils import (
get_module,
is_mllm_model,
llm_load_model,
+ load_model,
mllm_load_model,
parse_available_devices,
set_avg_auto_device_map,
@@ -152,7 +158,8 @@ class AutoSchemeWrapperLinear(WrapperLinear):
self.x_diff = None
return None
- qdq_x.register_hook(save_grad)
+ if qdq_x.requires_grad:
+ qdq_x.register_hook(save_grad)
return qdq_x, scale, zp
def _qdq_weight(self, value, min_scale, max_scale):
@@ -185,7 +192,8 @@ class AutoSchemeWrapperLinear(WrapperLinear):
self.mix_score = self.weight_score + self.act_score
return None
- qdq_w.register_hook(save_grad)
+ if qdq_w.requires_grad:
+ qdq_w.register_hook(save_grad)
return qdq_w, 1.0, None
@@ -304,7 +312,8 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):
self.x_diff = None
return None
- qdq_x.register_hook(save_grad)
+ if qdq_x.requires_grad:
+ qdq_x.register_hook(save_grad)
return qdq_x, scale, zp
def _qdq_weight(self, value, min_scale, max_scale):
@@ -607,11 +616,16 @@ def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_dev
# Enable gradients for the output of the last block
if module.tmp_name == block_names[-1]:
if isinstance(result, torch.Tensor):
- result = result.requires_grad_(True)
+ if result.is_floating_point():
+ result = result.requires_grad_(True)
elif isinstance(result, tuple):
- result = tuple(r.requires_grad_(True) if isinstance(r, torch.Tensor) else r for r in result)
+ result = tuple(
+ r.requires_grad_(True) if isinstance(r, torch.Tensor) and r.is_floating_point() else r
+ for r in result
+ )
- pbar.update(1)
+ if pbar is not None:
+ pbar.update(1)
return result
return new_forward
@@ -775,9 +789,12 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, sch
if isinstance(block_output, tuple):
# For tuple outputs, we usually care about the first element (hidden states)
main_output = block_output[0]
- main_output = main_output.requires_grad_(True)
- else:
+ if isinstance(main_output, torch.Tensor) and main_output.is_floating_point():
+ main_output = main_output.requires_grad_(True)
+ elif isinstance(block_output, torch.Tensor) and block_output.is_floating_point():
main_output = block_output.requires_grad_(True)
+ else:
+ main_output = block_output
# Backward pass for the current block
torch.autograd.backward(
@@ -789,7 +806,11 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, sch
# Extract gradients w.r.t. the block input
if block_input_args and isinstance(block_input_args[0], torch.Tensor):
- current_grad = block_input_args[0].grad.detach().clone()
+ if block_input_args[0].grad is not None:
+ current_grad = block_input_args[0].grad.detach().clone()
+ else:
+ logger.warning(f"No gradient found for input of {block_name}, stopping backward replay")
+ break
else:
logger.warning(f"No suitable input gradient found for {block_name}")
break
@@ -800,7 +821,8 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, sch
# clear_memory(device_list=major_device) # this one is very slow and seems does not affect max ram usage
memory_monitor.update()
- pbar.update(1)
+ if pbar is not None:
+ pbar.update(1)
_log_batch_avg_loss(
model,
@@ -867,6 +889,8 @@ def get_score_for_scheme(
if name in fixed_layer_scheme.keys():
continue
m = get_module(model, name)
+ if m is None:
+ raise RuntimeError(f"AutoScheme scoring layer {name!r} is missing after model preprocessing")
if not check_to_quantized(m):
layer_bits, _ = compute_layer_bits(m, ignore_scale_zp_bits)
scores_dict[name] = [layer_bits, 0.0]
@@ -895,7 +919,9 @@ def get_score_for_scheme(
device = major_device
else:
device = m.weight.device
- m.tuning_device = m.weight.device
+ # Replacement materialization may create fresh expert Linear modules
+ # without the metadata assigned on the pre-materialized tree.
+ m.tuning_device = device
new_m = WrapperLayer(
m,
@@ -1226,9 +1252,10 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = None
(layer_names, scheme) for each layer, or (None, None) if no feasible
solution exists.
"""
- # dp: total_params -> (accumulated_loss, chosen_path)
- # The path is stored as a tuple to avoid the high overhead of repeatedly
- # copying Python lists for every DP transition.
+ # dp: total_params -> (accumulated_loss, path_node)
+ # Each path node points to its parent. Tuple/list concatenation still copies
+ # the entire path on every transition, which becomes quadratic for large
+ # models; linked nodes keep each transition O(1) and are expanded only once.
dp: dict[int, tuple[float, tuple]] = {0: (0.0, ())}
for layer_name, opts in layers.items():
new_dp: dict[int, tuple[float, tuple]] = {}
@@ -1240,7 +1267,7 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = None
continue
new_loss = cur_loss + loss_cost
- new_path = cur_path + ((layer_names, scheme),)
+ new_path = (cur_path, layer_names, scheme)
# Keep the path with smaller loss for the same parameter budget
if np_total not in new_dp or new_loss < new_dp[np_total][0]:
@@ -1285,7 +1312,12 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = None
# Select the solution with the minimum loss
best_params = min(dp.keys(), key=lambda k: dp[k][0])
best_loss, best_path = dp[best_params]
- return best_loss, list(best_path)
+ path = []
+ while best_path:
+ best_path, layer_names, scheme = best_path
+ path.append((layer_names, scheme))
+ path.reverse()
+ return best_loss, path
def move_module_to_tuning_device(module, major_device="cpu"):
@@ -1321,7 +1353,7 @@ def move_module_to_tuning_device(module, major_device="cpu"):
for n, m in module.named_modules():
if hasattr(m, "orig_layer"):
- target = m.orig_layer.tuning_device
+ target = getattr(m.orig_layer, "tuning_device", getattr(m, "tuning_device", major_device))
m.to(target)
_move_own_tensors(m, target)
elif hasattr(m, "tuning_device"):
@@ -1432,6 +1464,388 @@ def _apply_head_trick(head_name, schemes, sorted_indices, target_bits, target_pa
total_scores[head_name] = filtered
+# ---------------------------------------------------------------------------
+# AutoScheme scoring cache helpers
+# ---------------------------------------------------------------------------
+
+
+def _scheme_repr(s):
+ """Normalize a scheme (str/QuantizationScheme/dict) to a hashable repr."""
+ if isinstance(s, str):
+ return s
+ if isinstance(s, QuantizationScheme):
+ return json.dumps(asdict(s), sort_keys=True, default=str)
+ if isinstance(s, dict):
+ return json.dumps(s, sort_keys=True, default=str)
+ return str(s)
+
+
+def _autoscheme_cache_key(
+ model_name,
+ dataset,
+ nsamples,
+ seqlen,
+ batch_size,
+ quant_layer_names,
+ fixed_layer_scheme,
+ scheme,
+ force_mllm,
+ low_gpu_mem_usage,
+):
+ """Return a 16-char hex digest that uniquely identifies a **single-scheme** scoring run.
+
+ The key covers every parameter that directly affects per-layer loss values
+ **except** ``avg_bits`` / ``target_bits`` (only drive the DP step).
+ Unlike the old version, this key is generated **per-scheme**, not per-run,
+ so caching is granular: adding/removing schemes doesn't invalidate cached
+ scores for unchanged schemes.
+ """
+ key_data = {
+ "model_name": model_name,
+ "dataset": dataset,
+ "nsamples": nsamples,
+ "seqlen": seqlen,
+ "batch_size": batch_size,
+ "quant_layer_names": sorted(quant_layer_names),
+ "fixed_layer_scheme": {k: v for k, v in sorted(fixed_layer_scheme.items())},
+ "scheme": _scheme_repr(scheme),
+ "force_mllm": force_mllm,
+ "low_gpu_mem_usage": low_gpu_mem_usage,
+ }
+ key_str = json.dumps(key_data, sort_keys=True, default=str)
+ return hashlib.sha256(key_str.encode()).hexdigest()[:16]
+
+
+def _autoscheme_cache_path(cache_key, scheme_index):
+ """Return the full path to the JSON cache file for a **single scheme**.
+
+ Each scheme gets its own cache file under ``{AR_WORK_SPACE}/auto_scheme_cache/``
+ to enable granular reuse: adding/removing schemes or changing non-scoring
+ parameters (e.g., target_bits) doesn't invalidate caches for unmodified schemes.
+
+ Args:
+ cache_key: Per-scheme cache key (includes model, dataset, scheme, etc.)
+ scheme_index: Index of the scheme (for human readability in filenames)
+ """
+ from auto_round import envs as _envs
+
+ cache_dir = os.path.join(_envs.AR_WORK_SPACE, "auto_scheme_cache")
+ os.makedirs(cache_dir, exist_ok=True)
+ return os.path.join(cache_dir, f"scheme_{scheme_index:02d}_{cache_key}.json")
+
+
+def _save_autoscheme_scores(
+ cache_path,
+ cache_key,
+ scheme_index,
+ scheme_dict,
+ layer_scores,
+ total_loss_for_scheme,
+ total_params,
+):
+ """Persist scoring results for **a single scheme** to *cache_path* as JSON.
+
+ Each scheme's scores are stored independently so that adding/removing schemes
+ or changing unrelated parameters (e.g., target_bits) doesn't invalidate cached
+ scores for unchanged schemes.
+
+ Schema version 3 (per-scheme, per-op cache)::
+
+ {
+ "version": 3,
+ "score_granularity": "per_op",
+ "cache_key": "<hex>",
+ "scheme_index": 0,
+ "scheme": { ... scheme dict ... },
+ "created_at": "<ISO datetime>",
+ "layer_scores": { layer_key: [bits, loss], ... },
+ "total_loss_for_scheme": 1.234,
+ "total_params": 12345
+ }
+ """
+ if cache_path is None or cache_key is None:
+ return
+ # Persist only per-layer independent scores. Grouping (e.g. shared_layers
+ # or MoE expert groups) is intentionally NOT stored here — callers should
+ # re-apply grouping when loading a cache so the on-disk format stays
+ # per-op and backward/forward compatible.
+ data = {
+ "version": 3,
+ "score_granularity": "per_op",
+ "cache_key": cache_key,
+ "scheme_index": scheme_index,
+ "scheme": scheme_dict,
+ "created_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
+ "layer_scores": layer_scores,
+ "total_loss_for_scheme": total_loss_for_scheme,
+ "total_params": total_params,
+ }
+ try:
+ with open(cache_path, "w", encoding="utf-8") as _f:
+ json.dump(data, _f, indent=2, default=str)
+ logger.info("AutoScheme: per-scheme cache saved → %s", cache_path)
+ except Exception as _exc: # noqa: BLE001
+ logger.warning("AutoScheme: failed to save per-scheme cache: %s", _exc)
+
+
+def _load_autoscheme_scores(cache_path):
+ """Load and validate a **single-scheme** per-op scoring cache file (version 3).
+
+ Returns the parsed dict with keys ``layer_scores``, ``total_loss_for_scheme``,
+ and ``total_params`` on success, or ``None`` if the file is missing, malformed,
+ or fails the version sanity check.
+ """
+ _required = ("layer_scores", "total_loss_for_scheme", "total_params")
+ try:
+ with open(cache_path, encoding="utf-8") as _f:
+ data = json.load(_f)
+ if data.get("version") != 3 or data.get("score_granularity") != "per_op":
+ logger.warning(
+ "AutoScheme: per-scheme cache schema mismatch "
+ "(expected version=3, score_granularity=per_op; got version=%s, score_granularity=%s)",
+ data.get("version"),
+ data.get("score_granularity"),
+ )
+ return None
+ for _k in _required:
+ if _k not in data:
+ logger.warning("AutoScheme: per-scheme cache missing required field %s", _k)
+ return None
+ return data
+ except Exception as _exc: # noqa: BLE001
+ logger.warning("AutoScheme: failed to read per-scheme cache %s: %s", cache_path, _exc)
+ return None
+
+
+def _is_per_op_cache_compatible(cached_data, quant_layer_names, fixed_layer_scheme):
+ """Return whether a cache contains exactly one score for every non-fixed quant layer."""
+ expected_layers = set(quant_layer_names) - set(fixed_layer_scheme)
+ return set(cached_data["layer_scores"]) == expected_layers
+
+
+def _refresh_cached_layer_bits(
+ model,
+ quant_layer_names,
+ fixed_layer_scheme,
+ scheme,
+ cached_layer_scores,
+ ignore_scale_zp_bits,
+):
+ """Recompute bit costs for the current accounting mode while preserving cached losses."""
+ apply_quant_scheme(
+ model,
+ quant_layer_names=quant_layer_names,
+ fixed_layer_scheme=fixed_layer_scheme,
+ scheme=scheme,
+ )
+ refreshed_scores = {}
+ for name, (_, loss) in cached_layer_scores.items():
+ bits, _ = compute_layer_bits(get_module(model, name), ignore_scale_zp_bits)
+ refreshed_scores[name] = [bits, loss]
+ return refreshed_scores
+
+
+def _assign_scheme_worker_devices(worker_count, available_devices):
+ """Assign workers round-robin within the devices selected by the caller."""
+ if not available_devices:
+ raise ValueError("available_devices must contain at least one device")
+ return [available_devices[worker_index % len(available_devices)] for worker_index in range(worker_count)]
+
+
+class _ProgressQueueProxy:
+ """Forward worker progress events to the parent process that owns tqdm."""
+
+ def __init__(self, progress_queue):
+ self.progress_queue = progress_queue
+
+ def update(self, steps=1):
+ self.progress_queue.put(("update", steps))
+
+ def write(self, message):
+ self.progress_queue.put(("write", message))
+
+
+def _drain_progress_queue(progress_queue, pbar):
+ """Apply all currently queued worker progress events to the parent tqdm instance."""
+ import queue
+
+ while True:
+ try:
+ event, payload = progress_queue.get_nowait()
+ except queue.Empty:
+ break
+ if event == "update":
+ pbar.update(payload)
+ elif event == "write":
+ pbar.write(payload)
+
+
+def _get_worker_memory_report(worker_device):
+ """Return this worker's peak RAM and VRAM for its assigned logical device."""
+ memory_monitor.update(device_list=worker_device)
+ device_key = str(worker_device).split(":")[-1]
+ return {
+ "device": device_key,
+ "peak_ram": memory_monitor.peak_ram,
+ "peak_vram": memory_monitor.peak_vram.get(device_key, 0.0),
+ }
+
+
+def _merge_worker_memory_reports(monitor, reports):
+ """Merge child-process RAM/VRAM peaks into the parent monitor."""
+ # Worker reports are process-local. Sum them to estimate the concurrent
+ # worker peak, then add the parent RSS after workers have exited. The
+ # parent's sampled peak may already contain the full live process tree,
+ # so retain whichever aggregate is larger rather than adding both.
+ worker_peak_ram = sum(report.get("peak_ram", 0.0) for report in reports)
+ parent_ram = monitor._process_tree_rss() if hasattr(monitor, "_process_tree_rss") else monitor.peak_ram
+ monitor.peak_ram = max(monitor.peak_ram, parent_ram + worker_peak_ram)
+
+ worker_peaks = {}
+ for report in reports:
+ device = str(report["device"])
+ worker_peaks[device] = worker_peaks.get(device, 0.0) + report["peak_vram"]
+ for device, peak_vram in worker_peaks.items():
+ monitor.peak_vram[device] = max(monitor.peak_vram.get(device, 0.0), peak_vram)
+
+
+def _get_scheme_worker_count(num_schemes, num_gpus):
+ """Use one worker per scoring scheme; workers may share a visible GPU."""
+ if num_gpus < 1:
+ raise ValueError("AutoScheme multiprocessing requires at least one GPU")
+ return num_schemes
+
+
+def _load_scheme_worker_model(model_name, use_model_replacements, low_cpu_mem_usage):
+ """Load an isolated worker model without an extra full-size CPU initialization copy."""
+ return load_model(
+ model_name,
+ device="cpu",
+ use_auto_mapping=False,
+ use_model_replacements=use_model_replacements,
+ low_cpu_mem_usage=low_cpu_mem_usage,
+ )
+
+
+def _score_scheme_worker(args):
+ """Score one scheme and return its index, scores, and worker VRAM peak."""
+ import traceback as _tb
+
+ (
+ index,
+ scheme,
+ model_name,
+ is_vlm,
+ quant_layer_names,
+ fixed_layer_scheme,
+ dataset,
+ nsamples,
+ seqlen,
+ batch_size,
+ need_weight_grad,
+ enable_torch_compile,
+ low_cpu_mem_usage,
+ low_gpu_mem_usage,
+ ignore_scale_zp_bits,
+ force_mllm,
+ use_model_replacements,
+ worker_device,
+ total_schemes,
+ progress_queue,
+ ) = args
+
+ from auto_round.auto_scheme.utils import _scheme_short_name as _short_name
+ from auto_round.auto_scheme.utils import apply_quant_scheme as _apply_quant_scheme
+ from auto_round.auto_scheme.utils import compute_layer_bits as _compute_layer_bits
+ from auto_round.utils import get_block_names as _get_block_names
+ from auto_round.utils import get_module as _get_module
+
+ try:
+ model, tokenizer, processor, _, _, is_vlm, _ = _load_scheme_worker_model(
+ model_name,
+ use_model_replacements,
+ low_cpu_mem_usage,
+ )
+ except Exception as exc:
+ raise RuntimeError(
+ f"_score_scheme_worker[{index}]: failed to load model {model_name!r}\n{_tb.format_exc()}"
+ ) from exc
+
+ safe_to_cpu_(model)
+ block_names = _get_block_names(model, quant_vision=force_mllm)[0]
+ for block_name in block_names:
+ block = _get_module(model, block_name)
+ block.in_block = True
+ for _, module in block.named_modules():
+ module.in_block = True
+ for _, module in model.named_modules():
+ if len(list(module.children())) == 0:
+ if not hasattr(module, "in_block"):
+ module.in_block = False
+ if not module.in_block and low_gpu_mem_usage:
+ module.to(worker_device)
+
+ from auto_round.modeling.fused_moe.replace_modules import materialize_model_
+
+ materialize_model_(model)
+ # MoE materialization can replace fused expert modules with newly-created
+ # Linear layers. Assign tuning metadata only after the final module tree exists.
+ for layer_name in quant_layer_names:
+ layer = _get_module(model, layer_name)
+ if layer is None:
+ raise RuntimeError(
+ f"_score_scheme_worker[{index}]: layer {layer_name!r} is missing after model preprocessing"
+ )
+ if not hasattr(layer, "tuning_device"):
+ layer.tuning_device = worker_device
+ layer.tmp_name = layer_name
+
+ _apply_quant_scheme(
+ model,
+ quant_layer_names=quant_layer_names,
+ fixed_layer_scheme=fixed_layer_scheme,
+ scheme=scheme,
+ )
+
+ is_bf16 = isinstance(scheme, str) and scheme.upper() == "BF16"
+ if isinstance(scheme, dict):
+ is_bf16 = scheme.get("bits", 16) >= 16 and scheme.get("act_bits", 16) >= 16
+ if is_bf16:
+ scores = {}
+ for layer_name in quant_layer_names:
+ if layer_name in fixed_layer_scheme:
+ continue
+ layer_bits, _ = _compute_layer_bits(_get_module(model, layer_name), ignore_scale_zp_bits)
+ scores[layer_name] = [layer_bits, 0.0]
+ return index, scores, _get_worker_memory_report(worker_device)
+
+ from auto_round.auto_scheme.delta_loss import get_score_for_scheme
+
+ scores = get_score_for_scheme(
+ model,
+ tokenizer,
+ quant_layer_names,
+ fixed_layer_scheme,
+ dataset,
+ ignore_scale_zp_bits=ignore_scale_zp_bits,
+ pbar=_ProgressQueueProxy(progress_queue),
+ nsamples=nsamples,
+ seqlen=seqlen,
+ need_weight_grad=need_weight_grad,
+ enable_torch_compile=enable_torch_compile,
+ low_gpu_mem_usage=low_gpu_mem_usage,
+ major_device=worker_device,
+ batch_size=batch_size,
+ offload_context=None,
+ processor=processor,
+ is_vlm=is_vlm,
+ force_mllm=force_mllm,
+ model_name=model_name,
+ scheme_tag=f"{index + 1}/{total_schemes} {_short_name(scheme)}",
+ )
+ return index, scores, _get_worker_memory_report(worker_device)
+
+
def _gen_layer_config(
auto_scheme: AutoScheme,
model: Union[str, torch.nn.Module],
@@ -1539,8 +1953,8 @@ def _gen_layer_config(
return True
if isinstance(scheme, QuantizationScheme):
scheme = asdict(scheme)
- if scheme["bits"] >= 16 and scheme["act_bits"] >= 16:
- return True
+ if isinstance(scheme, dict):
+ return scheme.get("bits", 16) >= 16 and scheme.get("act_bits", 16) >= 16
return False
from auto_round import envs as _envs
@@ -1670,108 +2084,387 @@ def _gen_layer_config(
layer_numel[_n] = _np
options_scores = []
+ pbar = None
+
+ # ---- Scoring cache (per-scheme) -------------------------------------------------- #
+ # Each scheme gets its own cache file so that adding/removing schemes or changing
+ # unrelated parameters (e.g., target_bits) doesn't invalidate cached scores for
+ # unchanged schemes.
+ _model_id_for_cache = model_name or getattr(getattr(model, "config", None), "_name_or_path", None)
+
+ # In per-scheme caching, each scheme is checked independently for cache hits.
+ # We always prepare the common resources (imatrix, pbar, etc.) since they're
+ # reusable across schemes and their setup cost is negligible compared to scoring.
+ if True:
+ if need_imatrix:
+ dataloader = get_dataloader(
+ tokenizer,
+ seqlen=max(seqlen * 2, 2048),
+ dataset_name=dataset,
+ seed=42,
+ bs=batch_size,
+ nsamples=min(nsamples, 128),
+ )
+ logger.info("start to compute imatrix in AutoScheme")
+ cal_imatrix(model, dataloader, major_device, low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage)
+ memory_monitor.update()
+ memory_monitor.log_summary()
+ logger.info("finish calculating imatrix")
- if need_imatrix:
- dataloader = get_dataloader(
- tokenizer,
- seqlen=max(seqlen * 2, 2048),
- dataset_name=dataset,
- seed=42,
- bs=batch_size,
- nsamples=min(nsamples, 128),
- )
- logger.info("start to compute imatrix in AutoScheme")
- cal_imatrix(model, dataloader, major_device, low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage)
- memory_monitor.update()
- memory_monitor.log_summary()
- logger.info("finish calculating imatrix")
-
- # Register hooks and clear all block weights before the scheme loop.
- # Hooks will transparently reload weights on demand during forward passes.
- if offload_context is not None:
- offload_context.add_offload_hooks(model, block_name)
+ # Register hooks and clear all block weights before the scheme loop.
+ # Hooks will transparently reload weights on demand during forward passes.
+ if offload_context is not None:
+ offload_context.add_offload_hooks(model, block_name)
+
+ pbar = tqdm(total=pbar_cnt, desc="Generating AutoScheme")
+ scored_layer_names = set(quant_layer_names + embedding_layers_names)
+
+ def _group_per_op_scores(index, per_op_scores):
+ """Apply the current shared-layer grouping without mutating cached per-op scores."""
+ grouped_scores = {}
+ remaining = {name: list(score) for name, score in per_op_scores.items()}
+ for share_layer in shared_layers:
+ param_bits = 0
+ tmp_loss = 0
+ name_list = []
+ for name in share_layer:
+ if name in remaining:
+ bits, loss = remaining.pop(name)
+ param_bits += bits
+ tmp_loss += loss
+ name_list.append(name)
+ if name_list:
+ grouped_scores[name_list[0]] = [index, param_bits, tmp_loss, name_list]
+ for name, (bits, loss) in remaining.items():
+ grouped_scores[name] = [index, bits, loss, [name]]
+ return grouped_scores
+
+ def _record_scheme_scores(index, per_op_scores):
+ grouped_scores = _group_per_op_scores(index, per_op_scores)
+ total_loss = sum(item[2] for item in grouped_scores.values())
+ for key, item in grouped_scores.items():
+ total_scores.setdefault(key, []).append(item)
+ options_scores.append(total_loss)
+ return total_loss
+
+ def _save_per_op_scores(index, scheme, cache_key, cache_path, per_op_scores):
+ if cache_key is None or cache_path is None:
+ return
+ if isinstance(scheme, str):
+ scheme_dict = asdict(preset_name_to_scheme(scheme))
+ elif isinstance(scheme, QuantizationScheme):
+ scheme_dict = asdict(scheme)
+ else:
+ scheme_dict = scheme if isinstance(scheme, dict) else {"preset": str(scheme)}
+ _save_autoscheme_scores(
+ cache_path=cache_path,
+ cache_key=cache_key,
+ scheme_index=index,
+ scheme_dict=scheme_dict,
+ layer_scores={name: list(score) for name, score in per_op_scores.items()},
+ total_loss_for_scheme=sum(score[1] for score in per_op_scores.values()),
+ total_params=sum(
+ (
+ module.weight.numel()
+ if hasattr(module, "weight") and module.weight is not None
+ else getattr(module, "_cached_weight_numel", 0)
+ )
+ for name, module in model.named_modules()
+ if name in scored_layer_names
+ ),
+ )
- pbar = tqdm(total=pbar_cnt, desc="Generating AutoScheme")
- for index, scheme in enumerate(schemes):
- scheme_tag = f"{index + 1}/{len(schemes)} {_scheme_short_name(scheme)}"
- logger.info(f"AutoScheme transition: switch to scheme {index + 1}/{len(schemes)} ({scheme})")
- apply_quant_scheme(
- model, quant_layer_names=quant_layer_names, fixed_layer_scheme=fixed_layer_scheme, scheme=scheme
- )
- scores = {} # name: bits, loss
- if check_bf16_scheme(scheme):
- for n in quant_layer_names:
- if n in fixed_layer_scheme.keys():
- continue
- m = get_module(model, n)
- bits, _ = compute_layer_bits(m, auto_scheme.ignore_scale_zp_bits)
- scores[n] = [bits, 0.0]
- else:
- scores = get_score_for_scheme(
- model,
- tokenizer,
- quant_layer_names,
- fixed_layer_scheme,
- dataset,
- ignore_scale_zp_bits=auto_scheme.ignore_scale_zp_bits,
- pbar=pbar,
+ scheme_cache_meta = []
+ for index, scheme in enumerate(schemes):
+ if check_bf16_scheme(scheme) or _model_id_for_cache is None:
+ scheme_cache_meta.append((None, None, None))
+ if check_bf16_scheme(scheme):
+ logger.info(
+ "AutoScheme: scheme %d/%d (%s) is a BF16 baseline; skipping scoring and cache lookup.",
+ index + 1,
+ len(schemes),
+ _scheme_short_name(scheme),
+ )
+ continue
+ cache_key = _autoscheme_cache_key(
+ model_name=_model_id_for_cache,
+ dataset=dataset,
nsamples=nsamples,
seqlen=seqlen,
- need_weight_grad=need_weight_grad,
- enable_torch_compile=enable_torch_compile,
- low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage,
- major_device=major_device,
batch_size=batch_size,
- offload_context=offload_context,
- processor=processor,
- is_vlm=is_vlm,
+ quant_layer_names=quant_layer_names,
+ fixed_layer_scheme=fixed_layer_scheme,
+ scheme=scheme,
force_mllm=force_mllm,
- model_name=model_name,
- scheme_tag=scheme_tag,
+ low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage,
)
- # Track peak RAM after each scheme scoring
- memory_monitor.update()
- memory_monitor.log_summary()
-
- new_scores = {}
- for share_layer in shared_layers:
- param_bits = 0
- tmp_loss = 0
- name_list = []
- for name in share_layer:
- if name in scores.keys():
- param_bits += scores[name][0]
- tmp_loss += scores[name][1]
- name_list.append(name)
- scores.pop(name)
- new_scores[name_list[0]] = [index, param_bits, tmp_loss, name_list]
- for name, item in scores.items():
- new_scores[name] = [index, item[0], item[1], [name]]
- options_total_loss = 0.0
- for key, item in new_scores.items():
- options_total_loss += item[2]
- if key in total_scores:
- total_scores[key].append(item)
- else:
- total_scores[key] = [item]
- options_scores.append(options_total_loss)
- logger.info(
- f"AutoScheme transition: scheme {index + 1}/{len(schemes)} "
- f"scoring finished (total_loss={options_total_loss:.6f})"
+ cache_path = _autoscheme_cache_path(cache_key, index)
+ cached_data = _load_autoscheme_scores(cache_path) if os.path.exists(cache_path) else None
+ if cached_data is not None and not _is_per_op_cache_compatible(
+ cached_data, quant_layer_names, fixed_layer_scheme
+ ):
+ logger.warning(
+ "AutoScheme: cache %s does not contain complete per-op scores; rescoring scheme %d",
+ cache_path,
+ index,
+ )
+ cached_data = None
+ scheme_cache_meta.append((cache_key, cache_path, cached_data))
+
+ uncached_indices = [
+ index
+ for index, (_, _, cached) in enumerate(scheme_cache_meta)
+ if not check_bf16_scheme(schemes[index]) and cached is None
+ ]
+ worker_device_pool = [device for device in device_list if str(device).startswith("cuda:")]
+ num_gpus = len(worker_device_pool)
+ parallel_enabled = _envs.AR_ENABLE_AUTO_SCHEME_PARALLEL
+ can_parallel = (
+ parallel_enabled
+ and _model_id_for_cache is not None
+ and num_gpus >= 1
+ and len(uncached_indices) >= 2
+ and not need_imatrix
)
- clear_memory(device_list=device_list)
+ if not parallel_enabled and len(uncached_indices) >= 2:
+ logger.info(
+ "AutoScheme: parallel scoring is disabled; set AR_ENABLE_AUTO_SCHEME_PARALLEL=1 to enable it. "
+ "scoring %d uncached non-BF16 schemes serially.",
+ len(uncached_indices),
+ )
- # Remove hooks and restore original weights from disk for final bit-budget computations
- if offload_context is not None:
- offload_context.remove_offload_hooks(model, block_name)
+ from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase
- total_params = 0
- for n, m in model.named_modules():
- if n in quant_layer_names + embedding_layers_names:
- n_param = m.weight.numel()
- if n_param == 0 and hasattr(m, "_cached_weight_numel"):
- n_param = m._cached_weight_numel
- total_params += n_param
+ use_model_replacements = any(isinstance(module, ReplacementModuleBase) for module in model.modules())
+
+ parallel_done = False
+ if can_parallel:
+ try:
+ import torch.multiprocessing as multiprocessing
+
+ def _serialize_scheme(scheme):
+ return asdict(scheme) if isinstance(scheme, QuantizationScheme) else scheme
+
+ num_workers = _get_scheme_worker_count(len(uncached_indices), num_gpus)
+ worker_devices = _assign_scheme_worker_devices(len(uncached_indices), worker_device_pool)
+ logger.info(
+ "AutoScheme: starting %d parallel scoring workers for %d uncached non-BF16 schemes "
+ "(devices=%s; workers may share devices)",
+ num_workers,
+ len(uncached_indices),
+ worker_device_pool,
+ )
+ logger.info(
+ "AutoScheme: if parallel scoring runs out of RAM/VRAM or automatic serial fallback cannot "
+ "recover, set AR_ENABLE_AUTO_SCHEME_PARALLEL=0 and rerun."
+ )
+ spawn_context = multiprocessing.get_context("spawn")
+ with spawn_context.Manager() as manager:
+ progress_queue = manager.Queue()
+ worker_args = [
+ (
+ index,
+ _serialize_scheme(schemes[index]),
+ _model_id_for_cache,
+ is_vlm,
+ list(quant_layer_names),
+ dict(fixed_layer_scheme),
+ dataset,
+ nsamples,
+ seqlen,
+ batch_size,
+ need_weight_grad,
+ enable_torch_compile,
+ auto_scheme.low_cpu_mem_usage,
+ auto_scheme.low_gpu_mem_usage,
+ auto_scheme.ignore_scale_zp_bits,
+ force_mllm,
+ use_model_replacements,
+ worker_devices[slot],
+ len(schemes),
+ progress_queue,
+ )
+ for slot, index in enumerate(uncached_indices)
+ ]
+ with spawn_context.Pool(processes=num_workers) as pool:
+ async_results = pool.map_async(_score_scheme_worker, worker_args)
+ while not async_results.ready():
+ _drain_progress_queue(progress_queue, pbar)
+ memory_monitor.update_cpu()
+ async_results.wait(timeout=0.1)
+ _drain_progress_queue(progress_queue, pbar)
+ worker_results = async_results.get()
+
+ parallel_results = {index: scores for index, scores, _ in worker_results}
+ _merge_worker_memory_reports(memory_monitor, [report for _, _, report in worker_results])
+ logger.info(
+ "AutoScheme parallel aggregate memory (parent + %d workers): %s",
+ len(worker_results),
+ memory_monitor.get_summary(),
+ )
+
+ for index, scheme in enumerate(schemes):
+ cache_key, cache_path, cached_data = scheme_cache_meta[index]
+ if check_bf16_scheme(scheme):
+ apply_quant_scheme(
+ model,
+ quant_layer_names=quant_layer_names,
+ fixed_layer_scheme=fixed_layer_scheme,
+ scheme=scheme,
+ )
+ per_op_scores = {}
+ for name in quant_layer_names:
+ if name in fixed_layer_scheme:
+ continue
+ bits, _ = compute_layer_bits(get_module(model, name), auto_scheme.ignore_scale_zp_bits)
+ per_op_scores[name] = [bits, 0.0]
+ elif cached_data is not None:
+ logger.info(
+ "AutoScheme: loading per-scheme cache for scheme %d from %s."
+ " Delete this file to disable reuse and rescore.",
+ index,
+ cache_path,
+ )
+ per_op_scores = _refresh_cached_layer_bits(
+ model,
+ quant_layer_names,
+ fixed_layer_scheme,
+ scheme,
+ cached_data["layer_scores"],
+ auto_scheme.ignore_scale_zp_bits,
+ )
+ if not check_bf16_scheme(scheme):
+ pbar.update(pbar_cnt // effective_scheme_num if effective_scheme_num > 0 else 1)
+ else:
+ per_op_scores = parallel_results[index]
+ _save_per_op_scores(index, scheme, cache_key, cache_path, per_op_scores)
+ total_loss = _record_scheme_scores(index, per_op_scores)
+ logger.info(
+ "AutoScheme transition: scheme %d/%d scoring finished (total_loss=%.6f)",
+ index + 1,
+ len(schemes),
+ total_loss,
+ )
+ parallel_done = True
+ logger.info("AutoScheme: parallel scoring completed.")
+ post_scoring_started = time.perf_counter()
+ except Exception as parallel_error: # noqa: BLE001
+ logger.warning(
+ "AutoScheme: parallel scoring failed, falling back to serial: %s. "
+ "If fallback cannot recover (for example after RAM/VRAM exhaustion), rerun with "
+ "AR_ENABLE_AUTO_SCHEME_PARALLEL=0.",
+ parallel_error,
+ )
+ total_scores.clear()
+ options_scores.clear()
+ pbar.reset(total=pbar_cnt)
+
+ if not parallel_done:
+ if uncached_indices:
+ from auto_round.modeling.fused_moe.replace_modules import materialize_model_
+
+ materialize_model_(model)
+ for index, scheme in enumerate(schemes):
+ scheme_tag = f"{index + 1}/{len(schemes)} {_scheme_short_name(scheme)}"
+ logger.info(f"AutoScheme transition: switch to scheme {index + 1}/{len(schemes)} ({scheme})")
+ cache_key, cache_path, cached_data = scheme_cache_meta[index]
+
+ if cached_data is not None:
+ logger.info(
+ "AutoScheme: loading per-scheme cache for scheme %d from %s."
+ " Delete this file to disable reuse and rescore.",
+ index,
+ cache_path,
+ )
+ per_op_scores = _refresh_cached_layer_bits(
+ model,
+ quant_layer_names,
+ fixed_layer_scheme,
+ scheme,
+ cached_data["layer_scores"],
+ auto_scheme.ignore_scale_zp_bits,
+ )
+ if not check_bf16_scheme(scheme):
+ pbar.update(pbar_cnt // effective_scheme_num if effective_scheme_num > 0 else 1)
+ else:
+ apply_quant_scheme(
+ model,
+ quant_layer_names=quant_layer_names,
+ fixed_layer_scheme=fixed_layer_scheme,
+ scheme=scheme,
+ )
+ if check_bf16_scheme(scheme):
+ per_op_scores = {}
+ for name in quant_layer_names:
+ if name in fixed_layer_scheme:
+ continue
+ bits, _ = compute_layer_bits(get_module(model, name), auto_scheme.ignore_scale_zp_bits)
+ per_op_scores[name] = [bits, 0.0]
+ else:
+ per_op_scores = get_score_for_scheme(
+ model,
+ tokenizer,
+ quant_layer_names,
+ fixed_layer_scheme,
+ dataset,
+ ignore_scale_zp_bits=auto_scheme.ignore_scale_zp_bits,
+ pbar=pbar,
+ nsamples=nsamples,
+ seqlen=seqlen,
+ need_weight_grad=need_weight_grad,
+ enable_torch_compile=enable_torch_compile,
+ low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage,
+ major_device=major_device,
+ batch_size=batch_size,
+ offload_context=offload_context,
+ processor=processor,
+ is_vlm=is_vlm,
+ force_mllm=force_mllm,
+ model_name=model_name,
+ scheme_tag=scheme_tag,
+ )
+ memory_monitor.update()
+ memory_monitor.log_summary()
+ if not check_bf16_scheme(scheme):
+ _save_per_op_scores(index, scheme, cache_key, cache_path, per_op_scores)
+
+ total_loss = _record_scheme_scores(index, per_op_scores)
+ logger.info(
+ "AutoScheme transition: scheme %d/%d scoring finished (total_loss=%.6f)",
+ index + 1,
+ len(schemes),
+ total_loss,
+ )
+ clear_memory(device_list=device_list)
+
+ # Serial scoring leaves the main model configured with the final scheme and
+ # applies fixed_layer_scheme as a side effect. Parallel workers and cache hits
+ # do not touch the main model, so restore that state before bit-budget math.
+ if schemes:
+ apply_quant_scheme(
+ model,
+ quant_layer_names=quant_layer_names,
+ fixed_layer_scheme=fixed_layer_scheme,
+ scheme=schemes[-1],
+ )
+
+ # Remove hooks and restore original weights from disk for final bit-budget computations
+ if offload_context is not None:
+ offload_context.remove_offload_hooks(model, block_name)
+
+ total_params = 0
+ for n, m in model.named_modules():
+ if n in quant_layer_names + embedding_layers_names:
+ n_param = m.weight.numel()
+ if n_param == 0 and hasattr(m, "_cached_weight_numel"):
+ n_param = m._cached_weight_numel
+ total_params += n_param
+
+ if parallel_done:
+ logger.info(
+ "AutoScheme post-scoring: model restore and parameter accounting took %.2fs",
+ time.perf_counter() - post_scoring_started,
+ )
target_params_cnt = int(total_params * target_bits)
sorted_indices = sorted(range(len(options_scores)), key=lambda i: options_scores[i])
@@ -1903,19 +2596,25 @@ def _gen_layer_config(
if target_params_cnt <= 0:
raise ValueError("Avg bits is too small")
+ cleanup_started = time.perf_counter()
remove_quant_scheme(model) # Must place after minus fixed_layer
memory_monitor.update()
memory_monitor.log_summary()
+ logger.info(
+ "AutoScheme post-scoring: scheme cleanup and memory accounting took %.2fs",
+ time.perf_counter() - cleanup_started,
+ )
+ dp_started = time.perf_counter()
best_loss, best_path = choose_bits_per_layer_with_path(total_scores, target_params_cnt)
+ logger.info(
+ "AutoScheme post-scoring: DP selection took %.2fs (layers=%d)",
+ time.perf_counter() - dp_started,
+ len(total_scores),
+ )
+
if best_path is None:
- min_dp_avg = min_dp_bits / max(sum(layer_numel.get(n, 0) for n in total_scores), 1)
- raise ValueError(
- f"target_bits is infeasible: after subtracting fixed/embedding layers the remaining "
- f"budget ({target_params_cnt} bits) is below the minimum the given options can reach "
- f"({min_dp_bits} bits, avg {min_dp_avg:.3f}). Raise target_bits, add a lower-bit option, "
- f"or lower the bits of fixed layers."
- )
+ raise ValueError("Avg bits is too small")
# print(best_loss, best_path) # TODO better log
layer_config = copy.deepcopy(fixed_layer_scheme)
@@ -1929,13 +2628,15 @@ def _gen_layer_config(
layer_scheme = options[item[1]]
for layer_name in layer_names:
layer_config[layer_name] = asdict(layer_scheme)
+ reporting_started = time.perf_counter()
_log_scheme_loss_matrix(total_scores, options, block_name, model=model, layer_numel=layer_numel)
_describe_layer_config(layer_config, total_scores, options, block_name, model=model)
+ logger.info("AutoScheme post-scoring: result reporting took %.2fs", time.perf_counter() - reporting_started)
if model_name is not None:
model = None
del model
else:
- model = model.to("cpu") # TODO this requires large ram
+ safe_to_cpu_(model)
if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
import accelerate
@@ -1960,7 +2661,8 @@ def _gen_layer_config(
low_cpu_str = "enabled" if auto_scheme.low_cpu_mem_usage else "disabled"
memory_monitor.log_summary(f"AutoScheme complete (low_cpu_mem_usage={low_cpu_str})")
- pbar.close()
+ if pbar is not None:
+ pbar.close()
return layer_config
@@ -1989,30 +2691,11 @@ def gen_layer_config(
is_vlm = False
if isinstance(model, str):
model_name = model
- # Detect VLM (Qwen-VL / Qwen3-VL / LLaVA / etc.) and load via the MLLM
- # path so we get a usable ``processor`` for the multimodal calibration
- # dataloader. The block walker auto-detects VLMs too and skips the
- # vision tower when scoring.
- is_vlm = is_mllm_model(model_name)
- if is_vlm:
- model, processor, tokenizer, _ = mllm_load_model(
- model_name,
- device="cpu",
- use_auto_mapping=False,
- )
- else:
- # Load model on CPU only; do not apply automatic device map or tuning-aware placement at load time.
- model, tokenizer, _ = llm_load_model(model_name, device_map="cpu")
+ model, tokenizer, processor, _, _, is_vlm, _ = load_model(model_name, device="cpu", use_auto_mapping=False)
else:
# Object passed in: still try to detect VLM so we can pick the right dataloader later.
- try:
- is_vlm = is_mllm_model(model)
- except Exception: # noqa: BLE001
- is_vlm = False
-
- from auto_round.modeling.fused_moe.replace_modules import materialize_model_
+ _, _, _, _, _, is_vlm, _ = load_model(model)
- materialize_model_(model)
# ---- Vision-tower scoring requires a full backward ---- #
# ``model_forward_low_gpu`` only walks the language tower (it uses
# ``get_block_names(model)[0]``, which excludes vision blocks by default)
@@ -2041,7 +2724,7 @@ def gen_layer_config(
else:
model = dispatch_model_by_all_available_devices(model, device_map)
else:
- model.to("cpu")
+ safe_to_cpu_(model)
if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
import accelerate
@@ -2112,7 +2795,7 @@ def gen_layer_config(
"Fallback to CPU for automatic scheme generation."
" Using multiple devices is strongly recommended (e.g., --device_map 0,1,2,3)."
)
- model.to("cpu")
+ safe_to_cpu_(model)
for n, m in model.named_modules():
if hasattr(m, "orig_layer"):
set_module(model, n, m.orig_layer)
diff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.py
index 7068b7fbc..a65fc05ba 100644
--- a/auto_round/auto_scheme/gen_auto_scheme.py
+++ b/auto_round/auto_scheme/gen_auto_scheme.py
@@ -27,7 +27,7 @@ from auto_round.utils import get_layer_features, get_module
@dataclass
class AutoScheme:
- avg_bits: float
+ avg_bits: Union[float, list[float]]
options: Union[str, list[Union[QuantizationScheme, str]], tuple[Union[QuantizationScheme, str], ...]]
shared_layers: Optional[Iterable[Iterable[str]]] = None
method: str = "default"
@@ -126,31 +126,41 @@ class GenScheme:
if not isinstance(self.dataset, str):
raise TypeError("`dataset` must be a string, got {type(self.dataset).__name__}.")
- target = self.auto_scheme.avg_bits
+ targets = (
+ self.auto_scheme.avg_bits if isinstance(self.auto_scheme.avg_bits, list) else [self.auto_scheme.avg_bits]
+ )
+ if not targets:
+ raise ValueError("`avg_bits` must not be an empty list.")
+
min_avg_bit = self.min_avg_bit
max_avg_bit = self.max_avg_bit
- logger.info("Average bits range: [%.3f, %.3f], target = %.3f", min_avg_bit, max_avg_bit, target)
- if abs(target - min_avg_bit) < 1e-3 or abs(target - max_avg_bit) < 1e-3:
+ normalized_targets = []
+ for target in targets:
+ logger.info("Average bits range: [%.3f, %.3f], target = %.3f", min_avg_bit, max_avg_bit, target)
if abs(target - min_avg_bit) < 1e-3:
target = min_avg_bit
- else:
+ elif abs(target - max_avg_bit) < 1e-3:
target = max_avg_bit
- self.auto_scheme.avg_bits = target
-
- if not (min_avg_bit <= target <= max_avg_bit):
- details = ", ".join(f"{opt}={bits:.3f}" for opt, bits in getattr(self, "_option_avg_bits", []))
- hint = (
- "Provide at least one option whose avg_bits is <= target and one whose avg_bits is >= target. "
- "Note that layers fixed to high-precision (e.g. lm_head/embed kept at bf16) inflate the lower bound; "
- "use `quant_lm_head=True`/adjust `layer_config`/`ignore_layers` if needed."
- )
- raise ValueError(
- f"Target avg_bits={target:.3f} is outside the achievable range "
- f"[{min_avg_bit:.3f}, {max_avg_bit:.3f}] for options={list(self.auto_scheme.options)} "
- f"(per-option avg_bits: {details}). {hint}"
- )
-
- def get_layer_config(self) -> dict[str, dict]:
+ normalized_targets.append(target)
+
+ if not (min_avg_bit <= target <= max_avg_bit):
+ details = ", ".join(f"{opt}={bits:.3f}" for opt, bits in getattr(self, "_option_avg_bits", []))
+ hint = (
+ "Provide at least one option whose avg_bits is <= target and one whose avg_bits is >= target. "
+ "Note that layers fixed to high-precision (e.g. lm_head/embed kept at bf16) "
+ "inflate the lower bound; use `quant_lm_head=True`/adjust `layer_config`/`ignore_layers` if needed."
+ )
+ raise ValueError(
+ f"Target avg_bits={target:.3f} is outside the achievable range "
+ f"[{min_avg_bit:.3f}, {max_avg_bit:.3f}] for options={list(self.auto_scheme.options)} "
+ f"(per-option avg_bits: {details}). {hint}"
+ )
+
+ self.auto_scheme.avg_bits = (
+ normalized_targets if isinstance(self.auto_scheme.avg_bits, list) else normalized_targets[0]
+ )
+
+ def get_layer_config(self) -> dict[str, dict] | dict[float, dict[str, dict]]:
method_name = self.auto_scheme.method
from auto_round import auto_scheme
@@ -158,6 +168,20 @@ class GenScheme:
if self.auto_scheme.low_gpu_mem_usage:
self.enable_torch_compile = False
+ target_bits = self.auto_scheme.avg_bits
+ if not isinstance(target_bits, list):
+ return self._get_layer_config(method_func)
+
+ layer_configs = {}
+ try:
+ for target_bit in target_bits:
+ self.auto_scheme.avg_bits = target_bit
+ layer_configs[target_bit] = self._get_layer_config(method_func)
+ finally:
+ self.auto_scheme.avg_bits = target_bits
+ return layer_configs
+
+ def _get_layer_config(self, method_func) -> dict[str, dict]:
layer_config = method_func(
self.auto_scheme,
self.model,
diff --git a/auto_round/auto_scheme/utils.py b/auto_round/auto_scheme/utils.py
index caa7c2151..7b5d869a1 100644
--- a/auto_round/auto_scheme/utils.py
+++ b/auto_round/auto_scheme/utils.py
@@ -636,16 +636,13 @@ def _log_batch_avg_loss(model, batch_idx: int, pbar=None, block_names=None, tota
)
-def _build_layer_config_header_rows(columns: list[str]) -> list[list[str]]:
+def _build_layer_config_header_rows(columns: list[str], has_expert_layers: bool = False) -> list[list[str]]:
"""Build a compact two-row header for the layer-config matrix.
The first row keeps a shared prefix for each grouped set of columns (for
example ``mlp`` or ``self_attn``), while the second row keeps the leaf
suffix (for example ``down_proj``).
"""
- if not columns:
- return [["block"], [""]]
-
leaves = []
first_row = []
prev_prefix = None
@@ -664,6 +661,10 @@ def _build_layer_config_header_rows(columns: list[str]) -> list[list[str]]:
first_row.append("")
prev_prefix = None
+ if has_expert_layers:
+ first_row.append("mlp")
+ leaves.append("experts")
+
header_rows = [["block"] + first_row]
header_rows.append([""] + leaves)
return header_rows
@@ -711,7 +712,7 @@ def _log_scheme_loss_matrix(total_scores, options, block_names, model=None, laye
has_expert_layers = any(block_expert_keys.get(b) for b in block_names)
block_display_names = [_short_summary_name(b) for b in block_names]
- header_rows = _build_layer_config_header_rows(columns)
+ header_rows = _build_layer_config_header_rows(columns, has_expert_layers)
# Fixed cell width for scientific-notation values: "1.234E-05" = 9 chars.
_CELL_W = 9
@@ -727,11 +728,8 @@ def _log_scheme_loss_matrix(total_scores, options, block_names, model=None, laye
widths[col] = max(widths[col], len(prefix))
header_keys = ["block"] + columns + (["experts"] if has_expert_layers else [])
- header_vals = [header_rows[0][0]] + header_rows[0][1:]
- header_vals_row2 = ([""]) + (header_rows[1][1:] if len(header_rows) > 1 else [""] * len(columns))
- if has_expert_layers:
- header_vals.append("experts")
- header_vals_row2.append("")
+ header_vals = header_rows[0]
+ header_vals_row2 = header_rows[1]
def _fmt_row(values: list[str], keys: list[str]) -> str:
return "|".join(v.ljust(widths[k]) for v, k in zip(values, keys))
@@ -862,7 +860,7 @@ def _describe_layer_config(layer_config, total_scores, options, block_names, mod
has_expert_layers = any(block_expert_layers.get(block_name) for block_name in block_names)
block_display_names = [_short_summary_name(block_name) for block_name in block_names]
- header_rows = _build_layer_config_header_rows(columns)
+ header_rows = _build_layer_config_header_rows(columns, has_expert_layers)
# Width is driven by the leaf name and cell content — NOT the full dotted column name,
# so columns stay tight (e.g. "down_proj" width, not "mlp.down_proj" width).
@@ -872,7 +870,9 @@ def _describe_layer_config(layer_config, total_scores, options, block_names, mod
max_cell_len = max((len(block_cells.get(b, {}).get(col, "-")) for b in block_names), default=1)
widths[col] = max(len(leaf), max_cell_len)
if has_expert_layers:
- widths["experts"] = max(len("experts"), max((len(v) for v in expert_text_by_block.values()), default=1))
+ widths["experts"] = max(
+ len("mlp"), len("experts"), max((len(v) for v in expert_text_by_block.values()), default=1)
+ )
# Ensure the first column of each prefix group is wide enough for the prefix text in header row 1.
for col, prefix in zip(columns, header_rows[0][1:]):
if prefix:
@@ -882,11 +882,8 @@ def _describe_layer_config(layer_config, total_scores, options, block_names, mod
return "|".join(v.ljust(widths[k]) for v, k in zip(values, keys))
header_keys = ["block"] + columns + (["experts"] if has_expert_layers else [])
- header_vals = [header_rows[0][0]] + header_rows[0][1:]
- if len(header_rows) > 1:
- header_vals_row2 = [header_rows[1][0]] + header_rows[1][1:]
- else:
- header_vals_row2 = [""] + [""] * len(columns)
+ header_vals = header_rows[0]
+ header_vals_row2 = header_rows[1]
logger.info("AutoScheme final layer_config matrix:")
logger.info("AutoScheme note: cell=`scheme`.")
diff --git a/auto_round/cli/main.py b/auto_round/cli/main.py
index a09467e3e..67d38c7ad 100644
--- a/auto_round/cli/main.py
+++ b/auto_round/cli/main.py
@@ -332,6 +332,38 @@ def tune(args):
from auto_round.auto_scheme import AutoScheme
+ # Normalize --options: accepts both space-separated (nargs="+" list) and comma-separated string.
+ # Examples: --options W4A16 W8A16 OR --options W4A16,W8A16
+ if args.options is not None:
+ flat = ",".join(args.options) # handles list; each element may itself contain commas
+ args.options = ",".join(p.strip() for p in flat.split(",") if p.strip())
+
+ # Normalize --shared_layers: supports three forms per invocation:
+ # - all bare tokens (no commas): treated as one group
+ # e.g. --shared_layers l1 l2 → [['l1', 'l2']]
+ # - comma-containing tokens: each token is its own group
+ # e.g. --shared_layers l1,l2 l3,l4 → [['l1','l2'], ['l3','l4']]
+ # - single comma token: one group
+ # e.g. --shared_layers l1,l2 → [['l1', 'l2']]
+ # Multiple --shared_layers flags always produce multiple groups (one per flag when each flag
+ # yields exactly one group, or more if a flag contains comma-tokens).
+ if args.shared_layers is not None:
+ normalized_groups = []
+ for invocation in args.shared_layers:
+ # invocation is a list of tokens from one --shared_layers flag (nargs="+")
+ if any("," in token for token in invocation):
+ # at least one comma token → each token becomes its own group
+ for token in invocation:
+ group = [p.strip() for p in token.split(",") if p.strip()]
+ if group:
+ normalized_groups.append(group)
+ else:
+ # all bare names → bundle into a single group
+ group = [p.strip() for p in invocation if p.strip()]
+ if group:
+ normalized_groups.append(group)
+ args.shared_layers = normalized_groups or None
+
if args.avg_bits is not None:
if args.options is None:
raise ValueError("please set --options for auto scheme")
diff --git a/auto_round/cli/parser.py b/auto_round/cli/parser.py
index 85f2bfdcc..ce68bf283 100644
--- a/auto_round/cli/parser.py
+++ b/auto_round/cli/parser.py
@@ -134,7 +134,13 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Argu
)
rt.add_argument("--output_dir", default="./tmp_autoround", type=str, help="Directory to save quantized artifacts.")
rt.add_argument("--avg_bits", "--target_bits", default=None, type=float, help="Average target bits for AutoScheme.")
- rt.add_argument("--options", default=None, type=str, help="AutoScheme options, for example 'W4A16,W8A16'.")
+ rt.add_argument(
+ "--options",
+ default=None,
+ type=str,
+ nargs="+",
+ help="AutoScheme options. Accepts comma-separated ('W4A16,W8A16') or space-separated (W4A16 W8A16).",
+ )
rt.add_argument(
"--low_gpu_mem_usage", action="store_true", help="Enable memory-efficient mode by offloading features to CPU."
)
@@ -157,7 +163,8 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Argu
nargs="+",
action="append",
default=None,
- help="Ensure listed layers share the same quantization data type.",
+ help="Ensure listed layers share the same quantization data type."
+ " Accepts space-separated (--shared_layers l1 l2) or comma-separated ('l1,l2') per group.",
)
rt.add_argument(
"--static_kv_dtype",
diff --git a/auto_round/compressors/entry.py b/auto_round/compressors/entry.py
index ce6b826e8..9cdff97af 100644
--- a/auto_round/compressors/entry.py
+++ b/auto_round/compressors/entry.py
@@ -369,7 +369,12 @@ class AutoRound(object):
# Model-free routing is now supported directly by the new entry path.
model_free_iters = 0 if isinstance(quant_config, RTNConfig) else getattr(quant_config, "iters", None)
model_free_disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", None)
- route_decision_kwargs = dict(route_kwargs, format=format)
+ route_decision_kwargs = dict(
+ route_kwargs,
+ format=format,
+ static_kv_dtype=base_kwargs.get("static_kv_dtype"),
+ static_attention_dtype=base_kwargs.get("static_attention_dtype"),
+ )
if is_model_free_route(model, scheme, model_free_iters, model_free_disable_opt_rtn, route_decision_kwargs):
from auto_round.compressors.model_free import ModelFreeCompressor
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.py
index 8f07b5138..8c171a82a 100644
--- a/auto_round/compressors/model_free.py
+++ b/auto_round/compressors/model_free.py
@@ -120,6 +120,8 @@ _BLOCK_NAME_TO_IGNORE = ["shared_expert_gate.", ".gate.", "embed", "conv"]
# Preset schemes that model-free mode can produce.
# INT presets use ``auto_round:auto_gptq`` packing; MXFP presets use
# ``mxfp4-pack-quantized`` or ``mxfp8-quantized`` (compressed-tensors) packing.
+# BF16 acts as a full-precision default — all layers stay in BF16 unless
+# overridden by layer_config.
#
# Note: ``W3A16`` (3-bit) is intentionally excluded. 3-bit packing requires
# in_features to be padded to a multiple of pack_factor=10, which the current
@@ -133,6 +135,7 @@ SUPPORTED_PRESET_SCHEMES: tuple[str, ...] = (
"W8A16",
"MXFP4",
"MXFP8",
+ "BF16",
)
# Allowed ``bits`` values for integer WOQ.
@@ -1048,14 +1051,13 @@ def _build_mxfp_quantization_config(
:mod:`auto_round.export.export_to_llmcompressor.export_to_fp`.
"""
from auto_round.export.export_to_llmcompressor.config import (
- check_compressed_tensors_supported,
initialize_quantization,
)
- check_compressed_tensors_supported(raise_error=True)
+ bits = default_scheme.get("bits", 4)
+ is_fp_default = (bits or 0) >= 16 # BF16/FP16 full-precision default
- bits = default_scheme["bits"]
- if bits not in _SUPPORTED_MXFP_BITS:
+ if not is_fp_default and bits not in _SUPPORTED_MXFP_BITS:
raise ValueError(f"Unsupported MXFP bits={bits} for model-free output.")
# Default ignore list: any layer present in ignored_layers (deduped) that
@@ -1077,13 +1079,24 @@ def _build_mxfp_quantization_config(
layer_bits = scheme.get("bits", bits) if scheme is not None else bits
scheme_groups.setdefault(layer_bits, []).append(layer)
else:
- scheme_groups[bits] = list(quantized_layers)
+ if not is_fp_default:
+ scheme_groups[bits] = list(quantized_layers)
+ # else: BF16 default with no layer_config → no MXFP layers; scheme_groups stays {}
if len(scheme_groups) <= 1:
- # Single scheme — existing behavior.
- scheme_name = "MXFP4" if bits == 4 else "MXFP8"
- fmt = "mxfp4-pack-quantized" if bits == 4 else "mxfp8-quantized"
+ # Single scheme — use the actual MXFP bits from the group, not the
+ # default bits (which may be 16 for a BF16 default scheme).
+ actual_bits = next(iter(scheme_groups.keys())) if scheme_groups else bits
+ if actual_bits not in _SUPPORTED_MXFP_BITS:
+ raise ValueError(f"Unsupported MXFP bits={actual_bits} for model-free output.")
+ scheme_name = "MXFP4" if actual_bits == 4 else "MXFP8"
+ fmt = "mxfp4-pack-quantized" if actual_bits == 4 else "mxfp8-quantized"
qconfig = initialize_quantization(scheme=scheme_name, ignore=ignore)
+ if is_fp_default and scheme_groups:
+ targets = list(quantized_layers)
+ if any(".experts." in name for name in quantized_layers):
+ targets.insert(0, "RoutedExperts")
+ qconfig.config_groups["group_0"].targets = targets
qconfig = qconfig.to_dict()
qconfig["format"] = fmt
qconfig["provider"] = "auto-round"
@@ -1128,6 +1141,217 @@ def _build_mxfp_quantization_config(
return full_dict
+def _build_mxfp_autoround_quantization_config(
+ default_scheme: dict,
+ quantized_layers: list[str],
+ ignored_layers: list[str],
+ layer_config: dict | None = None,
+) -> dict:
+ """Build an auto-round style quantization_config for MXFP4 / MXFP8.
+
+ Unlike :func:`_build_mxfp_quantization_config` which produces a
+ compressed-tensors / llm-compressor style config, this function produces
+ an ``auto-round`` style config (``quant_method="auto-round"``,
+ ``packing_format="auto_round:llm_compressor"``). The on-disk weight
+ tensors are identical; only the ``quantization_config`` metadata differs.
+
+ The generated config mirrors the layout of the regular AutoRound MXFP
+ export, including activation quantization fields (``act_bits`` /
+ ``act_data_type`` / …) from the scheme, ``enable_quanted_input``, and
+ ``lm_head`` in ``extra_config`` when it was quantized.
+ """
+ from collections import Counter
+
+ from auto_round.version import __version__
+
+ bits = default_scheme.get("bits", 4)
+ group_size = default_scheme.get("group_size", 32)
+ is_fp_default = (bits or 0) >= 16
+
+ # For BF16 default + MXFP layer_config overrides, derive the dominant
+ # MXFP bit-width from the layers that were actually quantized.
+ if is_fp_default and layer_config and quantized_layers:
+ temp_matcher = _PatternMatcher(
+ ignore_patterns=[],
+ layer_config=layer_config,
+ default_scheme=default_scheme,
+ )
+ mxfp_counter: Counter = Counter()
+ for layer in quantized_layers:
+ scheme = temp_matcher.resolve_scheme(f"{layer}.weight")
+ if scheme is None:
+ continue
+ lb = scheme.get("bits")
+ ldt = (scheme.get("data_type") or "").lower()
+ if lb and lb < 16 and is_mx_fp(ldt):
+ mxfp_counter[lb] += 1
+ if mxfp_counter:
+ bits, _ = mxfp_counter.most_common(1)[0]
+ group_size = 32
+
+ if (bits or 0) < 1 or bits not in _SUPPORTED_MXFP_BITS:
+ bits = 4 # safe fallback
+
+ qconfig: dict = {
+ "quant_method": "auto-round",
+ "packing_format": "auto_round:llm_compressor",
+ "bits": bits,
+ "group_size": group_size or 32,
+ "sym": True, # MXFP is always symmetric
+ "data_type": "mx_fp",
+ "iters": 0,
+ "model_free": True,
+ "autoround_version": __version__,
+ "enable_quanted_input": False,
+ }
+
+ # Carry activation quantization fields from the scheme (e.g. MXFP4 has
+ # act_bits=4, act_data_type="mx_fp", act_dynamic=True, act_group_size=32,
+ # act_sym=True). Including these keeps the config consistent with the
+ # output produced by the regular AutoRound MXFP export flow.
+ for act_key in ("act_bits", "act_data_type", "act_dynamic", "act_group_size", "act_sym"):
+ val = default_scheme.get(act_key)
+ if val is not None:
+ qconfig[act_key] = val
+
+ scheme_keys = [f.name for f in fields(QuantizationScheme)]
+ extra_config: dict = {}
+ non_linear_ops = ["embed", "conv"]
+ non_linear_re = re.compile("|".join(re.escape(op) for op in non_linear_ops))
+
+ if layer_config:
+ for layer_name, cfg in layer_config.items():
+ if not isinstance(cfg, dict):
+ continue
+ cfg_bits = cfg.get("bits", bits)
+ if cfg_bits >= 16:
+ extra_config[layer_name] = {k: cfg[k] for k in scheme_keys if cfg.get(k) is not None}
+ continue
+ differs = any(
+ cfg.get(key) is not None and cfg[key] != default_scheme.get(key)
+ for key in ("bits", "group_size", "sym", "data_type")
+ )
+ if differs:
+ extra_config[layer_name] = {k: cfg[k] for k in scheme_keys if cfg.get(k) is not None}
+
+ quantized_set = set(quantized_layers)
+ unique_ignored = list(dict.fromkeys(ignored_layers))
+ for layer_name in unique_ignored:
+ if layer_name in quantized_set or layer_name in extra_config:
+ continue
+ if non_linear_re.search(layer_name):
+ continue
+ extra_config[layer_name] = {"bits": 16, "data_type": "float"}
+
+ # lm_head: when explicitly quantized, record its full scheme in extra_config
+ # (mirrors the regular AutoRound MXFP export behavior).
+ if "lm_head" in quantized_set and "lm_head" not in extra_config:
+ lm_head_cfg = (layer_config or {}).get("lm_head", default_scheme)
+ extra_config["lm_head"] = {k: lm_head_cfg.get(k) for k in scheme_keys if lm_head_cfg.get(k) is not None}
+
+ if extra_config:
+ qconfig["extra_config"] = extra_config
+
+ return qconfig
+
+
+def _layer_config_has_mxfp(layer_config: dict | None) -> bool:
+ """Return True if any layer_config entry requests MXFP quantization.
+
+ Handles values that are plain strings (preset names), dicts (possibly
+ with a ``"scheme"`` key or a ``"data_type"`` key), or
+ :class:`QuantizationScheme` instances.
+ """
+ if not layer_config:
+ return False
+ for val in layer_config.values():
+ if isinstance(val, str):
+ try:
+ s = _normalize_scheme(val.upper())
+ if is_mx_fp((s.data_type or "").lower()):
+ return True
+ except Exception:
+ pass
+ elif isinstance(val, dict):
+ # Direct data_type field
+ if is_mx_fp((val.get("data_type") or "").lower()):
+ return True
+ # Nested 'scheme' key (e.g. {"scheme": "MXFP4"})
+ scheme_val = val.get("scheme")
+ if isinstance(scheme_val, str):
+ try:
+ s = _normalize_scheme(scheme_val.upper())
+ if is_mx_fp((s.data_type or "").lower()):
+ return True
+ except Exception:
+ pass
+ elif isinstance(val, QuantizationScheme):
+ if is_mx_fp((val.data_type or "").lower()):
+ return True
+ return False
+
+
+def _is_full_precision_default(scheme_input: Any) -> bool:
+ """Return True when *scheme_input* represents a full-precision default (bits >= 16).
+
+ Used to detect BF16/FP16 schemes where no weights are quantized by
+ default but layer_config overrides are still applied.
+ """
+ try:
+ s = _normalize_scheme(scheme_input)
+ return (s.bits or 0) >= 16 and (s.act_bits or 16) >= 16
+ except Exception:
+ return False
+
+
+def _derive_dominant_int_scheme(
+ quantized_layers: list[str],
+ layer_config: dict,
+ fallback: dict,
+) -> dict | None:
+ """Infer the dominant INT scheme from layers that were actually quantized.
+
+ Used when the default scheme is full-precision (BF16/FP16) to produce a
+ quantization_config where the most common INT scheme becomes the top-level
+ default and the BF16 layers appear as ``bits=16`` exceptions in
+ ``extra_config``. This matches the layout expected by AutoRound loaders
+ (dominant quantized bits at the top, full-precision overrides below).
+
+ Returns ``None`` when no INT layers were found in *quantized_layers*.
+ """
+ from collections import Counter
+
+ temp_matcher = _PatternMatcher(
+ ignore_patterns=[],
+ layer_config=layer_config,
+ default_scheme=fallback,
+ )
+ counter: "Counter[tuple]" = Counter()
+ for layer in quantized_layers:
+ scheme = temp_matcher.resolve_scheme(f"{layer}.weight")
+ if scheme is None:
+ continue
+ bits = scheme.get("bits")
+ if bits is None or bits >= 16:
+ continue
+ data_type = (scheme.get("data_type") or "int").lower()
+ if is_mx_fp(data_type):
+ continue # MXFP is handled by the dedicated path
+ key = (bits, scheme.get("group_size") or fallback.get("group_size"), bool(scheme.get("sym", True)), data_type)
+ counter[key] += 1
+
+ if not counter:
+ return None
+
+ (bits, group_size, sym, data_type), _ = counter.most_common(1)[0]
+ return {
+ "bits": bits,
+ "group_size": group_size,
+ "sym": sym,
+ "data_type": data_type,
+ }
+
+
def _build_quantization_config(
default_scheme: dict,
layer_config: dict,
@@ -1135,10 +1359,27 @@ def _build_quantization_config(
quantized_layers: list[str],
ignored_layers: list[str],
block_name_to_quantize: Optional[list[str]] = None,
+ format: str = "auto_round",
) -> dict:
"""Build a quantization_config dict compatible with auto-round format."""
- # MXFP (mx_fp) uses the llm-compressor / compressed-tensors style config.
- if is_mx_fp((default_scheme.get("data_type") or "int").lower()):
+ # MXFP (mx_fp) supports two output styles depending on *format*:
+ # - "llm_compressor" → compressed-tensors / llm-compressor style config
+ # (produced by _build_mxfp_quantization_config).
+ # - "auto_round" / "auto_round:auto_gptq" → auto-round style config
+ # (produced by _build_mxfp_autoround_quantization_config).
+ # Also route to MXFP config when the default is full-precision (BF16/FP16)
+ # but layer_config contains MXFP overrides.
+ data_type = (default_scheme.get("data_type") or "int").lower()
+ default_bits = default_scheme.get("bits", 4)
+ is_fp_default = (default_bits or 0) >= 16 and not is_mx_fp(data_type)
+ if is_mx_fp(data_type) or (is_fp_default and _layer_config_has_mxfp(layer_config)):
+ if format in ("auto_round", "auto_round:auto_gptq"):
+ return _build_mxfp_autoround_quantization_config(
+ default_scheme=default_scheme,
+ quantized_layers=quantized_layers,
+ ignored_layers=ignored_layers,
+ layer_config=layer_config,
+ )
return _build_mxfp_quantization_config(
default_scheme=default_scheme,
quantized_layers=quantized_layers,
@@ -1146,6 +1387,15 @@ def _build_quantization_config(
layer_config=layer_config,
)
+ # When the default is full-precision (BF16/FP16) but INT layers were
+ # quantized via layer_config, derive the dominant INT scheme and use it as
+ # the top-level defaults so the output config reads as
+ # "INT-quantized with BF16 exceptions" rather than the inverse.
+ if is_fp_default and quantized_layers:
+ dominant = _derive_dominant_int_scheme(quantized_layers, layer_config, default_scheme)
+ if dominant is not None:
+ default_scheme = dominant
+
from auto_round.version import __version__
scheme_keys = [f.name for f in fields(QuantizationScheme)]
@@ -1421,6 +1671,11 @@ def _validate_supported_scheme(
bits = scheme_obj.bits
act_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16
+ # Full-precision (BF16/FP16) default: all layers stay in full precision
+ # unless layer_config provides lower-bit overrides.
+ if (bits or 0) >= 16 and act_bits >= 16:
+ return
+
# MXFP weight-only path: accept mx_fp data type with bits in {4, 8}.
# Activation quantization for MXFP is dynamic at inference time, so the
# weight-only RTN path here is independent of act_bits.
@@ -1429,7 +1684,7 @@ def _validate_supported_scheme(
# name is provided. Variants such as MXFP4_RCEIL / MXFP8_RCEIL use a
# different activation format; silently mapping them to "MXFP4" /
# "MXFP8" in the output config would misrepresent the requested scheme.
- if isinstance(scheme_input, str) and scheme_input not in ("MXFP4", "MXFP8"):
+ if isinstance(scheme_input, str) and scheme_input.upper() not in ("MXFP4", "MXFP8"):
raise ValueError(
f"Model-free mode only supports MXFP preset names 'MXFP4' and 'MXFP8', "
f"but got '{scheme_input}'. "
@@ -1504,12 +1759,16 @@ def _looks_like_auto_scheme(scheme: Any) -> bool:
def _validate_auto_scheme_options(auto_scheme: Any) -> str:
- """Validate that every AutoScheme option is model-free-packable.
+ """Validate AutoScheme options for model-free packing.
+
+ Returns the single quantized data-type family shared by all options
+ (``"int"`` or ``"mx_fp"``). ``BF16`` (or any >=16-bit no-op scheme) is
+ allowed and treated as "keep layer in full precision" during conversion,
+ so it does not contribute to the returned family.
- Returns the single data-type family shared by all options
- (``"int"`` or ``"mx_fp"``). Raises ``ValueError`` when any option is
- unsupported or when INT and MXFP options are mixed (they use different
- packing formats and cannot be produced in one model-free run).
+ Raises ``ValueError`` when any non-BF16 option is unsupported or when INT
+ and MXFP options are mixed (they use different packing formats and cannot
+ be produced in one model-free run).
"""
options = list(getattr(auto_scheme, "options", []) or [])
if not options:
@@ -1530,6 +1789,14 @@ def _validate_auto_scheme_options(auto_scheme: Any) -> str:
else:
scheme_obj = None
+ # AutoScheme may include BF16/no-op options. In model-free flow these
+ # layers become ignore_layers entries (full precision), so they are
+ # allowed and excluded from family checks.
+ if scheme_obj is not None:
+ act_bits = scheme_obj.act_bits if scheme_obj.act_bits is not None else 16
+ if (scheme_obj.bits or 0) >= 16 and act_bits >= 16:
+ continue
+
# GGUF k-quants carry super_bits and are not packable by the model-free
# RTN kernel even though their data_type is nominally "int".
if scheme_obj is None or getattr(scheme_obj, "super_bits", None) is not None:
@@ -1584,9 +1851,33 @@ def _convert_auto_scheme_layer_config(
if not isinstance(cfg, dict):
continue
bits = cfg.get("bits")
+ data_type_raw = (cfg.get("data_type") or "").strip()
+
+ # Infer bits from compact dtype aliases that embed bit-width in the name,
+ # e.g. "mxfp8" → 8, "MXFP4" → 4, "mx_fp4" → 4.
+ if bits is None and data_type_raw:
+ dt_lower = data_type_raw.lower()
+ for prefix in ("mxfp", "mx_fp", "nvfp", "nv_fp"):
+ tail = dt_lower[len(prefix) :]
+ if dt_lower.startswith(prefix) and tail.isdigit():
+ bits = int(tail)
+ break
+
if bits is None:
continue
+
clean = {k: cfg[k] for k in scheme_keys if cfg.get(k) is not None}
+ clean["bits"] = bits # honour inferred value when not explicit
+
+ # Normalise compact dtype aliases to canonical forms understood by
+ # the quantization kernels ("mxfp8" / "MXFP4" → "mx_fp").
+ if data_type_raw:
+ dt_lower = data_type_raw.lower()
+ if dt_lower.startswith("mxfp") or dt_lower.startswith("mx_fp"):
+ clean["data_type"] = "mx_fp"
+ elif dt_lower.startswith("nvfp") or dt_lower.startswith("nv_fp"):
+ clean["data_type"] = "nv_fp"
+
if bits >= 16:
fp16_layers.append(name)
continue
@@ -2029,6 +2320,7 @@ class _ModelFreeCompressorCore:
ignore_patterns=self.ignore_patterns,
quantized_layers=self.all_quantized_layers,
ignored_layers=self.all_ignored_layers,
+ format=self.format,
)
self.config["quantization_config"] = quantization_config
@@ -2470,7 +2762,7 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
family = _validate_auto_scheme_options(self.scheme_input)
accepted_formats = {"auto_round", "auto_round:auto_gptq"}
if family == "mx_fp":
- accepted_formats = {"llm_compressor"}
+ accepted_formats = {"llm_compressor", "auto_round", "auto_round:auto_gptq"}
if format not in accepted_formats:
logger.warning(
@@ -2509,10 +2801,18 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
"auto_round",
"auto_round:auto_gptq",
}
- # MXFP only supports the llm_compressor format (INT string preset,
- # or an AutoScheme run whose options resolved to the MXFP family).
- if self.scheme_input in ["MXFP4", "MXFP8"] or self._auto_scheme_family == "mx_fp":
- _accepted_formats = ["llm_compressor"]
+ # MXFP supports both llm_compressor (compressed-tensors) and auto_round formats.
+ # The only difference is the quantization_config metadata; on-disk weights are identical.
+ normalized_scheme = (
+ _normalize_scheme(self.scheme_input) if not _looks_like_auto_scheme(self.scheme_input) else None
+ )
+ if (
+ normalized_scheme is not None and is_mx_fp((normalized_scheme.data_type or "").lower())
+ ) or self._auto_scheme_family == "mx_fp":
+ _accepted_formats = {"llm_compressor", "auto_round", "auto_round:auto_gptq"}
+ elif _is_full_precision_default(self.scheme_input) and _layer_config_has_mxfp(self.layer_config_input):
+ # BF16 default with MXFP layer_config overrides.
+ _accepted_formats = {"llm_compressor", "auto_round", "auto_round:auto_gptq"}
if format not in _accepted_formats:
logger.warning(
f"Format '{format}' is not supported by model-free mode for scheme '{self.scheme_input}'; "
@@ -2524,11 +2824,16 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
if self.user_scheme_overrides:
self.scheme_input = _apply_scheme_overrides(self.scheme_input, self.user_scheme_overrides)
- # Temporarily point output_dir at what the caller requested
- orig = self.output_dir
+ # Temporarily point output_dir and format at what the caller requested
+ orig_dir = self.output_dir
+ orig_fmt = self.format
self.output_dir = output_dir
- out_path = self.run()
- self.output_dir = orig
+ self.format = format
+ try:
+ out_path = self.run()
+ finally:
+ self.output_dir = orig_dir
+ self.format = orig_fmt
self.quantized = True
return None, out_path
diff --git a/auto_round/compressors/orchestrator.py b/auto_round/compressors/orchestrator.py
index c355cb566..6e0e5d12b 100644
--- a/auto_round/compressors/orchestrator.py
+++ b/auto_round/compressors/orchestrator.py
@@ -283,6 +283,7 @@ class CompressionOrchestrator(BaseOrchestrator):
if len(device_manager.device_list) > 1 and not self.model_context.is_diffusion:
accelerate.hooks.remove_hook_from_submodules(m)
mv_module_from_gpu(m)
+ clear_memory(device_list=device_manager.device_list)
memory_monitor.log_summary()
# ── Infrastructure: immediate_pack / shard write ──────────────────
diff --git a/auto_round/envs.py b/auto_round/envs.py
index 714eaacc6..0769754cf 100644
--- a/auto_round/envs.py
+++ b/auto_round/envs.py
@@ -26,6 +26,7 @@ if TYPE_CHECKING:
LLAMA_CPP_ROOT: Optional[str] = None
AR_AUTO_SCHEME_NSAMPLES: Optional[int] = None
AR_AUTO_SCHEME_BATCH_SIZE: Optional[int] = None
+ AR_ENABLE_AUTO_SCHEME_PARALLEL: bool = False
def _get_optional_positive_int_env(name: str) -> Optional[int]:
@@ -85,6 +86,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
# when ``AutoScheme.batch_size`` is not explicitly set.
# When unset, AutoScheme uses its built-in heuristic (8 for low GPU memory mode, 1 for normal mode).
"AR_AUTO_SCHEME_BATCH_SIZE": lambda: _get_optional_positive_int_env("AR_AUTO_SCHEME_BATCH_SIZE"),
+ # Enables AutoScheme to score schemes in parallel. Disabled by default to
+ # avoid multiple model-loading workers exhausting host RAM or device memory.
+ "AR_ENABLE_AUTO_SCHEME_PARALLEL": lambda: os.getenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "0").lower()
+ in ("1", "true", "yes"),
}
diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.py
index a276a5bff..af05f2f13 100644
--- a/auto_round/modeling/fused_moe/replace_modules.py
+++ b/auto_round/modeling/fused_moe/replace_modules.py
@@ -155,14 +155,12 @@ def release_original_module_(model: torch.nn.Module) -> None:
def safe_to_cpu_(model: torch.nn.Module) -> None:
- # If no replacement happened, move model to CPU directly
- if global_state.replaced_module_count == 0:
+ """Move a model to CPU unless it still contains unmaterialized meta tensors."""
+ has_meta_tensor = any(param.device.type == "meta" for param in model.parameters()) or any(
+ buffer.device.type == "meta" for buffer in model.buffers()
+ )
+ if not has_meta_tensor:
model.to("cpu")
- return
- else:
- # TODO: (yiliu30) there might be some edge cases where some modules are replaced
- # and we need to move them to CPU safely.
- pass
class ReplacementModuleBase(ABC, torch.nn.Module):
diff --git a/auto_round/utils/device.py b/auto_round/utils/device.py
index 5364a8280..c0b809388 100644
--- a/auto_round/utils/device.py
+++ b/auto_round/utils/device.py
@@ -21,7 +21,6 @@ import sys
import tempfile
from contextlib import ContextDecorator, contextmanager
from functools import lru_cache
-from itertools import combinations
from threading import Lock
from typing import Any, Callable, Optional, Union
@@ -1026,66 +1025,36 @@ def partition_dict_numbers(number_dict, n):
"""
Partition a dictionary of numbers into N groups with approximately equal sums
"""
- # Edge cases
- if n > len(number_dict):
- groups = []
- for key, value in number_dict.items():
- groups.append({key: value})
- for _ in range(n - len(number_dict)):
- groups.append({})
- return groups
-
- if n == len(number_dict):
- return [{key: value} for key, value in number_dict.items()]
-
- total_sum = sum(number_dict.values())
- # target = total_sum / n # Use float for better precision
+ # Quick handling of edge cases
+ if n <= 0:
+ return []
items = list(number_dict.items())
- result = []
- remaining = items.copy()
-
- def find_optimal_subset(arr, target):
- """Find subset with sum closest to target"""
- best_subset = []
- best_diff = float("inf")
-
- # Try all possible subset sizes
- for r in range(1, len(arr) + 1):
- for combo in combinations(arr, r):
- current_sum = sum(value for _, value in combo)
- current_diff = abs(current_sum - target)
-
- # If we found a perfect match, return immediately
- if current_diff == 0:
- return list(combo)
-
- # Update the best subset if this is better
- if current_diff < best_diff and current_sum <= total_sum:
- best_diff = current_diff
- best_subset = list(combo)
-
- return best_subset
-
- # Distribute items into n-1 groups
- for i in range(n - 1):
- if not remaining:
- break
+ m = len(items)
- # Calculate dynamic target based on remaining items
- remaining_target = sum(value for _, value in remaining) / (n - i)
- subset = find_optimal_subset(remaining, remaining_target)
+ if n >= m:
+ # Put each item in its own bucket, then pad empty buckets
+ groups = [{k: v} for k, v in items]
+ groups.extend({} for _ in range(n - m))
+ return groups
- result.append(dict(subset))
+ # Greedy largest-first balancing (Longest Processing Time / LPT-like):
+ # - Sort items by value descending
+ # - Assign each item to the group with the current smallest sum
+ # Complexity: O(m log n) which scales well for large m (layers)
+ groups_sums = [0.0] * n
+ groups = [dict() for _ in range(n)]
- # Remove allocated items
- for item in subset:
- remaining.remove(item)
+ # Sort items descending by size
+ items_sorted = sorted(items, key=lambda kv: kv[1], reverse=True)
- # Last group gets all remaining items
- result.append(dict(remaining))
+ for key, val in items_sorted:
+ # choose the group with minimum current sum
+ idx = min(range(n), key=lambda i: groups_sums[i])
+ groups[idx][key] = val
+ groups_sums[idx] += val
- return result
+ return groups
def dispatch_model_block_wise(model: torch.nn.Module, device_map: str, max_mem_ratio=0.9):
@@ -1338,13 +1307,24 @@ class MemoryMonitor:
self.peak_vram = {} # {device_id: peak_mb}
self.enabled = True
+ @staticmethod
+ def _process_tree_rss() -> float:
+ """Return the combined RSS of this process and all live descendants in GB."""
+ process = psutil.Process()
+ rss = process.memory_info().rss
+ for child in process.children(recursive=True):
+ try:
+ rss += child.memory_info().rss
+ except (psutil.NoSuchProcess, psutil.AccessDenied):
+ continue
+ return rss / 1024**3
+
def update(self, device_list=None):
"""Update current memory usage and track peaks."""
if not self.enabled:
return
# Track RAM
- process = psutil.Process()
- current_ram = process.memory_info().rss / 1024**3 # GB
+ current_ram = self._process_tree_rss()
self.peak_ram = max(self.peak_ram, current_ram)
if device_list is None: # TODO this has issue, wait for clean_memory all pass device_list
device_list = [0]
@@ -1386,8 +1366,7 @@ class MemoryMonitor:
def update_cpu(self):
if not self.enabled:
return
- process = psutil.Process()
- current_ram = process.memory_info().rss / 1024**3 # GB
+ current_ram = self._process_tree_rss()
self.peak_ram = max(self.peak_ram, current_ram)
def update_hpu(self, device_list=None):
diff --git a/auto_round/utils/device_manager.py b/auto_round/utils/device_manager.py
index 1e99a93cf..b3aa5be8f 100644
--- a/auto_round/utils/device_manager.py
+++ b/auto_round/utils/device_manager.py
@@ -410,9 +410,35 @@ class ARDevice:
return _DeviceIndexContext(self, index)
def total_memory(self, index: int = 0) -> int:
- fn = getattr(self._module, "get_memory_info", None)
+ return self.mem_get_info(index)[1]
- return fn(index)[1] if callable(fn) else None # pylint: disable=E1102
+ def mem_get_info(self, index: int = 0) -> tuple[int, int]:
+ """Return ``(free_bytes, total_bytes)`` for ``index``."""
+ modules = (self._module, self.get_device_module(self.type))
+ for module in dict.fromkeys(modules):
+ if module is None:
+ continue
+ for name in ("get_memory_info", "mem_get_info"):
+ fn = getattr(module, name, None)
+ if not callable(fn):
+ continue
+ try:
+ free, total = fn(index) # pylint: disable=E1102
+ if int(total) > 0:
+ return int(free), int(total)
+ except Exception:
+ continue
+
+ module = self.get_device_module(self.type)
+ get_properties = getattr(module, "get_device_properties", None)
+ if callable(get_properties):
+ try:
+ total = int(get_properties(index).total_memory)
+ if total > 0:
+ return max(0, total - self.memory_reserved(index)), total
+ except Exception:
+ pass
+ return 0, 0
def memory_reserved(self, index: int = 0) -> int:
if self._module is None:
@@ -432,17 +458,6 @@ class ARDevice:
except Exception:
return 0
- # def mem_get_info(self, index: int = 0) -> tuple[int, int]:
- # """Return ``(free_bytes, total_bytes)`` for ``index``.
- #
- # Falls back to ``total - reserved`` when the backend lacks a native
- # ``mem_get_info`` implementation.
- # """
- # module = self.get_device_module(self.type) if self._module is _accelerator_api() else self._module
- # fn = getattr(module, "get_memory_info", None)
- #
- # return fn(index) if callable(fn) else (0, 0) # pylint: disable=E1102
-
# -- numeric format / mixed-precision policy ---------------------------
def supports_bf16(self) -> bool:
"""Whether this backend can execute the ``bfloat16`` data type."""
@@ -1055,7 +1070,15 @@ def get_device_memory(i: int = 0) -> int:
dev_mgr = get_current_device_manager()
if not dev_mgr.is_available() or dev_mgr.type == "cpu":
raise RuntimeError("No supported device found (CUDA/XPU/HPU/...).")
- return dev_mgr.total_memory(i) / 1024 / 1024 / 1024
+
+ total_memory = dev_mgr.total_memory(i)
+ if total_memory is None:
+ raise RuntimeError(f"Failed to query total memory for device index {i} on backend {dev_mgr.type}.")
+ if total_memory <= 0:
+ raise RuntimeError(
+ f"Detected non-positive total memory ({total_memory}) for device index {i} on backend {dev_mgr.type}."
+ )
+ return total_memory / 1024**3
def _clear_memory_for_cpu_and_cuda(
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index cc2c761e1..d84b18b64 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -356,11 +356,22 @@ def llm_load_model(
if device_str is not None and "hpu" in device_str:
torch_dtype = torch.bfloat16
+ if "dtype" in kwargs:
+ torch_dtype = kwargs.pop("dtype")
+ kwargs.pop("torch_dtype", None)
+ elif "torch_dtype" in kwargs:
+ torch_dtype = kwargs.pop("torch_dtype")
+ dtype_key = (
+ "dtype"
+ if platform == "hf" and version.parse(transformers.__version__) >= version.parse("4.56.0")
+ else "torch_dtype"
+ )
load_kwargs = {
- "torch_dtype": torch_dtype,
+ dtype_key: torch_dtype,
"trust_remote_code": trust_remote_code,
"device_map": "auto" if use_auto_mapping else None,
}
+ load_kwargs.update(kwargs)
if version.parse(transformers.__version__) >= version.parse("5.0.0"):
is_mxfp4 = _is_mxfp4_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
@@ -925,6 +936,119 @@ def diffusion_load_model(
return pipe, model.to(device)
+def load_model(
+ pretrained_model_name_or_path: Union[str, torch.nn.Module],
+ platform: str = "hf",
+ model_dtype: str = None,
+ trust_remote_code: bool = True,
+ device: str = "cpu",
+ use_auto_mapping: bool = None,
+ use_model_replacements: bool = False,
+ **kwargs,
+) -> tuple:
+ """Unified model loader supporting LLM, MLLM (VLM), and diffusion models.
+
+ Automatically detects model type (text-only LLM / multimodal MLLM / diffusion
+ pipeline) via ``is_mllm_model`` / ``is_diffusion_model`` and dispatches to the
+ corresponding loader. When a pre-loaded ``torch.nn.Module`` is passed the
+ function skips loading and only performs type detection.
+
+ Args:
+ pretrained_model_name_or_path: HuggingFace model ID, local path, or an
+ already-loaded ``torch.nn.Module``.
+ platform: ``"hf"`` (default) or ``"model_scope"``.
+ model_dtype: Cast the model to this dtype after loading (e.g. ``"bf16"``).
+ trust_remote_code: Passed through to every HF ``from_pretrained`` call.
+ device: Device to load the model onto (default ``"cpu"``).
+ use_auto_mapping: Override ``device_map="auto"`` for MLLM/diffusion loaders.
+ ``None`` means each sub-loader decides its own default.
+ use_model_replacements: Apply AutoRound replacement modules and special-model
+ forward patches after loading. Intended for isolated quantization workers.
+ **kwargs: Extra keyword arguments forwarded verbatim to the active sub-loader.
+
+ Returns:
+ A 7-tuple ``(model, tokenizer, processor, image_processor, pipe, is_mllm,
+ is_diffusion)`` where unused fields are ``None`` / ``False``.
+ """
+
+ def _prepare_model(model):
+ if not use_model_replacements:
+ return model
+ from auto_round.special_model_handler import _handle_special_model, update_module
+
+ model = update_module(model, formats=None, cleanup_original=False)
+ return _handle_special_model(model)
+
+ model = None
+ tokenizer = None
+ processor = None
+ image_processor = None
+ pipe = None
+ _is_mllm = False
+ _is_diffusion = False
+
+ if isinstance(pretrained_model_name_or_path, str):
+ if is_mllm_model(pretrained_model_name_or_path, platform=platform):
+ _kwargs = dict(
+ platform=platform,
+ device=device,
+ model_dtype=model_dtype,
+ trust_remote_code=trust_remote_code,
+ )
+ if use_auto_mapping is not None:
+ _kwargs["use_auto_mapping"] = use_auto_mapping
+ _kwargs.update(kwargs)
+ model, processor, tokenizer, image_processor = mllm_load_model(pretrained_model_name_or_path, **_kwargs)
+ model = _prepare_model(model)
+ _is_mllm = True
+
+ elif is_diffusion_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code):
+ _kwargs = dict(
+ platform=platform,
+ device=device,
+ model_dtype=model_dtype,
+ trust_remote_code=trust_remote_code,
+ )
+ if use_auto_mapping is not None:
+ _kwargs["use_auto_mapping"] = use_auto_mapping
+ _kwargs.update(kwargs)
+ pipe, model = diffusion_load_model(pretrained_model_name_or_path, **_kwargs)
+ model = _prepare_model(model)
+ _is_diffusion = True
+
+ else:
+ # Plain text LLM
+ # Apply class-level block patches (e.g. unfused-MoE replacements) before
+ # ``from_pretrained`` so the model is loaded with the correct architecture.
+ from auto_round.modeling.unfused_moe import apply_model_monkey_patches
+
+ apply_model_monkey_patches(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
+
+ _kwargs = dict(
+ platform=platform,
+ device=device,
+ model_dtype=model_dtype,
+ trust_remote_code=trust_remote_code,
+ )
+ _kwargs.update(kwargs)
+ model, tokenizer = llm_load_model(pretrained_model_name_or_path, **_kwargs)
+ model = _prepare_model(model)
+ else:
+ # Already a loaded model object – detect type without triggering any file I/O.
+ try:
+ _is_mllm = is_mllm_model(pretrained_model_name_or_path)
+ except Exception:
+ pass
+ if not _is_mllm:
+ try:
+ _is_diffusion = is_diffusion_model(pretrained_model_name_or_path)
+ except Exception:
+ pass
+ model = _prepare_model(pretrained_model_name_or_path)
+
+ return model, tokenizer, processor, image_processor, pipe, _is_mllm, _is_diffusion
+
+
def is_pure_text_model(model):
"""verify on: phi-3.5, Mistral-Small-3.1, gemma-3, qwen2-vl,"""
if hasattr(model, "config") and hasattr(model.config, "vision_config"):
@@ -2310,6 +2434,11 @@ def is_model_free_route(
explicit = bool(kwargs.get("model_free", False))
disabled = bool(kwargs.get("disable_model_free", False))
+ has_static_attention_quantization = any(
+ kwargs.get(key) is not None for key in ("static_kv_dtype", "static_attention_dtype")
+ )
+ if has_static_attention_quantization:
+ return False
if explicit:
return True
# Only auto-route when format is auto_round (or not specified).
diff --git a/docs/environments.md b/docs/environments.md
index 296bf6c28..3ccd400c0 100644
--- a/docs/environments.md
+++ b/docs/environments.md
@@ -141,6 +141,16 @@ export AR_AUTO_SCHEME_NSAMPLES=1 # set 1 for quick execution
export AR_AUTO_SCHEME_BATCH_SIZE=1
```
+### AR_ENABLE_AUTO_SCHEME_PARALLEL
+- **Description**: Enables multiprocessing across AutoScheme candidates. Keep it disabled when parallel model-loading workers could exhaust host RAM or device memory.
+- **Default**: `False` (schemes are scored serially)
+- **Valid Values**: `"1"`, `"true"`, `"yes"` (case-insensitive) enable parallel scoring; any other value keeps parallel scoring disabled
+- **Usage**: Set this before running AutoScheme to enable parallel candidate scoring
+
+```bash
+export AR_ENABLE_AUTO_SCHEME_PARALLEL=1
+```
+
## Usage Examples
### Setting Environment Variables
diff --git a/docs/environments_CN.md b/docs/environments_CN.md
index c04c45568..e2100d60a 100644
--- a/docs/environments_CN.md
+++ b/docs/environments_CN.md
@@ -141,6 +141,16 @@ export AR_AUTO_SCHEME_NSAMPLES=1
export AR_AUTO_SCHEME_BATCH_SIZE=1
```
+### AR_ENABLE_AUTO_SCHEME_PARALLEL
+- **描述**:启用 AutoScheme 候选方案之间的多进程并行。当并行加载多个模型可能耗尽主机内存或显存时,请保持关闭。
+- **默认值**:`False`(串行评分各方案)
+- **有效值**:`"1"`、`"true"`、`"yes"`(不区分大小写)表示启用并行评分;其他值保持关闭
+- **用途**:运行 AutoScheme 前设置,以启用候选方案并行评分
+
+```bash
+export AR_ENABLE_AUTO_SCHEME_PARALLEL=1
+```
+
## 使用示例
### 设置环境变量
diff --git a/test/fixtures.py b/test/fixtures.py
index ffc628401..258b05456 100644
--- a/test/fixtures.py
+++ b/test/fixtures.py
@@ -401,7 +401,8 @@ def mock_fp8_capable_device():
def clean_tmp_model_folder():
yield
shutil.rmtree("./tmp", ignore_errors=True) # unittest default workspace
- shutil.rmtree("./tmp_autoround", ignore_errors=True) # autoround default workspace
+ shutil.rmtree("./ar_work_space", ignore_errors=True) # autoround default workspace
+ shutil.rmtree("./tmp_autoround", ignore_errors=True) # autoround default model output path
# Create objective fixtures for testing
diff --git a/test/test_cpu/export/test_llmc_format.py b/test/test_cpu/export/test_llmc_format.py
index 7982bf8bd..8a6e6d95f 100644
--- a/test/test_cpu/export/test_llmc_format.py
+++ b/test/test_cpu/export/test_llmc_format.py
@@ -105,6 +105,7 @@ class TestLLMC:
iters=0,
disable_opt_rtn=True,
scheme=scheme,
+ disable_model_free=True,
)
compressed_model, tmp_path = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")
tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_proj
diff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.py
index 1d25ba719..cb21213f7 100644
--- a/test/test_cpu/models/test_moe_model.py
+++ b/test/test_cpu/models/test_moe_model.py
@@ -22,6 +22,7 @@ def quantize_model(model, output_dir, scheme, iters=0, ignore_layers="self_attn,
seqlen=32,
ignore_layers=ignore_layers,
disable_opt_rtn=disable_opt_rtn,
+ disable_model_free=True,
)
quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)
return quantized_model, save_folder
@@ -98,6 +99,7 @@ def test_qwen3_vl_moe_mxfp(tiny_qwen3_vl_moe_model_path, tmp_path):
seqlen=32,
iters=0,
disable_opt_rtn=True,
+ disable_model_free=True,
ignore_layers="self_attn,lm_head, mlp.gate",
)
quantized_model, quantized_model_path = autoround.quantize_and_save(format="auto_round", output_dir=tmp_path)
diff --git a/test/test_cpu/quantization/test_model_free.py b/test/test_cpu/quantization/test_model_free.py
index bceede445..8849281b9 100644
--- a/test/test_cpu/quantization/test_model_free.py
+++ b/test/test_cpu/quantization/test_model_free.py
@@ -18,12 +18,40 @@ import json
import os
import pytest
+
+
+def test_quantize_and_save_restores_temporary_state_on_failure(monkeypatch):
+ from auto_round.compressors.model_free import ModelFreeCompressor
+
+ compressor = ModelFreeCompressor.__new__(ModelFreeCompressor)
+ compressor.scheme_input = "W4A16"
+ compressor.layer_config_input = None
+ compressor.user_scheme_overrides = None
+ compressor._auto_scheme_family = None
+ compressor.output_dir = "original-output"
+ compressor.format = "auto_round"
+ compressor.quantized = False
+
+ def fail_run():
+ raise RuntimeError("save failed")
+
+ monkeypatch.setattr(compressor, "run", fail_run)
+
+ with pytest.raises(RuntimeError, match="save failed"):
+ compressor.quantize_and_save("temporary-output", format="auto_round:auto_gptq")
+
+ assert compressor.output_dir == "original-output"
+ assert compressor.format == "auto_round"
+ assert compressor.quantized is False
+
+
import torch
from safetensors import safe_open
from safetensors.torch import save_file
from auto_round import AutoRound
from auto_round.compressors.model_free import (
+ _build_mxfp_autoround_quantization_config,
_build_mxfp_quantization_config,
_build_quantization_config,
_convert_auto_scheme_layer_config,
@@ -43,6 +71,7 @@ from auto_round.compressors.model_free import (
is_model_free_supported_scheme,
)
from auto_round.schemes import QuantizationScheme
+from auto_round.utils.model import is_model_free_route
from ...envs import require_compressed_tensors
@@ -50,6 +79,19 @@ from ...envs import require_compressed_tensors
# Helpers
# ---------------------------------------------------------------------------
+
[email protected]("scheme", ["MXFP8", "mxfp8"])
+def test_model_free_route_accepts_mxfp_scheme_case_insensitively(scheme):
+ assert is_model_free_route("model", scheme, 0, True, {})
+
+
[email protected]("dtype_key", ["static_kv_dtype", "static_attention_dtype"])
[email protected]("explicit", [False, True])
+def test_model_free_route_rejects_static_attention_quantization(dtype_key, explicit):
+ kwargs = {dtype_key: "fp8", "model_free": explicit}
+ assert not is_model_free_route("model", "W4A16", 0, True, kwargs)
+
+
_LLAMA_CFG = {"architectures": ["LlamaForCausalLM"], "model_type": "llama"}
_DEFAULT_SCHEME = {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"}
@@ -503,7 +545,9 @@ class TestModelFreeMXFP:
}
model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
output_dir = str(tmp_path / "output")
- _ModelFreeCompressorCore(model_name_or_path=model_dir, output_dir=output_dir, scheme=scheme).run()
+ _ModelFreeCompressorCore(
+ model_name_or_path=model_dir, output_dir=output_dir, scheme=scheme, format="llm_compressor"
+ ).run()
qc = _read_qconfig(output_dir)
assert qc["format"] == fmt
assert qc["quant_method"] == "compressed-tensors"
@@ -551,6 +595,20 @@ class TestModelFreeMXFP:
assert "lm_head" in cfg["ignore"]
assert len(cfg["config_groups"]) == 1
+ @require_compressed_tensors
+ def test_build_mxfp_config_bf16_default_targets_only_overrides(self):
+ """A single MXFP override must not describe all BF16 Linear layers as quantized."""
+ default = {"bits": 16, "group_size": -1, "sym": True, "data_type": "bf16"}
+ layer_config = {
+ "model.layers.0.self_attn.q_proj": {"bits": 4, "group_size": 32, "data_type": "mx_fp"},
+ }
+ quantized = ["model.layers.0.self_attn.q_proj"]
+
+ cfg = _build_mxfp_quantization_config(default, quantized, [], layer_config=layer_config)
+
+ assert cfg["format"] == "mxfp4-pack-quantized"
+ assert cfg["config_groups"]["group_0"]["targets"] == quantized
+
@require_compressed_tensors
def test_build_mxfp_mixed_config_two_groups(self):
"""Mixed MXFP4+MXFP8: override layers get explicit targets; default gets ["Linear"]."""
@@ -643,6 +701,7 @@ class TestModelFreeMXFP:
output_dir=output_dir,
scheme="MXFP4",
layer_config=layer_config,
+ format="llm_compressor",
).run()
qc = _read_qconfig(output_dir)
@@ -853,7 +912,9 @@ class TestLLMCompressorMXFPSource:
}
model_dir = _make_model_dir(tmp_path, _LLMCOMPRESSOR_MXFP_CFG_FP8, tensors)
output_dir = str(tmp_path / "output")
- _ModelFreeCompressorCore(model_name_or_path=model_dir, output_dir=output_dir, scheme="MXFP8").run()
+ _ModelFreeCompressorCore(
+ model_name_or_path=model_dir, output_dir=output_dir, scheme="MXFP8", format="llm_compressor"
+ ).run()
qc = _read_qconfig(output_dir)
assert qc["format"] == "mxfp8-quantized"
@@ -934,7 +995,9 @@ class TestLLMCompressorMXFPSource:
}
model_dir = _make_model_dir(tmp_path, _DEEPSEEK_V4_CFG, tensors)
output_dir = str(tmp_path / "output")
- _ModelFreeCompressorCore(model_name_or_path=model_dir, output_dir=output_dir, scheme="MXFP8").run()
+ _ModelFreeCompressorCore(
+ model_name_or_path=model_dir, output_dir=output_dir, scheme="MXFP8", format="llm_compressor"
+ ).run()
qc = _read_qconfig(output_dir)
assert qc["format"] == "mxfp8-quantized"
@@ -979,11 +1042,10 @@ class TestLLMCompressorMXFPSource:
# ===========================================================================
-_SUPPORTED = ["W2A16", "W2A16G32", "W2A16G64", "W4A16", "W4A16_MIXED", "W8A16", "MXFP4", "MXFP8"]
+_SUPPORTED = ["W2A16", "W2A16G32", "W2A16G64", "W4A16", "W4A16_MIXED", "W8A16", "MXFP4", "MXFP8", "BF16"]
_UNSUPPORTED = [
"W3A16",
"FPW8A16",
- "BF16",
"MXINT4",
"NVFP4",
"FP8_BLOCK",
@@ -1022,11 +1084,16 @@ class TestSchemeValidation:
layer_config=core.layer_config,
ignore_patterns=core.ignore_patterns,
)
- assert "model.layers.0.mlp.fc1" in quantized
- if name.startswith("MXFP"):
- assert "model.layers.0.mlp.fc1.weight_scale" in output
+ if name == "BF16":
+ # BF16 default: no weights quantized; the layer is kept in full precision.
+ assert len(quantized) == 0
+ assert "model.layers.0.mlp.fc1.weight" in output
else:
- assert "model.layers.0.mlp.fc1.qweight" in output
+ assert "model.layers.0.mlp.fc1" in quantized
+ if name.startswith("MXFP"):
+ assert "model.layers.0.mlp.fc1.weight_scale" in output
+ else:
+ assert "model.layers.0.mlp.fc1.qweight" in output
@pytest.mark.parametrize("name", _UNSUPPORTED)
def test_unsupported_raises(self, tmp_path, name):
@@ -1400,6 +1467,11 @@ class TestModelFreeAutoScheme:
assert _validate_auto_scheme_options(AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "W8A16"))) == "int"
+ def test_validate_options_int_family_with_bf16(self):
+ from auto_round import AutoScheme
+
+ assert _validate_auto_scheme_options(AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16"))) == "int"
+
def test_validate_options_mxfp_family(self):
from auto_round import AutoScheme
@@ -1440,6 +1512,22 @@ class TestModelFreeAutoScheme:
assert "model.embed_tokens" not in per_layer
assert fp16_layers == ["model.embed_tokens"]
+ def test_convert_layer_config_infers_mxfp_bits_from_dtype_alias(self):
+ generated = {
+ "model.layers.0.q_proj": {"group_size": 32, "sym": True, "data_type": "mxfp8"},
+ "model.layers.0.k_proj": {"group_size": 32, "sym": True, "data_type": "MXFP4"},
+ }
+
+ base_scheme, per_layer, fp16_layers = _convert_auto_scheme_layer_config(generated)
+
+ assert fp16_layers == []
+ assert per_layer["model.layers.0.q_proj"]["bits"] == 8
+ assert per_layer["model.layers.0.q_proj"]["data_type"] == "mx_fp"
+ assert per_layer["model.layers.0.k_proj"]["bits"] == 4
+ assert per_layer["model.layers.0.k_proj"]["data_type"] == "mx_fp"
+ assert base_scheme.data_type == "mx_fp"
+ assert base_scheme.bits in (4, 8)
+
def test_e2e_int_auto_scheme(self, tmp_path, tiny_opt_model_path):
from auto_round import AutoScheme
@@ -1468,3 +1556,269 @@ class TestModelFreeAutoScheme:
qc = _read_qconfig(output_dir)
assert qc["quant_method"] == "compressed-tensors"
assert qc["provider"] == "auto-round"
+
+
+# ===========================================================================
+# MXFP auto-round format (packing_format=auto_round:llm_compressor)
+# ===========================================================================
+
+
+class TestMXFPAutoRoundFormat:
+ """Tests for MXFP4/MXFP8 model-free quantization with format='auto_round'.
+
+ The weight tensors on disk are identical to the llm_compressor path;
+ only the ``quantization_config`` metadata differs.
+ """
+
+ # ------------------------------------------------------------------
+ # Unit tests: _build_mxfp_autoround_quantization_config
+ # ------------------------------------------------------------------
+
+ @require_compressed_tensors
+ def test_mxfp4_top_level_fields(self):
+ """MXFP4 scheme → all mandatory top-level fields are correct."""
+ from dataclasses import asdict
+
+ from auto_round.schemes import PRESET_SCHEMES
+
+ default = {k: v for k, v in asdict(PRESET_SCHEMES["MXFP4"]).items() if v is not None}
+ cfg = _build_mxfp_autoround_quantization_config(default, quantized_layers=["layer.fc1"], ignored_layers=[])
+ assert cfg["quant_method"] == "auto-round"
+ assert cfg["packing_format"] == "auto_round:llm_compressor"
+ assert cfg["bits"] == 4
+ assert cfg["group_size"] == 32
+ assert cfg["data_type"] == "mx_fp"
+ assert cfg["sym"] is True
+ assert cfg["enable_quanted_input"] is False
+ assert cfg["model_free"] is True
+ assert "autoround_version" in cfg
+
+ @require_compressed_tensors
+ def test_mxfp4_activation_fields(self):
+ """MXFP4 scheme carries act_* fields matching the real AutoRound output."""
+ from dataclasses import asdict
+
+ from auto_round.schemes import PRESET_SCHEMES
+
+ default = {k: v for k, v in asdict(PRESET_SCHEMES["MXFP4"]).items() if v is not None}
+ cfg = _build_mxfp_autoround_quantization_config(default, quantized_layers=["layer.fc1"], ignored_layers=[])
+ assert cfg["act_bits"] == 4
+ assert cfg["act_data_type"] == "mx_fp"
+ assert cfg["act_dynamic"] is True
+ assert cfg["act_group_size"] == 32
+ assert cfg["act_sym"] is True
+
+ @require_compressed_tensors
+ def test_mxfp8_bits(self):
+ """MXFP8 scheme → bits=8, act_bits=8."""
+ from dataclasses import asdict
+
+ from auto_round.schemes import PRESET_SCHEMES
+
+ default = {k: v for k, v in asdict(PRESET_SCHEMES["MXFP8"]).items() if v is not None}
+ cfg = _build_mxfp_autoround_quantization_config(default, quantized_layers=["layer.fc1"], ignored_layers=[])
+ assert cfg["bits"] == 8
+ assert cfg["act_bits"] == 8
+ assert cfg["packing_format"] == "auto_round:llm_compressor"
+
+ @require_compressed_tensors
+ def test_ignored_layers_in_extra_config(self):
+ """Ignored layers (non-embed/conv) go into extra_config as bits=16."""
+ default = {
+ "bits": 4,
+ "group_size": 32,
+ "sym": True,
+ "data_type": "mx_fp",
+ "act_bits": 4,
+ "act_data_type": "mx_fp",
+ "act_dynamic": True,
+ "act_group_size": 32,
+ "act_sym": True,
+ }
+ ignored = ["lm_head", "model.embed_tokens", "model.conv1", "model.layers.0.mlp.gate"]
+ cfg = _build_mxfp_autoround_quantization_config(
+ default,
+ quantized_layers=["model.layers.0.fc1"],
+ ignored_layers=ignored,
+ )
+ extra = cfg.get("extra_config", {})
+ assert extra.get("lm_head") == {"bits": 16, "data_type": "float"}
+ assert extra.get("model.layers.0.mlp.gate") == {"bits": 16, "data_type": "float"}
+ # embed / conv are non-Linear — filtered out
+ assert "model.embed_tokens" not in extra
+ assert "model.conv1" not in extra
+
+ @require_compressed_tensors
+ def test_quantized_lm_head_in_extra_config(self):
+ """When lm_head is quantized it must appear in extra_config with its scheme."""
+ default = {
+ "bits": 4,
+ "group_size": 32,
+ "sym": True,
+ "data_type": "mx_fp",
+ "act_bits": 4,
+ "act_data_type": "mx_fp",
+ "act_dynamic": True,
+ "act_group_size": 32,
+ "act_sym": True,
+ }
+ cfg = _build_mxfp_autoround_quantization_config(
+ default,
+ quantized_layers=["model.layers.0.fc1", "lm_head"],
+ ignored_layers=[],
+ )
+ extra = cfg.get("extra_config", {})
+ assert "lm_head" in extra
+ assert extra["lm_head"]["bits"] == 4
+ assert extra["lm_head"]["data_type"] == "mx_fp"
+
+ @require_compressed_tensors
+ def test_build_quantization_config_routes_mxfp_to_autoround(self):
+ """_build_quantization_config with format='auto_round' routes MXFP to auto-round style."""
+ default = {
+ "bits": 4,
+ "group_size": 32,
+ "sym": True,
+ "data_type": "mx_fp",
+ "act_bits": 4,
+ "act_data_type": "mx_fp",
+ "act_dynamic": True,
+ "act_group_size": 32,
+ "act_sym": True,
+ }
+ cfg = _build_quantization_config(
+ default_scheme=default,
+ layer_config={},
+ ignore_patterns=[],
+ quantized_layers=["layer.fc1"],
+ ignored_layers=[],
+ format="auto_round",
+ )
+ assert cfg["quant_method"] == "auto-round"
+ assert cfg["packing_format"] == "auto_round:llm_compressor"
+
+ @require_compressed_tensors
+ def test_build_quantization_config_mxfp_llmcompressor_path_unchanged(self):
+ """_build_quantization_config with format='llm_compressor' still produces compressed-tensors."""
+ default = {"bits": 4, "group_size": 32, "sym": True, "data_type": "mx_fp"}
+ cfg = _build_quantization_config(
+ default_scheme=default,
+ layer_config={},
+ ignore_patterns=[],
+ quantized_layers=["layer.fc1"],
+ ignored_layers=["lm_head"],
+ format="llm_compressor",
+ )
+ assert cfg["quant_method"] == "compressed-tensors"
+ assert cfg["format"] == "mxfp4-pack-quantized"
+
+ # ------------------------------------------------------------------
+ # End-to-end tests via AutoRound.quantize_and_save
+ # ------------------------------------------------------------------
+
+ @require_compressed_tensors
+ def test_e2e_mxfp4_config_fields(self, tmp_path):
+ """MXFP4 + format='auto_round': quantization_config matches real AutoRound output."""
+ tensors = {
+ "model.layers.0.self_attn.q_proj.weight": torch.randn(128, 128),
+ "model.layers.0.fc1.weight": torch.randn(512, 128),
+ "lm_head.weight": torch.randn(1000, 128),
+ }
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
+ output_dir = str(tmp_path / "output")
+ AutoRound(model=model_dir, scheme="MXFP4", model_free=True).quantize_and_save(output_dir, format="auto_round")
+ qc = _read_qconfig(output_dir)
+ assert qc["quant_method"] == "auto-round"
+ assert qc["packing_format"] == "auto_round:llm_compressor"
+ assert qc["bits"] == 4
+ assert qc["data_type"] == "mx_fp"
+ assert qc["act_bits"] == 4
+ assert qc["act_data_type"] == "mx_fp"
+ assert qc["enable_quanted_input"] is False
+ assert qc["model_free"] is True
+ # lm_head kept full-precision → extra_config entry
+ extra = qc.get("extra_config", {})
+ assert extra.get("lm_head") == {"bits": 16, "data_type": "float"}
+
+ @require_compressed_tensors
+ def test_e2e_mxfp4_weight_tensors(self, tmp_path):
+ """MXFP4 + format='auto_round': on-disk weight layout is unchanged (weight_packed + weight_scale)."""
+ tensors = {
+ "model.layers.0.fc1.weight": torch.randn(512, 128),
+ "lm_head.weight": torch.randn(1000, 128),
+ }
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
+ output_dir = str(tmp_path / "output")
+ AutoRound(model=model_dir, scheme="MXFP4", model_free=True).quantize_and_save(output_dir, format="auto_round")
+ keys = _read_output_keys(output_dir)
+ assert "model.layers.0.fc1.weight_packed" in keys
+ assert "model.layers.0.fc1.weight_scale" in keys
+ assert "lm_head.weight" in keys
+ assert "lm_head.weight_packed" not in keys
+
+ @require_compressed_tensors
+ def test_e2e_mxfp8_autoround_format(self, tmp_path):
+ """MXFP8 + format='auto_round': bits=8, weight layout uses float8 .weight."""
+ tensors = {
+ "model.layers.0.fc1.weight": torch.randn(128, 128),
+ "lm_head.weight": torch.randn(1000, 128),
+ }
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
+ output_dir = str(tmp_path / "output")
+ AutoRound(model=model_dir, scheme="MXFP8", model_free=True).quantize_and_save(output_dir, format="auto_round")
+ qc = _read_qconfig(output_dir)
+ assert qc["quant_method"] == "auto-round"
+ assert qc["packing_format"] == "auto_round:llm_compressor"
+ assert qc["bits"] == 8
+ assert qc["act_bits"] == 8
+ keys = _read_output_keys(output_dir)
+ assert "model.layers.0.fc1.weight" in keys
+ assert "model.layers.0.fc1.weight_scale" in keys
+
+ @require_compressed_tensors
+ def test_e2e_mxfp4_quant_lm_head_extra_config(self, tmp_path):
+ """When lm_head is quantized, it appears in extra_config with its full scheme."""
+ tensors = {
+ "model.layers.0.fc1.weight": torch.randn(128, 128),
+ "lm_head.weight": torch.randn(1000, 128),
+ }
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
+ output_dir = str(tmp_path / "output")
+ AutoRound(model=model_dir, scheme="MXFP4", model_free=True, quant_lm_head=True).quantize_and_save(
+ output_dir, format="auto_round"
+ )
+ qc = _read_qconfig(output_dir)
+ extra = qc.get("extra_config", {})
+ assert "lm_head" in extra
+ assert extra["lm_head"]["bits"] == 4
+ assert extra["lm_head"]["data_type"] == "mx_fp"
+ keys = _read_output_keys(output_dir)
+ assert "lm_head.weight_packed" in keys
+
+ @require_compressed_tensors
+ def test_e2e_mxfp4_autoround_format_same_weights_as_llmcompressor(self, tmp_path):
+ """auto_round and llm_compressor format produce identical weight bytes."""
+ tensors = {"model.layers.0.fc1.weight": torch.randn(64, 128)}
+ model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
+
+ out_ar = str(tmp_path / "out_ar")
+ out_llm = str(tmp_path / "out_llm")
+ AutoRound(model=model_dir, scheme="MXFP4", model_free=True).quantize_and_save(out_ar, format="auto_round")
+ AutoRound(model=model_dir, scheme="MXFP4", model_free=True).quantize_and_save(out_llm, format="llm_compressor")
+
+ # Weight bytes must match; only quantization_config differs.
+ def _load_tensor(directory, name):
+ for f in os.listdir(directory):
+ if f.endswith(".safetensors"):
+ with safe_open(os.path.join(directory, f), framework="pt") as sf:
+ if name in sf.keys():
+ return sf.get_tensor(name)
+ return None
+
+ wp_ar = _load_tensor(out_ar, "model.layers.0.fc1.weight_packed")
+ ws_ar = _load_tensor(out_ar, "model.layers.0.fc1.weight_scale")
+ wp_llm = _load_tensor(out_llm, "model.layers.0.fc1.weight_packed")
+ ws_llm = _load_tensor(out_llm, "model.layers.0.fc1.weight_scale")
+ assert wp_ar is not None and wp_llm is not None
+ assert torch.equal(wp_ar, wp_llm), "weight_packed must be identical"
+ assert torch.equal(ws_ar, ws_llm), "weight_scale must be identical"
diff --git a/test/test_cpu/schemes/test_auto_scheme.py b/test/test_cpu/schemes/test_auto_scheme.py
index 304b4d755..d6a2ce168 100644
--- a/test/test_cpu/schemes/test_auto_scheme.py
+++ b/test/test_cpu/schemes/test_auto_scheme.py
@@ -1,9 +1,209 @@
+import json
+import os
import shutil
import pytest
-from transformers import AutoRoundConfig
from auto_round import AutoRound, AutoScheme
+from auto_round.auto_scheme.utils import _build_layer_config_header_rows, _short_summary_name
+
+
+ "model_type, expected",
+ [
+ ("mllm", ("model", "tokenizer", "processor", "image_processor", None, True, False)),
+ ("diffusion", ("model", None, None, None, "pipe", False, True)),
+ ("llm", ("model", "tokenizer", None, None, None, False, False)),
+ ],
+)
+def test_load_model_returns_named_values_for_each_model_type(monkeypatch, model_type, expected):
+ import auto_round.utils.model as model_utils
+
+ monkeypatch.setattr(model_utils, "is_mllm_model", lambda *_args, **_kwargs: model_type == "mllm")
+ monkeypatch.setattr(model_utils, "is_diffusion_model", lambda *_args, **_kwargs: model_type == "diffusion")
+ monkeypatch.setattr(
+ model_utils,
+ "mllm_load_model",
+ lambda *_args, **_kwargs: ("model", "processor", "tokenizer", "image_processor"),
+ )
+ monkeypatch.setattr(model_utils, "diffusion_load_model", lambda *_args, **_kwargs: ("pipe", "model"))
+ monkeypatch.setattr(model_utils, "llm_load_model", lambda *_args, **_kwargs: ("model", "tokenizer"))
+ monkeypatch.setattr("auto_round.modeling.unfused_moe.apply_model_monkey_patches", lambda *_args, **_kwargs: None)
+
+ assert model_utils.load_model("model-id") == expected
+
+
+def test_env_ar_auto_scheme_nsamples_overrides_default(monkeypatch):
+ """AR_AUTO_SCHEME_NSAMPLES env var should override the built-in nsamples heuristic."""
+ import auto_round.envs as envs
+
+ monkeypatch.setenv("AR_AUTO_SCHEME_NSAMPLES", "7")
+ assert envs.AR_AUTO_SCHEME_NSAMPLES == 7
+
+
+def test_env_ar_auto_scheme_batch_size_overrides_default(monkeypatch):
+ """AR_AUTO_SCHEME_BATCH_SIZE env var should override the built-in batch_size default."""
+ import auto_round.envs as envs
+
+ monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "4")
+ assert envs.AR_AUTO_SCHEME_BATCH_SIZE == 4
+
+
+def test_env_ar_auto_scheme_batch_size_zero_raises(monkeypatch):
+ """Zero value for AR_AUTO_SCHEME_BATCH_SIZE should raise ValueError."""
+ import pytest
+
+ import auto_round.envs as envs
+
+ monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "0")
+ with pytest.raises(ValueError):
+ _ = envs.AR_AUTO_SCHEME_BATCH_SIZE
+
+
+def test_env_ar_enable_auto_scheme_parallel(monkeypatch):
+ import auto_round.envs as envs
+
+ monkeypatch.delenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", raising=False)
+ assert envs.AR_ENABLE_AUTO_SCHEME_PARALLEL is False
+ monkeypatch.setenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "1")
+ assert envs.AR_ENABLE_AUTO_SCHEME_PARALLEL is True
+
+
+def test_build_layer_config_header_rows_merges_adjacent_prefixes():
+ """Adjacent columns with the same prefix should be merged into one compact header cell."""
+ columns = ["mlp.down_proj", "mlp.gate_proj", "self_attn.q_proj", "self_attn.v_proj"]
+ assert _build_layer_config_header_rows(columns) == [
+ ["block", "mlp", "", "self_attn", ""],
+ ["", "down_proj", "gate_proj", "q_proj", "v_proj"],
+ ]
+
+
+def test_build_layer_config_header_rows_includes_experts_under_mlp():
+ columns = ["mlp.down_proj", "self_attn.q_proj"]
+ assert _build_layer_config_header_rows(columns, has_expert_layers=True) == [
+ ["block", "mlp", "self_attn", "mlp"],
+ ["", "down_proj", "q_proj", "experts"],
+ ]
+
+
+def test_short_summary_name_keeps_one_field_before_numeric_suffix():
+ """Numeric block suffixes should be shortened to keep the preceding field."""
+ assert _short_summary_name("model.layers.0") == "layers.0"
+
+
+def test_get_layer_config_supports_avg_bits_list(monkeypatch):
+ from auto_round import auto_scheme
+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme, GenScheme
+
+ scheme = AutoScheme(avg_bits=[2.5, 3.5], options=("W2A16", "W4A16"))
+ generator = GenScheme.__new__(GenScheme)
+ generator.auto_scheme = scheme
+ generator.model = object()
+ generator.quant_layer_names = []
+ generator.fixed_layer_scheme = {}
+ generator.dataset = "pile-10k"
+ generator.tokenizer = object()
+ generator.device_map = None
+ generator.enable_torch_compile = False
+ generator.min_avg_bit_scheme = "W2A16"
+ generator.processor = None
+
+ def method_func(auto_scheme, *args, **kwargs):
+ return {"layer": {"bits": auto_scheme.avg_bits}}
+
+ monkeypatch.setitem(auto_scheme.AUTO_SCHEME_METHODS, "default", method_func)
+
+ assert generator.get_layer_config() == {
+ 2.5: {"layer": {"bits": 2.5}},
+ 3.5: {"layer": {"bits": 3.5}},
+ }
+ assert scheme.avg_bits == [2.5, 3.5]
+
+
+def test_choose_bits_per_layer_reconstructs_optimal_path():
+ """DP parent pointers should preserve the optimal choices in layer order."""
+ from auto_round.auto_scheme.delta_loss import choose_bits_per_layer_with_path
+
+ layers = {
+ "layer.0": [(0, 2, 4.0, ["layer.0"]), (1, 4, 1.0, ["layer.0"])],
+ "layer.1": [(0, 2, 3.0, ["layer.1"]), (1, 4, 0.5, ["layer.1"])],
+ }
+
+ loss, path = choose_bits_per_layer_with_path(layers, P=6)
+
+ assert loss == 4.0
+ assert path == [(["layer.0"], 1), (["layer.1"], 0)]
+
+
+def test_build_expert_groups_groups_experts_per_block():
+ """Expert layers in the same block should be grouped together."""
+ import torch
+ from torch import nn
+
+ from auto_round.auto_scheme.utils import build_expert_groups
+
+ # Build a minimal MoE-like model with 2 blocks, each with 2 experts having 2 projections
+ class FakeModel(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = nn.Module()
+ self.model.layers = nn.ModuleList()
+ for i in range(2):
+ block = nn.Module()
+ block.mlp = nn.Module()
+ block.mlp.experts = nn.ModuleList()
+ for j in range(2):
+ expert = nn.Module()
+ expert.gate_proj = nn.Linear(8, 8, bias=False)
+ expert.up_proj = nn.Linear(8, 8, bias=False)
+ expert.down_proj = nn.Linear(8, 8, bias=False)
+ block.mlp.experts.append(expert)
+ block.self_attn = nn.Module()
+ block.self_attn.q_proj = nn.Linear(8, 8, bias=False)
+ self.model.layers.append(block)
+
+ model = FakeModel()
+ quant_layer_names = [n for n, m in model.named_modules() if isinstance(m, nn.Linear)]
+ fixed_layer_scheme = {}
+
+ groups = build_expert_groups(model, quant_layer_names, fixed_layer_scheme)
+ # Should have 2 groups (one per block), each containing all 6 expert projections
+ assert len(groups) == 2
+ for group in groups:
+ expert_layers = [n for n in group if "experts" in n]
+ assert len(expert_layers) == 6 # 2 experts * 3 projections
+ # Non-expert layers (q_proj) should NOT be in the group
+ assert all("self_attn" not in n for n in group)
+
+
+def test_build_expert_groups_skips_fixed_layers():
+ """Expert layers already in fixed_layer_scheme should not be grouped."""
+ import torch
+ from torch import nn
+
+ from auto_round.auto_scheme.utils import build_expert_groups
+
+ class FakeModel(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = nn.Module()
+ self.model.layers = nn.ModuleList()
+ block = nn.Module()
+ block.mlp = nn.Module()
+ block.mlp.experts = nn.ModuleList()
+ for j in range(2):
+ expert = nn.Module()
+ expert.gate_proj = nn.Linear(8, 8, bias=False)
+ block.mlp.experts.append(expert)
+ self.model.layers.append(block)
+
+ model = FakeModel()
+ quant_layer_names = [n for n, m in model.named_modules() if isinstance(m, nn.Linear)]
+ # Fix all expert layers
+ fixed_layer_scheme = {n: {} for n in quant_layer_names if "experts" in n}
+
+ groups = build_expert_groups(model, quant_layer_names, fixed_layer_scheme)
+ assert len(groups) == 0
class TestAutoScheme:
@@ -19,13 +219,28 @@ class TestAutoScheme:
def test_auto_scheme_export(self, tiny_opt_model_path):
model_name = tiny_opt_model_path
+ int_save_dir = os.path.join(self.save_dir, "int")
scheme = AutoScheme(avg_bits=2, options=("W2A16"), nsamples=1, ignore_scale_zp_bits=True)
ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)
- ar.quantize_and_save(self.save_dir)
+ _, int_model_path = ar.quantize_and_save(int_save_dir)
+
+ with open(os.path.join(int_model_path, "config.json")) as f:
+ int_config = json.load(f)["quantization_config"]
+ assert int_config["quant_method"] == "auto-round"
+ assert int_config["bits"] == 2
+ assert int_config["data_type"] == "int"
+ mxfp_save_dir = os.path.join(self.save_dir, "mxfp")
scheme = AutoScheme(avg_bits=4, options=("mxfp4"), nsamples=1, ignore_scale_zp_bits=True)
ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)
- ar.quantize_and_save(self.save_dir)
+ _, mxfp_model_path = ar.quantize_and_save(mxfp_save_dir)
+
+ with open(os.path.join(mxfp_model_path, "config.json")) as f:
+ mxfp_config = json.load(f)["quantization_config"]
+ assert mxfp_config["quant_method"] == "auto-round"
+ assert mxfp_config["bits"] == 4
+ assert mxfp_config["data_type"] == "mx_fp"
+ assert os.path.exists(os.path.join(int_model_path, "config.json"))
def test_gguf_user_fixed_embedding_budget(self, tiny_qwen_model_path):
"""Regression test: a user-fixed embedding must be budget-priced at its fixed bits.
@@ -127,3 +342,439 @@ class TestAutoScheme:
avg_bits, _ = compute_avg_bits_for_model(model)
print(avg_bits)
assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3
+
+ def test_cache_files_saved_with_correct_format(self, tiny_opt_model_path, tmp_path, monkeypatch):
+ """After AutoScheme runs, per-scheme JSON cache files must exist with individual layer scores."""
+ import glob
+ import json
+
+ from auto_round.auto_scheme.delta_loss import _load_autoscheme_scores
+
+ cache_dir = str(tmp_path / "ar_work_space")
+ monkeypatch.setenv("AR_WORK_SPACE", cache_dir)
+
+ scheme = AutoScheme(
+ avg_bits=3,
+ options=("W2A16", "W4A16"),
+ nsamples=1,
+ ignore_scale_zp_bits=True,
+ )
+ ar = AutoRound(model=tiny_opt_model_path, scheme=scheme, iters=0, nsamples=1)
+ _, layer_config = ar.quantize()
+
+ # Cache files must exist — one per scheme (2 schemes here)
+ cache_files = glob.glob(f"{cache_dir}/auto_scheme_cache/scheme_*.json")
+ assert (
+ len(cache_files) == 2
+ ), f"Expected 2 cache files (one per scheme), found {len(cache_files)}: {cache_files}"
+
+ for path in cache_files:
+ data = _load_autoscheme_scores(path)
+ assert data is not None, f"Cache file {path} could not be loaded"
+ assert data["version"] == 3
+ assert data["score_granularity"] == "per_op"
+ assert "layer_scores" in data
+ assert "total_loss_for_scheme" in data
+
+ # Every layer in layer_scores must have a [bits, loss] pair (individual, not merged)
+ for layer_name, score_pair in data["layer_scores"].items():
+ assert (
+ isinstance(score_pair, list) and len(score_pair) == 2
+ ), f"layer_scores[{layer_name!r}] should be [bits, loss], got {score_pair}"
+ bits, loss = score_pair
+ assert bits > 0, f"bits must be positive for {layer_name}"
+ assert loss >= 0, f"loss must be non-negative for {layer_name}"
+
+ # All non-fixed linear layers in layer_config should appear individually in cache
+ for layer_name in layer_config:
+ assert layer_name in data["layer_scores"], (
+ f"Layer {layer_name!r} missing from cache {path} — "
+ f"cache may have stored merged group scores instead of individual scores"
+ )
+
+ def test_different_avg_bits_produces_different_layer_config(self, tiny_opt_model_path):
+ """Changing avg_bits should change the resulting layer_config."""
+ scheme_low = AutoScheme(
+ avg_bits=2.5,
+ options=("W2A16", "W4A16"),
+ nsamples=1,
+ ignore_scale_zp_bits=True,
+ )
+ ar_low = AutoRound(model=tiny_opt_model_path, scheme=scheme_low, iters=0, nsamples=1)
+ _, config_low = ar_low.quantize()
+
+ scheme_high = AutoScheme(
+ avg_bits=3.5,
+ options=("W2A16", "W4A16"),
+ nsamples=1,
+ ignore_scale_zp_bits=True,
+ )
+ ar_high = AutoRound(model=tiny_opt_model_path, scheme=scheme_high, iters=0, nsamples=1)
+ _, config_high = ar_high.quantize()
+
+ low_avg = sum(v["bits"] for v in config_low.values() if "bits" in v) / max(
+ len([v for v in config_low.values() if "bits" in v]), 1
+ )
+ high_avg = sum(v["bits"] for v in config_high.values() if "bits" in v) / max(
+ len([v for v in config_high.values() if "bits" in v]), 1
+ )
+ assert high_avg > low_avg, (
+ f"avg_bits=4 should produce higher average bits than avg_bits=2, "
+ f"got low={low_avg:.2f} high={high_avg:.2f}"
+ )
+
+ def test_shared_layers_assigns_same_bits(self, tiny_opt_model_path):
+ """With shared_layers=[q_proj,k_proj,v_proj], all three must get the same bits per block."""
+ scheme = AutoScheme(
+ avg_bits=5,
+ options=("MXFP4", "MXFP8"),
+ nsamples=1,
+ ignore_scale_zp_bits=True,
+ shared_layers=[["fc1", "fc2"]],
+ )
+ ar = AutoRound(model=tiny_opt_model_path, scheme=scheme, iters=0, nsamples=1)
+ _, layer_config = ar.quantize()
+
+ scheme = AutoScheme(
+ avg_bits=5,
+ options=("MXFP4", "MXFP8"),
+ nsamples=1,
+ ignore_scale_zp_bits=True,
+ shared_layers=[["q_proj", "k_proj", "v_proj"]],
+ )
+ ar = AutoRound(model=tiny_opt_model_path, scheme=scheme, iters=0, nsamples=1)
+ _, layer_config = ar.quantize()
+
+ # Collect per-block q/k/v bits
+ block_qkv: dict[str, dict[str, int]] = {}
+ for name, cfg in layer_config.items():
+ if "bits" not in cfg:
+ continue
+ for proj in ("q_proj", "k_proj", "v_proj"):
+ if name.endswith("." + proj):
+ prefix = name[: -len(proj) - 1]
+ block_qkv.setdefault(prefix, {})[proj] = cfg["bits"]
+
+ assert block_qkv, "No q/k/v layers found in layer_config"
+ for prefix, proj_bits in block_qkv.items():
+ present = list(proj_bits.keys())
+ bits_values = list(proj_bits.values())
+ assert len(set(bits_values)) == 1, (
+ f"Block {prefix!r}: q/k/v should all have the same bits with shared_layers, "
+ f"got {dict(zip(present, bits_values))}"
+ )
+
+
+def test_autoscheme_cache_key_different_for_different_schemes():
+ """Per-scheme cache: different schemes should produce different cache keys."""
+ from auto_round.auto_scheme.delta_loss import _autoscheme_cache_key
+
+ key_w4 = _autoscheme_cache_key(
+ model_name="test-model",
+ dataset="pile-10k",
+ nsamples=16,
+ seqlen=256,
+ batch_size=8,
+ quant_layer_names=["layer.0"],
+ fixed_layer_scheme={},
+ scheme="W4A16",
+ force_mllm=False,
+ low_gpu_mem_usage=True,
+ )
+ key_w8 = _autoscheme_cache_key(
+ model_name="test-model",
+ dataset="pile-10k",
+ nsamples=16,
+ seqlen=256,
+ batch_size=8,
+ quant_layer_names=["layer.0"],
+ fixed_layer_scheme={},
+ scheme="W8A16",
+ force_mllm=False,
+ low_gpu_mem_usage=True,
+ )
+ assert key_w4 != key_w8
+ assert len(key_w4) == 16 # sha256 truncated to 16 chars
+
+
+def test_autoscheme_cache_key_insensitive_to_layer_order():
+ """Per-scheme cache: layer order should not affect the key (internally sorted)."""
+ from auto_round.auto_scheme.delta_loss import _autoscheme_cache_key
+
+ key1 = _autoscheme_cache_key(
+ model_name="test-model",
+ dataset="pile-10k",
+ nsamples=16,
+ seqlen=256,
+ batch_size=8,
+ quant_layer_names=["layer.0", "layer.1"],
+ fixed_layer_scheme={},
+ scheme="W4A16",
+ force_mllm=False,
+ low_gpu_mem_usage=True,
+ )
+ key2 = _autoscheme_cache_key(
+ model_name="test-model",
+ dataset="pile-10k",
+ nsamples=16,
+ seqlen=256,
+ batch_size=8,
+ quant_layer_names=["layer.1", "layer.0"], # Different order
+ fixed_layer_scheme={},
+ scheme="W4A16",
+ force_mllm=False,
+ low_gpu_mem_usage=True,
+ )
+ assert key1 == key2 # Should match after internal sorting
+
+
+def test_autoscheme_cache_key_changes_only_with_scoring_config():
+ """Execution settings that affect scoring invalidate the cache; bit accounting does not."""
+ from auto_round.auto_scheme.delta_loss import _autoscheme_cache_key
+
+ kwargs = {
+ "model_name": "test-model",
+ "dataset": "pile-10k",
+ "nsamples": 16,
+ "seqlen": 256,
+ "batch_size": 8,
+ "quant_layer_names": ["layer.0"],
+ "fixed_layer_scheme": {},
+ "scheme": "W4A16",
+ "force_mllm": False,
+ "low_gpu_mem_usage": True,
+ }
+ baseline = _autoscheme_cache_key(**kwargs)
+
+ assert baseline != _autoscheme_cache_key(**{**kwargs, "low_gpu_mem_usage": False})
+
+
+def test_refresh_cached_layer_bits_preserves_loss():
+ import torch
+ from torch import nn
+
+ from auto_round.auto_scheme.delta_loss import _refresh_cached_layer_bits
+
+ model = nn.Module()
+ model.layer = nn.Linear(8, 4, bias=False)
+ scheme = {"bits": 4, "group_size": 4, "sym": True, "data_type": "int", "act_bits": 16}
+ cached_scores = {"layer": [999, 1.25]}
+
+ with_overhead = _refresh_cached_layer_bits(model, ["layer"], {}, scheme, cached_scores, False)
+ without_overhead = _refresh_cached_layer_bits(model, ["layer"], {}, scheme, cached_scores, True)
+
+ assert with_overhead["layer"][0] > without_overhead["layer"][0]
+ assert without_overhead["layer"] == [model.layer.weight.numel() * 4, 1.25]
+ assert cached_scores == {"layer": [999, 1.25]}
+
+
+def test_autoscheme_cache_save_and_load(tmp_path):
+ """Per-scheme cache: scores can be saved and loaded correctly."""
+ from auto_round.auto_scheme.delta_loss import (
+ _load_autoscheme_scores,
+ _save_autoscheme_scores,
+ )
+
+ cache_key = "test_key_123"
+ cache_path = os.path.join(str(tmp_path), f"scheme_00_{cache_key}.json")
+
+ scheme_dict = {"bits": 4, "act_bits": 16}
+ layer_scores = {
+ "layer.0": [4, 1.2],
+ "layer.1": [4, 0.9],
+ }
+ total_loss = 2.1
+ total_params = 1000000
+
+ _save_autoscheme_scores(
+ cache_path,
+ cache_key,
+ 0,
+ scheme_dict,
+ layer_scores,
+ total_loss,
+ total_params,
+ )
+
+ loaded = _load_autoscheme_scores(cache_path)
+ assert loaded is not None
+ assert loaded["layer_scores"] == layer_scores
+ assert loaded["total_loss_for_scheme"] == total_loss
+ assert loaded["total_params"] == total_params
+
+
+def test_parallel_progress_events_are_applied_in_parent():
+ """Worker progress events should update only the parent-owned progress bar."""
+ import queue
+
+ from auto_round.auto_scheme.delta_loss import _drain_progress_queue, _ProgressQueueProxy
+
+ class FakeProgressBar:
+ def __init__(self):
+ self.steps = 0
+ self.messages = []
+
+ def update(self, steps):
+ self.steps += steps
+
+ def write(self, message):
+ self.messages.append(message)
+
+ progress_queue = queue.Queue()
+ worker_progress = _ProgressQueueProxy(progress_queue)
+ parent_progress = FakeProgressBar()
+
+ worker_progress.update(2)
+ worker_progress.write("scheme progress")
+ worker_progress.update()
+ _drain_progress_queue(progress_queue, parent_progress)
+
+ assert parent_progress.steps == 3
+ assert parent_progress.messages == ["scheme progress"]
+ assert progress_queue.empty()
+
+
+def test_assign_scheme_worker_devices_round_robin():
+ from auto_round.auto_scheme.delta_loss import _assign_scheme_worker_devices
+
+ assert _assign_scheme_worker_devices(5, ["cuda:2", "cuda:5"]) == [
+ "cuda:2",
+ "cuda:5",
+ "cuda:2",
+ "cuda:5",
+ "cuda:2",
+ ]
+
+
+def test_assign_scheme_worker_devices_shares_single_gpu():
+ from auto_round.auto_scheme.delta_loss import _assign_scheme_worker_devices
+
+ assert _assign_scheme_worker_devices(3, ["cuda:3"]) == ["cuda:3", "cuda:3", "cuda:3"]
+
+
+def test_assign_scheme_worker_devices_rejects_no_gpu():
+ from auto_round.auto_scheme.delta_loss import _assign_scheme_worker_devices
+
+ with pytest.raises(ValueError, match="at least one"):
+ _assign_scheme_worker_devices(1, [])
+
+
+def test_scheme_worker_count_allows_workers_to_share_gpus():
+ from auto_round.auto_scheme.delta_loss import _get_scheme_worker_count
+
+ assert _get_scheme_worker_count(5, 1) == 5
+ assert _get_scheme_worker_count(5, 2) == 5
+ assert _get_scheme_worker_count(1, 4) == 1
+
+
+def test_scheme_worker_count_rejects_no_gpu():
+ from auto_round.auto_scheme.delta_loss import _get_scheme_worker_count
+
+ with pytest.raises(ValueError, match="at least one GPU"):
+ _get_scheme_worker_count(2, 0)
+
+
+def test_opt_scheme_worker_uses_low_cpu_memory_loading(monkeypatch):
+ from test.helpers import opt_name_or_path
+
+ from auto_round.auto_scheme import delta_loss
+
+ captured = {}
+
+ def fake_load_model(model_name, **kwargs):
+ captured["model_name"] = model_name
+ captured.update(kwargs)
+ return ("model", "tokenizer", None, None, None, False, False)
+
+ monkeypatch.setattr(delta_loss, "load_model", fake_load_model)
+
+ result = delta_loss._load_scheme_worker_model(opt_name_or_path, True, True)
+
+ assert result[0] == "model"
+ assert captured == {
+ "model_name": opt_name_or_path,
+ "device": "cpu",
+ "use_auto_mapping": False,
+ "use_model_replacements": True,
+ "low_cpu_mem_usage": True,
+ }
+
+
+def test_per_op_cache_compatibility_rejects_grouped_scores():
+ from auto_round.auto_scheme.delta_loss import _is_per_op_cache_compatible
+
+ quant_layers = ["layer.0", "layer.1", "fixed"]
+ fixed_layers = {"fixed": {"bits": 8}}
+
+ assert _is_per_op_cache_compatible(
+ {"layer_scores": {"layer.0": [4, 1.0], "layer.1": [4, 2.0]}},
+ quant_layers,
+ fixed_layers,
+ )
+ assert not _is_per_op_cache_compatible(
+ {"layer_scores": {"layer.0": [8, 3.0]}},
+ quant_layers,
+ fixed_layers,
+ )
+
+
+def test_version_two_cache_is_rejected(tmp_path):
+ import json
+
+ from auto_round.auto_scheme.delta_loss import _load_autoscheme_scores
+
+ cache_path = tmp_path / "scheme_00_old.json"
+ cache_path.write_text(
+ json.dumps(
+ {
+ "version": 2,
+ "layer_scores": {"layer.0": [4, 1.0]},
+ "total_loss_for_scheme": 1.0,
+ "total_params": 4,
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ assert _load_autoscheme_scores(cache_path) is None
+
+
+def test_worker_memory_reports_cover_all_processes_and_devices():
+ from auto_round.auto_scheme.delta_loss import _merge_worker_memory_reports
+
+ class FakeMemoryMonitor:
+ peak_ram = 1.0
+ peak_vram = {"0": 1.0}
+
+ @staticmethod
+ def _process_tree_rss():
+ return 1.25
+
+ monitor = FakeMemoryMonitor()
+ _merge_worker_memory_reports(
+ monitor,
+ [
+ {"device": "0", "peak_ram": 0.5, "peak_vram": 2.0},
+ {"device": "0", "peak_ram": 0.75, "peak_vram": 3.0},
+ {"device": "1", "peak_ram": 0.25, "peak_vram": 4.0},
+ ],
+ )
+
+ assert monitor.peak_ram == 2.75
+ assert monitor.peak_vram == {"0": 5.0, "1": 4.0}
+
+
+def test_replacement_wrapper_without_tuning_device_uses_major_device():
+ import torch
+
+ from auto_round.auto_scheme.delta_loss import move_module_to_tuning_device
+
+ class ReplacementWrapper(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.orig_layer = torch.nn.Linear(2, 2)
+
+ wrapper = ReplacementWrapper()
+ move_module_to_tuning_device(wrapper, major_device="cpu")
+
+ assert wrapper.orig_layer.weight.device.type == "cpu"
diff --git a/test/test_cpu/schemes/test_auto_scheme_internal.py b/test/test_cpu/schemes/test_auto_scheme_internal.py
deleted file mode 100644
index d5080fa86..000000000
--- a/test/test_cpu/schemes/test_auto_scheme_internal.py
+++ /dev/null
@@ -1,116 +0,0 @@
-import os
-
-from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
-from auto_round.auto_scheme.utils import _build_layer_config_header_rows, _short_summary_name
-
-
-def test_env_ar_auto_scheme_nsamples_overrides_default(monkeypatch):
- """AR_AUTO_SCHEME_NSAMPLES env var should override the built-in nsamples heuristic."""
- import auto_round.envs as envs
-
- monkeypatch.setenv("AR_AUTO_SCHEME_NSAMPLES", "7")
- assert envs.AR_AUTO_SCHEME_NSAMPLES == 7
-
-
-def test_env_ar_auto_scheme_batch_size_overrides_default(monkeypatch):
- """AR_AUTO_SCHEME_BATCH_SIZE env var should override the built-in batch_size default."""
- import auto_round.envs as envs
-
- monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "4")
- assert envs.AR_AUTO_SCHEME_BATCH_SIZE == 4
-
-
-def test_env_ar_auto_scheme_batch_size_zero_raises(monkeypatch):
- """Zero value for AR_AUTO_SCHEME_BATCH_SIZE should raise ValueError."""
- import pytest
-
- import auto_round.envs as envs
-
- monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "0")
- with pytest.raises(ValueError):
- _ = envs.AR_AUTO_SCHEME_BATCH_SIZE
-
-
-def test_build_layer_config_header_rows_merges_adjacent_prefixes():
- """Adjacent columns with the same prefix should be merged into one compact header cell."""
- columns = ["mlp.down_proj", "mlp.gate_proj", "self_attn.q_proj", "self_attn.v_proj"]
- assert _build_layer_config_header_rows(columns) == [
- ["block", "mlp", "", "self_attn", ""],
- ["", "down_proj", "gate_proj", "q_proj", "v_proj"],
- ]
-
-
-def test_short_summary_name_keeps_one_field_before_numeric_suffix():
- """Numeric block suffixes should be shortened to keep the preceding field."""
- assert _short_summary_name("model.layers.0") == "layers.0"
-
-
-def test_build_expert_groups_groups_experts_per_block():
- """Expert layers in the same block should be grouped together."""
- import torch
- from torch import nn
-
- from auto_round.auto_scheme.utils import build_expert_groups
-
- # Build a minimal MoE-like model with 2 blocks, each with 2 experts having 2 projections
- class FakeModel(nn.Module):
- def __init__(self):
- super().__init__()
- self.model = nn.Module()
- self.model.layers = nn.ModuleList()
- for i in range(2):
- block = nn.Module()
- block.mlp = nn.Module()
- block.mlp.experts = nn.ModuleList()
- for j in range(2):
- expert = nn.Module()
- expert.gate_proj = nn.Linear(8, 8, bias=False)
- expert.up_proj = nn.Linear(8, 8, bias=False)
- expert.down_proj = nn.Linear(8, 8, bias=False)
- block.mlp.experts.append(expert)
- block.self_attn = nn.Module()
- block.self_attn.q_proj = nn.Linear(8, 8, bias=False)
- self.model.layers.append(block)
-
- model = FakeModel()
- quant_layer_names = [n for n, m in model.named_modules() if isinstance(m, nn.Linear)]
- fixed_layer_scheme = {}
-
- groups = build_expert_groups(model, quant_layer_names, fixed_layer_scheme)
- # Should have 2 groups (one per block), each containing all 6 expert projections
- assert len(groups) == 2
- for group in groups:
- expert_layers = [n for n in group if "experts" in n]
- assert len(expert_layers) == 6 # 2 experts * 3 projections
- # Non-expert layers (q_proj) should NOT be in the group
- assert all("self_attn" not in n for n in group)
-
-
-def test_build_expert_groups_skips_fixed_layers():
- """Expert layers already in fixed_layer_scheme should not be grouped."""
- import torch
- from torch import nn
-
- from auto_round.auto_scheme.utils import build_expert_groups
-
- class FakeModel(nn.Module):
- def __init__(self):
- super().__init__()
- self.model = nn.Module()
- self.model.layers = nn.ModuleList()
- block = nn.Module()
- block.mlp = nn.Module()
- block.mlp.experts = nn.ModuleList()
- for j in range(2):
- expert = nn.Module()
- expert.gate_proj = nn.Linear(8, 8, bias=False)
- block.mlp.experts.append(expert)
- self.model.layers.append(block)
-
- model = FakeModel()
- quant_layer_names = [n for n, m in model.named_modules() if isinstance(m, nn.Linear)]
- # Fix all expert layers
- fixed_layer_scheme = {n: {} for n in quant_layer_names if "experts" in n}
-
- groups = build_expert_groups(model, quant_layer_names, fixed_layer_scheme)
- assert len(groups) == 0
diff --git a/test/test_cpu/utils/test_cli_usage.py b/test/test_cpu/utils/test_cli_usage.py
index 81c490ad7..ac937ec92 100644
--- a/test/test_cpu/utils/test_cli_usage.py
+++ b/test/test_cpu/utils/test_cli_usage.py
@@ -249,3 +249,61 @@ def test_legacy_disable_flags_map_to_enable_bools():
assert args.enable_minmax_tuning is False
assert args.enable_quanted_input is False
+
+
+def _normalize_options(raw):
+ if raw is None:
+ return None
+ flat = ",".join(raw)
+ return ",".join(p.strip() for p in flat.split(",") if p.strip())
+
+
+def _normalize_shared_layers(raw):
+ if raw is None:
+ return None
+ normalized_groups = []
+ for invocation in raw:
+ if any("," in token for token in invocation):
+ for token in invocation:
+ group = [p.strip() for p in token.split(",") if p.strip()]
+ if group:
+ normalized_groups.append(group)
+ else:
+ group = [p.strip() for p in invocation if p.strip()]
+ if group:
+ normalized_groups.append(group)
+ return normalized_groups or None
+
+
+def test_options_comma_space_separated():
+ """--options accepts comma-separated and space-separated values."""
+ from auto_round.cli.parser import build_quantize_parser
+
+ p = build_quantize_parser()
+ assert _normalize_options(p.parse_args(["--avg_bits", "4", "--options", "W4A16,W8A16"]).options) == "W4A16,W8A16"
+ assert _normalize_options(p.parse_args(["--avg_bits", "4", "--options", "W4A16", "W8A16"]).options) == "W4A16,W8A16"
+ assert p.parse_args(["--model", "dummy"]).options is None
+
+
+def test_shared_layers_normalize():
+ """--shared_layers: bare tokens → one group; comma tokens → one group each; a,b c,d → two groups."""
+ from auto_round.cli.parser import build_quantize_parser
+
+ p = build_quantize_parser()
+ # bare tokens → one group
+ assert _normalize_shared_layers(
+ p.parse_args(["--model", "dummy", "--shared_layers", "l1", "l2"]).shared_layers
+ ) == [["l1", "l2"]]
+ # comma token → one group
+ assert _normalize_shared_layers(p.parse_args(["--model", "dummy", "--shared_layers", "l1,l2"]).shared_layers) == [
+ ["l1", "l2"]
+ ]
+ # a,b c,d → two groups (replaces --shared_layers a,b --shared_layers c,d)
+ assert _normalize_shared_layers(
+ p.parse_args(["--model", "dummy", "--shared_layers", "l1,l2", "l3,l4"]).shared_layers
+ ) == [["l1", "l2"], ["l3", "l4"]]
+ # multiple flags → multiple groups
+ assert _normalize_shared_layers(
+ p.parse_args(["--model", "dummy", "--shared_layers", "l1", "l2", "--shared_layers", "l3,l4"]).shared_layers
+ ) == [["l1", "l2"], ["l3", "l4"]]
+ assert p.parse_args(["--model", "dummy"]).shared_layers is None
diff --git a/test/test_cuda/models/test_fp8_model.py b/test/test_cuda/models/test_fp8_model.py
index 91599e1cf..7fc32a0f0 100644
--- a/test/test_cuda/models/test_fp8_model.py
+++ b/test/test_cuda/models/test_fp8_model.py
@@ -43,6 +43,7 @@ class TestAutoRound:
iters=0,
disable_opt_rtn=True,
low_cpu_mem_usage=False,
+ disable_model_free=True,
)
quantized_model, quantized_model_path = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)
assert quantized_model is not None, "Quantized model should not be None."