Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[Experimental]Add MLX format export support and AutoScheme for vlm support (#1732)

Wenhua Cheng0def9d632043
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index d7fde248a..8cb79004d 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -616,8 +616,11 @@ def tune(args):
if args.quant_lm_head:
for format in formats:
- if "auto_round" not in format and "fake" not in format:
- auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]
+ # MLX (native ``mlx`` and ``auto_round:mlx``) supports per-layer
+ # mixed-bit quantization including lm_head; treat it the same as
+ # auto_round / fake here.
+ if "auto_round" not in format and "fake" not in format and "mlx" not in format:
+ auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round") or s == "mlx"]
raise ValueError(
f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}"
)
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index 9a751968f..a7e2bdf90 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -48,12 +48,15 @@ from auto_round.utils import (
get_block_names,
get_major_device,
get_module,
+ is_mllm_model,
llm_load_model,
+ mllm_load_model,
parse_available_devices,
set_avg_auto_device_map,
set_module,
set_non_auto_device_map,
to_device,
+ to_dtype,
)
from auto_round.utils.device import MemoryMonitor
from auto_round.utils.offload import OffloadManager
@@ -377,6 +380,51 @@ def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_dev
module.forward = wrap_forward(module, block_name)
+def _prepare_mllm_inputs(data, model):
+ """Normalize one batch from a (possibly mllm) dataloader into a form the
+ model can be called on.
+
+ Crucially this casts ``images`` / ``pixel_values`` / ``pixel_values_videos``
+ to ``model.dtype`` — without it, vision tensors arrive as float32 while
+ the vision tower is bf16/fp16, and several HF VLM implementations silently
+ bypass the vision branch on dtype mismatch (which manifests downstream as
+ "vision grad = 0").
+
+ Returns ``(prepared, kind)`` where ``kind`` is ``"tensor" | "seq" | "dict"``
+ so the caller knows whether to use ``model(x)`` / ``model(*x)`` /
+ ``model(**x)``.
+ """
+ _img_keys = ("images", "image", "pixel_values", "pixel_values_videos", "pixel_values_images", "image_pixel_values")
+
+ if isinstance(data, torch.Tensor):
+ return data.to(model.device), "tensor"
+
+ if isinstance(data, (tuple, list)):
+ return to_device(data, model.device), "seq"
+
+ # Plain dict (the common path: HF VLM ``data_collator`` outputs).
+ new = {}
+ for key, value in data.items():
+ t = to_device(value, model.device)
+ if key in _img_keys:
+ t = to_dtype(t, model.dtype)
+ new[key] = t
+ return new, "dict"
+
+
+def mllm_model_forward(model, data, **forward_kwargs):
+ """Single entry point for "run a (possibly multimodal) batch through the
+ model". Used by both AutoScheme paths so that ``pixel_values`` / ``images``
+ are cast to ``model.dtype`` (otherwise VLMs silently skip the vision tower
+ → vision grad = 0)."""
+ prepared, kind = _prepare_mllm_inputs(data, model)
+ if kind == "tensor":
+ return model(prepared, **forward_kwargs), prepared
+ if kind == "seq":
+ return model(*prepared, **forward_kwargs), prepared
+ return model(**prepared, **forward_kwargs), prepared
+
+
def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):
block_inputs = {}
@@ -401,7 +449,17 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):
last_block_backward_hook = last_block.register_full_backward_pre_hook(backward_pre_hook)
data = to_device(data, model.device)
- output = model.forward(**data, labels=data["input_ids"], use_cache=False)
+ # VLM datasets often already include ``labels``; LLM ones don't. Strip
+ # any pre-existing ``labels`` from kwargs so we don't pass it twice.
+ labels = data["labels"] if isinstance(data, dict) and "labels" in data else data["input_ids"]
+ if isinstance(data, dict):
+ data_for_forward = {k: v for k, v in data.items() if k != "labels"}
+ else:
+ data_for_forward = data
+ # Route through the unified mllm forward so ``pixel_values`` /
+ # ``images`` get cast to ``model.dtype`` (otherwise the vision tower
+ # is silently bypassed on dtype mismatch and vision grad stays 0).
+ output, _prepared = mllm_model_forward(model, data_for_forward, labels=labels, use_cache=False)
try:
# Backward pass (will be interrupted by the hook)
@@ -434,12 +492,7 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):
for n, m in block_module.named_modules():
if hasattr(m, "grad_mode"):
m.grad_mode = True
- if hasattr(m, "orig_layer"):
- m.to(m.orig_layer.tuning_device)
- elif hasattr(m, "tuning_device"):
- m.to(m.tuning_device)
- elif len(list(m.children())) == 0:
- m.to(major_device)
+ move_module_to_tuning_device(block_module, major_device=major_device)
# Set the block to eval mode while enabling gradient computation
block_module.eval()
@@ -494,6 +547,10 @@ def get_score_for_scheme(
batch_size=1,
disable_opt_rtn=True,
offload_context: Optional[OffloadManager] = None,
+ processor=None,
+ is_vlm: bool = False,
+ force_mllm: bool = False,
+ model_name: Optional[str] = None,
):
scores_dict = {} # Key=name,Val=[quant_total_bits, loss]
for n, m in model.named_modules():
@@ -557,23 +614,242 @@ def get_score_for_scheme(
set_module(model, name, new_m)
if offload_context is not None:
offload_context.flush_loaded(model)
- if low_gpu_mem_usage:
- dataloader = get_dataloader(tokenizer, seqlen, dataset_name=dataset, seed=42, bs=batch_size, nsamples=nsamples)
- model_forward_low_gpu(model, dataloader, major_device=major_device, pbar=pbar)
+ # ---- Memory: only wrapper.orig_layer.weight needs ``requires_grad`` ---- #
+ # AutoScheme scoring uses ``iters=0`` (RTN), so we never UPDATE any
+ # parameter. The only parameters that need to participate in autograd
+ # are the wrappers' ``orig_layer.weight`` (so qdq_w in the backward
+ # graph can trace back to them and the weight-grad hook fires).
+ # All other parameters (norms, non-wrapped linears, vision-tower
+ # layers when ``--quant_nontext_module`` is off, …) just waste a full
+ # ``.grad`` buffer (~one model-worth of VRAM) during ``loss.backward()``.
+ # This is the biggest single VRAM win for the non-low_gpu_mem_usage
+ # path used to score VLMs.
+ wrapper_weight_ids = set()
+ for _, _m in model.named_modules():
+ if hasattr(_m, "orig_layer") and hasattr(_m.orig_layer, "weight") and _m.orig_layer.weight is not None:
+ wrapper_weight_ids.add(id(_m.orig_layer.weight))
+ _trimmed = 0
+ for _p in model.parameters():
+ if id(_p) not in wrapper_weight_ids and _p.requires_grad:
+ _p.requires_grad_(False)
+ _trimmed += 1
+ if _trimmed:
+ logger.info(
+ "AutoScheme: disabled requires_grad on %d non-wrapper parameters "
+ "(only wrapper.orig_layer.weight needs grad for scoring; saves "
+ "~one model-worth of grad buffer during backward).",
+ _trimmed,
+ )
+
+ # When scoring vision-tower layers, keep the autograd chain alive end-to-end:
+ # (1) every wrapper's orig weight must require grad — otherwise the STE
+ # output of W-only-low-bit wrappers has no grad path and act-score
+ # hooks never fire.
+ # (2) every vision sub-tree leaf param must require grad — so the very
+ # first vision op (patch_embed / first conv) enters autograd; its
+ # input ``pixel_values`` is a plain tensor with no grad.
+ if force_mllm:
+ _re_enabled_w = 0
+ for _, _m in model.named_modules():
+ if (
+ hasattr(_m, "orig_layer")
+ and hasattr(_m.orig_layer, "weight")
+ and _m.orig_layer.weight is not None
+ and not _m.orig_layer.weight.requires_grad
+ ):
+ _m.orig_layer.weight.requires_grad_(True)
+ _re_enabled_w += 1
+
+ _vision_markers = ("vision", "visual", "image_encoder", "img_encoder", "patch_embed")
+ _re_enabled_v = 0
+ _seen = set()
+ for _mod_name, _mod in model.named_modules():
+ if not any(mk in _mod_name.lower() for mk in _vision_markers):
+ continue
+ for _p in _mod.parameters(recurse=False):
+ if id(_p) in _seen:
+ continue
+ _seen.add(id(_p))
+ if not _p.requires_grad:
+ _p.requires_grad_(True)
+ _re_enabled_v += 1
+
+ logger.info(
+ "AutoScheme(force_mllm): kept requires_grad on %d wrapper weights, " "%d vision-side params.",
+ _re_enabled_w,
+ _re_enabled_v,
+ )
+
+ def _build_calib_dataloader():
+ """Pick the calibration dataloader.
+
+ Since AutoScheme only scores the language tower (``get_block_names``
+ already skips the vision/audio sub-trees on VLMs), a pure-text
+ calibration dataset is sufficient and far cheaper for VLMs too — most
+ VLMs accept a text-only forward and simply skip the vision encoder.
+ We therefore use ``get_dataloader`` (text-only) by default and only
+ fall back to the multimodal ``get_mllm_dataloader`` if a VLM truly
+ rejects text-only inputs (caller can detect that in the calling loop).
+ """
+ return get_dataloader(tokenizer, seqlen, dataset_name=dataset, seed=42, bs=batch_size, nsamples=nsamples)
+
+ def _build_mllm_calib_dataloader():
+ """Build the multimodal calibration dataloader (image + text).
+
+ Returns ``None`` if we can't build one (no processor / template /
+ dataset issue) so the caller can surface a clearer error.
+ """
+ if processor is None:
+ return None
+ import os as _os
+
+ from auto_round.compressors.mllm.dataset import MLLM_DATASET, get_mllm_dataloader
+
+ template = None
+ if hasattr(model, "config") and hasattr(model.config, "model_type"):
+ template = model.config.model_type
+
+ # Decide the effective dataset.
+ # ``get_mllm_dataloader`` only treats ``dataset`` as multimodal when it
+ # is either a local file OR a key registered in ``MLLM_DATASET``.
+ # Otherwise it silently falls back to ``get_dataloader`` (text-only),
+ # which produces batches with NO ``pixel_values`` -> the vision tower
+ # is never invoked -> every vision score / grad collapses to 0. We
+ # explicitly catch that case here and override to a known-good
+ # multimodal dataset so the user doesn't end up with silent garbage.
+ ds = dataset
+ _is_real_mllm = isinstance(ds, str) and (_os.path.isfile(ds) or ds in MLLM_DATASET.keys())
+ if not _is_real_mllm:
+ _fallback = "liuhaotian/llava_conv_58k"
+ logger.warning_once(
+ "AutoScheme(force_mllm): dataset=%r is text-only, " "overriding to %r.",
+ ds,
+ _fallback,
+ )
+ ds = _fallback
+
+ try:
+ loader, _, _, _ = get_mllm_dataloader(
+ template=template,
+ model=model,
+ tokenizer=tokenizer,
+ processor=processor,
+ dataset=ds,
+ seqlen=seqlen,
+ bs=batch_size,
+ nsamples=nsamples,
+ # If, for any reason, get_mllm_dataloader still falls back to
+ # text-only, force it to hard-error rather than silently
+ # producing image-less batches.
+ quant_nontext_module=True,
+ )
+ return loader
+ except Exception as exc: # noqa: BLE001
+ logger.warning(f"Failed to build mllm calibration dataloader: {exc}")
+ return None
+
+ if low_gpu_mem_usage:
+ if force_mllm:
+ mllm_loader = _build_mllm_calib_dataloader()
+ if mllm_loader is None:
+ raise RuntimeError(
+ "AutoScheme(force_mllm): cannot build mllm dataloader. "
+ "Provide a `processor` and a multimodal `dataset`."
+ )
+ model_forward_low_gpu(model, mllm_loader, major_device=major_device, pbar=pbar)
+ else:
+ try:
+ dataloader = _build_calib_dataloader()
+ model_forward_low_gpu(model, dataloader, major_device=major_device, pbar=pbar)
+ except Exception as exc: # noqa: BLE001
+ if not is_vlm:
+ raise
+ logger.warning(
+ f"Text-only calibration failed on VLM ({exc}); "
+ f"falling back to multimodal calibration dataloader."
+ )
+ mllm_loader = _build_mllm_calib_dataloader()
+ if mllm_loader is None:
+ raise
+ model_forward_low_gpu(model, mllm_loader, major_device=major_device, pbar=pbar)
else:
for n, m in model.named_modules():
if hasattr(m, "grad_mode"):
m.grad_mode = True
- dataloader = get_dataloader(tokenizer, seqlen, dataset_name=dataset, seed=42, bs=batch_size, nsamples=nsamples)
- for data in dataloader:
- data = to_device(data, model.device)
- output = model.forward(**data, labels=data["input_ids"], use_cache=False)
- output.loss.backward()
- for n, m in model.named_parameters(): # This should be kept to reduce VRAM footprint
- m.grad = None
- if pbar is not None:
- pbar.update(1)
+
+ def _run_forward_loop(loader):
+ _checked_pixel = False
+ _pixel_keys = (
+ "pixel_values",
+ "pixel_values_videos",
+ "pixel_values_images",
+ "image_pixel_values",
+ "images",
+ "image",
+ )
+ for data in loader:
+ # Pull labels out of the batch (VLM datasets often carry them;
+ # LLM ones don't) before mllm_model_forward casts dtypes.
+ _src = data if isinstance(data, dict) else None
+ labels = (
+ _src["labels"]
+ if _src is not None and "labels" in _src
+ else (_src["input_ids"] if _src is not None and "input_ids" in _src else None)
+ )
+ if _src is not None and "labels" in _src:
+ data_for_forward = {k: v for k, v in _src.items() if k != "labels"}
+ else:
+ data_for_forward = data
+
+ # Unified mllm-aware forward (casts pixel_values/images to
+ # model.dtype, handles dict-with-text/str/tuple paths the same
+ # way AutoRoundMLLM.calib does).
+ output, _prepared = mllm_model_forward(model, data_for_forward, labels=labels, use_cache=False)
+ output.loss.backward()
+
+ # One-shot sanity check: when scoring vision layers, the batch
+ # MUST carry image data, otherwise the vision tower is bypassed
+ # by the model and every vision score is silently 0.
+ if not _checked_pixel and force_mllm:
+ _checked_pixel = True
+ _has_pixel = isinstance(_prepared, dict) and any(k in _prepared for k in _pixel_keys)
+ if not _has_pixel:
+ _keys = list(_prepared.keys()) if isinstance(_prepared, dict) else type(_prepared).__name__
+ raise RuntimeError(
+ f"AutoScheme(force_mllm) batch has no pixel_values "
+ f"(keys: {_keys}). Vision scores would all be 0. "
+ f"Use a real multimodal dataset (e.g. "
+ f"liuhaotian/llava_conv_58k) and pass a processor."
+ )
+
+ for _, m in model.named_parameters(): # zero grads to keep VRAM low
+ m.grad = None
+ if pbar is not None:
+ pbar.update(1)
+
+ if force_mllm:
+ mllm_loader = _build_mllm_calib_dataloader()
+ if mllm_loader is None:
+ raise RuntimeError(
+ "AutoScheme(force_mllm): cannot build mllm dataloader. "
+ "Provide a `processor` and a multimodal `dataset`."
+ )
+ _run_forward_loop(mllm_loader)
+ else:
+ try:
+ _run_forward_loop(_build_calib_dataloader())
+ except Exception as exc: # noqa: BLE001
+ if not is_vlm:
+ raise
+ logger.warning(
+ f"Text-only calibration failed on VLM ({exc}); "
+ f"falling back to multimodal calibration dataloader."
+ )
+ mllm_loader = _build_mllm_calib_dataloader()
+ if mllm_loader is None:
+ raise
+ _run_forward_loop(mllm_loader)
for n, m in model.named_parameters():
m.grad = None
@@ -647,13 +923,40 @@ def choose_bits_per_layer_with_path(layers: dict, P: int):
def move_module_to_tuning_device(module, major_device="cpu"):
+ def _normalize(dev):
+ return dev if isinstance(dev, torch.device) else torch.device(dev)
+
+ def _move_own_tensors(m, device):
+ # Cover non-leaf modules that directly hold nn.Parameter / buffers
+ # (e.g. Mamba/GDN linear_attn with A_log & dt_bias). Also relocate
+ # p.grad together with p.data — otherwise the next backward's grad
+ # accumulation hits a cuda/cpu device mismatch.
+ target = _normalize(device)
+ for p in m.parameters(recurse=False):
+ if p.device != target:
+ p.data = p.data.to(target)
+ if p.grad is not None and p.grad.device != target:
+ p.grad.data = p.grad.data.to(target)
+ for b_name, b in list(m.named_buffers(recurse=False)):
+ if b is None:
+ continue
+ if b.device != target:
+ m._buffers[b_name] = b.to(target)
+
for n, m in module.named_modules():
if hasattr(m, "orig_layer"):
- m.to(m.orig_layer.tuning_device)
+ target = m.orig_layer.tuning_device
+ m.to(target)
+ _move_own_tensors(m, target)
elif hasattr(m, "tuning_device"):
- m.to(m.tuning_device)
+ target = m.tuning_device
+ m.to(target)
+ _move_own_tensors(m, target)
elif len(list(m.children())) == 0:
m.to(major_device)
+ _move_own_tensors(m, major_device)
+ else:
+ _move_own_tensors(m, major_device)
def _gen_layer_config(
@@ -670,6 +973,8 @@ def _gen_layer_config(
model_name=None,
major_device="cpu",
device_list=None,
+ processor=None,
+ is_vlm: bool = False,
):
# Initialize memory tracking for AutoScheme
memory_monitor = MemoryMonitor()
@@ -687,7 +992,33 @@ def _gen_layer_config(
offload_context = OffloadManager(enabled=True, mode="clean", model_dir=_model_dir, cache_numel=True)
target_bits = auto_scheme.avg_bits
- model.eval()
+ # HF gates gradient checkpointing on ``self.training`` — it's a no-op in eval mode.
+ # In the non-low_gpu path we run a full forward+backward through the whole model,
+ # so we want checkpointing to actually kick in (train mode). In the low_gpu path
+ # we drive the blocks manually and don't want dropout / training-only side effects,
+ # so keep eval mode.
+ if auto_scheme.low_gpu_mem_usage:
+ model.eval()
+ else:
+ # To trigger gradient checkpoint, but it will enable dropout, batchnorm, which is not good for accuracy
+ model.train()
+
+ # Filter out embedding layers from the scoring set (they aren't linear
+ # quantization targets in any of our schemes).
+ embedding_layers_names = []
+ for name in quant_layer_names:
+ module = get_module(model, name)
+ if isinstance(module, torch.nn.Embedding):
+ embedding_layers_names.append(name)
+ quant_layer_names = list(set(quant_layer_names) - set(embedding_layers_names))
+
+ # Decide whether AutoScheme has to score vision-tower layers (typically
+ # because the user passed ``--quant_nontext_module``). Used below to
+ # clamp batch_size to 1 (image sizes vary) and to pick the multimodal
+ # dataloader. The actual switch from low_gpu to full forward+backward
+ # is done upstream in ``gen_layer_config``.
+ vision_markers = ("vision", "visual", "image", "img")
+ force_mllm = is_vlm and any(any(marker in n.lower() for marker in vision_markers) for n in quant_layer_names)
block_name = get_block_names(model)[0] # TODO need change to support vlm
for name in block_name:
@@ -732,7 +1063,6 @@ def _gen_layer_config(
nsamples = 64
else:
nsamples = 16
-
seqlen = auto_scheme.seqlen if auto_scheme.seqlen is not None else 256
if auto_scheme.batch_size is not None:
@@ -743,6 +1073,16 @@ def _gen_layer_config(
else:
batch_size = 1
+ # ------------------------------------------------------------------ #
+ # Multimodal calibration: ``batch_size`` must be 1 because image #
+ # sizes differ across samples (the multimodal collator can't stack #
+ # them otherwise). #
+ # ------------------------------------------------------------------ #
+ if force_mllm:
+ if batch_size != 1:
+ logger.info("AutoScheme(force_mllm): clamping batch_size %d -> 1.", batch_size)
+ batch_size = 1
+
pbar_cnt = 0
need_weight_grad = False
for index, scheme in enumerate(schemes):
@@ -762,13 +1102,6 @@ def _gen_layer_config(
pbar_cnt += len(block_name) * 2 * ((nsamples + batch_size - 1) // batch_size) # forward backward
shared_layers = parse_shared_layers(model, auto_scheme.shared_layers)
- embedding_layers_names = []
- for name in quant_layer_names:
- module = get_module(model, name)
- if isinstance(module, torch.nn.Embedding):
- embedding_layers_names.append(name)
- quant_layer_names = list(set(quant_layer_names) - set(embedding_layers_names))
-
options_scores = []
if auto_scheme.low_gpu_mem_usage and not disable_opt_rtn:
logger.warning("low_gpu_mem_usage is enabled, disable_opt_rtn will be set to True automatically")
@@ -785,7 +1118,7 @@ def _gen_layer_config(
if need_imatrix:
break
if need_imatrix: # TODO change to block way in low_gpu_mem_usage
- dataloader = get_dataloader(tokenizer, seqlen=256, dataset_name=dataset, seed=42, bs=8, nsamples=16)
+ dataloader = get_dataloader(tokenizer, seqlen=256, dataset_name=dataset, seed=42, bs=8, nsamples=nsamples)
logger.info("start to compute imatrix for GGUF-K quantization in AutoScheme")
cal_imatrix(model, dataloader)
logger.info("finish calculating imatrix")
@@ -826,6 +1159,10 @@ def _gen_layer_config(
batch_size=batch_size,
disable_opt_rtn=auto_scheme.disable_opt_rtn,
offload_context=offload_context,
+ processor=processor,
+ is_vlm=is_vlm,
+ force_mllm=force_mllm,
+ model_name=model_name,
)
# Track peak RAM after each scheme scoring
memory_monitor.update_cpu()
@@ -970,13 +1307,54 @@ def gen_layer_config(
disable_opt_rtn=True,
low_gpu_mem_usage=True,
min_avg_bit_scheme=None,
+ processor=None,
**kwargs,
):
model_name = None
+ is_vlm = False
if isinstance(model, str):
model_name = model
- # Load model on CPU only; do not apply automatic device map or tuning-aware placement at load time.
- model, tokenizer, _ = llm_load_model(model_name, device_map="cpu")
+ # Detect VLM (Qwen-VL / Qwen3-VL / LLaVA / etc.) and load via the MLLM
+ # path so we get a usable ``processor`` for the multimodal calibration
+ # dataloader. The block walker auto-detects VLMs too and skips the
+ # vision tower when scoring.
+ is_vlm = is_mllm_model(model_name)
+ if is_vlm:
+ model, processor, tokenizer, _ = mllm_load_model(
+ model_name,
+ device="cpu",
+ use_auto_mapping=False,
+ )
+ else:
+ # Load model on CPU only; do not apply automatic device map or tuning-aware placement at load time.
+ model, tokenizer, _ = llm_load_model(model_name, device_map="cpu")
+ else:
+ # Object passed in: still try to detect VLM so we can pick the right dataloader later.
+ try:
+ is_vlm = is_mllm_model(model)
+ except Exception: # noqa: BLE001
+ is_vlm = False
+
+ # ---- Vision-tower scoring requires a full backward ---- #
+ # ``model_forward_low_gpu`` only walks the language tower (it uses
+ # ``get_block_names(model)[0]``, which excludes vision blocks by default)
+ # and interrupts ``loss.backward()`` at the LAST language block via
+ # ``backward_pre_hook`` -> ``MyCustomError``. As a result, gradient never
+ # propagates into the vision tower and any AutoScheme score for vision
+ # layers comes out as 0. If the caller asked us to score vision layers
+ # (typically because ``--quant_nontext_module`` was passed), force a
+ # full forward+backward instead.
+ vision_markers = ("vision", "visual", "image", "img")
+ force_mllm_for_vision = is_vlm and any(
+ any(marker in n.lower() for marker in vision_markers) for n in quant_layer_names
+ )
+ if force_mllm_for_vision and low_gpu_mem_usage:
+ logger.warning("AutoScheme: scoring vision layers requires full backward; " "disabling low_gpu_mem_usage.")
+ low_gpu_mem_usage = False
+ try:
+ auto_scheme.low_gpu_mem_usage = False
+ except Exception: # noqa: BLE001
+ pass
# Get major device
major_device = get_major_device(device_map)
if not low_gpu_mem_usage:
@@ -1002,9 +1380,30 @@ def gen_layer_config(
device_list = parse_available_devices(device_map)
- # Enable gradient checkpointing if supported
- if model.supports_gradient_checkpointing:
- model.gradient_checkpointing_enable()
+ # Enable gradient checkpointing if supported.
+ #
+ # IMPORTANT: we must use ``use_reentrant=False``. The reentrant
+ # implementation requires the inputs that enter the checkpointed region to
+ # have ``requires_grad=True`` — otherwise its backward sees "no grad-
+ # requiring input" and returns ``None`` for the input gradient, which kills
+ # the autograd chain *before* the checkpoint boundary. In AutoScheme we
+ # aggressively turn off ``requires_grad`` on every non-wrapper parameter
+ # (token embeddings, norms, vision-tower non-linear layers, patch embeds,
+ # …), so ``inputs_embeds`` entering the first text decoder block often does
+ # NOT require grad. With reentrant=True that means gradient never flows
+ # back into the vision tower → vision wrapper hooks see grad=0.
+ # ``use_reentrant=False`` (saved-tensor-hooks impl) does not have this
+ # restriction.
+ def _enable_gc(mod):
+ if not getattr(mod, "supports_gradient_checkpointing", False):
+ return
+ try:
+ mod.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})
+ except TypeError:
+ # Older transformers without the kwargs argument.
+ mod.gradient_checkpointing_enable()
+
+ _enable_gc(model)
for name in quant_layer_names:
m = get_module(model, name)
@@ -1025,6 +1424,8 @@ def gen_layer_config(
major_device=major_device,
device_list=device_list,
min_avg_bit_scheme=min_avg_bit_scheme,
+ processor=processor,
+ is_vlm=is_vlm,
)
except torch.OutOfMemoryError:
logger.warning(
@@ -1055,6 +1456,8 @@ def gen_layer_config(
major_device=major_device,
device_list=device_list,
min_avg_bit_scheme=min_avg_bit_scheme,
+ processor=processor,
+ is_vlm=is_vlm,
)
return res
diff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.py
index 55ec4e205..737d7d43e 100644
--- a/auto_round/auto_scheme/gen_auto_scheme.py
+++ b/auto_round/auto_scheme/gen_auto_scheme.py
@@ -61,10 +61,12 @@ class GenScheme:
device_map: Union[str, torch.device, int, dict, None] = None,
tokenizer=None,
enable_torch_compile=False,
+ processor=None,
):
self.auto_scheme = auto_scheme
self.model = model
self.tokenizer = tokenizer
+ self.processor = processor
self.quant_layer_names = quant_layer_names
self.fixed_layer_scheme = fixed_layer_scheme
self.dataset = dataset
@@ -98,8 +100,16 @@ class GenScheme:
self.auto_scheme.avg_bits = target
if not (min_avg_bit <= target <= max_avg_bit):
+ details = ", ".join(f"{opt}={bits:.3f}" for opt, bits in getattr(self, "_option_avg_bits", []))
+ hint = (
+ "Provide at least one option whose avg_bits is <= target and one whose avg_bits is >= target. "
+ "Note that layers fixed to high-precision (e.g. lm_head/embed kept at bf16) inflate the lower bound; "
+ "use `quant_lm_head=True`/adjust `layer_config`/`ignore_layers` if needed."
+ )
raise ValueError(
- f"Target avg_bits={target:.3f} is outside the valid range " f"[{min_avg_bit:.3f}, {max_avg_bit:.3f}]."
+ f"Target avg_bits={target:.3f} is outside the achievable range "
+ f"[{min_avg_bit:.3f}, {max_avg_bit:.3f}] for options={list(self.auto_scheme.options)} "
+ f"(per-option avg_bits: {details}). {hint}"
)
def get_layer_config(self) -> dict[str, dict]:
@@ -122,6 +132,7 @@ class GenScheme:
disable_opt_rtn=self.disable_opt_rtn,
low_gpu_mem_usage=self.auto_scheme.low_gpu_mem_usage,
min_avg_bit_scheme=self.min_avg_bit_scheme,
+ processor=self.processor,
)
layer_config = self.fallback_gguf_layer_config(layer_config)
return layer_config
@@ -183,6 +194,10 @@ class GenScheme:
)[0]
for option in self.auto_scheme.options
]
+ # Cache per-option avg bits so log/error messages can reference them.
+ self._option_avg_bits = list(zip(list(self.auto_scheme.options), avg_bits))
+ for opt, bits in self._option_avg_bits:
+ logger.info("AutoScheme option %s -> avg_bits=%.4f", opt, bits)
min_avg_bit, max_avg_bit = min(avg_bits), max(avg_bits)
min_avg_bit_scheme = self.auto_scheme.options[avg_bits.index(min_avg_bit)]
return min_avg_bit, max_avg_bit, min_avg_bit_scheme
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 238df50e5..69fbb2b57 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -603,8 +603,15 @@ class BaseCompressor(object):
def _gen_auto_scheme(self) -> dict[str, dict]:
if self.mllm:
- logger.info("AutoScheme is not yet supported for multimodal LLMs.")
- sys.exit(-1)
+ # AutoScheme on a VLM only scores the language tower (the block
+ # walker in delta_loss already skips vision/audio sub-trees) and
+ # uses a pure-text calibration dataset by default, falling back to
+ # the multimodal dataloader if the VLM rejects text-only forward.
+ logger.info(
+ "AutoScheme on multimodal LLM: scoring the language tower only "
+ "with text-only calibration (multimodal dataloader will be used "
+ "as a fallback if needed)."
+ )
if is_quantized_input_module(self.model):
logger.info("AutoScheme does not currently support quantized input models (e.g., FP8).")
@@ -654,6 +661,71 @@ class BaseCompressor(object):
is_mllm=self.mllm,
)
quant_layer_names = layer_config.keys()
+
+ # ---- VLM: peel non-text sub-trees AutoScheme should not score ---- #
+ # Behavior matches ``--quant_nontext_module``:
+ # * default (False): peel BOTH vision and audio; AutoScheme only
+ # scores the language tower (vision/audio kept at their original
+ # 16-bit configuration).
+ # * True: peel only audio; AutoScheme also scores the vision tower
+ # (audio is always peeled because we don't ship an audio
+ # calibration dataset, so any score for it would be unreliable).
+ nontext_skipped_layers: dict[str, dict] = {}
+ if self.mllm:
+ from auto_round.utils import get_block_names
+
+ quant_nontext = getattr(self, "quant_nontext_module", False)
+ # ``_get_vlm_block_names`` always drops audio blocks regardless of
+ # ``quant_vision``.
+ scoreable_blocks = get_block_names(self.model, quant_vision=quant_nontext)
+ scoreable_block_prefixes = tuple(blk for group in scoreable_blocks for blk in group)
+ if quant_nontext:
+ # vision stays in; only audio gets peeled.
+ peel_markers = ("audio", "speech", "wav", "waveform")
+ tower_label = "language+vision"
+ peel_label = "audio/speech"
+ else:
+ # default: peel both vision and audio.
+ peel_markers = (
+ "vision",
+ "visual",
+ "image",
+ "img",
+ "audio",
+ "speech",
+ "wav",
+ "waveform",
+ )
+ tower_label = "language"
+ peel_label = "vision/audio"
+
+ def _is_scoreable_layer(name: str) -> bool:
+ # Quick prefix test against detected scoreable blocks.
+ if any(name == p or name.startswith(p + ".") for p in scoreable_block_prefixes):
+ return True
+ # Fallback: drop anything matching a peel marker.
+ lname = name.lower()
+ return not any(marker in lname for marker in peel_markers)
+
+ scoreable_layer_config = {}
+ for name, cfg in layer_config.items():
+ if _is_scoreable_layer(name):
+ scoreable_layer_config[name] = cfg
+ else:
+ nontext_skipped_layers[name] = cfg
+
+ if nontext_skipped_layers:
+ logger.info(
+ "AutoScheme (VLM): scoring %d %s-tower layers; "
+ "%d %s-tower layers kept at their original 16-bit configuration.",
+ len(scoreable_layer_config),
+ tower_label,
+ len(nontext_skipped_layers),
+ peel_label,
+ )
+ layer_config = scoreable_layer_config
+ quant_layer_names = layer_config.keys()
+
scheme_keys = {f.name for f in fields(QuantizationScheme)}
fixed_layer_scheme_new = {
k: {key: v[key] for key in scheme_keys & v.keys()}
@@ -675,8 +747,26 @@ class BaseCompressor(object):
device_map=self.device_map,
tokenizer=self.tokenizer,
enable_torch_compile=self.enable_torch_compile,
+ processor=getattr(self, "processor", None),
)
layer_config = self.scheme_generator.get_layer_config()
+ # Re-attach vision/audio-tower layers we peeled off earlier so the
+ # downstream quantization pipeline sees the complete layer map.
+ # NOTE: ``nontext_skipped_layers`` was populated *after* the first
+ # ``set_layer_config`` call, so each cfg already carries internal-only
+ # keys such as ``in_blocks`` that the second ``set_layer_config`` call
+ # in ``configure_layer_config`` would reject via its ``normalize_item``
+ # validator. Strip those keys (keep only the allowed scheme fields)
+ # before merging.
+ if nontext_skipped_layers:
+ allowed_keys = {f.name for f in fields(QuantizationScheme)} | {
+ "fixed_by_user",
+ "scale_dtype",
+ "scheme",
+ }
+ for name, cfg in nontext_skipped_layers.items():
+ clean_cfg = {k: v for k, v in cfg.items() if k in allowed_keys} if isinstance(cfg, dict) else cfg
+ layer_config.setdefault(name, clean_cfg)
return layer_config
def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None:
diff --git a/auto_round/compressors/mllm/template.py b/auto_round/compressors/mllm/template.py
index 78abc6424..eb4a56e24 100644
--- a/auto_round/compressors/mllm/template.py
+++ b/auto_round/compressors/mllm/template.py
@@ -183,7 +183,7 @@ def get_template(
template = TEMPLATES[template_or_path]
else:
if not quiet:
- logger.warning(f"Unable to recognize {template_or_path}, using default template instead.")
+ logger.warning_once(f"Unable to recognize {template_or_path}, using default template instead.")
template = TEMPLATES["default"]
template.model_type = template_or_path
diff --git a/auto_round/export/export_to_mlx/__init__.py b/auto_round/export/export_to_mlx/__init__.py
new file mode 100644
index 000000000..5c6c18356
--- /dev/null
+++ b/auto_round/export/export_to_mlx/__init__.py
@@ -0,0 +1,17 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .export import pack_layer, save_quantized_as_mlx
+
+__all__ = ["pack_layer", "save_quantized_as_mlx"]
diff --git a/auto_round/export/export_to_mlx/export.py b/auto_round/export/export_to_mlx/export.py
new file mode 100644
index 000000000..c65eef80d
--- /dev/null
+++ b/auto_round/export/export_to_mlx/export.py
@@ -0,0 +1,728 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+MLX Format Exporter for AutoRound
+
+Exports quantized models to MLX-compatible format that can be loaded
+directly by mlx-lm for inference on Apple Silicon.
+
+MLX QuantizedLinear dequantization (affine mode):
+ w_float = scale * w_int + bias
+
+where w_int is packed into uint32, each holding (32 // bits) elements,
+and scale/bias (called "biases" in MLX) have shape [out_features, num_groups].
+
+Supported schemes: W2A16, W3A16, W4A16, W5A16, W6A16, W8A16
+"""
+
+import copy
+import json
+import os
+from typing import Callable, Union
+
+import torch
+import torch.nn as nn
+import transformers
+
+from auto_round.export.utils import unsupported_meta_device
+from auto_round.logger import logger
+from auto_round.utils import (
+ check_to_quantized,
+ copy_python_files_from_model_cache,
+ get_module,
+ get_packing_device,
+ set_module,
+)
+
+SUPPORTED_LAYER_TYPES = [nn.Linear, nn.Conv2d, transformers.pytorch_utils.Conv1D]
+
+
+def _is_mlx_quantizable(module: nn.Module, group_size: int) -> bool:
+ """Predicate matching mlx-lm's default quantization predicate.
+
+ A layer is considered quantizable by mlx-lm if its inner dim is a multiple
+ of ``group_size`` and its outer dim is a multiple of 64. Embeddings are also
+ eligible. Layers that fail this predicate are skipped at load time, so we do
+ NOT need to emit ``false`` entries for them.
+ """
+ if isinstance(module, nn.Linear):
+ in_dim, out_dim = module.in_features, module.out_features
+ elif isinstance(module, nn.Embedding):
+ in_dim, out_dim = module.embedding_dim, module.num_embeddings
+ else:
+ return False
+ return (in_dim % group_size == 0) and (out_dim % 64 == 0)
+
+
+# Sub-modules whose layers we never want to auto-quantize on the mlx side.
+# Vision / audio towers in VLMs are conventionally kept in fp16 by mlx-vlm,
+# so emitting per-layer ``false`` for hundreds of vision linears just bloats
+# the config. We simply skip these subtrees.
+_DEFAULT_SKIP_PREFIXES = (
+ "vision_tower",
+ "vision_model",
+ "visual",
+ "audio_tower",
+ "audio_model",
+ "image_newline",
+ "multi_modal_projector",
+)
+
+
+def _build_mlx_quantization_config(
+ model: nn.Module,
+ default_bits: int,
+ default_group_size: int,
+ skip_prefixes: tuple = _DEFAULT_SKIP_PREFIXES,
+) -> dict:
+ """Build the ``quantization`` dict for an MLX ``config.json``.
+
+ Mirrors the mlx-community mixed-bit format (see e.g.
+ ``mlx-community/GLM-4.7-REAP-50-mixed-3-4-bits``)::
+
+ "quantization": {
+ "group_size": 64,
+ "bits": 4,
+ "model.layers.0.self_attn.q_proj": {"group_size": 64, "bits": 3},
+ "model.layers.0.mlp.down_proj": false,
+ ...
+ }
+
+ * Top-level ``group_size`` / ``bits`` give the default scheme.
+ * Per-layer ``{group_size, bits}`` overrides differing layers.
+ * Per-layer ``false`` marks an mlx-quantizable layer that should be kept
+ in fp16 (e.g. ``lm_head`` left at 16 bits in our ``layer_config``).
+
+ For VLMs, ``skip_prefixes`` lists sub-trees (vision/audio towers, projectors)
+ whose un-quantized layers we do NOT emit ``false`` entries for. Quantized
+ layers inside those sub-trees are still recorded faithfully.
+
+ Note: AutoRound may leave non-quantized layers wrapped in ``WrapperLinear``
+ (``.orig_layer`` points to the real ``nn.Linear``); we unwrap before testing
+ so that e.g. ``lm_head`` with ``bits=16`` is correctly emitted as ``false``.
+ """
+ quant_cfg: dict = {"group_size": default_group_size, "bits": default_bits}
+
+ def _is_skipped(layer_name: str) -> bool:
+ return any(layer_name == p or layer_name.startswith(p + ".") for p in skip_prefixes)
+
+ for name, module in model.named_modules():
+ if isinstance(module, _MLXPackedLayer):
+ # Always record packed layers, even if inside a skipped sub-tree
+ # (user explicitly asked for them to be quantized).
+ quant_cfg[name] = {
+ "group_size": int(module.group_size) if module.group_size is not None else default_group_size,
+ "bits": int(module.bits) if module.bits is not None else default_bits,
+ }
+ continue
+
+ if _is_skipped(name):
+ continue
+
+ # Unwrap auto-round WrapperLinear (and similar) before the predicate test.
+ target = module
+ while hasattr(target, "orig_layer") and target.orig_layer is not None:
+ target = target.orig_layer
+
+ if _is_mlx_quantizable(target, default_group_size):
+ # mlx-lm would auto-quantize this layer using the default scheme,
+ # but since we deliberately left it un-quantized, mark it explicitly.
+ quant_cfg[name] = False
+ return quant_cfg
+
+
+# Sub-config keys commonly used by HF multi-modal / dual-tower configs that
+# carry their own ``rope_parameters`` and may want a ``quantization`` block.
+_TEXT_SUB_CONFIG_KEYS = ("text_config", "language_config", "thinker_config")
+
+
+def _flatten_rope_parameters_recursive(cfg: dict) -> None:
+ """Pop ``rope_parameters`` and surface its keys to the same level.
+
+ mlx-lm / mlx-vlm both expect ``rope_theta`` (and friends) at the same level
+ as the rest of the model config — i.e. on the top-level config for LLMs and
+ on ``text_config`` for VLMs. We walk every nested dict so both cases work.
+
+ Two ``rope_parameters`` shapes appear in the wild and both are handled:
+ 1. ``{"rope_theta": 1e6, "rope_type": "default", ...}`` (flat)
+ 2. ``{"default": {"rope_theta": 1e6, "rope_type": "default", ...}, ...}``
+ (newer HF "by-mode" layout — we surface keys from the ``"default"``
+ entry, falling back to the first sub-dict if absent).
+ """
+ if not isinstance(cfg, dict):
+ return
+ rope_params = cfg.pop("rope_parameters", None)
+ if isinstance(rope_params, dict):
+ # If the dict is nested by mode, prefer "default", else any sub-dict.
+ sub_dicts = {k: v for k, v in rope_params.items() if isinstance(v, dict)}
+ if sub_dicts:
+ chosen = sub_dicts.get("default") or next(iter(sub_dicts.values()))
+ for k, v in chosen.items():
+ cfg.setdefault(k, v)
+ else:
+ for k, v in rope_params.items():
+ cfg.setdefault(k, v)
+ for v in cfg.values():
+ if isinstance(v, dict):
+ _flatten_rope_parameters_recursive(v)
+
+
+def _extract_rope_theta_from_obj(config_obj) -> "float | None":
+ """Best-effort extraction of ``rope_theta`` from a HF config object.
+
+ Newer transformers versions stop exposing ``rope_theta`` as a direct
+ attribute on the config and only expose ``rope_parameters`` (which can be
+ a dict, a typed object, or a by-mode mapping like
+ ``{"default": <obj-or-dict>}``). This helper tries every shape we've
+ encountered in the wild.
+ """
+ if config_obj is None:
+ return None
+ direct = getattr(config_obj, "rope_theta", None)
+ if direct is not None:
+ return direct
+
+ rope_params = getattr(config_obj, "rope_parameters", None)
+ if rope_params is None:
+ return None
+
+ # Case A: dict-like
+ if isinstance(rope_params, dict):
+ if "rope_theta" in rope_params:
+ return rope_params["rope_theta"]
+ # by-mode: {"default": {...}, ...}
+ sub_dicts = [v for v in rope_params.values() if isinstance(v, (dict,)) or hasattr(v, "rope_theta")]
+ chosen = None
+ if "default" in rope_params and (
+ isinstance(rope_params["default"], dict) or hasattr(rope_params["default"], "rope_theta")
+ ):
+ chosen = rope_params["default"]
+ elif sub_dicts:
+ chosen = sub_dicts[0]
+ if chosen is not None:
+ return _extract_rope_theta_from_obj(chosen) if not isinstance(chosen, dict) else chosen.get("rope_theta")
+ return None
+
+ # Case B: object with attributes
+ inner = getattr(rope_params, "rope_theta", None)
+ if inner is not None:
+ return inner
+ # by-mode object: try .default
+ inner_default = getattr(rope_params, "default", None)
+ if inner_default is not None:
+ return _extract_rope_theta_from_obj(inner_default)
+ return None
+
+
+def _ensure_rope_theta_from_config_obj(cfg: dict, config_obj) -> None:
+ """Backfill ``rope_theta`` on every level by probing the live HF config object.
+
+ HF model configs may carry ``rope_theta`` only inside ``rope_parameters`` (or
+ not at all on the serialized JSON when ``rope_parameters`` is nested by mode
+ and the JSON-side flatten couldn't reach a usable value). Walk the config
+ object alongside the JSON dict and copy ``rope_theta`` over wherever it's
+ still missing — at top level for LLMs, and on
+ ``text_config`` / ``language_config`` / ``thinker_config`` for VLMs.
+ """
+ if not isinstance(cfg, dict) or config_obj is None:
+ return
+ if "rope_theta" not in cfg:
+ rope_theta = _extract_rope_theta_from_obj(config_obj)
+ if rope_theta is not None:
+ cfg["rope_theta"] = rope_theta
+ for sub_key in _TEXT_SUB_CONFIG_KEYS + ("vision_config", "audio_config"):
+ sub_cfg = cfg.get(sub_key)
+ sub_obj = getattr(config_obj, sub_key, None)
+ if isinstance(sub_cfg, dict) and sub_obj is not None:
+ _ensure_rope_theta_from_config_obj(sub_cfg, sub_obj)
+
+
+# Sub-config keys whose ``model_type`` we want to preserve verbatim from the
+# *original* checkpoint's ``config.json``. Newer ``transformers`` versions
+# have a bug where they rewrite a sub-config's ``model_type`` on save (e.g.
+# ``vision_config.model_type`` becomes ``qwen3_5_vision`` while the original
+# checkpoint had ``qwen3_5``). We snapshot the original JSON and restore.
+_PRESERVE_MODEL_TYPE_SUB_KEYS = (
+ "text_config",
+ "language_config",
+ "thinker_config",
+ "vision_config",
+ "audio_config",
+)
+
+
+def _load_original_config_json(model: nn.Module) -> "dict | None":
+ """Best-effort load of the *raw* ``config.json`` from the original checkpoint.
+
+ HF's in-memory ``model.config`` is unreliable for preserving ``model_type``
+ because newer ``transformers`` versions auto-rewrite sub-config
+ ``model_type`` values based on the registered child config class
+ (e.g. an original ``qwen3_5`` becomes ``qwen3_5_text`` /
+ ``qwen3_5_vision`` once the model is instantiated). The on-disk
+ ``config.json`` from the source checkpoint is the only faithful source.
+
+ We try ``model.config._name_or_path`` first, then ``model.name_or_path``.
+ Returns the parsed JSON dict, or ``None`` if it can't be located/read.
+ """
+ candidates = []
+ cfg_obj = getattr(model, "config", None)
+ if cfg_obj is not None:
+ nop = getattr(cfg_obj, "_name_or_path", None) or getattr(cfg_obj, "name_or_path", None)
+ if nop:
+ candidates.append(nop)
+ nop2 = getattr(model, "name_or_path", None)
+ if nop2 and nop2 not in candidates:
+ candidates.append(nop2)
+
+ for path in candidates:
+ try:
+ if isinstance(path, str) and os.path.isdir(path):
+ cfg_path = os.path.join(path, "config.json")
+ elif isinstance(path, str) and os.path.isfile(path) and path.endswith(".json"):
+ cfg_path = path
+ else:
+ continue
+ if not os.path.isfile(cfg_path):
+ continue
+ with open(cfg_path, "r", encoding="utf-8") as f:
+ return json.load(f)
+ except Exception as e:
+ logger.warning(f"[mlx-export] failed to read original config.json at {path!r}: {e}")
+ return None
+
+
+def _snapshot_original_model_types(model: nn.Module) -> "dict | None":
+ """Snapshot ``model_type`` for top-level and known sub-configs.
+
+ Prefer the original on-disk ``config.json`` (the only faithful source),
+ falling back to the in-memory ``model.config`` if the file isn't reachable.
+ Newer ``transformers`` versions rewrite sub-config ``model_type`` values
+ on instantiation (e.g. ``qwen3_5`` -> ``qwen3_5_text`` /
+ ``qwen3_5_vision``), so the in-memory copy can't be trusted for this.
+
+ Returns a plain dict shaped like::
+
+ {"model_type": "qwen3_5",
+ "vision_config": {"model_type": "qwen3_5"},
+ ...}
+
+ or ``None`` if no config is available.
+ """
+ cfg_dict = _load_original_config_json(model)
+ source = "on-disk config.json"
+ if cfg_dict is None:
+ cfg_obj = getattr(model, "config", None)
+ if cfg_obj is None:
+ logger.warning("[mlx-export] model has no .config; skipping model_type preservation")
+ return None
+ try:
+ cfg_dict = cfg_obj.to_dict() if hasattr(cfg_obj, "to_dict") else dict(cfg_obj.__dict__)
+ source = "in-memory model.config (fallback; may be HF-rewritten)"
+ except Exception as e:
+ logger.warning(f"[mlx-export] failed to snapshot model.config: {e}")
+ return None
+
+ snapshot: dict = {}
+ top_model_type = cfg_dict.get("model_type")
+ if top_model_type is not None:
+ snapshot["model_type"] = top_model_type
+ # Force every known sub-config to share the top-level ``model_type``.
+ # Different ``transformers`` versions disagree on what sub-config
+ # ``model_type`` should be (some leave it equal to the parent, some append
+ # ``_text`` / ``_vision`` / ``_audio``). To keep the exported MLX config
+ # stable across versions we just propagate the top-level value down.
+ for sub_key in _PRESERVE_MODEL_TYPE_SUB_KEYS:
+ sub = cfg_dict.get(sub_key)
+ if isinstance(sub, dict) and top_model_type is not None:
+ snapshot[sub_key] = {"model_type": top_model_type}
+ logger.info(f"[mlx-export] snapshotted original model_types from {source}: {snapshot!r}")
+ return snapshot
+
+
+def _preserve_original_model_types(new_cfg: dict, orig_cfg: "dict | None") -> None:
+ """Make ``new_cfg``'s ``model_type`` fields exactly match ``orig_cfg``.
+
+ For the top-level config and each known sub-config (``text_config``,
+ ``vision_config``, etc.):
+ * If the original had a ``model_type`` value, write it back.
+ * If the original did **not** have ``model_type`` at all, delete the key
+ from the new config (so HF can't sneak in a renamed default like
+ ``qwen3_5`` → ``qwen3_5_vision``).
+ * If the original sub-config doesn't exist in the new config, nothing to do.
+
+ No-op if ``orig_cfg`` is ``None``.
+ """
+ if not isinstance(new_cfg, dict):
+ logger.warning("[mlx-export] new config is not a dict; skip model_type preservation")
+ return
+ if not isinstance(orig_cfg, dict):
+ # Already warned in _load_original_config_json.
+ return
+
+ def _align(sub_new: dict, sub_orig: dict, label: str) -> None:
+ new_val = sub_new.get("model_type", "<missing>")
+ if "model_type" in sub_orig:
+ orig_val = sub_orig["model_type"]
+ if new_val != orig_val:
+ logger.info(f"[mlx-export] restoring {label}.model_type: {new_val!r} -> {orig_val!r}")
+ sub_new["model_type"] = orig_val
+ else:
+ if "model_type" in sub_new:
+ logger.info(
+ f"[mlx-export] removing {label}.model_type ({new_val!r}); " f"original config did not declare one"
+ )
+ sub_new.pop("model_type", None)
+
+ _align(new_cfg, orig_cfg, "top-level")
+ for sub_key in _PRESERVE_MODEL_TYPE_SUB_KEYS:
+ sub_new = new_cfg.get(sub_key)
+ sub_orig = orig_cfg.get(sub_key)
+ if isinstance(sub_new, dict) and isinstance(sub_orig, dict):
+ _align(sub_new, sub_orig, sub_key)
+
+
+def _strip_prefix(name: str, prefix: str) -> str:
+ """Return ``name`` with ``prefix.`` removed, or ``name`` if no match."""
+ if name == prefix:
+ return name
+ pref = prefix + "."
+ return name.removeprefix(pref)
+
+
+def _build_text_subconfig_quantization(quant_cfg: dict, text_prefix: str) -> dict:
+ """Re-key a top-level quantization dict for placement under ``text_config``.
+
+ mlx-vlm's language-model loader queries ``text_config["quantization"]`` with
+ keys *relative to the language model* (e.g. ``model.layers.0...``), so we
+ strip the outer ``language_model.`` / ``text_model.`` prefix from each entry.
+ Entries that don't belong to the language model are dropped.
+ """
+ sub: dict = {"group_size": quant_cfg["group_size"], "bits": quant_cfg["bits"]}
+ pref = text_prefix + "."
+ for k, v in quant_cfg.items():
+ if k in ("group_size", "bits"):
+ continue
+ if k == text_prefix or k.startswith(pref):
+ sub[_strip_prefix(k, text_prefix)] = v
+ return sub
+
+
+def _detect_text_module_prefix(model: nn.Module) -> str:
+ """Best-effort detection of the language-model sub-module name in a VLM.
+
+ Returns an empty string if the model is text-only.
+ """
+ for cand in ("language_model", "text_model", "thinker"):
+ if hasattr(model, cand):
+ return cand
+ return ""
+
+
+def _pack_weight_mlx(intweight, bits):
+ """Pack integer weights into uint32 in MLX format.
+
+ MLX packs elements as a contiguous bit stream across uint32 boundaries.
+ For bits that evenly divide 32 (2, 4, 8), each uint32 holds 32//bits elements.
+ For other bit widths (e.g. 3, 5, 6, 7), every 32 elements are packed into
+ `bits` uint32s (32 * bits = bits * 32 bits total).
+
+ Args:
+ intweight: shape [out_features, in_features], values in [0, 2^bits - 1]
+ bits: quantization bits (2, 3, 4, 5, 6, 7, 8)
+
+ Returns:
+ packed: uint32 tensor, shape [out_features, in_features * bits / 32]
+ """
+ out_features, in_features = intweight.shape
+
+ if 32 % bits == 0:
+ # Simple case: bits evenly divides 32 (2, 4, 8)
+ elems_per_int = 32 // bits
+ assert (
+ in_features % elems_per_int == 0
+ ), f"in_features ({in_features}) must be divisible by {elems_per_int} for {bits}-bit packing"
+
+ intweight = intweight.to(torch.int32)
+ intweight = intweight.reshape(out_features, -1, elems_per_int)
+ shifts = torch.arange(elems_per_int, device=intweight.device, dtype=torch.int32) * bits
+ packed = (intweight << shifts).sum(dim=-1).to(torch.int32)
+ return packed.view(torch.uint32)
+ else:
+ # Cross-word packing: 32 elements → `bits` uint32s
+ # MLX packs as a contiguous bit stream across uint32 boundaries
+ assert in_features % 32 == 0, f"in_features ({in_features}) must be divisible by 32 for {bits}-bit packing"
+
+ intweight = intweight.to(torch.int64)
+ num_groups = in_features // 32
+ # Reshape to [out_features, num_groups, 32]
+ elems = intweight.reshape(out_features, num_groups, 32)
+
+ # For each element i in [0..31], it contributes `bits` bits starting at bit position i*bits
+ # in a 32*bits bit stream packed into `bits` uint32s.
+ # We process each bit b of each element i:
+ # absolute_bit = i * bits + b
+ # word_idx = absolute_bit // 32
+ # bit_pos = absolute_bit % 32
+ packed = torch.zeros(out_features, num_groups, bits, dtype=torch.int64, device=intweight.device)
+
+ for b in range(bits):
+ # Extract bit b from all elements: shape [out, num_groups, 32]
+ bit_vals = (elems >> b) & 1
+ for i in range(32):
+ abs_bit = i * bits + b
+ word_idx = abs_bit // 32
+ bit_pos = abs_bit % 32
+ packed[:, :, word_idx] |= bit_vals[:, :, i] << bit_pos
+
+ packed = packed.to(torch.int32).reshape(out_features, num_groups * bits)
+ return packed.view(torch.uint32)
+
+
+class _MLXPackedLayer(nn.Module):
+ """Holds MLX-packed quantized tensors for serialization.
+
+ Tensor names match MLX convention: weight, scales, biases, bias.
+ The ``bits`` / ``group_size`` attributes are kept so that mixed-bit
+ quantization can be reflected in the exported ``config.json``.
+ """
+
+ def __init__(self, weight, scales, biases, bias, bits=None, group_size=None):
+ super().__init__()
+ self.register_buffer("weight", weight) # uint32
+ self.register_buffer("scales", scales) # float16
+ self.register_buffer("biases", biases) # float16
+ if bias is not None:
+ self.register_buffer("bias", bias)
+ else:
+ self.bias = None
+ self.bits = bits
+ self.group_size = group_size
+
+
+def pack_layer(name, model, device=None, **kwargs):
+ """Pack a single layer into MLX quantized format.
+
+ Reads layer attributes set by auto-round quantization:
+ layer.weight - float weight [out_features, in_features]
+ layer.scale - [out_features, num_groups]
+ layer.zp - [out_features, num_groups] or scalar
+ layer.bits, layer.group_size, layer.sym
+
+ Replaces the layer with _MLXPackedLayer containing:
+ weight - uint32 packed ints [out_features, in_features * bits / 32]
+ scales - float16 [out_features, num_groups]
+ biases - float16 [out_features, num_groups]
+ bias - original linear bias (if any)
+ """
+ layer = get_module(model, name)
+ if hasattr(layer, "orig_layer"):
+ layer = layer.orig_layer
+
+ if type(layer) not in SUPPORTED_LAYER_TYPES:
+ return
+
+ if not check_to_quantized(layer):
+ return
+
+ device = get_packing_device(device)
+
+ bits = int(layer.bits)
+ group_size = int(layer.group_size)
+ sym = bool(getattr(layer, "sym", False))
+ scale = layer.scale # [out_features, num_groups]
+ zp = layer.zp # [out_features, num_groups] or scalar (asym only)
+
+ # Get weight in [out_features, in_features] layout
+ W = layer.weight.data.to(device).clone().float()
+ if type(layer) == nn.Conv2d:
+ W = W.flatten(1)
+ if type(layer) == transformers.pytorch_utils.Conv1D:
+ W = W.t()
+
+ out_features, in_features = W.shape
+ if group_size == -1:
+ group_size = in_features
+
+ maxq = 2**bits - 1
+
+ # Quantize: w_int = round(W / scale + zp), clamped to [0, maxq].
+ # For symmetric mode we deliberately IGNORE layer.zp (it may be a tensor /
+ # carry rounding noise / be device-dependent) and use the fixed integer
+ # zero point ``2**(bits-1)`` (e.g. 8 for 4-bit, 4 for 3-bit, 128 for 8-bit).
+ # This matches QuantLinearMLX.from_gptq's sym branch and the GPTQ sym
+ # convention used everywhere else in auto-round.
+ scale_dev = scale.to(device).float()
+ repeat_scales = scale_dev.repeat_interleave(group_size, dim=1)[:, :in_features]
+
+ if sym:
+ zp_dev = float(2 ** (bits - 1))
+ repeat_zp = zp_dev
+ elif isinstance(zp, torch.Tensor):
+ zp_dev = zp.to(device).float()
+ repeat_zp = zp_dev.repeat_interleave(group_size, dim=1)[:, :in_features]
+ else:
+ zp_dev = float(zp)
+ repeat_zp = zp_dev
+
+ intweight = torch.round(W / repeat_scales + repeat_zp).clamp(0, maxq).to(torch.int32)
+
+ # Pack weights into uint32
+ packed_weight = _pack_weight_mlx(intweight, bits)
+
+ # MLX dequant: w_float = mlx_scale * w_int + mlx_bias
+ # auto-round: w_float = scale * (w_int - zp) = scale * w_int - scale * zp
+ # So: mlx_scale = scale, mlx_bias = -scale * zp
+ mlx_scales = scale_dev.contiguous().to(torch.float16)
+ mlx_biases = (-scale_dev * zp_dev).contiguous().to(torch.float16)
+
+ # Preserve original bias
+ orig_bias = None
+ if layer.bias is not None:
+ orig_bias = layer.bias.clone().to(torch.float16).cpu()
+
+ new_layer = _MLXPackedLayer(
+ packed_weight.cpu(),
+ mlx_scales.cpu(),
+ mlx_biases.cpu(),
+ orig_bias,
+ bits=bits,
+ group_size=group_size,
+ )
+ set_module(model, name, new_layer)
+ logger.debug(f"Packed layer {name} for MLX format (bits={bits}, group_size={group_size})")
+
+
+def save_quantized_as_mlx(
+ output_dir: str,
+ model: nn.Module = None,
+ tokenizer: Callable = None,
+ layer_config: dict = None,
+ inplace: bool = True,
+ device: Union[str, torch.device] = "cpu",
+ serialization_dict: dict = None,
+ **kwargs,
+) -> nn.Module:
+ """Save quantized model in MLX-compatible format.
+
+ The output can be loaded by mlx-lm::
+
+ from mlx_lm import load, generate
+ model, tokenizer = load("output_dir")
+ response = generate(model, tokenizer, prompt="Hello", max_tokens=100)
+ """
+ if not inplace:
+ model = copy.deepcopy(model.to("cpu"))
+
+ if not unsupported_meta_device(model):
+ model = model.to("cpu")
+
+ os.makedirs(output_dir, exist_ok=True)
+
+ bits = serialization_dict.get("bits", 4) if serialization_dict else 4
+ group_size = serialization_dict.get("group_size", 128) if serialization_dict else 128
+
+ # Pack all quantized layers (skip if already packed by immediate_pack)
+ if layer_config:
+ for layer_name in layer_config:
+ pack_layer(layer_name, model, device=device)
+
+ # Snapshot ``model_type`` (top-level + sub-configs) from the live
+ # ``model.config`` BEFORE HF's ``save_pretrained`` runs. The in-memory
+ # config still holds the original strings — newer ``transformers`` only
+ # corrupts them on serialize.
+ orig_model_types = _snapshot_original_model_types(model)
+
+ # Save model weights (uint32 packed weights are saved directly by safetensors)
+ if not unsupported_meta_device(model):
+ model.save_pretrained(output_dir, safe_serialization=True)
+
+ # Write config.json with MLX-compatible quantization info
+ config_path = os.path.join(output_dir, "config.json")
+ if hasattr(model, "config") and not os.path.exists(config_path):
+ model.config.save_pretrained(output_dir)
+
+ if os.path.exists(config_path):
+ with open(config_path, "r") as f:
+ config = json.load(f)
+ # Build mixed-bit aware quantization dict (mlx-community format).
+ quant_cfg = _build_mlx_quantization_config(model, default_bits=bits, default_group_size=group_size)
+ config["quantization"] = quant_cfg
+ autoround_format = kwargs.get("autoround_format", False)
+ quant_cfg_full = None
+ if autoround_format:
+ quant_cfg_full = {
+ "quant_method": "auto-round",
+ "packing_format": "mlx",
+ "bits": bits,
+ "group_size": group_size,
+ "sym": serialization_dict.get("sym", True) if serialization_dict else True,
+ "data_type": serialization_dict.get("data_type", "int") if serialization_dict else "int",
+ }
+ config["quantization_config"] = quant_cfg_full
+ else:
+ config["quantization_config"] = quant_cfg
+
+ # ---- VLM support ------------------------------------------------ #
+ # If the model has a language-model sub-module (e.g. Qwen-VL family),
+ # mlx-vlm's language loader queries the quantization dict under
+ # ``text_config`` (or its alias) with keys relative to that sub-module.
+ # Mirror our config there so both code paths work.
+ text_prefix = _detect_text_module_prefix(model)
+ if text_prefix:
+ sub_quant = _build_text_subconfig_quantization(quant_cfg, text_prefix)
+ for sub_key in _TEXT_SUB_CONFIG_KEYS:
+ sub_cfg = config.get(sub_key)
+ if isinstance(sub_cfg, dict):
+ sub_cfg["quantization"] = sub_quant
+ if autoround_format and quant_cfg_full is not None:
+ sub_cfg["quantization_config"] = quant_cfg_full
+
+ # Flatten rope_parameters recursively so mlx-lm/mlx-vlm find rope_theta
+ # at the level it expects (top-level for LLMs, text_config for VLMs).
+ _flatten_rope_parameters_recursive(config)
+
+ # Backfill rope_theta from the live model.config object — required by
+ # HF's model __init__ for the ``auto_round:mlx`` packing format (the
+ # checkpoint is loaded through transformers.AutoModelForCausalLM, which
+ # raises KeyError for missing rope_theta on Qwen-family models).
+ _ensure_rope_theta_from_config_obj(config, getattr(model, "config", None))
+
+ # Restore ``model_type`` (top-level + sub-configs) from the in-memory
+ # snapshot so that newer ``transformers`` versions don't silently
+ # rename them on save (e.g. vision_config: qwen3_5 -> qwen3_5_vision).
+ _preserve_original_model_types(config, orig_model_types)
+
+ with open(config_path, "w") as f:
+ json.dump(config, f, indent=2)
+
+ # Save tokenizer
+ if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):
+ tokenizer.save_pretrained(output_dir)
+
+ # Copy processor if available
+ processor = kwargs.get("processor", None)
+ if processor is not None and hasattr(processor, "save_pretrained"):
+ processor.save_pretrained(output_dir)
+
+ # Try to copy Python files from model cache
+ try:
+ copy_python_files_from_model_cache(model, output_dir)
+ except Exception as e:
+ logger.warning(f"Failed to copy Python files from model cache: {e}")
+
+ logger.info(f"Model saved to {output_dir} in MLX format (bits={bits}, group_size={group_size})")
+ logger.info(f"Load with: from mlx_lm import load, generate; " f"model, tokenizer = load('{output_dir}')")
+
+ return model
diff --git a/auto_round/formats.py b/auto_round/formats.py
index 68d392d48..cd7f076ea 100644
--- a/auto_round/formats.py
+++ b/auto_round/formats.py
@@ -1084,11 +1084,67 @@ class FP8Format(OutputFormat):
)
+class MLXFormat(OutputFormat):
+ support_schemes = ["W2A16", "W2A16G32", "W2A16G64", "W3A16", "W4A16", "W5A16", "W6A16", "W8A16", "BF16"]
+ format_name = "mlx"
+
+ @classmethod
+ def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:
+ error_logs = []
+ if scheme.bits not in [2, 3, 4, 5, 6, 8, 16]:
+ error_logs.append(f"bits={scheme.bits}")
+ if scheme.act_bits != 16:
+ error_logs.append(f"act_bits={scheme.act_bits}")
+ if not re.search("int", scheme.data_type):
+ error_logs.append(f"data_type={scheme.data_type}")
+ if scheme.super_bits:
+ error_logs.append(f"super_bits={scheme.super_bits}")
+ if scheme.super_group_size:
+ error_logs.append(f"super_group_size={scheme.super_group_size}")
+ if error_logs:
+ raise ValueError(
+ f"MLX format support quantization scheme with {','.join(cls.support_schemes)} "
+ f"but got {', '.join(error_logs)}, please have a check."
+ )
+ return True
+
+ def pack_layer(self, layer_name, model, device=None, **kwargs):
+ from auto_round.export.export_to_mlx.export import pack_layer
+
+ pack_layer(layer_name, model, device=device, **kwargs)
+
+ def save_quantized(
+ self,
+ output_dir: str,
+ model: torch.nn.Module = None,
+ tokenizer: Callable = None,
+ layer_config: dict = None,
+ inplace: bool = True,
+ device: Union[str, torch.device] = "cpu",
+ serialization_dict: dict = None,
+ **kwargs,
+ ) -> torch.nn.Module:
+ from auto_round.export.export_to_mlx.export import save_quantized_as_mlx
+
+ return save_quantized_as_mlx(
+ output_dir=output_dir,
+ model=model,
+ tokenizer=tokenizer,
+ layer_config=layer_config,
+ inplace=inplace,
+ device=device,
+ serialization_dict=serialization_dict,
+ **kwargs,
+ )
+
+
@OutputFormat.register("auto_round")
@OutputFormat.register("auto_round:auto_awq")
@OutputFormat.register("auto_round:llm_compressor")
@OutputFormat.register("auto_round:gptqmodel", "auto_round:auto_gptq")
@OutputFormat.register("auto_round:fp8")
[email protected]("auto_round:mlx")
class AutoRoundFormat(OutputFormat):
support_schemes = [
"W4A16",
@@ -1143,10 +1199,15 @@ class AutoRoundFormat(OutputFormat):
)
# for auto_round:fp8_static, auto_round:nv_fp etc.
elif not format.startswith("auto_round"):
- if format.upper() not in list(AutoRoundExportFormat.__members__.keys()):
+ if format == "mlx":
+ self.backend = MLXFormat("mlx", ar)
+ elif format.upper() not in list(AutoRoundExportFormat.__members__.keys()):
raise KeyError(f"Unsupported backend format auto_round:{format}, please check")
- self.output_format = f"auto_round:{format}"
- self.backend = None
+ else:
+ self.output_format = f"auto_round:{format}"
+ self.backend = None
+ elif format == "auto_round:mlx":
+ self.backend = MLXFormat("mlx", ar)
else:
backend = format.split(":")[1] if ":" in format else None
self.backend = self._format_list.get(backend)(format, ar) if backend else None
@@ -1219,6 +1280,9 @@ class AutoRoundFormat(OutputFormat):
**kwargs,
) -> torch.nn.Module:
if self.backend is not None:
+ extra_kwargs = {}
+ if isinstance(self.backend, MLXFormat):
+ extra_kwargs["autoround_format"] = True
return self.backend.save_quantized(
output_dir=output_dir,
model=model,
@@ -1227,6 +1291,7 @@ class AutoRoundFormat(OutputFormat):
inplace=inplace,
device=device,
serialization_dict=serialization_dict,
+ **extra_kwargs,
**kwargs,
)
backend = self.get_backend_name()
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index b36234c8c..d643ff659 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -198,6 +198,11 @@ AWQ_FORMAT = ["auto_round:auto_awq"]
LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor"]
WOQ_DEFAULT_ACT_BITS = [None, 16, 32]
+# CPU backends that target Intel/x86 (ark / ipex / auto_round_kernel) cannot
+# run on Apple Silicon. Restrict them so the MLX backend wins on Darwin and we
+# don't try to load auto-round-lib / intel-extension-for-pytorch on macOS.
+_NON_DARWIN_SYSTEMS = ["linux", "windows"]
+
# AutoGPTQ is no longer maintained, supports transformers < 5.0.0
BackendInfos["auto_gptq:exllamav2"] = BackendInfo(
device=["cuda"],
@@ -483,7 +488,7 @@ BackendInfos["gptqmodel:awq_gemm"] = BackendInfo(
packing_format=AWQ_FORMAT,
bits=[4],
group_size=[-1, 16, 32, 64, 128],
- priority=3,
+ priority=4,
compute_dtype=["float16"],
data_type=["int"],
act_bits=WOQ_DEFAULT_ACT_BITS,
@@ -514,7 +519,7 @@ BackendInfos["auto_awq:gemm"] = BackendInfo(
packing_format=AWQ_FORMAT,
bits=[4],
group_size=None,
- priority=4,
+ priority=3,
compute_dtype=["float16"],
data_type=["int"],
act_bits=WOQ_DEFAULT_ACT_BITS,
@@ -535,6 +540,7 @@ BackendInfos["auto_round_kernel"] = BackendInfo(
data_type=["int"],
act_bits=WOQ_DEFAULT_ACT_BITS,
requirements=["torch>=2.8.0", "auto-round-lib"],
+ systems=_NON_DARWIN_SYSTEMS, # auto-round-lib targets x86; not for Apple Silicon
)
BackendInfos["auto_round_kernel_xpu"] = BackendInfo(
@@ -565,6 +571,7 @@ BackendInfos["auto_round_kernel_zp"] = BackendInfo(
data_type=["int"],
act_bits=WOQ_DEFAULT_ACT_BITS,
requirements=["torch>=2.8.0", "auto-round-lib"],
+ systems=_NON_DARWIN_SYSTEMS,
)
BackendInfos["auto_round_kernel_zp_xpu"] = BackendInfo(
@@ -595,6 +602,7 @@ BackendInfos["auto_round_kernel_awq"] = BackendInfo(
data_type=["int"],
act_bits=WOQ_DEFAULT_ACT_BITS,
requirements=["torch>=2.8.0", "auto-round-lib"],
+ systems=_NON_DARWIN_SYSTEMS,
)
BackendInfos["auto_round_kernel_awq_xpu"] = BackendInfo(
@@ -625,6 +633,7 @@ BackendInfos["ipex_gptq_cpu"] = BackendInfo(
act_bits=WOQ_DEFAULT_ACT_BITS,
alias=["ipex"],
requirements=["torch<2.9", "intel-extension-for-pytorch>=2.5"],
+ systems=_NON_DARWIN_SYSTEMS, # intel-extension-for-pytorch is Intel x86 only
)
BackendInfos["ipex_gptq"] = BackendInfo(
@@ -655,6 +664,7 @@ BackendInfos["ipex_awq_cpu"] = BackendInfo(
act_bits=WOQ_DEFAULT_ACT_BITS,
alias=["ipex"],
requirements=["torch<2.9", "intel-extension-for-pytorch>=2.5"],
+ systems=_NON_DARWIN_SYSTEMS,
)
BackendInfos["ipex_awq"] = BackendInfo(
@@ -696,6 +706,22 @@ BackendInfos["hpu_zp"] = BackendInfo(
priority=0,
)
+# MLX Backend for Apple Silicon (M1, M2, M3, etc.)
+BackendInfos["mlx"] = BackendInfo(
+ device=["cpu", "mps"], # MLX runs on Apple Silicon (CPU or MPS)
+ sym=[True, False],
+ packing_format=["mlx"] + GPTQ_FORMAT + GPTQ_FORMAT_NO_ZP,
+ bits=[2, 3, 4, 5, 6, 8],
+ compute_dtype=["float32", "float16", "bfloat16"],
+ data_type=["int"],
+ act_bits=WOQ_DEFAULT_ACT_BITS,
+ group_size=[-1, 32, 64, 128, 256, 512],
+ priority=5, # High priority for Apple Silicon users
+ alias=["mlx"],
+ requirements=["mlx>=0.16.0"],
+ systems=["darwin"], # Only on macOS
+)
+
def check_compatible(
backend_name: str,
@@ -770,16 +796,16 @@ def check_compatible(
return True
-def dynamic_import_inference_linear(backend, config):
+def dynamic_import_inference_linear(backend, config, packing_format=None):
"""Dynamically imports and returns the appropriate QuantLinear class based on the given backend.
This function dynamically loads the correct `QuantLinear` class based on the backend and quantization
- configuration (e.g., ark, marlin, hpu, gptq, awq). It imports specific modules or raises
+ configuration (e.g., ark, marlin, hpu, gptq, awq, mlx). It imports specific modules or raises
errors if the required packages are not installed or the environment is not set up.
Args:
backend (str):
- The backend to be used for quantization (e.g., 'ark', 'marlin', 'hpu', 'gptq', 'awq').
+ The backend to be used for quantization (e.g., 'ark', 'marlin', 'hpu', 'gptq', 'awq', 'mlx').
config (QuantizationScheme):
The quantization configuration containing parameters like bits, group_size, and sym.
@@ -792,6 +818,19 @@ def dynamic_import_inference_linear(backend, config):
If required modules are missing for a backend (e.g., ark, GPTQ, auto_awq).
"""
bits, group_size, sym = config["bits"], config["group_size"], config["sym"]
+
+ # MLX backend
+ if "mlx" in backend:
+ # If loading a GPTQ-format checkpoint, use GPTQ QuantLinear for buffer name compatibility.
+ # The layers will be converted to MLX in post_init.
+ if packing_format and "mlx" not in packing_format:
+ from auto_round_extension.torch.qlinear_torch import QuantLinear
+
+ return QuantLinear
+ from auto_round_extension.mlx.qlinear_mlx import QuantLinearMLX
+
+ return QuantLinearMLX
+
if "torch_fp8_static" in backend:
return ar_qmodules.WeightFP8ActFP8StaticQuantLinear
if "torch_mxfp8" in backend:
@@ -1154,9 +1193,16 @@ def get_layer_backend(
def get_highest_priority_backend(
quantization_config: "AutoRoundConfig", device: str, packing_format: str
) -> str | None:
+ current_system = platform.system().lower()
supported_backends = []
for key in BackendInfos.keys():
backend = BackendInfos[key]
+ # Filter by operating system (e.g. MLX is Darwin-only; ark/ipex CPU
+ # backends are non-Darwin only).
+ if backend.systems is not None:
+ if current_system not in [s.lower() for s in backend.systems]:
+ continue
+
# Check if device is supported by the backend
if device not in backend.device:
continue
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 4ae6405f0..70bb311f7 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -45,7 +45,7 @@ from auto_round.utils import (
set_module,
)
-supported_devices = ("cpu", "hpu", "xpu", "cuda")
+supported_devices = ("cpu", "hpu", "xpu", "cuda", "mps")
def flatten_list(nested_list):
@@ -174,6 +174,9 @@ def get_available_devices():
if hasattr(torch, "xpu") and torch.xpu.is_available():
devices.append("xpu")
+ if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+ devices.append("mps")
+
devices.append("cpu") # Always available
return devices
@@ -268,7 +271,12 @@ def get_layer_config(model, quantization_config):
# Determine the quantization block list
quant_block_list = getattr(quantization_config, "quant_block_list", None)
- if quant_block_list is None:
+ if quant_block_list is not None:
+ # Handle nested list format: [[block1, block2, ...], ...] -> [prefix1, ...]
+ if quant_block_list and isinstance(quant_block_list[0], (list, tuple)):
+ for i in range(len(quant_block_list)):
+ quant_block_list[i] = os.path.commonprefix(quant_block_list[i]).rstrip(".")
+ elif quant_block_list is None:
to_quant_block_names = getattr(quantization_config, "block_name_to_quantize", None) # Prioritize this parameter
if to_quant_block_names is None:
to_quant_block_names = getattr(quantization_config, "to_quant_block_names", None)
@@ -418,7 +426,7 @@ def _replace_by_quant_layers(
logger.debug(f"{layer_name}: {layer_backend} backend is used")
# Create and replace layer
- new_layer = _create_quant_layer(layer, layer_backend, config, in_features, out_features)
+ new_layer = _create_quant_layer(layer, layer_backend, config, in_features, out_features, packing_format)
set_module(module, layer_name, new_layer)
return used_backends
@@ -445,11 +453,21 @@ def _import_exllamav2_kernels():
logger.warning_once("try to fallback to other autogptq backends for now")
-def _create_quant_layer(layer, layer_backend, config, in_features, out_features):
+def _create_quant_layer(layer, layer_backend, config, in_features, out_features, packing_format=None):
"""Creates a quantized layer using the appropriate class."""
- QuantLinear = dynamic_import_inference_linear(layer_backend, config)
+ QuantLinear = dynamic_import_inference_linear(layer_backend, config, packing_format=packing_format)
bias = layer.bias is not None
+ # MLX backend with MLX packing format (native MLX checkpoint)
+ if "mlx" in layer_backend and (packing_format is None or "mlx" in packing_format):
+ return QuantLinear(
+ bits=config["bits"],
+ group_size=config["group_size"],
+ infeatures=in_features,
+ outfeatures=out_features,
+ bias=bias,
+ )
+
# Special handling for auto-round-lib AWQ layers
from auto_round_extension.ark.qlinear import QuantLinearAWQ
@@ -550,6 +568,71 @@ def convert_gptq_v1_to_v2_format(model: nn.Module):
logger.warning_once("Converting gptq v1 to v2 format")
+def _maybe_convert_gptq_to_mlx(model: nn.Module, used_backends: list[str]) -> None:
+ """On macOS with MLX available, convert GPTQ-format QuantLinear layers to MLX QuantLinearMLX.
+
+ This is the MLX equivalent of the IPEX/ARK post_init step: when an MLX backend was
+ selected but the checkpoint layers were materialized in GPTQ packing format, we
+ re-pack them into the MLX format so that ``mx.quantized_matmul`` can be used for
+ hardware-accelerated inference on Apple Silicon. All conversion logic lives in
+ :meth:`QuantLinearMLX.from_gptq`.
+ """
+ import platform
+
+ if platform.system() != "Darwin":
+ return
+
+ # Only run if an MLX-related backend was selected for some layer.
+ if not any("mlx" in b for b in used_backends):
+ return
+
+ try:
+ import mlx.core as mx # noqa: F401
+ except ImportError:
+ logger.debug("MLX not installed, skipping GPTQ-to-MLX conversion")
+ return
+
+ from auto_round_extension.mlx.qlinear_mlx import QuantLinearMLX
+
+ # Collect GPTQ-style layers that need re-packing into MLX format.
+ layers = []
+ for name, module in model.named_modules():
+ if not hasattr(module, "QUANT_TYPE"):
+ continue
+ if getattr(module, "QUANT_TYPE", None) == "mlx":
+ continue
+ if not hasattr(module, "qweight"):
+ continue
+ layers.append((name, module))
+
+ if not layers:
+ return
+
+ # Get sym flag from model config (once, outside loop)
+ quant_config = getattr(model.config, "quantization_config", {})
+ if hasattr(quant_config, "to_dict"):
+ quant_config = quant_config.to_dict()
+ sym = quant_config.get("sym", False)
+
+ converted = 0
+ skipped = 0
+ for name, module in tqdm(layers, desc="repacking to MLX format", total=len(layers), leave=True):
+ try:
+ mlx_layer = QuantLinearMLX.from_gptq(module, sym=sym)
+ set_module(model, name, mlx_layer)
+ converted += 1
+
+ except Exception as e:
+ logger.warning(f"Failed to convert layer '{name}' to MLX format: {e}")
+ skipped += 1
+ continue
+
+ if converted > 0:
+ logger.info(f"Auto-converted {converted} GPTQ layers to MLX format for Apple Silicon acceleration")
+ if skipped > 0:
+ logger.warning(f"Skipped {skipped} layers during GPTQ-to-MLX conversion")
+
+
def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:
"""Performs post-initialization for different quantization backends.
@@ -631,6 +714,9 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:
if used_gptq_exllamav2:
_import_exllamav2_kernels()
+ # On macOS (Apple Silicon), auto-convert GPTQ layers to MLX for faster inference
+ _maybe_convert_gptq_to_mlx(model, used_backends)
+
# Determine common data type across backends
data_types = [set(BackendInfos[b].compute_dtype) for b in used_backends]
common_dtypes = set.intersection(*data_types) if data_types else set()
@@ -719,6 +805,8 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.M
packing_format = "auto_round:auto_awq"
elif packing_format == "auto_round:gptq":
packing_format = "auto_round:auto_gptq"
+ elif packing_format in ("mlx", "auto_round:mlx"):
+ pass # keep as-is for MLX backend selection
is_applied = apply_modeling_patch(model)
if not is_applied:
# Preprocess model before replace layers
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
index 0e9b922ba..c3bec7ebf 100644
--- a/auto_round/schemes.py
+++ b/auto_round/schemes.py
@@ -142,6 +142,26 @@ W4A16 = QuantizationScheme.from_dict(
}
)
+W5A16 = QuantizationScheme.from_dict(
+ {
+ "bits": 5,
+ "sym": True,
+ "group_size": 128,
+ "data_type": "int",
+ "act_bits": 16,
+ }
+)
+
+W6A16 = QuantizationScheme.from_dict(
+ {
+ "bits": 6,
+ "sym": True,
+ "group_size": 128,
+ "data_type": "int",
+ "act_bits": 16,
+ }
+)
+
W2A16 = QuantizationScheme.from_dict(
{
"bits": 2,
@@ -373,6 +393,8 @@ PRESET_SCHEMES = {
"W4A16": W4A16,
"W2A16": W2A16,
"W3A16": W3A16,
+ "W5A16": W5A16,
+ "W6A16": W6A16,
"W8A16": W8A16,
"MXFP4": MXFP4,
"MXFP6": MXFP6,
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.py
index 30c9edc0c..03a98dc12 100644
--- a/auto_round/utils/common.py
+++ b/auto_round/utils/common.py
@@ -443,6 +443,8 @@ class SupportedFormats:
"llm_compressor",
"fp8",
"auto_round:fp8",
+ "mlx",
+ "auto_round:mlx",
)
self._gguf_format = tuple(sorted(GGUF_CONFIG.keys()))
self._support_list = self._support_format + self._gguf_format
diff --git a/auto_round_extension/mlx/__init__.py b/auto_round_extension/mlx/__init__.py
new file mode 100644
index 000000000..4a8119fce
--- /dev/null
+++ b/auto_round_extension/mlx/__init__.py
@@ -0,0 +1,17 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from auto_round_extension.mlx.qlinear_mlx import QuantLinearMLX, MLX_AVAILABLE
+
+__all__ = ["QuantLinearMLX", "MLX_AVAILABLE"]
diff --git a/auto_round_extension/mlx/qlinear_mlx.py b/auto_round_extension/mlx/qlinear_mlx.py
new file mode 100644
index 000000000..81f1e81e4
--- /dev/null
+++ b/auto_round_extension/mlx/qlinear_mlx.py
@@ -0,0 +1,341 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+MLX format QuantLinear for inference via MLX kernels on Apple Silicon.
+
+Uses mx.quantized_matmul for hardware-accelerated quantized inference.
+Falls back to PyTorch dequantization if MLX is not available.
+
+Tensor naming follows MLX convention:
+ - weight: uint32 packed integers [out_features, packed_in_features]
+ - scales: float16 [out_features, num_groups]
+ - biases: float16 [out_features, num_groups] (MLX calls zero-point-derived term "biases")
+ - bias: optional linear bias
+"""
+
+import math
+
+import torch
+import torch.nn as nn
+
+try:
+ import mlx.core as mx
+
+ MLX_AVAILABLE = True
+except ImportError:
+ MLX_AVAILABLE = False
+
+
+def _torch_to_mlx(t):
+ """Convert a PyTorch tensor to MLX array."""
+ # Accept either a numpy array or a torch.Tensor.
+ try:
+ import numpy as _np
+ except Exception:
+ _np = None
+
+ # If it's a torch tensor, try to use DLPack for zero-copy when supported by MLX.
+ if isinstance(t, torch.Tensor):
+ # Ensure tensor is contiguous
+ tt = t.contiguous()
+ try:
+ from torch.utils.dlpack import to_dlpack
+
+ if hasattr(mx, "from_dlpack"):
+ return mx.from_dlpack(to_dlpack(tt))
+ except Exception:
+ # fall back to numpy
+ pass
+
+ return mx.array(tt.cpu().numpy())
+
+ # Otherwise assume it's a numpy-like object
+ return mx.array(t)
+
+
+def _mlx_to_torch(a, dtype=None):
+ """Convert an MLX array to PyTorch tensor."""
+ import numpy as np
+
+ t = torch.from_numpy(np.array(a))
+ if dtype is not None:
+ t = t.to(dtype)
+ return t
+
+
+class QuantLinearMLX(nn.Module):
+ """Quantized linear layer using MLX kernels for fast inference on Apple Silicon.
+
+ Falls back to pure PyTorch dequantization when MLX is not available.
+ """
+
+ QUANT_TYPE = "mlx"
+
+ def __init__(self, bits, group_size, infeatures, outfeatures, bias, **kwargs):
+ super().__init__()
+ if bits not in [2, 3, 4, 5, 6, 8]:
+ raise NotImplementedError("Only 2, 3, 4, 5, 6, 8 bits are supported.")
+ self.infeatures = infeatures
+ self.outfeatures = outfeatures
+ self.bits = bits
+ self.group_size = group_size if group_size != -1 else infeatures
+ self.maxq = 2**self.bits - 1
+
+ # MLX packing: [out_features, in_features * bits / 32]
+ packed_dim = infeatures * bits // 32
+ num_groups = math.ceil(infeatures / self.group_size)
+
+ self.register_buffer(
+ "weight",
+ torch.zeros((outfeatures, packed_dim), dtype=torch.uint32),
+ )
+ self.register_buffer(
+ "scales",
+ torch.zeros((outfeatures, num_groups), dtype=torch.float16),
+ )
+ self.register_buffer(
+ "biases",
+ torch.zeros((outfeatures, num_groups), dtype=torch.float16),
+ )
+ if bias:
+ self.register_buffer("bias", torch.zeros(outfeatures, dtype=torch.float16))
+ else:
+ self.bias = None
+
+ # Cache MLX arrays after first forward
+ self._mlx_weight = None
+ self._mlx_scales = None
+ self._mlx_biases = None
+
+ def _ensure_mlx_cache(self):
+ """Convert PyTorch buffers to MLX arrays, cached for reuse."""
+ if self._mlx_weight is None:
+ # Avoid unnecessary device transfers when buffers are already on CPU.
+ weight_for_mx = self.weight if self.weight.device.type == "cpu" else self.weight.cpu()
+ scales_for_mx = self.scales if self.scales.device.type == "cpu" else self.scales.cpu()
+ biases_for_mx = self.biases if self.biases.device.type == "cpu" else self.biases.cpu()
+
+ self._mlx_weight = _torch_to_mlx(weight_for_mx)
+ self._mlx_scales = _torch_to_mlx(scales_for_mx.float())
+ self._mlx_biases = _torch_to_mlx(biases_for_mx.float())
+
+ def _forward_mlx(self, x):
+ """Forward pass using MLX quantized_matmul kernel."""
+ self._ensure_mlx_cache()
+
+ input_dtype = x.dtype
+ # Avoid an unnecessary copy when input is already on CPU.
+ if x.device.type == "cpu":
+ x_cpu = x.detach()
+ else:
+ # For MPS or other devices, move to CPU for MLX bridge.
+ # Use .to('cpu') which may be optimized on Apple Silicon unified memory.
+ x_cpu = x.detach().to("cpu")
+
+ # Prepare contiguous float32 tensor for MLX consumption; _torch_to_mlx
+ # will attempt a zero-copy DLPack transfer if supported by MLX.
+ x_for_mx = x_cpu.float().contiguous()
+ x_mlx = _torch_to_mlx(x_for_mx)
+
+ # mx.quantized_matmul: x @ w.T with quantized w
+ out_mlx = mx.quantized_matmul(
+ x_mlx,
+ self._mlx_weight,
+ scales=self._mlx_scales,
+ biases=self._mlx_biases,
+ transpose=True,
+ group_size=self.group_size,
+ bits=self.bits,
+ )
+ mx.eval(out_mlx)
+
+ out = _mlx_to_torch(out_mlx, dtype=input_dtype)
+
+ if self.bias is not None:
+ out = out + self.bias.to(input_dtype)
+
+ return out.to(x.device)
+
+ def _forward_torch(self, x):
+ """Fallback forward pass using pure PyTorch dequantization."""
+ input_dtype = x.dtype
+ input_device = x.device
+ # Move to CPU for bitwise unpack ops (MPS may not support all int ops)
+ intweight = self._unpack_weight_torch(device=torch.device("cpu")).float()
+ scales = self.scales.cpu().float().repeat_interleave(self.group_size, dim=1)[:, : self.infeatures]
+ biases = self.biases.cpu().float().repeat_interleave(self.group_size, dim=1)[:, : self.infeatures]
+ weight = scales * intweight + biases
+ # NOTE: self.bias is registered as float16 but ``weight`` here is float32 because
+ # we promoted intweight via .float(). F.linear requires bias dtype to match weight
+ # dtype, otherwise it raises RuntimeError. Cast explicitly.
+ bias = self.bias.cpu().to(weight.dtype) if self.bias is not None else None
+ out = torch.nn.functional.linear(x.cpu().to(weight.dtype), weight, bias)
+ return out.to(input_dtype).to(input_device)
+
+ def _unpack_weight_torch(self, device=None):
+ """Unpack uint32 weight tensor to integer values (PyTorch fallback)."""
+ weight = self.weight.view(torch.int32)
+ if device is not None:
+ weight = weight.to(device)
+ out_features, packed_dim = weight.shape
+
+ if 32 % self.bits == 0:
+ elems_per_int = 32 // self.bits
+ shifts = torch.arange(elems_per_int, device=weight.device, dtype=torch.int32) * self.bits
+ mask = (1 << self.bits) - 1
+ unpacked = (weight.unsqueeze(-1) >> shifts) & mask
+ unpacked = unpacked.reshape(out_features, -1)
+ else:
+ num_groups_32 = packed_dim // self.bits
+ weight_reshaped = weight.reshape(out_features, num_groups_32, self.bits).to(torch.int64)
+ unpacked = torch.zeros(out_features, num_groups_32, 32, dtype=torch.int64, device=weight.device)
+ mask = (1 << self.bits) - 1
+ for i in range(32):
+ val = 0
+ for b in range(self.bits):
+ abs_bit = i * self.bits + b
+ word_idx = abs_bit // 32
+ bit_pos = abs_bit % 32
+ val |= ((weight_reshaped[:, :, word_idx] >> bit_pos) & 1) << b
+ unpacked[:, :, i] = val & mask
+ unpacked = unpacked.reshape(out_features, -1).to(torch.int32)
+
+ return unpacked[:, : self.infeatures]
+
+ def forward(self, x):
+ if MLX_AVAILABLE:
+ return self._forward_mlx(x)
+ return self._forward_torch(x)
+
+ @classmethod
+ def from_gptq(cls, module: nn.Module, sym: bool = False) -> "QuantLinearMLX":
+ """Convert a GPTQ-format QuantLinear module into a QuantLinearMLX module.
+
+ Repacks qweight/qzeros/scales (GPTQ format) into weight/scales/biases (MLX format)
+ so that ``mx.quantized_matmul`` can be used for hardware-accelerated inference on
+ Apple Silicon.
+
+ Args:
+ module: The source GPTQ-style quantized linear module (must expose
+ ``qweight``, ``scales``, ``bits``, ``group_size`` and optionally ``qzeros``).
+ sym: Whether symmetric quantization was used. For symmetric quantization the
+ zero-point is fixed to ``2**(bits-1)`` which avoids the GPTQ ``qzeros`` ±1
+ offset ambiguity.
+
+ Returns:
+ A fully initialized ``QuantLinearMLX`` module ready for inference.
+ """
+ # Use CPU for conversion to avoid MPS compatibility issues with bitwise ops
+ convert_device = torch.device("cpu")
+
+ bits = module.bits
+ group_size = module.group_size
+ if bits not in [2, 3, 4, 8]:
+ raise NotImplementedError(f"Unsupported bits={bits} for GPTQ->MLX conversion.")
+
+ qweight = module.qweight.to(convert_device) # [in//32*bits, out] int32
+ scales_gptq = module.scales.to(convert_device) # [groups, out] float16
+
+ out_features = scales_gptq.shape[1]
+ in_features = qweight.shape[0] * 32 // bits
+
+ # Unpack GPTQ qweight -> int weight [in_features, out_features]
+ if bits in [2, 4, 8]:
+ elems_per_int = 32 // bits
+ wf = torch.arange(0, 32, bits, dtype=torch.int32, device=convert_device)
+ weight_int = torch.bitwise_right_shift(
+ qweight.unsqueeze(1).expand(-1, elems_per_int, -1),
+ wf.unsqueeze(-1),
+ ).to(torch.int16 if bits == 8 else torch.int8)
+ weight_int = torch.bitwise_and(weight_int, (1 << bits) - 1)
+ weight_int = weight_int.reshape(-1, out_features)
+ else: # bits == 3
+ from auto_round_extension.torch.qlinear_torch import get_wf_3bits_tensor
+
+ wf = get_wf_3bits_tensor(convert_device)
+ weight_int = qweight.reshape(qweight.shape[0] // 3, 3, 1, qweight.shape[1]).expand(-1, -1, 12, -1)
+ weight_int = (weight_int >> wf.unsqueeze(-1)) & 0x7
+ weight_int[:, 0, 10] = (weight_int[:, 0, 10] & 0x3) | ((weight_int[:, 1, 0] << 2) & 0x4)
+ weight_int[:, 1, 11] = (weight_int[:, 1, 11] & 0x1) | ((weight_int[:, 2, 0] << 1) & 0x6)
+ weight_int = weight_int & 0x7
+ weight_int = torch.cat([weight_int[:, 0, :11], weight_int[:, 1, 1:12], weight_int[:, 2, 1:11]], dim=1)
+ weight_int = weight_int.reshape(-1, out_features)
+
+ # Determine zeros
+ # NOTE on GPTQ qzeros offset:
+ # GPTQ packs qzeros as ``(actual_zero - 1) & maxq`` (a quirk of the
+ # original GPTQ kernel). When unpacking we MUST add 1 (mod 2**bits)
+ # to recover the true zero point. AutoRound's own asymmetric W4A16
+ # actually exports to AWQ format, but third-party GPTQ checkpoints
+ # loaded via ``auto_gptq`` / ``gptqmodel`` (packing format
+ # ``auto_round:auto_gptq``) hit this code path and require the +1.
+ maxq = (1 << bits) - 1
+ if sym:
+ zero_val = 2 ** (bits - 1)
+ zeros = torch.full(scales_gptq.shape, zero_val, dtype=torch.int32, device=convert_device)
+ else:
+ if not hasattr(module, "qzeros") or module.qzeros is None:
+ zero_val = 2 ** (bits - 1)
+ zeros = torch.full(scales_gptq.shape, zero_val, dtype=torch.int32, device=convert_device)
+ else:
+ qzeros = module.qzeros.to(convert_device)
+ if bits in [2, 4, 8]:
+ elems_per_int = 32 // bits
+ wf = torch.arange(0, 32, bits, dtype=torch.int32, device=convert_device)
+ zeros = torch.bitwise_right_shift(
+ qzeros.unsqueeze(2).expand(-1, -1, elems_per_int),
+ wf.unsqueeze(0),
+ ).to(torch.int16 if bits == 8 else torch.int8)
+ zeros = torch.bitwise_and(zeros, maxq)
+ zeros = zeros.reshape(scales_gptq.shape).to(torch.int32)
+ else: # bits == 3
+ from auto_round_extension.torch.qlinear_torch import get_wf_3bits_tensor
+
+ wf = get_wf_3bits_tensor(convert_device)
+ zeros = qzeros.reshape(qzeros.shape[0], qzeros.shape[1] // 3, 3, 1).expand(-1, -1, -1, 12)
+ zeros = zeros >> wf.unsqueeze(0)
+ zeros[:, :, 0, 10] = (zeros[:, :, 0, 10] & 0x3) | ((zeros[:, :, 1, 0] << 2) & 0x4)
+ zeros[:, :, 1, 11] = (zeros[:, :, 1, 11] & 0x1) | ((zeros[:, :, 2, 0] << 1) & 0x6)
+ zeros = zeros & 0x7
+ zeros = torch.cat([zeros[:, :, 0, :11], zeros[:, :, 1, 1:12], zeros[:, :, 2, 1:11]], dim=2)
+ zeros = zeros.reshape(scales_gptq.shape).to(torch.int32)
+
+ # GPTQ qzeros are stored off-by-one; recover the true zero point.
+ zeros = (zeros + 1) & maxq
+
+ # GPTQ: w = scale * (w_int - zero)
+ # MLX: w = mlx_scale * w_int + mlx_bias
+ # => mlx_scale = scale, mlx_bias = -scale * zero
+ from auto_round.export.export_to_mlx.export import _pack_weight_mlx
+
+ weight_int_t = weight_int.t().contiguous().to(torch.int32)
+ scales_mlx = scales_gptq.t().contiguous().to(torch.float16)
+ biases_mlx = (-scales_gptq.float().t().contiguous() * zeros.float().t().contiguous()).to(torch.float16)
+ packed_weight = _pack_weight_mlx(weight_int_t, bits)
+
+ has_bias = getattr(module, "bias", None) is not None
+ mlx_layer = cls(
+ bits=bits,
+ group_size=group_size,
+ infeatures=in_features,
+ outfeatures=out_features,
+ bias=has_bias,
+ )
+ mlx_layer.weight.copy_(packed_weight)
+ mlx_layer.scales.copy_(scales_mlx)
+ mlx_layer.biases.copy_(biases_mlx)
+ if has_bias:
+ mlx_layer.bias.copy_(module.bias.to(torch.float16))
+ return mlx_layer
diff --git a/auto_round_extension/torch/qlinear_mlx.py b/auto_round_extension/torch/qlinear_mlx.py
new file mode 100644
index 000000000..d506ffb12
--- /dev/null
+++ b/auto_round_extension/torch/qlinear_mlx.py
@@ -0,0 +1,17 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Backward compatibility shim. Use auto_round_extension.mlx.qlinear_mlx instead."""
+
+from auto_round_extension.mlx.qlinear_mlx import MLX_AVAILABLE, QuantLinearMLX # noqa: F401
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 53acef0bc..dfb55e7a8 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -119,6 +119,8 @@ AutoRound supports several Schemes:
- **W4A16**(bits:4,group_size:128,sym:True,act_bits:16)
- **W8A16**(bits:8,group_size:128,sym:True,act_bits:16)
+- **W6A16**(bits:6,group_size:128,sym:True,act_bits:16) — `mlx` format only
+- **W5A16**(bits:5,group_size:128,sym:True,act_bits:16) — `mlx` format only
- **W3A16**(bits:3,group_size:128,sym:True,act_bits:16)
- **W2A16**(bits:2,group_size:128,sym:True,act_bits:16)
- **GGUF:Q4_K_M**(all Q*_K,Q*_0,Q*_1 provided by llamacpp are supported)
@@ -152,6 +154,11 @@ adopted within the community, **only 4-bits quantization is supported**. Please
**LLM-Compressor Format**: **NVFP4, MXFP4(kernel in WIP), MXFP8 are supported**. Please set `--format llm_compressor`
+**MLX Format**: This format targets Apple Silicon (M1/M2/M3/...) and is loaded directly by [`mlx-lm`](https://github.com/ml-explore/mlx-lm) (text-only LLM) or [`mlx-vlm`](https://github.com/Blaizzy/mlx-vlm) (vision/audio + language).
+- Supports **2, 3, 4, 5, 6, 8 bits** (5/6 bits are MLX-exclusive — GPTQ/AWQ have no standard packing for them).
+- Native **mixed-bit / mixed-group_size** via `layer_config` or AutoScheme (`--target_bits 3.5 --options "..."`);
+- Use `--format mlx` for a native MLX checkpoint; use `--format auto_round:mlx` if you want HuggingFace `transformers` + AutoRound to load it (post-init repacks each layer into MLX `QuantLinear` on Darwin).
+- Limitation: embedding layer quantization has not supported
#### Format and scheme support matrix
> Gray indicates the absence of a kernel or the presence of only an inefficient/reference kernel. BF16 is mainly for AutoScheme
@@ -162,13 +169,14 @@ adopted within the community, **only 4-bits quantization is supported**. Please
| **auto_awq** | W4A16, BF16 |
| **auto_gptq** | W4A16, W2A16, W3A16, W8A16,W2A16G64, W2A16G32, BF16 |
| **llm_compressor** | NVFP4, `MXFP4`, `MXFP8`, `FPW8A16`, `FP8_STATIC`, FP8_BLOCK |
+| **mlx** / **auto_round:mlx** | W2A16, W3A16, W4A16, W5A16, W6A16, W8A16, BF16, mixed-bit / mixed-group_size (Apple Silicon only) |
| **gguf** | GGUF:Q4_K_M, GGUF:Q2_K_S, GGUF:Q3_K_S, GGUF:Q3_K_M, GGUF:Q3_K_L, GGUF:Q4_K_S, GGUF:Q5_K_S, GGUF:Q5_K_M, GGUF:Q6_K, GGUF:Q4_0, GGUF:Q4_1, GGUF:Q5_0, GGUF:Q5_1,GGUF:Q8_0 |
| **fp8** | FP8_BLOCK |
| **fake** | `all schemes (only for research)` |
### Hardware Compatibility
-CPU, Intel GPU, HPU and CUDA for both quantization and inference.
+CPU, Intel GPU, HPU and CUDA for both quantization and inference. The **MLX format** is exclusive to **Apple Silicon (macOS / Darwin)** at inference time; quantization (export) itself can be done on any platform.
### Environment Configuration
diff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.md
index 5eddd22d3..e04021a03 100644
--- a/docs/step_by_step_CN.md
+++ b/docs/step_by_step_CN.md
@@ -113,6 +113,8 @@ pip install auto-round
AutoRound 支持多种量化配置:
- **W4A16**(bits:4, group_size:128, sym:True, act_bits:16) # 4位权重,分组大小为128,对称量化,16位激活,
- **W8A16**(bits:8, group_size:128, sym:True, act_bits:16)
+- **W6A16**(bits:6, group_size:128, sym:True, act_bits:16) — 仅 `mlx` 格式支持
+- **W5A16**(bits:5, group_size:128, sym:True, act_bits:16) — 仅 `mlx` 格式支持
- **W3A16**(bits:3, group_size:128, sym:True, act_bits:16)
- **W2A16**(bits:2, group_size:128, sym:True, act_bits:16)
- **GGUF:Q4_K_M**(支持 llamacpp 提供的所有 Q*_K、Q*_0、Q*_1 量化类型)
@@ -142,6 +144,12 @@ AutoRound 支持多种量化配置:
**LLM-Compressor 格式**:**支持 NVFP4、MXFP4(kernel 开发中)、MXFP8** 等。需设置 `--format llm_compressor`。
+**MLX 格式**:面向 Apple Silicon (M1/M2/M3/...),可直接被 [`mlx-lm`](https://github.com/ml-explore/mlx-lm)(纯文本 LLM)或 [`mlx-vlm`](https://github.com/Blaizzy/mlx-vlm)(多模态 VLM)加载推理。
+- 支持 **2、3、4、5、6、8 bits**(其中 5/6 bits 是 MLX 独有,GPTQ/AWQ 没有标准打包格式)。
+- 原生支持 **混合 bit / 混合 group_size**:通过 `layer_config` 或 AutoScheme(如 `--target_bits 3.5 --options "..."`),按层覆盖会写入 `config.json["quantization"]`,
+- `--format mlx` 导出原生 MLX checkpoint;`--format auto_round:mlx` 则让 HuggingFace `transformers` + AutoRound 加载它(在 Darwin 上 post-init 会把每层重新打包成 MLX 的 `QuantLinear`)。
+- 已经问题: 没有支持嵌入层的量化
+
#### 格式与方案支持对照表
> 灰色背景的 schemes 表示它没有专门优化的内核,或只有效率极低的参考内核。
@@ -152,13 +160,14 @@ AutoRound 支持多种量化配置:
| **auto_awq** | W4A16、BF16 |
| **auto_gptq** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、BF16 |
| **llm_compressor** | NVFP4、`MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC`、FP8_STATIC |
+| **mlx** / **auto_round:mlx** | W2A16、W3A16、W4A16、W5A16、W6A16、W8A16、BF16、混合 bit / 混合 group_size(仅 Apple Silicon) |
| **gguf** | GGUF:Q4_K_M、GGUF:Q2_K_S、GGUF:Q3_K_S、GGUF:Q3_K_M、GGUF:Q3_K_L、GGUF:Q4_K_S、GGUF:Q5_K_S、GGUF:Q5_K_M、GGUF:Q6_K、GGUF:Q4_0、GGUF:Q4_1、GGUF:Q5_0、GGUF:Q5_1、GGUF:Q8_0 |
| **fp8** | FP8_BLOCK |
| **fake** | `所有方案(仅用于研究场景)` |
### 硬件兼容性
-量化和推理均支持 CPU、英特尔 GPU、HPU 和 CUDA。
+量化和推理均支持 CPU、英特尔 GPU、HPU 和 CUDA。**MLX 格式**的推理仅支持 **Apple Silicon (macOS / Darwin)**,但量化(导出)阶段在任意平台均可进行。
### 环境参数配置
diff --git a/test/helpers.py b/test/helpers.py
index f2a670d4c..b568c137a 100644
--- a/test/helpers.py
+++ b/test/helpers.py
@@ -132,6 +132,7 @@ deepseek_v2_name_or_path = get_model_path("deepseek-ai/DeepSeek-V2-Lite")
qwen_moe_name_or_path = get_model_path("Qwen/Qwen1.5-MoE-A2.7B")
qwen_vl_name_or_path = get_model_path("Qwen/Qwen2-VL-2B-Instruct")
qwen_2_5_vl_name_or_path = get_model_path("Qwen/Qwen2.5-VL-3B-Instruct")
+qwen_3_vl_9b_name_or_path = get_model_path("Qwen/Qwen3.5-9B")
gemma_name_or_path = get_model_path("benzart/gemma-2b-it-fine-tuning-for-code-test")
qwen2_5_omni_name_or_path = get_model_path("Qwen/Qwen2.5-Omni-3B")
qwen3_omni_name_or_path = get_model_path("Qwen/Qwen3-Omni-30B-A3B-Instruct")
diff --git a/test/test_cuda/algorithms/test_auto_scheme.py b/test/test_cuda/algorithms/test_auto_scheme.py
index 0967f7e01..9cde148d7 100644
--- a/test/test_cuda/algorithms/test_auto_scheme.py
+++ b/test/test_cuda/algorithms/test_auto_scheme.py
@@ -280,7 +280,7 @@ class TestAutoScheme:
quantization.
"""
target_bits = 2.5
- ref_threshold = 0.58
+ ref_threshold = 0.56
common_kwargs = dict(
iters=0,
disable_opt_rtn=True,
diff --git a/test/test_mlx/__init__.py b/test/test_mlx/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/test_mlx/test_mlx_format.py b/test/test_mlx/test_mlx_format.py
new file mode 100644
index 000000000..81106ca6d
--- /dev/null
+++ b/test/test_mlx/test_mlx_format.py
@@ -0,0 +1,496 @@
+# Copyright (c) 2025 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+
+"""Unified MLX format tests for AutoRound on Qwen3-0.6B (RTN mode).
+
+Two export formats are covered:
+
+ * ``mlx`` — native MLX checkpoint, loaded and run through ``mlx_lm``
+ on Apple Silicon.
+ * ``auto_round`` — GPTQ-style packing. On Darwin the loader post-init
+ re-packs each layer into :class:`QuantLinearMLX` via
+ :meth:`QuantLinearMLX.from_gptq`; inference uses the
+ HuggingFace ``transformers`` + AutoRound pipeline.
+
+Each format is exercised over different bit-widths (2/3/4/8) and a mixed-bit
+configuration via ``layer_config``. All runs use RTN (``iters=0``) for speed.
+Platform/runtime-specific inference tests are skipped automatically on
+non-Darwin platforms or when the required packages are missing.
+"""
+
+import importlib.util
+import json
+import os
+import platform
+import shutil
+from pathlib import Path
+from test.helpers import qwen_3_vl_9b_name_or_path, qwen_name_or_path
+
+import pytest
+import torch
+
+from auto_round import AutoRound
+
+MLX_AVAILABLE = importlib.util.find_spec("mlx") is not None
+MLX_LM_AVAILABLE = importlib.util.find_spec("mlx_lm") is not None
+MLX_VLM_AVAILABLE = importlib.util.find_spec("mlx_vlm") is not None
+IS_DARWIN = platform.system() == "Darwin"
+
+# Native mlx checkpoint needs Darwin + mlx_lm to actually load & run.
+requires_mlx_lm_runtime = pytest.mark.skipif(
+ not (IS_DARWIN and MLX_LM_AVAILABLE),
+ reason="Native MLX inference requires macOS (Darwin) with 'mlx_lm' installed.",
+)
+# auto_round GPTQ -> QuantLinearMLX post-init needs Darwin + mlx.
+requires_mlx_runtime = pytest.mark.skipif(
+ not (IS_DARWIN and MLX_AVAILABLE),
+ reason="QuantLinearMLX post-init requires macOS (Darwin) with 'mlx' installed.",
+)
+# Native mlx VLM checkpoint needs Darwin + mlx_vlm to actually load & run.
+requires_mlx_vlm_runtime = pytest.mark.skipif(
+ not (IS_DARWIN and MLX_VLM_AVAILABLE),
+ reason="Native MLX VLM inference requires macOS (Darwin) with 'mlx_vlm' installed.",
+)
+# Skip large 9B VLM tests when the model isn't materialized locally — we don't
+# want CI to silently kick off a ~18 GB HuggingFace download.
+_qwen3_vl_9b_local = os.path.isdir(qwen_3_vl_9b_name_or_path)
+requires_qwen3_vl_9b_local = pytest.mark.skipif(
+ not _qwen3_vl_9b_local,
+ reason=(
+ f"Qwen3-VL-9B model not found locally at '{qwen_3_vl_9b_name_or_path}'. "
+ "Place the weights under one of the test model paths to enable this test."
+ ),
+)
+
+
+# --------------------------------------------------------------------------- #
+# Helpers
+# --------------------------------------------------------------------------- #
+def _quantize_and_save(
+ output_dir: str,
+ scheme: str,
+ bits: int,
+ fmt: str,
+ sym: bool = True,
+ group_size: int = 128,
+ layer_config: dict = None,
+):
+ """Run AutoRound RTN quantization on Qwen3-0.6B and export to ``fmt``."""
+ ar = AutoRound(
+ model=qwen_name_or_path,
+ scheme=scheme,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=0, # RTN
+ nsamples=1,
+ seqlen=32,
+ disable_opt_rtn=True,
+ layer_config=layer_config,
+ )
+ ar.quantize_and_save(output_dir=output_dir, format=fmt)
+
+
+def _read_quant_config(output_dir: str) -> dict:
+ """Return the quantization_config dict from the exported model."""
+ config_path = Path(output_dir) / "config.json"
+ if config_path.exists():
+ with open(config_path, "r", encoding="utf-8") as f:
+ cfg = json.load(f)
+ if "quantization_config" in cfg:
+ return cfg["quantization_config"]
+ standalone = Path(output_dir) / "quantization_config.json"
+ if standalone.exists():
+ with open(standalone, "r", encoding="utf-8") as f:
+ return json.load(f)
+ return {}
+
+
+def _count_mlx_layers(model: torch.nn.Module) -> int:
+ """Count QuantLinearMLX instances in a loaded HF model."""
+ from auto_round_extension.mlx.qlinear_mlx import QuantLinearMLX
+
+ return sum(1 for m in model.modules() if isinstance(m, QuantLinearMLX))
+
+
+def _mlx_lm_generate(output_dir: str, prompt: str = "The capital of France is", max_tokens: int = 8) -> str:
+ """Load a native MLX checkpoint through mlx_lm and generate text."""
+ from mlx_lm import generate, load
+
+ model, tokenizer = load(output_dir)
+ return generate(model, tokenizer, prompt=prompt, max_tokens=max_tokens)
+
+
+def _hf_generate(output_dir: str, prompt: str = "The capital of France is", max_new_tokens: int = 5):
+ """Load an auto_round checkpoint through HF transformers and generate text.
+
+ Returns:
+ Tuple of (model, decoded_text).
+ """
+ from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizer
+
+ model = AutoModelForCausalLM.from_pretrained(
+ output_dir,
+ device_map="cpu",
+ torch_dtype="auto",
+ trust_remote_code=True,
+ quantization_config=AutoRoundConfig(),
+ )
+ tokenizer = AutoTokenizer.from_pretrained(output_dir, trust_remote_code=True)
+ inputs = tokenizer(prompt, return_tensors="pt")
+ with torch.no_grad():
+ out = model.generate(**inputs, max_new_tokens=max_new_tokens)
+ return model, tokenizer.decode(out[0], skip_special_tokens=True)
+
+
+# --------------------------------------------------------------------------- #
+# Tests
+# --------------------------------------------------------------------------- #
+class TestMLXFormat:
+ @pytest.fixture(autouse=True)
+ def _save_dir(self, tmp_path):
+ self.save_dir = str(tmp_path / "mlx_saved")
+ yield
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+ @classmethod
+ def teardown_class(cls):
+ shutil.rmtree("runs", ignore_errors=True)
+
+ # ---- 1) Native MLX export: sweep bits ------------------------------- #
+ @pytest.mark.parametrize(
+ "bits, scheme",
+ [
+ (2, "W2A16"),
+ (3, "W3A16"),
+ (4, "W4A16"),
+ (5, "W5A16"),
+ (6, "W6A16"),
+ (8, "W8A16"),
+ ],
+ )
+ def test_mlx_native_export_bits(self, bits, scheme):
+ """Export Qwen3-0.6B as a native MLX checkpoint at 2/3/4/8 bits."""
+ _quantize_and_save(
+ output_dir=self.save_dir,
+ scheme=scheme,
+ bits=bits,
+ fmt="mlx",
+ sym=True,
+ group_size=128,
+ )
+ assert os.path.exists(os.path.join(self.save_dir, "config.json"))
+ qcfg = _read_quant_config(self.save_dir)
+ assert qcfg, f"No quantization_config found for scheme={scheme}"
+ assert qcfg.get("bits") == bits, f"expected bits={bits}, got {qcfg.get('bits')}"
+
+ @requires_mlx_lm_runtime
+ @pytest.mark.parametrize(
+ "bits, scheme",
+ [
+ (2, "W2A16"),
+ (3, "W3A16"),
+ (4, "W4A16"),
+ (5, "W5A16"),
+ (6, "W6A16"),
+ (8, "W8A16"),
+ ],
+ )
+ def test_mlx_native_inference_with_mlx_lm(self, bits, scheme):
+ """Native MLX checkpoint loads and runs end-to-end through mlx_lm."""
+ _quantize_and_save(
+ output_dir=self.save_dir,
+ scheme=scheme,
+ bits=bits,
+ fmt="mlx",
+ sym=True,
+ group_size=128,
+ )
+ text = _mlx_lm_generate(self.save_dir)
+ assert isinstance(text, str) and len(text) > 0, f"mlx_lm generated empty output for bits={bits}"
+
+ # ---- 2) auto_round export: sweep bits + sym/asym -------------------- #
+ @pytest.mark.parametrize(
+ "bits, scheme, sym",
+ [
+ (2, "W2A16", True),
+ (3, "W3A16", True),
+ (4, "W4A16", True),
+ (4, "W4A16", False), # asymmetric -> exercises qzeros path
+ (8, "W8A16", True),
+ ],
+ )
+ def test_auto_round_export_bits(self, bits, scheme, sym):
+ """Export Qwen3-0.6B as auto_round (GPTQ-style) across bits/sym."""
+ _quantize_and_save(
+ output_dir=self.save_dir,
+ scheme=scheme,
+ bits=bits,
+ fmt="auto_round",
+ sym=sym,
+ group_size=128,
+ )
+ assert os.path.exists(os.path.join(self.save_dir, "config.json"))
+ qcfg = _read_quant_config(self.save_dir)
+ assert qcfg.get("bits") == bits
+ assert qcfg.get("sym") == sym
+
+ @requires_mlx_runtime
+ @pytest.mark.parametrize(
+ "bits, scheme, sym",
+ [
+ (4, "W4A16", True),
+ (4, "W4A16", False),
+ (8, "W8A16", True),
+ ],
+ )
+ def test_auto_round_post_init_to_mlx(self, bits, scheme, sym):
+ """On Darwin, auto_round layers must be post-init converted to QuantLinearMLX."""
+ _quantize_and_save(
+ output_dir=self.save_dir,
+ scheme=scheme,
+ bits=bits,
+ fmt="auto_round:auto_gptq",
+ sym=sym,
+ group_size=128,
+ )
+ model, text = _hf_generate(self.save_dir)
+ assert _count_mlx_layers(model) > 0, (
+ f"Expected auto_round->MLX post-init conversion to produce QuantLinearMLX "
+ f"layers on Darwin, but found none (bits={bits}, sym={sym})."
+ )
+ assert isinstance(text, str) and len(text) > 0
+
+ # ---- 3) Mixed-bit via layer_config ---------------------------------- #
+ def test_mixed_bits_mlx_export(self):
+ """Mixed-bit layer_config exported in native MLX format.
+
+ Verifies the resulting ``config.json`` follows the mlx-community
+ mixed-bit convention: per-layer ``{group_size, bits}`` overrides for
+ layers quantized with a non-default scheme, and ``false`` for layers
+ we explicitly kept in fp16 (e.g. ``lm_head``).
+ """
+ layer_config = {
+ "lm_head": {"bits": 16},
+ "model.layers.0.self_attn.q_proj": {"bits": 8, "group_size": 128, "sym": True},
+ "model.layers.0.self_attn.k_proj": {"bits": 8, "group_size": 128, "sym": True},
+ "model.layers.0.mlp.down_proj": {"bits": 2, "group_size": 128, "sym": True},
+ }
+ _quantize_and_save(
+ output_dir=self.save_dir,
+ scheme="W4A16",
+ bits=4,
+ fmt="mlx",
+ sym=True,
+ group_size=128,
+ layer_config=layer_config,
+ )
+ assert os.path.exists(os.path.join(self.save_dir, "config.json"))
+
+ with open(os.path.join(self.save_dir, "config.json"), "r", encoding="utf-8") as f:
+ cfg = json.load(f)
+ quant = cfg.get("quantization", {})
+ # global defaults
+ assert quant.get("bits") == 4
+ assert quant.get("group_size") == 128
+ # per-layer overrides
+ assert quant.get("model.layers.0.self_attn.q_proj") == {"group_size": 128, "bits": 8}
+ assert quant.get("model.layers.0.self_attn.k_proj") == {"group_size": 128, "bits": 8}
+ assert quant.get("model.layers.0.mlp.down_proj") == {"group_size": 128, "bits": 2}
+ # lm_head kept at fp16 must be marked False so mlx-lm does not auto-quantize it
+ assert quant.get("lm_head") is False, f"expected lm_head=False, got {quant.get('lm_head')!r}"
+
+ def test_mixed_bits_auto_round_export(self):
+ """Mixed-bit layer_config exported as auto_round."""
+ layer_config = {
+ "lm_head": {"bits": 16},
+ "model.layers.0.self_attn.v_proj": {"bits": 8, "group_size": 128, "sym": True},
+ "model.layers.0.mlp.gate_proj": {"bits": 2, "group_size": 128, "sym": True},
+ }
+ _quantize_and_save(
+ output_dir=self.save_dir,
+ scheme="W4A16",
+ bits=4,
+ fmt="auto_round",
+ sym=True,
+ group_size=128,
+ layer_config=layer_config,
+ )
+ qcfg = _read_quant_config(self.save_dir)
+ assert qcfg.get("bits") == 4
+
+ @requires_mlx_lm_runtime
+ def test_mixed_bits_mlx_inference_with_mlx_lm(self):
+ """Mixed-bit native MLX checkpoint runs end-to-end under mlx_lm."""
+ layer_config = {
+ "lm_head": {"bits": 16},
+ "model.layers.0.self_attn.q_proj": {"bits": 8, "group_size": 128, "sym": True},
+ "model.layers.0.mlp.down_proj": {"bits": 2, "group_size": 128, "sym": True},
+ }
+ _quantize_and_save(
+ output_dir=self.save_dir,
+ scheme="W4A16",
+ bits=4,
+ fmt="mlx",
+ sym=True,
+ group_size=128,
+ layer_config=layer_config,
+ )
+ text = _mlx_lm_generate(self.save_dir)
+ assert isinstance(text, str) and len(text) > 0
+
+ @requires_mlx_runtime
+ def test_mixed_bits_auto_round_inference_on_darwin(self):
+ """Mixed-bit auto_round export, loaded via HF + post-init MLX repack."""
+ layer_config = {
+ "lm_head": {"bits": 16},
+ "model.layers.0.self_attn.q_proj": {"bits": 8, "group_size": 128, "sym": True},
+ }
+ _quantize_and_save(
+ output_dir=self.save_dir,
+ scheme="W4A16",
+ bits=4,
+ fmt="auto_round",
+ sym=True,
+ group_size=128,
+ layer_config=layer_config,
+ )
+ model, text = _hf_generate(self.save_dir)
+ assert _count_mlx_layers(model) > 0
+ assert isinstance(text, str) and len(text) > 0
+
+ @requires_qwen3_vl_9b_local
+ def test_qwen3_vl_9b_mlx_export_config(self, tmp_path):
+ """Quantize Qwen3-VL-9B (RTN W4A16) and validate the VLM-aware MLX config.
+
+ Verifies the four VLM-specific guarantees of the MLX exporter:
+ 1. Top-level ``quantization`` exists with global defaults.
+ 2. Language-tower quantized layers are recorded with their full path.
+ 3. ``text_config.quantization`` exists and uses paths *relative* to
+ the language sub-module (i.e. without the ``language_model.``
+ prefix), matching mlx-vlm's loader expectations.
+ 4. ``vision_tower``/``visual`` un-quantized layers do NOT bloat the
+ config with ``false`` entries.
+ 5. ``rope_parameters`` has been flattened (``rope_theta`` surfaced).
+ """
+ from auto_round import AutoRoundMLLM
+
+ save_dir = str(tmp_path / "qwen3_vl_9b_mlx")
+ try:
+ ar = AutoRoundMLLM(
+ qwen_3_vl_9b_name_or_path,
+ scheme="W4A16",
+ bits=4,
+ group_size=128,
+ sym=True,
+ iters=0,
+ nsamples=1,
+ seqlen=32,
+ disable_opt_rtn=True,
+ )
+ ar.quantize_and_save(output_dir=save_dir, format="mlx")
+
+ cfg_path = os.path.join(save_dir, "config.json")
+ assert os.path.exists(cfg_path), "config.json missing after VLM mlx export"
+ with open(cfg_path, "r", encoding="utf-8") as f:
+ cfg = json.load(f)
+
+ # 1) top-level defaults
+ quant = cfg.get("quantization", {})
+ assert quant.get("bits") == 4
+ assert quant.get("group_size") == 128
+
+ # 2) at least some language-tower layers are recorded with their full path
+ lang_keys = [
+ k
+ for k in quant
+ if isinstance(k, str)
+ and k.startswith(("language_model.", "model.language_model."))
+ and isinstance(quant[k], dict)
+ and quant[k].get("bits") == 4
+ ]
+ assert lang_keys, "No language_model.* quantized entries in top-level quantization dict"
+
+ # 3) text_config.quantization mirrors them with the prefix stripped
+ sub = None
+ for sub_key in ("text_config", "language_config", "thinker_config"):
+ cand = cfg.get(sub_key)
+ if isinstance(cand, dict) and "quantization" in cand:
+ sub = cand["quantization"]
+ break
+ assert sub is not None, "expected text_config.quantization to be injected for VLM"
+ assert sub.get("bits") == 4 and sub.get("group_size") == 128
+
+ # Pick one full-path key and verify its prefix-stripped form is in sub
+ sample = lang_keys[0]
+ for prefix in ("language_model.", "model.language_model."):
+ if sample.startswith(prefix):
+ stripped = sample[len(prefix) :]
+ break
+ assert stripped in sub, (
+ f"text_config.quantization missing prefix-stripped key '{stripped}' "
+ f"(derived from top-level '{sample}')"
+ )
+
+ # 4) vision tower must NOT pollute the config with ``false`` entries
+ vision_false_keys = [
+ k
+ for k, v in quant.items()
+ if isinstance(k, str)
+ and any(k.startswith(p + ".") for p in ("vision_tower", "vision_model", "visual"))
+ and v is False
+ ]
+ assert not vision_false_keys, (
+ f"Expected vision_tower un-quantized layers to be skipped, "
+ f"but found {len(vision_false_keys)} ``false`` entries (e.g. {vision_false_keys[:3]})"
+ )
+
+ # 5) rope_parameters flattened — either at top level or inside text_config
+ assert "rope_parameters" not in cfg, "top-level rope_parameters should be flattened"
+ for sub_key in ("text_config", "language_config", "thinker_config"):
+ sc = cfg.get(sub_key)
+ if isinstance(sc, dict):
+ assert "rope_parameters" not in sc, f"{sub_key}.rope_parameters should have been flattened"
+ finally:
+ shutil.rmtree(save_dir, ignore_errors=True)
+
+ @requires_qwen3_vl_9b_local
+ @requires_mlx_vlm_runtime
+ def test_qwen3_vl_9b_mlx_inference_with_mlx_vlm(self, tmp_path):
+ """End-to-end: quantize Qwen3-VL-9B → mlx → load & generate via mlx_vlm."""
+ from mlx_vlm import generate, load
+ from mlx_vlm.prompt_utils import apply_chat_template
+ from mlx_vlm.utils import load_config
+
+ from auto_round import AutoRoundMLLM
+
+ save_dir = str(tmp_path / "qwen3_vl_9b_mlx_run")
+ try:
+ ar = AutoRoundMLLM(
+ qwen_3_vl_9b_name_or_path,
+ scheme="W4A16",
+ bits=4,
+ group_size=128,
+ sym=True,
+ iters=0,
+ nsamples=1,
+ seqlen=32,
+ disable_opt_rtn=True,
+ )
+ ar.quantize_and_save(output_dir=save_dir, format="mlx")
+
+ model, processor = load(save_dir)
+ mlx_cfg = load_config(save_dir)
+ prompt_text = "Describe this image in one sentence."
+ formatted = apply_chat_template(processor, mlx_cfg, prompt_text, num_images=1)
+ # Use a public example image so the test does not need local assets.
+ image_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"
+ output = generate(model, processor, formatted, image=[image_url], max_tokens=32).text
+ assert isinstance(output, str) and len(output) > 0
+ finally:
+ shutil.rmtree(save_dir, ignore_errors=True)
diff --git a/test_mlx_export.py b/test_mlx_export.py
new file mode 100644
index 000000000..c7920a80b
--- /dev/null
+++ b/test_mlx_export.py
@@ -0,0 +1,192 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Test script for MLX format export with AutoRound.
+Tests quantization with W4A16 scheme on Qwen3-0.6B model.
+"""
+
+import os
+import sys
+from pathlib import Path
+
+import torch
+
+# Add project root to path
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from auto_round import AutoRound
+
+
+def test_mlx_export():
+ """Test MLX format export with W4A16 quantization."""
+ print("=" * 80)
+ print("AutoRound MLX Format Export Test")
+ print("=" * 80)
+
+ # Model configuration
+ model_name = "Qwen/Qwen3-0.6B"
+ output_dir = "./mlx_model_w4a16"
+
+ print(f"\n[1/4] Loading model: {model_name}")
+ try:
+ ar = AutoRound(
+ model_name,
+ scheme="W4A16",
+ bits=4,
+ group_size=128,
+ sym=True,
+ iters=0, # Fast RTN mode
+ disable_opt_rtn=True, # Disable optimization for faster quantization
+ nsamples=32, # Use fewer samples for testing
+ )
+ print("✓ Model loaded successfully")
+ except Exception as e:
+ print(f"✗ Failed to load model: {e}")
+ return False
+
+ print("\n[2/4] Quantizing model to MLX format...")
+ try:
+ ar.quantize_and_save(output_dir=output_dir, format="mlx")
+ print(f"✓ Model quantized and saved to {output_dir}")
+ except Exception as e:
+ print(f"✗ Failed to quantize and save model: {e}")
+ import traceback
+
+ traceback.print_exc()
+ return False
+
+ print("\n[3/4] Verifying output files...")
+ try:
+ # Check if required files exist
+ required_files = [
+ "config.json",
+ "quantization_config.json",
+ "mlx_metadata.json",
+ ]
+
+ output_path = Path(output_dir)
+ for file_name in required_files:
+ file_path = output_path / file_name
+ if file_path.exists():
+ print(f" ✓ {file_name} exists")
+ else:
+ print(f" ✗ {file_name} NOT found")
+ return False
+
+ # Print quantization config
+ import json
+
+ quantization_config_path = output_path / "quantization_config.json"
+ with open(quantization_config_path, "r") as f:
+ config = json.load(f)
+ print("\n Quantization Config:")
+ print(f" - Format: {config.get('format')}")
+ print(f" - Quant Method: {config.get('quant_method')}")
+ print(f" - Bits: {config.get('bits')}")
+ print(f" - Group Size: {config.get('group_size')}")
+ print(f" - Symmetric: {config.get('sym')}")
+
+ except Exception as e:
+ print(f"✗ Failed to verify output files: {e}")
+ return False
+
+ print("\n[4/4] Testing model loading from MLX format...")
+ try:
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+
+ # Load the quantized model
+ model = AutoModelForCausalLM.from_pretrained(output_dir, device_map="auto", torch_dtype="auto")
+ tokenizer = AutoTokenizer.from_pretrained(output_dir)
+ print("✓ Model loaded successfully from MLX format")
+
+ # Simple inference test
+ prompt = "Hello, my name is"
+ inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+ with torch.no_grad():
+ outputs = model.generate(**inputs, max_new_tokens=20)
+ result = tokenizer.decode(outputs[0], skip_special_tokens=True)
+ print(f" Generated text: {result}")
+ print("✓ Inference test passed")
+
+ except Exception as e:
+ print(f"⚠ Warning: Model loading or inference test failed (this is expected if MLX not installed): {e}")
+
+ print("\n" + "=" * 80)
+ print("✓ MLX format export test completed successfully!")
+ print("=" * 80)
+ return True
+
+
+def test_mlx_export_w3a16():
+ """Test MLX format export with W3A16 quantization."""
+ print("\n" + "=" * 80)
+ print("AutoRound MLX Format Export Test (W3A16)")
+ print("=" * 80)
+
+ # Model configuration
+ model_name = "Qwen/Qwen3-0.6B"
+ output_dir = "./mlx_model_w3a16"
+
+ print("\n[1/3] Loading and quantizing model with W3A16...")
+ try:
+ ar = AutoRound(
+ model_name,
+ scheme="W3A16",
+ bits=3,
+ group_size=128,
+ sym=True,
+ iters=0,
+ disable_opt_rtn=True,
+ nsamples=32,
+ )
+ print("✓ Model loaded successfully")
+ except Exception as e:
+ print(f"✗ Failed to load model: {e}")
+ return False
+
+ print("\n[2/3] Saving model in MLX format...")
+ try:
+ ar.quantize_and_save(output_dir=output_dir, format="mlx")
+ print(f"✓ Model saved to {output_dir}")
+ except Exception as e:
+ print(f"✗ Failed to save model: {e}")
+ return False
+
+ print("\n[3/3] Verifying MLX format files...")
+ try:
+ output_path = Path(output_dir)
+ quantization_config_path = output_path / "quantization_config.json"
+
+ import json
+
+ with open(quantization_config_path, "r") as f:
+ config = json.load(f)
+
+ assert config.get("bits") == 3, f"Expected bits=3, got {config.get('bits')}"
+ print(" ✓ Bits set correctly to 3")
+ print(" ✓ All verifications passed")
+
+ except Exception as e:
+ print(f"✗ Verification failed: {e}")
+ return False
+
+ print("\n✓ W3A16 export test completed successfully!")
+ return True
+
+
+if __name__ == "__main__":
+ print("\n🚀 Starting AutoRound MLX Format Tests\n")
+
+ # Run W4A16 test
+ success1 = test_mlx_export()
+
+ # Run W3A16 test
+ success2 = test_mlx_export_w3a16()
+
+ if success1 and success2:
+ print("\n✅ All tests passed!")
+ sys.exit(0)
+ else:
+ print("\n❌ Some tests failed!")
+ sys.exit(1)