PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[Experimental]Add MLX format export support and AutoScheme for vlm support (#1732)
Wenhua Cheng0def9d632043
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex d7fde248a..8cb79004d 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -616,8 +616,11 @@ def tune(args):if args.quant_lm_head:for format in formats:- if "auto_round" not in format and "fake" not in format:- auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]+ # MLX (native ``mlx`` and ``auto_round:mlx``) supports per-layer+ # mixed-bit quantization including lm_head; treat it the same as+ # auto_round / fake here.+ if "auto_round" not in format and "fake" not in format and "mlx" not in format:+ auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round") or s == "mlx"]raise ValueError(f"{format} is not supported for lm-head quantization, please change to {auto_round_formats}")diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex 9a751968f..a7e2bdf90 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -48,12 +48,15 @@ from auto_round.utils import (get_block_names,get_major_device,get_module,+ is_mllm_model,llm_load_model,+ mllm_load_model,parse_available_devices,set_avg_auto_device_map,set_module,set_non_auto_device_map,to_device,+ to_dtype,)from auto_round.utils.device import MemoryMonitorfrom auto_round.utils.offload import OffloadManager@@ -377,6 +380,51 @@ def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_devmodule.forward = wrap_forward(module, block_name)+def _prepare_mllm_inputs(data, model):+ """Normalize one batch from a (possibly mllm) dataloader into a form the+ model can be called on.++ Crucially this casts ``images`` / ``pixel_values`` / ``pixel_values_videos``+ to ``model.dtype`` — without it, vision tensors arrive as float32 while+ the vision tower is bf16/fp16, and several HF VLM implementations silently+ bypass the vision branch on dtype mismatch (which manifests downstream as+ "vision grad = 0").++ Returns ``(prepared, kind)`` where ``kind`` is ``"tensor" | "seq" | "dict"``+ so the caller knows whether to use ``model(x)`` / ``model(*x)`` /+ ``model(**x)``.+ """+ _img_keys = ("images", "image", "pixel_values", "pixel_values_videos", "pixel_values_images", "image_pixel_values")++ if isinstance(data, torch.Tensor):+ return data.to(model.device), "tensor"++ if isinstance(data, (tuple, list)):+ return to_device(data, model.device), "seq"++ # Plain dict (the common path: HF VLM ``data_collator`` outputs).+ new = {}+ for key, value in data.items():+ t = to_device(value, model.device)+ if key in _img_keys:+ t = to_dtype(t, model.dtype)+ new[key] = t+ return new, "dict"+++def mllm_model_forward(model, data, **forward_kwargs):+ """Single entry point for "run a (possibly multimodal) batch through the+ model". Used by both AutoScheme paths so that ``pixel_values`` / ``images``+ are cast to ``model.dtype`` (otherwise VLMs silently skip the vision tower+ → vision grad = 0)."""+ prepared, kind = _prepare_mllm_inputs(data, model)+ if kind == "tensor":+ return model(prepared, **forward_kwargs), prepared+ if kind == "seq":+ return model(*prepared, **forward_kwargs), prepared+ return model(**prepared, **forward_kwargs), prepared++def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):block_inputs = {}@@ -401,7 +449,17 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):last_block_backward_hook = last_block.register_full_backward_pre_hook(backward_pre_hook)data = to_device(data, model.device)- output = model.forward(**data, labels=data["input_ids"], use_cache=False)+ # VLM datasets often already include ``labels``; LLM ones don't. Strip+ # any pre-existing ``labels`` from kwargs so we don't pass it twice.+ labels = data["labels"] if isinstance(data, dict) and "labels" in data else data["input_ids"]+ if isinstance(data, dict):+ data_for_forward = {k: v for k, v in data.items() if k != "labels"}+ else:+ data_for_forward = data+ # Route through the unified mllm forward so ``pixel_values`` /+ # ``images`` get cast to ``model.dtype`` (otherwise the vision tower+ # is silently bypassed on dtype mismatch and vision grad stays 0).+ output, _prepared = mllm_model_forward(model, data_for_forward, labels=labels, use_cache=False)try:# Backward pass (will be interrupted by the hook)@@ -434,12 +492,7 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None):for n, m in block_module.named_modules():if hasattr(m, "grad_mode"):m.grad_mode = True- if hasattr(m, "orig_layer"):- m.to(m.orig_layer.tuning_device)- elif hasattr(m, "tuning_device"):- m.to(m.tuning_device)- elif len(list(m.children())) == 0:- m.to(major_device)+ move_module_to_tuning_device(block_module, major_device=major_device)# Set the block to eval mode while enabling gradient computationblock_module.eval()@@ -494,6 +547,10 @@ def get_score_for_scheme(batch_size=1,disable_opt_rtn=True,offload_context: Optional[OffloadManager] = None,+ processor=None,+ is_vlm: bool = False,+ force_mllm: bool = False,+ model_name: Optional[str] = None,):scores_dict = {} # Key=name,Val=[quant_total_bits, loss]for n, m in model.named_modules():@@ -557,23 +614,242 @@ def get_score_for_scheme(set_module(model, name, new_m)if offload_context is not None:offload_context.flush_loaded(model)- if low_gpu_mem_usage:- dataloader = get_dataloader(tokenizer, seqlen, dataset_name=dataset, seed=42, bs=batch_size, nsamples=nsamples)- model_forward_low_gpu(model, dataloader, major_device=major_device, pbar=pbar)+ # ---- Memory: only wrapper.orig_layer.weight needs ``requires_grad`` ---- #+ # AutoScheme scoring uses ``iters=0`` (RTN), so we never UPDATE any+ # parameter. The only parameters that need to participate in autograd+ # are the wrappers' ``orig_layer.weight`` (so qdq_w in the backward+ # graph can trace back to them and the weight-grad hook fires).+ # All other parameters (norms, non-wrapped linears, vision-tower+ # layers when ``--quant_nontext_module`` is off, …) just waste a full+ # ``.grad`` buffer (~one model-worth of VRAM) during ``loss.backward()``.+ # This is the biggest single VRAM win for the non-low_gpu_mem_usage+ # path used to score VLMs.+ wrapper_weight_ids = set()+ for _, _m in model.named_modules():+ if hasattr(_m, "orig_layer") and hasattr(_m.orig_layer, "weight") and _m.orig_layer.weight is not None:+ wrapper_weight_ids.add(id(_m.orig_layer.weight))+ _trimmed = 0+ for _p in model.parameters():+ if id(_p) not in wrapper_weight_ids and _p.requires_grad:+ _p.requires_grad_(False)+ _trimmed += 1+ if _trimmed:+ logger.info(+ "AutoScheme: disabled requires_grad on %d non-wrapper parameters "+ "(only wrapper.orig_layer.weight needs grad for scoring; saves "+ "~one model-worth of grad buffer during backward).",+ _trimmed,+ )++ # When scoring vision-tower layers, keep the autograd chain alive end-to-end:+ # (1) every wrapper's orig weight must require grad — otherwise the STE+ # output of W-only-low-bit wrappers has no grad path and act-score+ # hooks never fire.+ # (2) every vision sub-tree leaf param must require grad — so the very+ # first vision op (patch_embed / first conv) enters autograd; its+ # input ``pixel_values`` is a plain tensor with no grad.+ if force_mllm:+ _re_enabled_w = 0+ for _, _m in model.named_modules():+ if (+ hasattr(_m, "orig_layer")+ and hasattr(_m.orig_layer, "weight")+ and _m.orig_layer.weight is not None+ and not _m.orig_layer.weight.requires_grad+ ):+ _m.orig_layer.weight.requires_grad_(True)+ _re_enabled_w += 1++ _vision_markers = ("vision", "visual", "image_encoder", "img_encoder", "patch_embed")+ _re_enabled_v = 0+ _seen = set()+ for _mod_name, _mod in model.named_modules():+ if not any(mk in _mod_name.lower() for mk in _vision_markers):+ continue+ for _p in _mod.parameters(recurse=False):+ if id(_p) in _seen:+ continue+ _seen.add(id(_p))+ if not _p.requires_grad:+ _p.requires_grad_(True)+ _re_enabled_v += 1++ logger.info(+ "AutoScheme(force_mllm): kept requires_grad on %d wrapper weights, " "%d vision-side params.",+ _re_enabled_w,+ _re_enabled_v,+ )++ def _build_calib_dataloader():+ """Pick the calibration dataloader.++ Since AutoScheme only scores the language tower (``get_block_names``+ already skips the vision/audio sub-trees on VLMs), a pure-text+ calibration dataset is sufficient and far cheaper for VLMs too — most+ VLMs accept a text-only forward and simply skip the vision encoder.+ We therefore use ``get_dataloader`` (text-only) by default and only+ fall back to the multimodal ``get_mllm_dataloader`` if a VLM truly+ rejects text-only inputs (caller can detect that in the calling loop).+ """+ return get_dataloader(tokenizer, seqlen, dataset_name=dataset, seed=42, bs=batch_size, nsamples=nsamples)++ def _build_mllm_calib_dataloader():+ """Build the multimodal calibration dataloader (image + text).++ Returns ``None`` if we can't build one (no processor / template /+ dataset issue) so the caller can surface a clearer error.+ """+ if processor is None:+ return None+ import os as _os++ from auto_round.compressors.mllm.dataset import MLLM_DATASET, get_mllm_dataloader++ template = None+ if hasattr(model, "config") and hasattr(model.config, "model_type"):+ template = model.config.model_type++ # Decide the effective dataset.+ # ``get_mllm_dataloader`` only treats ``dataset`` as multimodal when it+ # is either a local file OR a key registered in ``MLLM_DATASET``.+ # Otherwise it silently falls back to ``get_dataloader`` (text-only),+ # which produces batches with NO ``pixel_values`` -> the vision tower+ # is never invoked -> every vision score / grad collapses to 0. We+ # explicitly catch that case here and override to a known-good+ # multimodal dataset so the user doesn't end up with silent garbage.+ ds = dataset+ _is_real_mllm = isinstance(ds, str) and (_os.path.isfile(ds) or ds in MLLM_DATASET.keys())+ if not _is_real_mllm:+ _fallback = "liuhaotian/llava_conv_58k"+ logger.warning_once(+ "AutoScheme(force_mllm): dataset=%r is text-only, " "overriding to %r.",+ ds,+ _fallback,+ )+ ds = _fallback++ try:+ loader, _, _, _ = get_mllm_dataloader(+ template=template,+ model=model,+ tokenizer=tokenizer,+ processor=processor,+ dataset=ds,+ seqlen=seqlen,+ bs=batch_size,+ nsamples=nsamples,+ # If, for any reason, get_mllm_dataloader still falls back to+ # text-only, force it to hard-error rather than silently+ # producing image-less batches.+ quant_nontext_module=True,+ )+ return loader+ except Exception as exc: # noqa: BLE001+ logger.warning(f"Failed to build mllm calibration dataloader: {exc}")+ return None++ if low_gpu_mem_usage:+ if force_mllm:+ mllm_loader = _build_mllm_calib_dataloader()+ if mllm_loader is None:+ raise RuntimeError(+ "AutoScheme(force_mllm): cannot build mllm dataloader. "+ "Provide a `processor` and a multimodal `dataset`."+ )+ model_forward_low_gpu(model, mllm_loader, major_device=major_device, pbar=pbar)+ else:+ try:+ dataloader = _build_calib_dataloader()+ model_forward_low_gpu(model, dataloader, major_device=major_device, pbar=pbar)+ except Exception as exc: # noqa: BLE001+ if not is_vlm:+ raise+ logger.warning(+ f"Text-only calibration failed on VLM ({exc}); "+ f"falling back to multimodal calibration dataloader."+ )+ mllm_loader = _build_mllm_calib_dataloader()+ if mllm_loader is None:+ raise+ model_forward_low_gpu(model, mllm_loader, major_device=major_device, pbar=pbar)else:for n, m in model.named_modules():if hasattr(m, "grad_mode"):m.grad_mode = True- dataloader = get_dataloader(tokenizer, seqlen, dataset_name=dataset, seed=42, bs=batch_size, nsamples=nsamples)- for data in dataloader:- data = to_device(data, model.device)- output = model.forward(**data, labels=data["input_ids"], use_cache=False)- output.loss.backward()- for n, m in model.named_parameters(): # This should be kept to reduce VRAM footprint- m.grad = None- if pbar is not None:- pbar.update(1)++ def _run_forward_loop(loader):+ _checked_pixel = False+ _pixel_keys = (+ "pixel_values",+ "pixel_values_videos",+ "pixel_values_images",+ "image_pixel_values",+ "images",+ "image",+ )+ for data in loader:+ # Pull labels out of the batch (VLM datasets often carry them;+ # LLM ones don't) before mllm_model_forward casts dtypes.+ _src = data if isinstance(data, dict) else None+ labels = (+ _src["labels"]+ if _src is not None and "labels" in _src+ else (_src["input_ids"] if _src is not None and "input_ids" in _src else None)+ )+ if _src is not None and "labels" in _src:+ data_for_forward = {k: v for k, v in _src.items() if k != "labels"}+ else:+ data_for_forward = data++ # Unified mllm-aware forward (casts pixel_values/images to+ # model.dtype, handles dict-with-text/str/tuple paths the same+ # way AutoRoundMLLM.calib does).+ output, _prepared = mllm_model_forward(model, data_for_forward, labels=labels, use_cache=False)+ output.loss.backward()++ # One-shot sanity check: when scoring vision layers, the batch+ # MUST carry image data, otherwise the vision tower is bypassed+ # by the model and every vision score is silently 0.+ if not _checked_pixel and force_mllm:+ _checked_pixel = True+ _has_pixel = isinstance(_prepared, dict) and any(k in _prepared for k in _pixel_keys)+ if not _has_pixel:+ _keys = list(_prepared.keys()) if isinstance(_prepared, dict) else type(_prepared).__name__+ raise RuntimeError(+ f"AutoScheme(force_mllm) batch has no pixel_values "+ f"(keys: {_keys}). Vision scores would all be 0. "+ f"Use a real multimodal dataset (e.g. "+ f"liuhaotian/llava_conv_58k) and pass a processor."+ )++ for _, m in model.named_parameters(): # zero grads to keep VRAM low+ m.grad = None+ if pbar is not None:+ pbar.update(1)++ if force_mllm:+ mllm_loader = _build_mllm_calib_dataloader()+ if mllm_loader is None:+ raise RuntimeError(+ "AutoScheme(force_mllm): cannot build mllm dataloader. "+ "Provide a `processor` and a multimodal `dataset`."+ )+ _run_forward_loop(mllm_loader)+ else:+ try:+ _run_forward_loop(_build_calib_dataloader())+ except Exception as exc: # noqa: BLE001+ if not is_vlm:+ raise+ logger.warning(+ f"Text-only calibration failed on VLM ({exc}); "+ f"falling back to multimodal calibration dataloader."+ )+ mllm_loader = _build_mllm_calib_dataloader()+ if mllm_loader is None:+ raise+ _run_forward_loop(mllm_loader)for n, m in model.named_parameters():m.grad = None@@ -647,13 +923,40 @@ def choose_bits_per_layer_with_path(layers: dict, P: int):def move_module_to_tuning_device(module, major_device="cpu"):+ def _normalize(dev):+ return dev if isinstance(dev, torch.device) else torch.device(dev)++ def _move_own_tensors(m, device):+ # Cover non-leaf modules that directly hold nn.Parameter / buffers+ # (e.g. Mamba/GDN linear_attn with A_log & dt_bias). Also relocate+ # p.grad together with p.data — otherwise the next backward's grad+ # accumulation hits a cuda/cpu device mismatch.+ target = _normalize(device)+ for p in m.parameters(recurse=False):+ if p.device != target:+ p.data = p.data.to(target)+ if p.grad is not None and p.grad.device != target:+ p.grad.data = p.grad.data.to(target)+ for b_name, b in list(m.named_buffers(recurse=False)):+ if b is None:+ continue+ if b.device != target:+ m._buffers[b_name] = b.to(target)+for n, m in module.named_modules():if hasattr(m, "orig_layer"):- m.to(m.orig_layer.tuning_device)+ target = m.orig_layer.tuning_device+ m.to(target)+ _move_own_tensors(m, target)elif hasattr(m, "tuning_device"):- m.to(m.tuning_device)+ target = m.tuning_device+ m.to(target)+ _move_own_tensors(m, target)elif len(list(m.children())) == 0:m.to(major_device)+ _move_own_tensors(m, major_device)+ else:+ _move_own_tensors(m, major_device)def _gen_layer_config(@@ -670,6 +973,8 @@ def _gen_layer_config(model_name=None,major_device="cpu",device_list=None,+ processor=None,+ is_vlm: bool = False,):# Initialize memory tracking for AutoSchemememory_monitor = MemoryMonitor()@@ -687,7 +992,33 @@ def _gen_layer_config(offload_context = OffloadManager(enabled=True, mode="clean", model_dir=_model_dir, cache_numel=True)target_bits = auto_scheme.avg_bits- model.eval()+ # HF gates gradient checkpointing on ``self.training`` — it's a no-op in eval mode.+ # In the non-low_gpu path we run a full forward+backward through the whole model,+ # so we want checkpointing to actually kick in (train mode). In the low_gpu path+ # we drive the blocks manually and don't want dropout / training-only side effects,+ # so keep eval mode.+ if auto_scheme.low_gpu_mem_usage:+ model.eval()+ else:+ # To trigger gradient checkpoint, but it will enable dropout, batchnorm, which is not good for accuracy+ model.train()++ # Filter out embedding layers from the scoring set (they aren't linear+ # quantization targets in any of our schemes).+ embedding_layers_names = []+ for name in quant_layer_names:+ module = get_module(model, name)+ if isinstance(module, torch.nn.Embedding):+ embedding_layers_names.append(name)+ quant_layer_names = list(set(quant_layer_names) - set(embedding_layers_names))++ # Decide whether AutoScheme has to score vision-tower layers (typically+ # because the user passed ``--quant_nontext_module``). Used below to+ # clamp batch_size to 1 (image sizes vary) and to pick the multimodal+ # dataloader. The actual switch from low_gpu to full forward+backward+ # is done upstream in ``gen_layer_config``.+ vision_markers = ("vision", "visual", "image", "img")+ force_mllm = is_vlm and any(any(marker in n.lower() for marker in vision_markers) for n in quant_layer_names)block_name = get_block_names(model)[0] # TODO need change to support vlmfor name in block_name:@@ -732,7 +1063,6 @@ def _gen_layer_config(nsamples = 64else:nsamples = 16-seqlen = auto_scheme.seqlen if auto_scheme.seqlen is not None else 256if auto_scheme.batch_size is not None:@@ -743,6 +1073,16 @@ def _gen_layer_config(else:batch_size = 1+ # ------------------------------------------------------------------ #+ # Multimodal calibration: ``batch_size`` must be 1 because image #+ # sizes differ across samples (the multimodal collator can't stack #+ # them otherwise). #+ # ------------------------------------------------------------------ #+ if force_mllm:+ if batch_size != 1:+ logger.info("AutoScheme(force_mllm): clamping batch_size %d -> 1.", batch_size)+ batch_size = 1+pbar_cnt = 0need_weight_grad = Falsefor index, scheme in enumerate(schemes):@@ -762,13 +1102,6 @@ def _gen_layer_config(pbar_cnt += len(block_name) * 2 * ((nsamples + batch_size - 1) // batch_size) # forward backwardshared_layers = parse_shared_layers(model, auto_scheme.shared_layers)- embedding_layers_names = []- for name in quant_layer_names:- module = get_module(model, name)- if isinstance(module, torch.nn.Embedding):- embedding_layers_names.append(name)- quant_layer_names = list(set(quant_layer_names) - set(embedding_layers_names))-options_scores = []if auto_scheme.low_gpu_mem_usage and not disable_opt_rtn:logger.warning("low_gpu_mem_usage is enabled, disable_opt_rtn will be set to True automatically")@@ -785,7 +1118,7 @@ def _gen_layer_config(if need_imatrix:breakif need_imatrix: # TODO change to block way in low_gpu_mem_usage- dataloader = get_dataloader(tokenizer, seqlen=256, dataset_name=dataset, seed=42, bs=8, nsamples=16)+ dataloader = get_dataloader(tokenizer, seqlen=256, dataset_name=dataset, seed=42, bs=8, nsamples=nsamples)logger.info("start to compute imatrix for GGUF-K quantization in AutoScheme")cal_imatrix(model, dataloader)logger.info("finish calculating imatrix")@@ -826,6 +1159,10 @@ def _gen_layer_config(batch_size=batch_size,disable_opt_rtn=auto_scheme.disable_opt_rtn,offload_context=offload_context,+ processor=processor,+ is_vlm=is_vlm,+ force_mllm=force_mllm,+ model_name=model_name,)# Track peak RAM after each scheme scoringmemory_monitor.update_cpu()@@ -970,13 +1307,54 @@ def gen_layer_config(disable_opt_rtn=True,low_gpu_mem_usage=True,min_avg_bit_scheme=None,+ processor=None,**kwargs,):model_name = None+ is_vlm = Falseif isinstance(model, str):model_name = model- # Load model on CPU only; do not apply automatic device map or tuning-aware placement at load time.- model, tokenizer, _ = llm_load_model(model_name, device_map="cpu")+ # Detect VLM (Qwen-VL / Qwen3-VL / LLaVA / etc.) and load via the MLLM+ # path so we get a usable ``processor`` for the multimodal calibration+ # dataloader. The block walker auto-detects VLMs too and skips the+ # vision tower when scoring.+ is_vlm = is_mllm_model(model_name)+ if is_vlm:+ model, processor, tokenizer, _ = mllm_load_model(+ model_name,+ device="cpu",+ use_auto_mapping=False,+ )+ else:+ # Load model on CPU only; do not apply automatic device map or tuning-aware placement at load time.+ model, tokenizer, _ = llm_load_model(model_name, device_map="cpu")+ else:+ # Object passed in: still try to detect VLM so we can pick the right dataloader later.+ try:+ is_vlm = is_mllm_model(model)+ except Exception: # noqa: BLE001+ is_vlm = False++ # ---- Vision-tower scoring requires a full backward ---- #+ # ``model_forward_low_gpu`` only walks the language tower (it uses+ # ``get_block_names(model)[0]``, which excludes vision blocks by default)+ # and interrupts ``loss.backward()`` at the LAST language block via+ # ``backward_pre_hook`` -> ``MyCustomError``. As a result, gradient never+ # propagates into the vision tower and any AutoScheme score for vision+ # layers comes out as 0. If the caller asked us to score vision layers+ # (typically because ``--quant_nontext_module`` was passed), force a+ # full forward+backward instead.+ vision_markers = ("vision", "visual", "image", "img")+ force_mllm_for_vision = is_vlm and any(+ any(marker in n.lower() for marker in vision_markers) for n in quant_layer_names+ )+ if force_mllm_for_vision and low_gpu_mem_usage:+ logger.warning("AutoScheme: scoring vision layers requires full backward; " "disabling low_gpu_mem_usage.")+ low_gpu_mem_usage = False+ try:+ auto_scheme.low_gpu_mem_usage = False+ except Exception: # noqa: BLE001+ pass# Get major devicemajor_device = get_major_device(device_map)if not low_gpu_mem_usage:@@ -1002,9 +1380,30 @@ def gen_layer_config(device_list = parse_available_devices(device_map)- # Enable gradient checkpointing if supported- if model.supports_gradient_checkpointing:- model.gradient_checkpointing_enable()+ # Enable gradient checkpointing if supported.+ #+ # IMPORTANT: we must use ``use_reentrant=False``. The reentrant+ # implementation requires the inputs that enter the checkpointed region to+ # have ``requires_grad=True`` — otherwise its backward sees "no grad-+ # requiring input" and returns ``None`` for the input gradient, which kills+ # the autograd chain *before* the checkpoint boundary. In AutoScheme we+ # aggressively turn off ``requires_grad`` on every non-wrapper parameter+ # (token embeddings, norms, vision-tower non-linear layers, patch embeds,+ # …), so ``inputs_embeds`` entering the first text decoder block often does+ # NOT require grad. With reentrant=True that means gradient never flows+ # back into the vision tower → vision wrapper hooks see grad=0.+ # ``use_reentrant=False`` (saved-tensor-hooks impl) does not have this+ # restriction.+ def _enable_gc(mod):+ if not getattr(mod, "supports_gradient_checkpointing", False):+ return+ try:+ mod.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})+ except TypeError:+ # Older transformers without the kwargs argument.+ mod.gradient_checkpointing_enable()++ _enable_gc(model)for name in quant_layer_names:m = get_module(model, name)@@ -1025,6 +1424,8 @@ def gen_layer_config(major_device=major_device,device_list=device_list,min_avg_bit_scheme=min_avg_bit_scheme,+ processor=processor,+ is_vlm=is_vlm,)except torch.OutOfMemoryError:logger.warning(@@ -1055,6 +1456,8 @@ def gen_layer_config(major_device=major_device,device_list=device_list,min_avg_bit_scheme=min_avg_bit_scheme,+ processor=processor,+ is_vlm=is_vlm,)return resdiff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.pyindex 55ec4e205..737d7d43e 100644--- a/auto_round/auto_scheme/gen_auto_scheme.py+++ b/auto_round/auto_scheme/gen_auto_scheme.py@@ -61,10 +61,12 @@ class GenScheme:device_map: Union[str, torch.device, int, dict, None] = None,tokenizer=None,enable_torch_compile=False,+ processor=None,):self.auto_scheme = auto_schemeself.model = modelself.tokenizer = tokenizer+ self.processor = processorself.quant_layer_names = quant_layer_namesself.fixed_layer_scheme = fixed_layer_schemeself.dataset = dataset@@ -98,8 +100,16 @@ class GenScheme:self.auto_scheme.avg_bits = targetif not (min_avg_bit <= target <= max_avg_bit):+ details = ", ".join(f"{opt}={bits:.3f}" for opt, bits in getattr(self, "_option_avg_bits", []))+ hint = (+ "Provide at least one option whose avg_bits is <= target and one whose avg_bits is >= target. "+ "Note that layers fixed to high-precision (e.g. lm_head/embed kept at bf16) inflate the lower bound; "+ "use `quant_lm_head=True`/adjust `layer_config`/`ignore_layers` if needed."+ )raise ValueError(- f"Target avg_bits={target:.3f} is outside the valid range " f"[{min_avg_bit:.3f}, {max_avg_bit:.3f}]."+ f"Target avg_bits={target:.3f} is outside the achievable range "+ f"[{min_avg_bit:.3f}, {max_avg_bit:.3f}] for options={list(self.auto_scheme.options)} "+ f"(per-option avg_bits: {details}). {hint}")def get_layer_config(self) -> dict[str, dict]:@@ -122,6 +132,7 @@ class GenScheme:disable_opt_rtn=self.disable_opt_rtn,low_gpu_mem_usage=self.auto_scheme.low_gpu_mem_usage,min_avg_bit_scheme=self.min_avg_bit_scheme,+ processor=self.processor,)layer_config = self.fallback_gguf_layer_config(layer_config)return layer_config@@ -183,6 +194,10 @@ class GenScheme:)[0]for option in self.auto_scheme.options]+ # Cache per-option avg bits so log/error messages can reference them.+ self._option_avg_bits = list(zip(list(self.auto_scheme.options), avg_bits))+ for opt, bits in self._option_avg_bits:+ logger.info("AutoScheme option %s -> avg_bits=%.4f", opt, bits)min_avg_bit, max_avg_bit = min(avg_bits), max(avg_bits)min_avg_bit_scheme = self.auto_scheme.options[avg_bits.index(min_avg_bit)]return min_avg_bit, max_avg_bit, min_avg_bit_schemediff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 238df50e5..69fbb2b57 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -603,8 +603,15 @@ class BaseCompressor(object):def _gen_auto_scheme(self) -> dict[str, dict]:if self.mllm:- logger.info("AutoScheme is not yet supported for multimodal LLMs.")- sys.exit(-1)+ # AutoScheme on a VLM only scores the language tower (the block+ # walker in delta_loss already skips vision/audio sub-trees) and+ # uses a pure-text calibration dataset by default, falling back to+ # the multimodal dataloader if the VLM rejects text-only forward.+ logger.info(+ "AutoScheme on multimodal LLM: scoring the language tower only "+ "with text-only calibration (multimodal dataloader will be used "+ "as a fallback if needed)."+ )if is_quantized_input_module(self.model):logger.info("AutoScheme does not currently support quantized input models (e.g., FP8).")@@ -654,6 +661,71 @@ class BaseCompressor(object):is_mllm=self.mllm,)quant_layer_names = layer_config.keys()++ # ---- VLM: peel non-text sub-trees AutoScheme should not score ---- #+ # Behavior matches ``--quant_nontext_module``:+ # * default (False): peel BOTH vision and audio; AutoScheme only+ # scores the language tower (vision/audio kept at their original+ # 16-bit configuration).+ # * True: peel only audio; AutoScheme also scores the vision tower+ # (audio is always peeled because we don't ship an audio+ # calibration dataset, so any score for it would be unreliable).+ nontext_skipped_layers: dict[str, dict] = {}+ if self.mllm:+ from auto_round.utils import get_block_names++ quant_nontext = getattr(self, "quant_nontext_module", False)+ # ``_get_vlm_block_names`` always drops audio blocks regardless of+ # ``quant_vision``.+ scoreable_blocks = get_block_names(self.model, quant_vision=quant_nontext)+ scoreable_block_prefixes = tuple(blk for group in scoreable_blocks for blk in group)+ if quant_nontext:+ # vision stays in; only audio gets peeled.+ peel_markers = ("audio", "speech", "wav", "waveform")+ tower_label = "language+vision"+ peel_label = "audio/speech"+ else:+ # default: peel both vision and audio.+ peel_markers = (+ "vision",+ "visual",+ "image",+ "img",+ "audio",+ "speech",+ "wav",+ "waveform",+ )+ tower_label = "language"+ peel_label = "vision/audio"++ def _is_scoreable_layer(name: str) -> bool:+ # Quick prefix test against detected scoreable blocks.+ if any(name == p or name.startswith(p + ".") for p in scoreable_block_prefixes):+ return True+ # Fallback: drop anything matching a peel marker.+ lname = name.lower()+ return not any(marker in lname for marker in peel_markers)++ scoreable_layer_config = {}+ for name, cfg in layer_config.items():+ if _is_scoreable_layer(name):+ scoreable_layer_config[name] = cfg+ else:+ nontext_skipped_layers[name] = cfg++ if nontext_skipped_layers:+ logger.info(+ "AutoScheme (VLM): scoring %d %s-tower layers; "+ "%d %s-tower layers kept at their original 16-bit configuration.",+ len(scoreable_layer_config),+ tower_label,+ len(nontext_skipped_layers),+ peel_label,+ )+ layer_config = scoreable_layer_config+ quant_layer_names = layer_config.keys()+scheme_keys = {f.name for f in fields(QuantizationScheme)}fixed_layer_scheme_new = {k: {key: v[key] for key in scheme_keys & v.keys()}@@ -675,8 +747,26 @@ class BaseCompressor(object):device_map=self.device_map,tokenizer=self.tokenizer,enable_torch_compile=self.enable_torch_compile,+ processor=getattr(self, "processor", None),)layer_config = self.scheme_generator.get_layer_config()+ # Re-attach vision/audio-tower layers we peeled off earlier so the+ # downstream quantization pipeline sees the complete layer map.+ # NOTE: ``nontext_skipped_layers`` was populated *after* the first+ # ``set_layer_config`` call, so each cfg already carries internal-only+ # keys such as ``in_blocks`` that the second ``set_layer_config`` call+ # in ``configure_layer_config`` would reject via its ``normalize_item``+ # validator. Strip those keys (keep only the allowed scheme fields)+ # before merging.+ if nontext_skipped_layers:+ allowed_keys = {f.name for f in fields(QuantizationScheme)} | {+ "fixed_by_user",+ "scale_dtype",+ "scheme",+ }+ for name, cfg in nontext_skipped_layers.items():+ clean_cfg = {k: v for k, v in cfg.items() if k in allowed_keys} if isinstance(cfg, dict) else cfg+ layer_config.setdefault(name, clean_cfg)return layer_configdef _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None:diff --git a/auto_round/compressors/mllm/template.py b/auto_round/compressors/mllm/template.pyindex 78abc6424..eb4a56e24 100644--- a/auto_round/compressors/mllm/template.py+++ b/auto_round/compressors/mllm/template.py@@ -183,7 +183,7 @@ def get_template(template = TEMPLATES[template_or_path]else:if not quiet:- logger.warning(f"Unable to recognize {template_or_path}, using default template instead.")+ logger.warning_once(f"Unable to recognize {template_or_path}, using default template instead.")template = TEMPLATES["default"]template.model_type = template_or_pathdiff --git a/auto_round/export/export_to_mlx/__init__.py b/auto_round/export/export_to_mlx/__init__.pynew file mode 100644index 000000000..5c6c18356--- /dev/null+++ b/auto_round/export/export_to_mlx/__init__.py@@ -0,0 +1,17 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from .export import pack_layer, save_quantized_as_mlx++__all__ = ["pack_layer", "save_quantized_as_mlx"]diff --git a/auto_round/export/export_to_mlx/export.py b/auto_round/export/export_to_mlx/export.pynew file mode 100644index 000000000..c65eef80d--- /dev/null+++ b/auto_round/export/export_to_mlx/export.py@@ -0,0 +1,728 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""+MLX Format Exporter for AutoRound++Exports quantized models to MLX-compatible format that can be loaded+directly by mlx-lm for inference on Apple Silicon.++MLX QuantizedLinear dequantization (affine mode):+ w_float = scale * w_int + bias++where w_int is packed into uint32, each holding (32 // bits) elements,+and scale/bias (called "biases" in MLX) have shape [out_features, num_groups].++Supported schemes: W2A16, W3A16, W4A16, W5A16, W6A16, W8A16+"""++import copy+import json+import os+from typing import Callable, Union++import torch+import torch.nn as nn+import transformers++from auto_round.export.utils import unsupported_meta_device+from auto_round.logger import logger+from auto_round.utils import (+ check_to_quantized,+ copy_python_files_from_model_cache,+ get_module,+ get_packing_device,+ set_module,+)++SUPPORTED_LAYER_TYPES = [nn.Linear, nn.Conv2d, transformers.pytorch_utils.Conv1D]+++def _is_mlx_quantizable(module: nn.Module, group_size: int) -> bool:+ """Predicate matching mlx-lm's default quantization predicate.++ A layer is considered quantizable by mlx-lm if its inner dim is a multiple+ of ``group_size`` and its outer dim is a multiple of 64. Embeddings are also+ eligible. Layers that fail this predicate are skipped at load time, so we do+ NOT need to emit ``false`` entries for them.+ """+ if isinstance(module, nn.Linear):+ in_dim, out_dim = module.in_features, module.out_features+ elif isinstance(module, nn.Embedding):+ in_dim, out_dim = module.embedding_dim, module.num_embeddings+ else:+ return False+ return (in_dim % group_size == 0) and (out_dim % 64 == 0)+++# Sub-modules whose layers we never want to auto-quantize on the mlx side.+# Vision / audio towers in VLMs are conventionally kept in fp16 by mlx-vlm,+# so emitting per-layer ``false`` for hundreds of vision linears just bloats+# the config. We simply skip these subtrees.+_DEFAULT_SKIP_PREFIXES = (+ "vision_tower",+ "vision_model",+ "visual",+ "audio_tower",+ "audio_model",+ "image_newline",+ "multi_modal_projector",+)+++def _build_mlx_quantization_config(+ model: nn.Module,+ default_bits: int,+ default_group_size: int,+ skip_prefixes: tuple = _DEFAULT_SKIP_PREFIXES,+) -> dict:+ """Build the ``quantization`` dict for an MLX ``config.json``.++ Mirrors the mlx-community mixed-bit format (see e.g.+ ``mlx-community/GLM-4.7-REAP-50-mixed-3-4-bits``)::++ "quantization": {+ "group_size": 64,+ "bits": 4,+ "model.layers.0.self_attn.q_proj": {"group_size": 64, "bits": 3},+ "model.layers.0.mlp.down_proj": false,+ ...+ }++ * Top-level ``group_size`` / ``bits`` give the default scheme.+ * Per-layer ``{group_size, bits}`` overrides differing layers.+ * Per-layer ``false`` marks an mlx-quantizable layer that should be kept+ in fp16 (e.g. ``lm_head`` left at 16 bits in our ``layer_config``).++ For VLMs, ``skip_prefixes`` lists sub-trees (vision/audio towers, projectors)+ whose un-quantized layers we do NOT emit ``false`` entries for. Quantized+ layers inside those sub-trees are still recorded faithfully.++ Note: AutoRound may leave non-quantized layers wrapped in ``WrapperLinear``+ (``.orig_layer`` points to the real ``nn.Linear``); we unwrap before testing+ so that e.g. ``lm_head`` with ``bits=16`` is correctly emitted as ``false``.+ """+ quant_cfg: dict = {"group_size": default_group_size, "bits": default_bits}++ def _is_skipped(layer_name: str) -> bool:+ return any(layer_name == p or layer_name.startswith(p + ".") for p in skip_prefixes)++ for name, module in model.named_modules():+ if isinstance(module, _MLXPackedLayer):+ # Always record packed layers, even if inside a skipped sub-tree+ # (user explicitly asked for them to be quantized).+ quant_cfg[name] = {+ "group_size": int(module.group_size) if module.group_size is not None else default_group_size,+ "bits": int(module.bits) if module.bits is not None else default_bits,+ }+ continue++ if _is_skipped(name):+ continue++ # Unwrap auto-round WrapperLinear (and similar) before the predicate test.+ target = module+ while hasattr(target, "orig_layer") and target.orig_layer is not None:+ target = target.orig_layer++ if _is_mlx_quantizable(target, default_group_size):+ # mlx-lm would auto-quantize this layer using the default scheme,+ # but since we deliberately left it un-quantized, mark it explicitly.+ quant_cfg[name] = False+ return quant_cfg+++# Sub-config keys commonly used by HF multi-modal / dual-tower configs that+# carry their own ``rope_parameters`` and may want a ``quantization`` block.+_TEXT_SUB_CONFIG_KEYS = ("text_config", "language_config", "thinker_config")+++def _flatten_rope_parameters_recursive(cfg: dict) -> None:+ """Pop ``rope_parameters`` and surface its keys to the same level.++ mlx-lm / mlx-vlm both expect ``rope_theta`` (and friends) at the same level+ as the rest of the model config — i.e. on the top-level config for LLMs and+ on ``text_config`` for VLMs. We walk every nested dict so both cases work.++ Two ``rope_parameters`` shapes appear in the wild and both are handled:+ 1. ``{"rope_theta": 1e6, "rope_type": "default", ...}`` (flat)+ 2. ``{"default": {"rope_theta": 1e6, "rope_type": "default", ...}, ...}``+ (newer HF "by-mode" layout — we surface keys from the ``"default"``+ entry, falling back to the first sub-dict if absent).+ """+ if not isinstance(cfg, dict):+ return+ rope_params = cfg.pop("rope_parameters", None)+ if isinstance(rope_params, dict):+ # If the dict is nested by mode, prefer "default", else any sub-dict.+ sub_dicts = {k: v for k, v in rope_params.items() if isinstance(v, dict)}+ if sub_dicts:+ chosen = sub_dicts.get("default") or next(iter(sub_dicts.values()))+ for k, v in chosen.items():+ cfg.setdefault(k, v)+ else:+ for k, v in rope_params.items():+ cfg.setdefault(k, v)+ for v in cfg.values():+ if isinstance(v, dict):+ _flatten_rope_parameters_recursive(v)+++def _extract_rope_theta_from_obj(config_obj) -> "float | None":+ """Best-effort extraction of ``rope_theta`` from a HF config object.++ Newer transformers versions stop exposing ``rope_theta`` as a direct+ attribute on the config and only expose ``rope_parameters`` (which can be+ a dict, a typed object, or a by-mode mapping like+ ``{"default": <obj-or-dict>}``). This helper tries every shape we've+ encountered in the wild.+ """+ if config_obj is None:+ return None+ direct = getattr(config_obj, "rope_theta", None)+ if direct is not None:+ return direct++ rope_params = getattr(config_obj, "rope_parameters", None)+ if rope_params is None:+ return None++ # Case A: dict-like+ if isinstance(rope_params, dict):+ if "rope_theta" in rope_params:+ return rope_params["rope_theta"]+ # by-mode: {"default": {...}, ...}+ sub_dicts = [v for v in rope_params.values() if isinstance(v, (dict,)) or hasattr(v, "rope_theta")]+ chosen = None+ if "default" in rope_params and (+ isinstance(rope_params["default"], dict) or hasattr(rope_params["default"], "rope_theta")+ ):+ chosen = rope_params["default"]+ elif sub_dicts:+ chosen = sub_dicts[0]+ if chosen is not None:+ return _extract_rope_theta_from_obj(chosen) if not isinstance(chosen, dict) else chosen.get("rope_theta")+ return None++ # Case B: object with attributes+ inner = getattr(rope_params, "rope_theta", None)+ if inner is not None:+ return inner+ # by-mode object: try .default+ inner_default = getattr(rope_params, "default", None)+ if inner_default is not None:+ return _extract_rope_theta_from_obj(inner_default)+ return None+++def _ensure_rope_theta_from_config_obj(cfg: dict, config_obj) -> None:+ """Backfill ``rope_theta`` on every level by probing the live HF config object.++ HF model configs may carry ``rope_theta`` only inside ``rope_parameters`` (or+ not at all on the serialized JSON when ``rope_parameters`` is nested by mode+ and the JSON-side flatten couldn't reach a usable value). Walk the config+ object alongside the JSON dict and copy ``rope_theta`` over wherever it's+ still missing — at top level for LLMs, and on+ ``text_config`` / ``language_config`` / ``thinker_config`` for VLMs.+ """+ if not isinstance(cfg, dict) or config_obj is None:+ return+ if "rope_theta" not in cfg:+ rope_theta = _extract_rope_theta_from_obj(config_obj)+ if rope_theta is not None:+ cfg["rope_theta"] = rope_theta+ for sub_key in _TEXT_SUB_CONFIG_KEYS + ("vision_config", "audio_config"):+ sub_cfg = cfg.get(sub_key)+ sub_obj = getattr(config_obj, sub_key, None)+ if isinstance(sub_cfg, dict) and sub_obj is not None:+ _ensure_rope_theta_from_config_obj(sub_cfg, sub_obj)+++# Sub-config keys whose ``model_type`` we want to preserve verbatim from the+# *original* checkpoint's ``config.json``. Newer ``transformers`` versions+# have a bug where they rewrite a sub-config's ``model_type`` on save (e.g.+# ``vision_config.model_type`` becomes ``qwen3_5_vision`` while the original+# checkpoint had ``qwen3_5``). We snapshot the original JSON and restore.+_PRESERVE_MODEL_TYPE_SUB_KEYS = (+ "text_config",+ "language_config",+ "thinker_config",+ "vision_config",+ "audio_config",+)+++def _load_original_config_json(model: nn.Module) -> "dict | None":+ """Best-effort load of the *raw* ``config.json`` from the original checkpoint.++ HF's in-memory ``model.config`` is unreliable for preserving ``model_type``+ because newer ``transformers`` versions auto-rewrite sub-config+ ``model_type`` values based on the registered child config class+ (e.g. an original ``qwen3_5`` becomes ``qwen3_5_text`` /+ ``qwen3_5_vision`` once the model is instantiated). The on-disk+ ``config.json`` from the source checkpoint is the only faithful source.++ We try ``model.config._name_or_path`` first, then ``model.name_or_path``.+ Returns the parsed JSON dict, or ``None`` if it can't be located/read.+ """+ candidates = []+ cfg_obj = getattr(model, "config", None)+ if cfg_obj is not None:+ nop = getattr(cfg_obj, "_name_or_path", None) or getattr(cfg_obj, "name_or_path", None)+ if nop:+ candidates.append(nop)+ nop2 = getattr(model, "name_or_path", None)+ if nop2 and nop2 not in candidates:+ candidates.append(nop2)++ for path in candidates:+ try:+ if isinstance(path, str) and os.path.isdir(path):+ cfg_path = os.path.join(path, "config.json")+ elif isinstance(path, str) and os.path.isfile(path) and path.endswith(".json"):+ cfg_path = path+ else:+ continue+ if not os.path.isfile(cfg_path):+ continue+ with open(cfg_path, "r", encoding="utf-8") as f:+ return json.load(f)+ except Exception as e:+ logger.warning(f"[mlx-export] failed to read original config.json at {path!r}: {e}")+ return None+++def _snapshot_original_model_types(model: nn.Module) -> "dict | None":+ """Snapshot ``model_type`` for top-level and known sub-configs.++ Prefer the original on-disk ``config.json`` (the only faithful source),+ falling back to the in-memory ``model.config`` if the file isn't reachable.+ Newer ``transformers`` versions rewrite sub-config ``model_type`` values+ on instantiation (e.g. ``qwen3_5`` -> ``qwen3_5_text`` /+ ``qwen3_5_vision``), so the in-memory copy can't be trusted for this.++ Returns a plain dict shaped like::++ {"model_type": "qwen3_5",+ "vision_config": {"model_type": "qwen3_5"},+ ...}++ or ``None`` if no config is available.+ """+ cfg_dict = _load_original_config_json(model)+ source = "on-disk config.json"+ if cfg_dict is None:+ cfg_obj = getattr(model, "config", None)+ if cfg_obj is None:+ logger.warning("[mlx-export] model has no .config; skipping model_type preservation")+ return None+ try:+ cfg_dict = cfg_obj.to_dict() if hasattr(cfg_obj, "to_dict") else dict(cfg_obj.__dict__)+ source = "in-memory model.config (fallback; may be HF-rewritten)"+ except Exception as e:+ logger.warning(f"[mlx-export] failed to snapshot model.config: {e}")+ return None++ snapshot: dict = {}+ top_model_type = cfg_dict.get("model_type")+ if top_model_type is not None:+ snapshot["model_type"] = top_model_type+ # Force every known sub-config to share the top-level ``model_type``.+ # Different ``transformers`` versions disagree on what sub-config+ # ``model_type`` should be (some leave it equal to the parent, some append+ # ``_text`` / ``_vision`` / ``_audio``). To keep the exported MLX config+ # stable across versions we just propagate the top-level value down.+ for sub_key in _PRESERVE_MODEL_TYPE_SUB_KEYS:+ sub = cfg_dict.get(sub_key)+ if isinstance(sub, dict) and top_model_type is not None:+ snapshot[sub_key] = {"model_type": top_model_type}+ logger.info(f"[mlx-export] snapshotted original model_types from {source}: {snapshot!r}")+ return snapshot+++def _preserve_original_model_types(new_cfg: dict, orig_cfg: "dict | None") -> None:+ """Make ``new_cfg``'s ``model_type`` fields exactly match ``orig_cfg``.++ For the top-level config and each known sub-config (``text_config``,+ ``vision_config``, etc.):+ * If the original had a ``model_type`` value, write it back.+ * If the original did **not** have ``model_type`` at all, delete the key+ from the new config (so HF can't sneak in a renamed default like+ ``qwen3_5`` → ``qwen3_5_vision``).+ * If the original sub-config doesn't exist in the new config, nothing to do.++ No-op if ``orig_cfg`` is ``None``.+ """+ if not isinstance(new_cfg, dict):+ logger.warning("[mlx-export] new config is not a dict; skip model_type preservation")+ return+ if not isinstance(orig_cfg, dict):+ # Already warned in _load_original_config_json.+ return++ def _align(sub_new: dict, sub_orig: dict, label: str) -> None:+ new_val = sub_new.get("model_type", "<missing>")+ if "model_type" in sub_orig:+ orig_val = sub_orig["model_type"]+ if new_val != orig_val:+ logger.info(f"[mlx-export] restoring {label}.model_type: {new_val!r} -> {orig_val!r}")+ sub_new["model_type"] = orig_val+ else:+ if "model_type" in sub_new:+ logger.info(+ f"[mlx-export] removing {label}.model_type ({new_val!r}); " f"original config did not declare one"+ )+ sub_new.pop("model_type", None)++ _align(new_cfg, orig_cfg, "top-level")+ for sub_key in _PRESERVE_MODEL_TYPE_SUB_KEYS:+ sub_new = new_cfg.get(sub_key)+ sub_orig = orig_cfg.get(sub_key)+ if isinstance(sub_new, dict) and isinstance(sub_orig, dict):+ _align(sub_new, sub_orig, sub_key)+++def _strip_prefix(name: str, prefix: str) -> str:+ """Return ``name`` with ``prefix.`` removed, or ``name`` if no match."""+ if name == prefix:+ return name+ pref = prefix + "."+ return name.removeprefix(pref)+++def _build_text_subconfig_quantization(quant_cfg: dict, text_prefix: str) -> dict:+ """Re-key a top-level quantization dict for placement under ``text_config``.++ mlx-vlm's language-model loader queries ``text_config["quantization"]`` with+ keys *relative to the language model* (e.g. ``model.layers.0...``), so we+ strip the outer ``language_model.`` / ``text_model.`` prefix from each entry.+ Entries that don't belong to the language model are dropped.+ """+ sub: dict = {"group_size": quant_cfg["group_size"], "bits": quant_cfg["bits"]}+ pref = text_prefix + "."+ for k, v in quant_cfg.items():+ if k in ("group_size", "bits"):+ continue+ if k == text_prefix or k.startswith(pref):+ sub[_strip_prefix(k, text_prefix)] = v+ return sub+++def _detect_text_module_prefix(model: nn.Module) -> str:+ """Best-effort detection of the language-model sub-module name in a VLM.++ Returns an empty string if the model is text-only.+ """+ for cand in ("language_model", "text_model", "thinker"):+ if hasattr(model, cand):+ return cand+ return ""+++def _pack_weight_mlx(intweight, bits):+ """Pack integer weights into uint32 in MLX format.++ MLX packs elements as a contiguous bit stream across uint32 boundaries.+ For bits that evenly divide 32 (2, 4, 8), each uint32 holds 32//bits elements.+ For other bit widths (e.g. 3, 5, 6, 7), every 32 elements are packed into+ `bits` uint32s (32 * bits = bits * 32 bits total).++ Args:+ intweight: shape [out_features, in_features], values in [0, 2^bits - 1]+ bits: quantization bits (2, 3, 4, 5, 6, 7, 8)++ Returns:+ packed: uint32 tensor, shape [out_features, in_features * bits / 32]+ """+ out_features, in_features = intweight.shape++ if 32 % bits == 0:+ # Simple case: bits evenly divides 32 (2, 4, 8)+ elems_per_int = 32 // bits+ assert (+ in_features % elems_per_int == 0+ ), f"in_features ({in_features}) must be divisible by {elems_per_int} for {bits}-bit packing"++ intweight = intweight.to(torch.int32)+ intweight = intweight.reshape(out_features, -1, elems_per_int)+ shifts = torch.arange(elems_per_int, device=intweight.device, dtype=torch.int32) * bits+ packed = (intweight << shifts).sum(dim=-1).to(torch.int32)+ return packed.view(torch.uint32)+ else:+ # Cross-word packing: 32 elements → `bits` uint32s+ # MLX packs as a contiguous bit stream across uint32 boundaries+ assert in_features % 32 == 0, f"in_features ({in_features}) must be divisible by 32 for {bits}-bit packing"++ intweight = intweight.to(torch.int64)+ num_groups = in_features // 32+ # Reshape to [out_features, num_groups, 32]+ elems = intweight.reshape(out_features, num_groups, 32)++ # For each element i in [0..31], it contributes `bits` bits starting at bit position i*bits+ # in a 32*bits bit stream packed into `bits` uint32s.+ # We process each bit b of each element i:+ # absolute_bit = i * bits + b+ # word_idx = absolute_bit // 32+ # bit_pos = absolute_bit % 32+ packed = torch.zeros(out_features, num_groups, bits, dtype=torch.int64, device=intweight.device)++ for b in range(bits):+ # Extract bit b from all elements: shape [out, num_groups, 32]+ bit_vals = (elems >> b) & 1+ for i in range(32):+ abs_bit = i * bits + b+ word_idx = abs_bit // 32+ bit_pos = abs_bit % 32+ packed[:, :, word_idx] |= bit_vals[:, :, i] << bit_pos++ packed = packed.to(torch.int32).reshape(out_features, num_groups * bits)+ return packed.view(torch.uint32)+++class _MLXPackedLayer(nn.Module):+ """Holds MLX-packed quantized tensors for serialization.++ Tensor names match MLX convention: weight, scales, biases, bias.+ The ``bits`` / ``group_size`` attributes are kept so that mixed-bit+ quantization can be reflected in the exported ``config.json``.+ """++ def __init__(self, weight, scales, biases, bias, bits=None, group_size=None):+ super().__init__()+ self.register_buffer("weight", weight) # uint32+ self.register_buffer("scales", scales) # float16+ self.register_buffer("biases", biases) # float16+ if bias is not None:+ self.register_buffer("bias", bias)+ else:+ self.bias = None+ self.bits = bits+ self.group_size = group_size+++def pack_layer(name, model, device=None, **kwargs):+ """Pack a single layer into MLX quantized format.++ Reads layer attributes set by auto-round quantization:+ layer.weight - float weight [out_features, in_features]+ layer.scale - [out_features, num_groups]+ layer.zp - [out_features, num_groups] or scalar+ layer.bits, layer.group_size, layer.sym++ Replaces the layer with _MLXPackedLayer containing:+ weight - uint32 packed ints [out_features, in_features * bits / 32]+ scales - float16 [out_features, num_groups]+ biases - float16 [out_features, num_groups]+ bias - original linear bias (if any)+ """+ layer = get_module(model, name)+ if hasattr(layer, "orig_layer"):+ layer = layer.orig_layer++ if type(layer) not in SUPPORTED_LAYER_TYPES:+ return++ if not check_to_quantized(layer):+ return++ device = get_packing_device(device)++ bits = int(layer.bits)+ group_size = int(layer.group_size)+ sym = bool(getattr(layer, "sym", False))+ scale = layer.scale # [out_features, num_groups]+ zp = layer.zp # [out_features, num_groups] or scalar (asym only)++ # Get weight in [out_features, in_features] layout+ W = layer.weight.data.to(device).clone().float()+ if type(layer) == nn.Conv2d:+ W = W.flatten(1)+ if type(layer) == transformers.pytorch_utils.Conv1D:+ W = W.t()++ out_features, in_features = W.shape+ if group_size == -1:+ group_size = in_features++ maxq = 2**bits - 1++ # Quantize: w_int = round(W / scale + zp), clamped to [0, maxq].+ # For symmetric mode we deliberately IGNORE layer.zp (it may be a tensor /+ # carry rounding noise / be device-dependent) and use the fixed integer+ # zero point ``2**(bits-1)`` (e.g. 8 for 4-bit, 4 for 3-bit, 128 for 8-bit).+ # This matches QuantLinearMLX.from_gptq's sym branch and the GPTQ sym+ # convention used everywhere else in auto-round.+ scale_dev = scale.to(device).float()+ repeat_scales = scale_dev.repeat_interleave(group_size, dim=1)[:, :in_features]++ if sym:+ zp_dev = float(2 ** (bits - 1))+ repeat_zp = zp_dev+ elif isinstance(zp, torch.Tensor):+ zp_dev = zp.to(device).float()+ repeat_zp = zp_dev.repeat_interleave(group_size, dim=1)[:, :in_features]+ else:+ zp_dev = float(zp)+ repeat_zp = zp_dev++ intweight = torch.round(W / repeat_scales + repeat_zp).clamp(0, maxq).to(torch.int32)++ # Pack weights into uint32+ packed_weight = _pack_weight_mlx(intweight, bits)++ # MLX dequant: w_float = mlx_scale * w_int + mlx_bias+ # auto-round: w_float = scale * (w_int - zp) = scale * w_int - scale * zp+ # So: mlx_scale = scale, mlx_bias = -scale * zp+ mlx_scales = scale_dev.contiguous().to(torch.float16)+ mlx_biases = (-scale_dev * zp_dev).contiguous().to(torch.float16)++ # Preserve original bias+ orig_bias = None+ if layer.bias is not None:+ orig_bias = layer.bias.clone().to(torch.float16).cpu()++ new_layer = _MLXPackedLayer(+ packed_weight.cpu(),+ mlx_scales.cpu(),+ mlx_biases.cpu(),+ orig_bias,+ bits=bits,+ group_size=group_size,+ )+ set_module(model, name, new_layer)+ logger.debug(f"Packed layer {name} for MLX format (bits={bits}, group_size={group_size})")+++def save_quantized_as_mlx(+ output_dir: str,+ model: nn.Module = None,+ tokenizer: Callable = None,+ layer_config: dict = None,+ inplace: bool = True,+ device: Union[str, torch.device] = "cpu",+ serialization_dict: dict = None,+ **kwargs,+) -> nn.Module:+ """Save quantized model in MLX-compatible format.++ The output can be loaded by mlx-lm::++ from mlx_lm import load, generate+ model, tokenizer = load("output_dir")+ response = generate(model, tokenizer, prompt="Hello", max_tokens=100)+ """+ if not inplace:+ model = copy.deepcopy(model.to("cpu"))++ if not unsupported_meta_device(model):+ model = model.to("cpu")++ os.makedirs(output_dir, exist_ok=True)++ bits = serialization_dict.get("bits", 4) if serialization_dict else 4+ group_size = serialization_dict.get("group_size", 128) if serialization_dict else 128++ # Pack all quantized layers (skip if already packed by immediate_pack)+ if layer_config:+ for layer_name in layer_config:+ pack_layer(layer_name, model, device=device)++ # Snapshot ``model_type`` (top-level + sub-configs) from the live+ # ``model.config`` BEFORE HF's ``save_pretrained`` runs. The in-memory+ # config still holds the original strings — newer ``transformers`` only+ # corrupts them on serialize.+ orig_model_types = _snapshot_original_model_types(model)++ # Save model weights (uint32 packed weights are saved directly by safetensors)+ if not unsupported_meta_device(model):+ model.save_pretrained(output_dir, safe_serialization=True)++ # Write config.json with MLX-compatible quantization info+ config_path = os.path.join(output_dir, "config.json")+ if hasattr(model, "config") and not os.path.exists(config_path):+ model.config.save_pretrained(output_dir)++ if os.path.exists(config_path):+ with open(config_path, "r") as f:+ config = json.load(f)+ # Build mixed-bit aware quantization dict (mlx-community format).+ quant_cfg = _build_mlx_quantization_config(model, default_bits=bits, default_group_size=group_size)+ config["quantization"] = quant_cfg+ autoround_format = kwargs.get("autoround_format", False)+ quant_cfg_full = None+ if autoround_format:+ quant_cfg_full = {+ "quant_method": "auto-round",+ "packing_format": "mlx",+ "bits": bits,+ "group_size": group_size,+ "sym": serialization_dict.get("sym", True) if serialization_dict else True,+ "data_type": serialization_dict.get("data_type", "int") if serialization_dict else "int",+ }+ config["quantization_config"] = quant_cfg_full+ else:+ config["quantization_config"] = quant_cfg++ # ---- VLM support ------------------------------------------------ #+ # If the model has a language-model sub-module (e.g. Qwen-VL family),+ # mlx-vlm's language loader queries the quantization dict under+ # ``text_config`` (or its alias) with keys relative to that sub-module.+ # Mirror our config there so both code paths work.+ text_prefix = _detect_text_module_prefix(model)+ if text_prefix:+ sub_quant = _build_text_subconfig_quantization(quant_cfg, text_prefix)+ for sub_key in _TEXT_SUB_CONFIG_KEYS:+ sub_cfg = config.get(sub_key)+ if isinstance(sub_cfg, dict):+ sub_cfg["quantization"] = sub_quant+ if autoround_format and quant_cfg_full is not None:+ sub_cfg["quantization_config"] = quant_cfg_full++ # Flatten rope_parameters recursively so mlx-lm/mlx-vlm find rope_theta+ # at the level it expects (top-level for LLMs, text_config for VLMs).+ _flatten_rope_parameters_recursive(config)++ # Backfill rope_theta from the live model.config object — required by+ # HF's model __init__ for the ``auto_round:mlx`` packing format (the+ # checkpoint is loaded through transformers.AutoModelForCausalLM, which+ # raises KeyError for missing rope_theta on Qwen-family models).+ _ensure_rope_theta_from_config_obj(config, getattr(model, "config", None))++ # Restore ``model_type`` (top-level + sub-configs) from the in-memory+ # snapshot so that newer ``transformers`` versions don't silently+ # rename them on save (e.g. vision_config: qwen3_5 -> qwen3_5_vision).+ _preserve_original_model_types(config, orig_model_types)++ with open(config_path, "w") as f:+ json.dump(config, f, indent=2)++ # Save tokenizer+ if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):+ tokenizer.save_pretrained(output_dir)++ # Copy processor if available+ processor = kwargs.get("processor", None)+ if processor is not None and hasattr(processor, "save_pretrained"):+ processor.save_pretrained(output_dir)++ # Try to copy Python files from model cache+ try:+ copy_python_files_from_model_cache(model, output_dir)+ except Exception as e:+ logger.warning(f"Failed to copy Python files from model cache: {e}")++ logger.info(f"Model saved to {output_dir} in MLX format (bits={bits}, group_size={group_size})")+ logger.info(f"Load with: from mlx_lm import load, generate; " f"model, tokenizer = load('{output_dir}')")++ return modeldiff --git a/auto_round/formats.py b/auto_round/formats.pyindex 68d392d48..cd7f076ea 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -1084,11 +1084,67 @@ class FP8Format(OutputFormat):)[email protected]("mlx")+class MLXFormat(OutputFormat):+ support_schemes = ["W2A16", "W2A16G32", "W2A16G64", "W3A16", "W4A16", "W5A16", "W6A16", "W8A16", "BF16"]+ format_name = "mlx"++ @classmethod+ def check_scheme_args(cls: OutputFormat, scheme: QuantizationScheme) -> bool:+ error_logs = []+ if scheme.bits not in [2, 3, 4, 5, 6, 8, 16]:+ error_logs.append(f"bits={scheme.bits}")+ if scheme.act_bits != 16:+ error_logs.append(f"act_bits={scheme.act_bits}")+ if not re.search("int", scheme.data_type):+ error_logs.append(f"data_type={scheme.data_type}")+ if scheme.super_bits:+ error_logs.append(f"super_bits={scheme.super_bits}")+ if scheme.super_group_size:+ error_logs.append(f"super_group_size={scheme.super_group_size}")+ if error_logs:+ raise ValueError(+ f"MLX format support quantization scheme with {','.join(cls.support_schemes)} "+ f"but got {', '.join(error_logs)}, please have a check."+ )+ return True++ def pack_layer(self, layer_name, model, device=None, **kwargs):+ from auto_round.export.export_to_mlx.export import pack_layer++ pack_layer(layer_name, model, device=device, **kwargs)++ def save_quantized(+ self,+ output_dir: str,+ model: torch.nn.Module = None,+ tokenizer: Callable = None,+ layer_config: dict = None,+ inplace: bool = True,+ device: Union[str, torch.device] = "cpu",+ serialization_dict: dict = None,+ **kwargs,+ ) -> torch.nn.Module:+ from auto_round.export.export_to_mlx.export import save_quantized_as_mlx++ return save_quantized_as_mlx(+ output_dir=output_dir,+ model=model,+ tokenizer=tokenizer,+ layer_config=layer_config,+ inplace=inplace,+ device=device,+ serialization_dict=serialization_dict,+ **kwargs,+ )++@OutputFormat.register("auto_round")@OutputFormat.register("auto_round:auto_awq")@OutputFormat.register("auto_round:llm_compressor")@OutputFormat.register("auto_round:gptqmodel", "auto_round:auto_gptq")@OutputFormat.register("auto_round:fp8")[email protected]("auto_round:mlx")class AutoRoundFormat(OutputFormat):support_schemes = ["W4A16",@@ -1143,10 +1199,15 @@ class AutoRoundFormat(OutputFormat):)# for auto_round:fp8_static, auto_round:nv_fp etc.elif not format.startswith("auto_round"):- if format.upper() not in list(AutoRoundExportFormat.__members__.keys()):+ if format == "mlx":+ self.backend = MLXFormat("mlx", ar)+ elif format.upper() not in list(AutoRoundExportFormat.__members__.keys()):raise KeyError(f"Unsupported backend format auto_round:{format}, please check")- self.output_format = f"auto_round:{format}"- self.backend = None+ else:+ self.output_format = f"auto_round:{format}"+ self.backend = None+ elif format == "auto_round:mlx":+ self.backend = MLXFormat("mlx", ar)else:backend = format.split(":")[1] if ":" in format else Noneself.backend = self._format_list.get(backend)(format, ar) if backend else None@@ -1219,6 +1280,9 @@ class AutoRoundFormat(OutputFormat):**kwargs,) -> torch.nn.Module:if self.backend is not None:+ extra_kwargs = {}+ if isinstance(self.backend, MLXFormat):+ extra_kwargs["autoround_format"] = Truereturn self.backend.save_quantized(output_dir=output_dir,model=model,@@ -1227,6 +1291,7 @@ class AutoRoundFormat(OutputFormat):inplace=inplace,device=device,serialization_dict=serialization_dict,+ **extra_kwargs,**kwargs,)backend = self.get_backend_name()diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex b36234c8c..d643ff659 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -198,6 +198,11 @@ AWQ_FORMAT = ["auto_round:auto_awq"]LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor"]WOQ_DEFAULT_ACT_BITS = [None, 16, 32]+# CPU backends that target Intel/x86 (ark / ipex / auto_round_kernel) cannot+# run on Apple Silicon. Restrict them so the MLX backend wins on Darwin and we+# don't try to load auto-round-lib / intel-extension-for-pytorch on macOS.+_NON_DARWIN_SYSTEMS = ["linux", "windows"]+# AutoGPTQ is no longer maintained, supports transformers < 5.0.0BackendInfos["auto_gptq:exllamav2"] = BackendInfo(device=["cuda"],@@ -483,7 +488,7 @@ BackendInfos["gptqmodel:awq_gemm"] = BackendInfo(packing_format=AWQ_FORMAT,bits=[4],group_size=[-1, 16, 32, 64, 128],- priority=3,+ priority=4,compute_dtype=["float16"],data_type=["int"],act_bits=WOQ_DEFAULT_ACT_BITS,@@ -514,7 +519,7 @@ BackendInfos["auto_awq:gemm"] = BackendInfo(packing_format=AWQ_FORMAT,bits=[4],group_size=None,- priority=4,+ priority=3,compute_dtype=["float16"],data_type=["int"],act_bits=WOQ_DEFAULT_ACT_BITS,@@ -535,6 +540,7 @@ BackendInfos["auto_round_kernel"] = BackendInfo(data_type=["int"],act_bits=WOQ_DEFAULT_ACT_BITS,requirements=["torch>=2.8.0", "auto-round-lib"],+ systems=_NON_DARWIN_SYSTEMS, # auto-round-lib targets x86; not for Apple Silicon)BackendInfos["auto_round_kernel_xpu"] = BackendInfo(@@ -565,6 +571,7 @@ BackendInfos["auto_round_kernel_zp"] = BackendInfo(data_type=["int"],act_bits=WOQ_DEFAULT_ACT_BITS,requirements=["torch>=2.8.0", "auto-round-lib"],+ systems=_NON_DARWIN_SYSTEMS,)BackendInfos["auto_round_kernel_zp_xpu"] = BackendInfo(@@ -595,6 +602,7 @@ BackendInfos["auto_round_kernel_awq"] = BackendInfo(data_type=["int"],act_bits=WOQ_DEFAULT_ACT_BITS,requirements=["torch>=2.8.0", "auto-round-lib"],+ systems=_NON_DARWIN_SYSTEMS,)BackendInfos["auto_round_kernel_awq_xpu"] = BackendInfo(@@ -625,6 +633,7 @@ BackendInfos["ipex_gptq_cpu"] = BackendInfo(act_bits=WOQ_DEFAULT_ACT_BITS,alias=["ipex"],requirements=["torch<2.9", "intel-extension-for-pytorch>=2.5"],+ systems=_NON_DARWIN_SYSTEMS, # intel-extension-for-pytorch is Intel x86 only)BackendInfos["ipex_gptq"] = BackendInfo(@@ -655,6 +664,7 @@ BackendInfos["ipex_awq_cpu"] = BackendInfo(act_bits=WOQ_DEFAULT_ACT_BITS,alias=["ipex"],requirements=["torch<2.9", "intel-extension-for-pytorch>=2.5"],+ systems=_NON_DARWIN_SYSTEMS,)BackendInfos["ipex_awq"] = BackendInfo(@@ -696,6 +706,22 @@ BackendInfos["hpu_zp"] = BackendInfo(priority=0,)+# MLX Backend for Apple Silicon (M1, M2, M3, etc.)+BackendInfos["mlx"] = BackendInfo(+ device=["cpu", "mps"], # MLX runs on Apple Silicon (CPU or MPS)+ sym=[True, False],+ packing_format=["mlx"] + GPTQ_FORMAT + GPTQ_FORMAT_NO_ZP,+ bits=[2, 3, 4, 5, 6, 8],+ compute_dtype=["float32", "float16", "bfloat16"],+ data_type=["int"],+ act_bits=WOQ_DEFAULT_ACT_BITS,+ group_size=[-1, 32, 64, 128, 256, 512],+ priority=5, # High priority for Apple Silicon users+ alias=["mlx"],+ requirements=["mlx>=0.16.0"],+ systems=["darwin"], # Only on macOS+)+def check_compatible(backend_name: str,@@ -770,16 +796,16 @@ def check_compatible(return True-def dynamic_import_inference_linear(backend, config):+def dynamic_import_inference_linear(backend, config, packing_format=None):"""Dynamically imports and returns the appropriate QuantLinear class based on the given backend.This function dynamically loads the correct `QuantLinear` class based on the backend and quantization- configuration (e.g., ark, marlin, hpu, gptq, awq). It imports specific modules or raises+ configuration (e.g., ark, marlin, hpu, gptq, awq, mlx). It imports specific modules or raiseserrors if the required packages are not installed or the environment is not set up.Args:backend (str):- The backend to be used for quantization (e.g., 'ark', 'marlin', 'hpu', 'gptq', 'awq').+ The backend to be used for quantization (e.g., 'ark', 'marlin', 'hpu', 'gptq', 'awq', 'mlx').config (QuantizationScheme):The quantization configuration containing parameters like bits, group_size, and sym.@@ -792,6 +818,19 @@ def dynamic_import_inference_linear(backend, config):If required modules are missing for a backend (e.g., ark, GPTQ, auto_awq)."""bits, group_size, sym = config["bits"], config["group_size"], config["sym"]++ # MLX backend+ if "mlx" in backend:+ # If loading a GPTQ-format checkpoint, use GPTQ QuantLinear for buffer name compatibility.+ # The layers will be converted to MLX in post_init.+ if packing_format and "mlx" not in packing_format:+ from auto_round_extension.torch.qlinear_torch import QuantLinear++ return QuantLinear+ from auto_round_extension.mlx.qlinear_mlx import QuantLinearMLX++ return QuantLinearMLX+if "torch_fp8_static" in backend:return ar_qmodules.WeightFP8ActFP8StaticQuantLinearif "torch_mxfp8" in backend:@@ -1154,9 +1193,16 @@ def get_layer_backend(def get_highest_priority_backend(quantization_config: "AutoRoundConfig", device: str, packing_format: str) -> str | None:+ current_system = platform.system().lower()supported_backends = []for key in BackendInfos.keys():backend = BackendInfos[key]+ # Filter by operating system (e.g. MLX is Darwin-only; ark/ipex CPU+ # backends are non-Darwin only).+ if backend.systems is not None:+ if current_system not in [s.lower() for s in backend.systems]:+ continue+# Check if device is supported by the backendif device not in backend.device:continuediff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 4ae6405f0..70bb311f7 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -45,7 +45,7 @@ from auto_round.utils import (set_module,)-supported_devices = ("cpu", "hpu", "xpu", "cuda")+supported_devices = ("cpu", "hpu", "xpu", "cuda", "mps")def flatten_list(nested_list):@@ -174,6 +174,9 @@ def get_available_devices():if hasattr(torch, "xpu") and torch.xpu.is_available():devices.append("xpu")+ if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():+ devices.append("mps")+devices.append("cpu") # Always availablereturn devices@@ -268,7 +271,12 @@ def get_layer_config(model, quantization_config):# Determine the quantization block listquant_block_list = getattr(quantization_config, "quant_block_list", None)- if quant_block_list is None:+ if quant_block_list is not None:+ # Handle nested list format: [[block1, block2, ...], ...] -> [prefix1, ...]+ if quant_block_list and isinstance(quant_block_list[0], (list, tuple)):+ for i in range(len(quant_block_list)):+ quant_block_list[i] = os.path.commonprefix(quant_block_list[i]).rstrip(".")+ elif quant_block_list is None:to_quant_block_names = getattr(quantization_config, "block_name_to_quantize", None) # Prioritize this parameterif to_quant_block_names is None:to_quant_block_names = getattr(quantization_config, "to_quant_block_names", None)@@ -418,7 +426,7 @@ def _replace_by_quant_layers(logger.debug(f"{layer_name}: {layer_backend} backend is used")# Create and replace layer- new_layer = _create_quant_layer(layer, layer_backend, config, in_features, out_features)+ new_layer = _create_quant_layer(layer, layer_backend, config, in_features, out_features, packing_format)set_module(module, layer_name, new_layer)return used_backends@@ -445,11 +453,21 @@ def _import_exllamav2_kernels():logger.warning_once("try to fallback to other autogptq backends for now")-def _create_quant_layer(layer, layer_backend, config, in_features, out_features):+def _create_quant_layer(layer, layer_backend, config, in_features, out_features, packing_format=None):"""Creates a quantized layer using the appropriate class."""- QuantLinear = dynamic_import_inference_linear(layer_backend, config)+ QuantLinear = dynamic_import_inference_linear(layer_backend, config, packing_format=packing_format)bias = layer.bias is not None+ # MLX backend with MLX packing format (native MLX checkpoint)+ if "mlx" in layer_backend and (packing_format is None or "mlx" in packing_format):+ return QuantLinear(+ bits=config["bits"],+ group_size=config["group_size"],+ infeatures=in_features,+ outfeatures=out_features,+ bias=bias,+ )+# Special handling for auto-round-lib AWQ layersfrom auto_round_extension.ark.qlinear import QuantLinearAWQ@@ -550,6 +568,71 @@ def convert_gptq_v1_to_v2_format(model: nn.Module):logger.warning_once("Converting gptq v1 to v2 format")+def _maybe_convert_gptq_to_mlx(model: nn.Module, used_backends: list[str]) -> None:+ """On macOS with MLX available, convert GPTQ-format QuantLinear layers to MLX QuantLinearMLX.++ This is the MLX equivalent of the IPEX/ARK post_init step: when an MLX backend was+ selected but the checkpoint layers were materialized in GPTQ packing format, we+ re-pack them into the MLX format so that ``mx.quantized_matmul`` can be used for+ hardware-accelerated inference on Apple Silicon. All conversion logic lives in+ :meth:`QuantLinearMLX.from_gptq`.+ """+ import platform++ if platform.system() != "Darwin":+ return++ # Only run if an MLX-related backend was selected for some layer.+ if not any("mlx" in b for b in used_backends):+ return++ try:+ import mlx.core as mx # noqa: F401+ except ImportError:+ logger.debug("MLX not installed, skipping GPTQ-to-MLX conversion")+ return++ from auto_round_extension.mlx.qlinear_mlx import QuantLinearMLX++ # Collect GPTQ-style layers that need re-packing into MLX format.+ layers = []+ for name, module in model.named_modules():+ if not hasattr(module, "QUANT_TYPE"):+ continue+ if getattr(module, "QUANT_TYPE", None) == "mlx":+ continue+ if not hasattr(module, "qweight"):+ continue+ layers.append((name, module))++ if not layers:+ return++ # Get sym flag from model config (once, outside loop)+ quant_config = getattr(model.config, "quantization_config", {})+ if hasattr(quant_config, "to_dict"):+ quant_config = quant_config.to_dict()+ sym = quant_config.get("sym", False)++ converted = 0+ skipped = 0+ for name, module in tqdm(layers, desc="repacking to MLX format", total=len(layers), leave=True):+ try:+ mlx_layer = QuantLinearMLX.from_gptq(module, sym=sym)+ set_module(model, name, mlx_layer)+ converted += 1++ except Exception as e:+ logger.warning(f"Failed to convert layer '{name}' to MLX format: {e}")+ skipped += 1+ continue++ if converted > 0:+ logger.info(f"Auto-converted {converted} GPTQ layers to MLX format for Apple Silicon acceleration")+ if skipped > 0:+ logger.warning(f"Skipped {skipped} layers during GPTQ-to-MLX conversion")++def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:"""Performs post-initialization for different quantization backends.@@ -631,6 +714,9 @@ def post_init(model: torch.nn.Module, used_backends: list[str]) -> None:if used_gptq_exllamav2:_import_exllamav2_kernels()+ # On macOS (Apple Silicon), auto-convert GPTQ layers to MLX for faster inference+ _maybe_convert_gptq_to_mlx(model, used_backends)+# Determine common data type across backendsdata_types = [set(BackendInfos[b].compute_dtype) for b in used_backends]common_dtypes = set.intersection(*data_types) if data_types else set()@@ -719,6 +805,8 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.Mpacking_format = "auto_round:auto_awq"elif packing_format == "auto_round:gptq":packing_format = "auto_round:auto_gptq"+ elif packing_format in ("mlx", "auto_round:mlx"):+ pass # keep as-is for MLX backend selectionis_applied = apply_modeling_patch(model)if not is_applied:# Preprocess model before replace layersdiff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex 0e9b922ba..c3bec7ebf 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -142,6 +142,26 @@ W4A16 = QuantizationScheme.from_dict(})+W5A16 = QuantizationScheme.from_dict(+ {+ "bits": 5,+ "sym": True,+ "group_size": 128,+ "data_type": "int",+ "act_bits": 16,+ }+)++W6A16 = QuantizationScheme.from_dict(+ {+ "bits": 6,+ "sym": True,+ "group_size": 128,+ "data_type": "int",+ "act_bits": 16,+ }+)+W2A16 = QuantizationScheme.from_dict({"bits": 2,@@ -373,6 +393,8 @@ PRESET_SCHEMES = {"W4A16": W4A16,"W2A16": W2A16,"W3A16": W3A16,+ "W5A16": W5A16,+ "W6A16": W6A16,"W8A16": W8A16,"MXFP4": MXFP4,"MXFP6": MXFP6,diff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex 30c9edc0c..03a98dc12 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -443,6 +443,8 @@ class SupportedFormats:"llm_compressor","fp8","auto_round:fp8",+ "mlx",+ "auto_round:mlx",)self._gguf_format = tuple(sorted(GGUF_CONFIG.keys()))self._support_list = self._support_format + self._gguf_formatdiff --git a/auto_round_extension/mlx/__init__.py b/auto_round_extension/mlx/__init__.pynew file mode 100644index 000000000..4a8119fce--- /dev/null+++ b/auto_round_extension/mlx/__init__.py@@ -0,0 +1,17 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from auto_round_extension.mlx.qlinear_mlx import QuantLinearMLX, MLX_AVAILABLE++__all__ = ["QuantLinearMLX", "MLX_AVAILABLE"]diff --git a/auto_round_extension/mlx/qlinear_mlx.py b/auto_round_extension/mlx/qlinear_mlx.pynew file mode 100644index 000000000..81f1e81e4--- /dev/null+++ b/auto_round_extension/mlx/qlinear_mlx.py@@ -0,0 +1,341 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""+MLX format QuantLinear for inference via MLX kernels on Apple Silicon.++Uses mx.quantized_matmul for hardware-accelerated quantized inference.+Falls back to PyTorch dequantization if MLX is not available.++Tensor naming follows MLX convention:+ - weight: uint32 packed integers [out_features, packed_in_features]+ - scales: float16 [out_features, num_groups]+ - biases: float16 [out_features, num_groups] (MLX calls zero-point-derived term "biases")+ - bias: optional linear bias+"""++import math++import torch+import torch.nn as nn++try:+ import mlx.core as mx++ MLX_AVAILABLE = True+except ImportError:+ MLX_AVAILABLE = False+++def _torch_to_mlx(t):+ """Convert a PyTorch tensor to MLX array."""+ # Accept either a numpy array or a torch.Tensor.+ try:+ import numpy as _np+ except Exception:+ _np = None++ # If it's a torch tensor, try to use DLPack for zero-copy when supported by MLX.+ if isinstance(t, torch.Tensor):+ # Ensure tensor is contiguous+ tt = t.contiguous()+ try:+ from torch.utils.dlpack import to_dlpack++ if hasattr(mx, "from_dlpack"):+ return mx.from_dlpack(to_dlpack(tt))+ except Exception:+ # fall back to numpy+ pass++ return mx.array(tt.cpu().numpy())++ # Otherwise assume it's a numpy-like object+ return mx.array(t)+++def _mlx_to_torch(a, dtype=None):+ """Convert an MLX array to PyTorch tensor."""+ import numpy as np++ t = torch.from_numpy(np.array(a))+ if dtype is not None:+ t = t.to(dtype)+ return t+++class QuantLinearMLX(nn.Module):+ """Quantized linear layer using MLX kernels for fast inference on Apple Silicon.++ Falls back to pure PyTorch dequantization when MLX is not available.+ """++ QUANT_TYPE = "mlx"++ def __init__(self, bits, group_size, infeatures, outfeatures, bias, **kwargs):+ super().__init__()+ if bits not in [2, 3, 4, 5, 6, 8]:+ raise NotImplementedError("Only 2, 3, 4, 5, 6, 8 bits are supported.")+ self.infeatures = infeatures+ self.outfeatures = outfeatures+ self.bits = bits+ self.group_size = group_size if group_size != -1 else infeatures+ self.maxq = 2**self.bits - 1++ # MLX packing: [out_features, in_features * bits / 32]+ packed_dim = infeatures * bits // 32+ num_groups = math.ceil(infeatures / self.group_size)++ self.register_buffer(+ "weight",+ torch.zeros((outfeatures, packed_dim), dtype=torch.uint32),+ )+ self.register_buffer(+ "scales",+ torch.zeros((outfeatures, num_groups), dtype=torch.float16),+ )+ self.register_buffer(+ "biases",+ torch.zeros((outfeatures, num_groups), dtype=torch.float16),+ )+ if bias:+ self.register_buffer("bias", torch.zeros(outfeatures, dtype=torch.float16))+ else:+ self.bias = None++ # Cache MLX arrays after first forward+ self._mlx_weight = None+ self._mlx_scales = None+ self._mlx_biases = None++ def _ensure_mlx_cache(self):+ """Convert PyTorch buffers to MLX arrays, cached for reuse."""+ if self._mlx_weight is None:+ # Avoid unnecessary device transfers when buffers are already on CPU.+ weight_for_mx = self.weight if self.weight.device.type == "cpu" else self.weight.cpu()+ scales_for_mx = self.scales if self.scales.device.type == "cpu" else self.scales.cpu()+ biases_for_mx = self.biases if self.biases.device.type == "cpu" else self.biases.cpu()++ self._mlx_weight = _torch_to_mlx(weight_for_mx)+ self._mlx_scales = _torch_to_mlx(scales_for_mx.float())+ self._mlx_biases = _torch_to_mlx(biases_for_mx.float())++ def _forward_mlx(self, x):+ """Forward pass using MLX quantized_matmul kernel."""+ self._ensure_mlx_cache()++ input_dtype = x.dtype+ # Avoid an unnecessary copy when input is already on CPU.+ if x.device.type == "cpu":+ x_cpu = x.detach()+ else:+ # For MPS or other devices, move to CPU for MLX bridge.+ # Use .to('cpu') which may be optimized on Apple Silicon unified memory.+ x_cpu = x.detach().to("cpu")++ # Prepare contiguous float32 tensor for MLX consumption; _torch_to_mlx+ # will attempt a zero-copy DLPack transfer if supported by MLX.+ x_for_mx = x_cpu.float().contiguous()+ x_mlx = _torch_to_mlx(x_for_mx)++ # mx.quantized_matmul: x @ w.T with quantized w+ out_mlx = mx.quantized_matmul(+ x_mlx,+ self._mlx_weight,+ scales=self._mlx_scales,+ biases=self._mlx_biases,+ transpose=True,+ group_size=self.group_size,+ bits=self.bits,+ )+ mx.eval(out_mlx)++ out = _mlx_to_torch(out_mlx, dtype=input_dtype)++ if self.bias is not None:+ out = out + self.bias.to(input_dtype)++ return out.to(x.device)++ def _forward_torch(self, x):+ """Fallback forward pass using pure PyTorch dequantization."""+ input_dtype = x.dtype+ input_device = x.device+ # Move to CPU for bitwise unpack ops (MPS may not support all int ops)+ intweight = self._unpack_weight_torch(device=torch.device("cpu")).float()+ scales = self.scales.cpu().float().repeat_interleave(self.group_size, dim=1)[:, : self.infeatures]+ biases = self.biases.cpu().float().repeat_interleave(self.group_size, dim=1)[:, : self.infeatures]+ weight = scales * intweight + biases+ # NOTE: self.bias is registered as float16 but ``weight`` here is float32 because+ # we promoted intweight via .float(). F.linear requires bias dtype to match weight+ # dtype, otherwise it raises RuntimeError. Cast explicitly.+ bias = self.bias.cpu().to(weight.dtype) if self.bias is not None else None+ out = torch.nn.functional.linear(x.cpu().to(weight.dtype), weight, bias)+ return out.to(input_dtype).to(input_device)++ def _unpack_weight_torch(self, device=None):+ """Unpack uint32 weight tensor to integer values (PyTorch fallback)."""+ weight = self.weight.view(torch.int32)+ if device is not None:+ weight = weight.to(device)+ out_features, packed_dim = weight.shape++ if 32 % self.bits == 0:+ elems_per_int = 32 // self.bits+ shifts = torch.arange(elems_per_int, device=weight.device, dtype=torch.int32) * self.bits+ mask = (1 << self.bits) - 1+ unpacked = (weight.unsqueeze(-1) >> shifts) & mask+ unpacked = unpacked.reshape(out_features, -1)+ else:+ num_groups_32 = packed_dim // self.bits+ weight_reshaped = weight.reshape(out_features, num_groups_32, self.bits).to(torch.int64)+ unpacked = torch.zeros(out_features, num_groups_32, 32, dtype=torch.int64, device=weight.device)+ mask = (1 << self.bits) - 1+ for i in range(32):+ val = 0+ for b in range(self.bits):+ abs_bit = i * self.bits + b+ word_idx = abs_bit // 32+ bit_pos = abs_bit % 32+ val |= ((weight_reshaped[:, :, word_idx] >> bit_pos) & 1) << b+ unpacked[:, :, i] = val & mask+ unpacked = unpacked.reshape(out_features, -1).to(torch.int32)++ return unpacked[:, : self.infeatures]++ def forward(self, x):+ if MLX_AVAILABLE:+ return self._forward_mlx(x)+ return self._forward_torch(x)++ @classmethod+ def from_gptq(cls, module: nn.Module, sym: bool = False) -> "QuantLinearMLX":+ """Convert a GPTQ-format QuantLinear module into a QuantLinearMLX module.++ Repacks qweight/qzeros/scales (GPTQ format) into weight/scales/biases (MLX format)+ so that ``mx.quantized_matmul`` can be used for hardware-accelerated inference on+ Apple Silicon.++ Args:+ module: The source GPTQ-style quantized linear module (must expose+ ``qweight``, ``scales``, ``bits``, ``group_size`` and optionally ``qzeros``).+ sym: Whether symmetric quantization was used. For symmetric quantization the+ zero-point is fixed to ``2**(bits-1)`` which avoids the GPTQ ``qzeros`` ±1+ offset ambiguity.++ Returns:+ A fully initialized ``QuantLinearMLX`` module ready for inference.+ """+ # Use CPU for conversion to avoid MPS compatibility issues with bitwise ops+ convert_device = torch.device("cpu")++ bits = module.bits+ group_size = module.group_size+ if bits not in [2, 3, 4, 8]:+ raise NotImplementedError(f"Unsupported bits={bits} for GPTQ->MLX conversion.")++ qweight = module.qweight.to(convert_device) # [in//32*bits, out] int32+ scales_gptq = module.scales.to(convert_device) # [groups, out] float16++ out_features = scales_gptq.shape[1]+ in_features = qweight.shape[0] * 32 // bits++ # Unpack GPTQ qweight -> int weight [in_features, out_features]+ if bits in [2, 4, 8]:+ elems_per_int = 32 // bits+ wf = torch.arange(0, 32, bits, dtype=torch.int32, device=convert_device)+ weight_int = torch.bitwise_right_shift(+ qweight.unsqueeze(1).expand(-1, elems_per_int, -1),+ wf.unsqueeze(-1),+ ).to(torch.int16 if bits == 8 else torch.int8)+ weight_int = torch.bitwise_and(weight_int, (1 << bits) - 1)+ weight_int = weight_int.reshape(-1, out_features)+ else: # bits == 3+ from auto_round_extension.torch.qlinear_torch import get_wf_3bits_tensor++ wf = get_wf_3bits_tensor(convert_device)+ weight_int = qweight.reshape(qweight.shape[0] // 3, 3, 1, qweight.shape[1]).expand(-1, -1, 12, -1)+ weight_int = (weight_int >> wf.unsqueeze(-1)) & 0x7+ weight_int[:, 0, 10] = (weight_int[:, 0, 10] & 0x3) | ((weight_int[:, 1, 0] << 2) & 0x4)+ weight_int[:, 1, 11] = (weight_int[:, 1, 11] & 0x1) | ((weight_int[:, 2, 0] << 1) & 0x6)+ weight_int = weight_int & 0x7+ weight_int = torch.cat([weight_int[:, 0, :11], weight_int[:, 1, 1:12], weight_int[:, 2, 1:11]], dim=1)+ weight_int = weight_int.reshape(-1, out_features)++ # Determine zeros+ # NOTE on GPTQ qzeros offset:+ # GPTQ packs qzeros as ``(actual_zero - 1) & maxq`` (a quirk of the+ # original GPTQ kernel). When unpacking we MUST add 1 (mod 2**bits)+ # to recover the true zero point. AutoRound's own asymmetric W4A16+ # actually exports to AWQ format, but third-party GPTQ checkpoints+ # loaded via ``auto_gptq`` / ``gptqmodel`` (packing format+ # ``auto_round:auto_gptq``) hit this code path and require the +1.+ maxq = (1 << bits) - 1+ if sym:+ zero_val = 2 ** (bits - 1)+ zeros = torch.full(scales_gptq.shape, zero_val, dtype=torch.int32, device=convert_device)+ else:+ if not hasattr(module, "qzeros") or module.qzeros is None:+ zero_val = 2 ** (bits - 1)+ zeros = torch.full(scales_gptq.shape, zero_val, dtype=torch.int32, device=convert_device)+ else:+ qzeros = module.qzeros.to(convert_device)+ if bits in [2, 4, 8]:+ elems_per_int = 32 // bits+ wf = torch.arange(0, 32, bits, dtype=torch.int32, device=convert_device)+ zeros = torch.bitwise_right_shift(+ qzeros.unsqueeze(2).expand(-1, -1, elems_per_int),+ wf.unsqueeze(0),+ ).to(torch.int16 if bits == 8 else torch.int8)+ zeros = torch.bitwise_and(zeros, maxq)+ zeros = zeros.reshape(scales_gptq.shape).to(torch.int32)+ else: # bits == 3+ from auto_round_extension.torch.qlinear_torch import get_wf_3bits_tensor++ wf = get_wf_3bits_tensor(convert_device)+ zeros = qzeros.reshape(qzeros.shape[0], qzeros.shape[1] // 3, 3, 1).expand(-1, -1, -1, 12)+ zeros = zeros >> wf.unsqueeze(0)+ zeros[:, :, 0, 10] = (zeros[:, :, 0, 10] & 0x3) | ((zeros[:, :, 1, 0] << 2) & 0x4)+ zeros[:, :, 1, 11] = (zeros[:, :, 1, 11] & 0x1) | ((zeros[:, :, 2, 0] << 1) & 0x6)+ zeros = zeros & 0x7+ zeros = torch.cat([zeros[:, :, 0, :11], zeros[:, :, 1, 1:12], zeros[:, :, 2, 1:11]], dim=2)+ zeros = zeros.reshape(scales_gptq.shape).to(torch.int32)++ # GPTQ qzeros are stored off-by-one; recover the true zero point.+ zeros = (zeros + 1) & maxq++ # GPTQ: w = scale * (w_int - zero)+ # MLX: w = mlx_scale * w_int + mlx_bias+ # => mlx_scale = scale, mlx_bias = -scale * zero+ from auto_round.export.export_to_mlx.export import _pack_weight_mlx++ weight_int_t = weight_int.t().contiguous().to(torch.int32)+ scales_mlx = scales_gptq.t().contiguous().to(torch.float16)+ biases_mlx = (-scales_gptq.float().t().contiguous() * zeros.float().t().contiguous()).to(torch.float16)+ packed_weight = _pack_weight_mlx(weight_int_t, bits)++ has_bias = getattr(module, "bias", None) is not None+ mlx_layer = cls(+ bits=bits,+ group_size=group_size,+ infeatures=in_features,+ outfeatures=out_features,+ bias=has_bias,+ )+ mlx_layer.weight.copy_(packed_weight)+ mlx_layer.scales.copy_(scales_mlx)+ mlx_layer.biases.copy_(biases_mlx)+ if has_bias:+ mlx_layer.bias.copy_(module.bias.to(torch.float16))+ return mlx_layerdiff --git a/auto_round_extension/torch/qlinear_mlx.py b/auto_round_extension/torch/qlinear_mlx.pynew file mode 100644index 000000000..d506ffb12--- /dev/null+++ b/auto_round_extension/torch/qlinear_mlx.py@@ -0,0 +1,17 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Backward compatibility shim. Use auto_round_extension.mlx.qlinear_mlx instead."""++from auto_round_extension.mlx.qlinear_mlx import MLX_AVAILABLE, QuantLinearMLX # noqa: F401diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 53acef0bc..dfb55e7a8 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -119,6 +119,8 @@ AutoRound supports several Schemes:- **W4A16**(bits:4,group_size:128,sym:True,act_bits:16)- **W8A16**(bits:8,group_size:128,sym:True,act_bits:16)+- **W6A16**(bits:6,group_size:128,sym:True,act_bits:16) — `mlx` format only+- **W5A16**(bits:5,group_size:128,sym:True,act_bits:16) — `mlx` format only- **W3A16**(bits:3,group_size:128,sym:True,act_bits:16)- **W2A16**(bits:2,group_size:128,sym:True,act_bits:16)- **GGUF:Q4_K_M**(all Q*_K,Q*_0,Q*_1 provided by llamacpp are supported)@@ -152,6 +154,11 @@ adopted within the community, **only 4-bits quantization is supported**. Please**LLM-Compressor Format**: **NVFP4, MXFP4(kernel in WIP), MXFP8 are supported**. Please set `--format llm_compressor`+**MLX Format**: This format targets Apple Silicon (M1/M2/M3/...) and is loaded directly by [`mlx-lm`](https://github.com/ml-explore/mlx-lm) (text-only LLM) or [`mlx-vlm`](https://github.com/Blaizzy/mlx-vlm) (vision/audio + language).+- Supports **2, 3, 4, 5, 6, 8 bits** (5/6 bits are MLX-exclusive — GPTQ/AWQ have no standard packing for them).+- Native **mixed-bit / mixed-group_size** via `layer_config` or AutoScheme (`--target_bits 3.5 --options "..."`);+- Use `--format mlx` for a native MLX checkpoint; use `--format auto_round:mlx` if you want HuggingFace `transformers` + AutoRound to load it (post-init repacks each layer into MLX `QuantLinear` on Darwin).+- Limitation: embedding layer quantization has not supported#### Format and scheme support matrix> Gray indicates the absence of a kernel or the presence of only an inefficient/reference kernel. BF16 is mainly for AutoScheme@@ -162,13 +169,14 @@ adopted within the community, **only 4-bits quantization is supported**. Please| **auto_awq** | W4A16, BF16 || **auto_gptq** | W4A16, W2A16, W3A16, W8A16,W2A16G64, W2A16G32, BF16 || **llm_compressor** | NVFP4, `MXFP4`, `MXFP8`, `FPW8A16`, `FP8_STATIC`, FP8_BLOCK |+| **mlx** / **auto_round:mlx** | W2A16, W3A16, W4A16, W5A16, W6A16, W8A16, BF16, mixed-bit / mixed-group_size (Apple Silicon only) || **gguf** | GGUF:Q4_K_M, GGUF:Q2_K_S, GGUF:Q3_K_S, GGUF:Q3_K_M, GGUF:Q3_K_L, GGUF:Q4_K_S, GGUF:Q5_K_S, GGUF:Q5_K_M, GGUF:Q6_K, GGUF:Q4_0, GGUF:Q4_1, GGUF:Q5_0, GGUF:Q5_1,GGUF:Q8_0 || **fp8** | FP8_BLOCK || **fake** | `all schemes (only for research)` |### Hardware Compatibility-CPU, Intel GPU, HPU and CUDA for both quantization and inference.+CPU, Intel GPU, HPU and CUDA for both quantization and inference. The **MLX format** is exclusive to **Apple Silicon (macOS / Darwin)** at inference time; quantization (export) itself can be done on any platform.### Environment Configurationdiff --git a/docs/step_by_step_CN.md b/docs/step_by_step_CN.mdindex 5eddd22d3..e04021a03 100644--- a/docs/step_by_step_CN.md+++ b/docs/step_by_step_CN.md@@ -113,6 +113,8 @@ pip install auto-roundAutoRound 支持多种量化配置:- **W4A16**(bits:4, group_size:128, sym:True, act_bits:16) # 4位权重,分组大小为128,对称量化,16位激活,- **W8A16**(bits:8, group_size:128, sym:True, act_bits:16)+- **W6A16**(bits:6, group_size:128, sym:True, act_bits:16) — 仅 `mlx` 格式支持+- **W5A16**(bits:5, group_size:128, sym:True, act_bits:16) — 仅 `mlx` 格式支持- **W3A16**(bits:3, group_size:128, sym:True, act_bits:16)- **W2A16**(bits:2, group_size:128, sym:True, act_bits:16)- **GGUF:Q4_K_M**(支持 llamacpp 提供的所有 Q*_K、Q*_0、Q*_1 量化类型)@@ -142,6 +144,12 @@ AutoRound 支持多种量化配置:**LLM-Compressor 格式**:**支持 NVFP4、MXFP4(kernel 开发中)、MXFP8** 等。需设置 `--format llm_compressor`。+**MLX 格式**:面向 Apple Silicon (M1/M2/M3/...),可直接被 [`mlx-lm`](https://github.com/ml-explore/mlx-lm)(纯文本 LLM)或 [`mlx-vlm`](https://github.com/Blaizzy/mlx-vlm)(多模态 VLM)加载推理。+- 支持 **2、3、4、5、6、8 bits**(其中 5/6 bits 是 MLX 独有,GPTQ/AWQ 没有标准打包格式)。+- 原生支持 **混合 bit / 混合 group_size**:通过 `layer_config` 或 AutoScheme(如 `--target_bits 3.5 --options "..."`),按层覆盖会写入 `config.json["quantization"]`,+- `--format mlx` 导出原生 MLX checkpoint;`--format auto_round:mlx` 则让 HuggingFace `transformers` + AutoRound 加载它(在 Darwin 上 post-init 会把每层重新打包成 MLX 的 `QuantLinear`)。+- 已经问题: 没有支持嵌入层的量化+#### 格式与方案支持对照表> 灰色背景的 schemes 表示它没有专门优化的内核,或只有效率极低的参考内核。@@ -152,13 +160,14 @@ AutoRound 支持多种量化配置:| **auto_awq** | W4A16、BF16 || **auto_gptq** | W4A16、W2A16、W3A16、W8A16、W2A16G64、W2A16G32、BF16 || **llm_compressor** | NVFP4、`MXFP4`、`MXFP8`、`FPW8A16`、`FP8_STATIC`、FP8_STATIC |+| **mlx** / **auto_round:mlx** | W2A16、W3A16、W4A16、W5A16、W6A16、W8A16、BF16、混合 bit / 混合 group_size(仅 Apple Silicon) || **gguf** | GGUF:Q4_K_M、GGUF:Q2_K_S、GGUF:Q3_K_S、GGUF:Q3_K_M、GGUF:Q3_K_L、GGUF:Q4_K_S、GGUF:Q5_K_S、GGUF:Q5_K_M、GGUF:Q6_K、GGUF:Q4_0、GGUF:Q4_1、GGUF:Q5_0、GGUF:Q5_1、GGUF:Q8_0 || **fp8** | FP8_BLOCK || **fake** | `所有方案(仅用于研究场景)` |### 硬件兼容性-量化和推理均支持 CPU、英特尔 GPU、HPU 和 CUDA。+量化和推理均支持 CPU、英特尔 GPU、HPU 和 CUDA。**MLX 格式**的推理仅支持 **Apple Silicon (macOS / Darwin)**,但量化(导出)阶段在任意平台均可进行。### 环境参数配置diff --git a/test/helpers.py b/test/helpers.pyindex f2a670d4c..b568c137a 100644--- a/test/helpers.py+++ b/test/helpers.py@@ -132,6 +132,7 @@ deepseek_v2_name_or_path = get_model_path("deepseek-ai/DeepSeek-V2-Lite")qwen_moe_name_or_path = get_model_path("Qwen/Qwen1.5-MoE-A2.7B")qwen_vl_name_or_path = get_model_path("Qwen/Qwen2-VL-2B-Instruct")qwen_2_5_vl_name_or_path = get_model_path("Qwen/Qwen2.5-VL-3B-Instruct")+qwen_3_vl_9b_name_or_path = get_model_path("Qwen/Qwen3.5-9B")gemma_name_or_path = get_model_path("benzart/gemma-2b-it-fine-tuning-for-code-test")qwen2_5_omni_name_or_path = get_model_path("Qwen/Qwen2.5-Omni-3B")qwen3_omni_name_or_path = get_model_path("Qwen/Qwen3-Omni-30B-A3B-Instruct")diff --git a/test/test_cuda/algorithms/test_auto_scheme.py b/test/test_cuda/algorithms/test_auto_scheme.pyindex 0967f7e01..9cde148d7 100644--- a/test/test_cuda/algorithms/test_auto_scheme.py+++ b/test/test_cuda/algorithms/test_auto_scheme.py@@ -280,7 +280,7 @@ class TestAutoScheme:quantization."""target_bits = 2.5- ref_threshold = 0.58+ ref_threshold = 0.56common_kwargs = dict(iters=0,disable_opt_rtn=True,diff --git a/test/test_mlx/__init__.py b/test/test_mlx/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/test/test_mlx/test_mlx_format.py b/test/test_mlx/test_mlx_format.pynew file mode 100644index 000000000..81106ca6d--- /dev/null+++ b/test/test_mlx/test_mlx_format.py@@ -0,0 +1,496 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0++"""Unified MLX format tests for AutoRound on Qwen3-0.6B (RTN mode).++Two export formats are covered:++ * ``mlx`` — native MLX checkpoint, loaded and run through ``mlx_lm``+ on Apple Silicon.+ * ``auto_round`` — GPTQ-style packing. On Darwin the loader post-init+ re-packs each layer into :class:`QuantLinearMLX` via+ :meth:`QuantLinearMLX.from_gptq`; inference uses the+ HuggingFace ``transformers`` + AutoRound pipeline.++Each format is exercised over different bit-widths (2/3/4/8) and a mixed-bit+configuration via ``layer_config``. All runs use RTN (``iters=0``) for speed.+Platform/runtime-specific inference tests are skipped automatically on+non-Darwin platforms or when the required packages are missing.+"""++import importlib.util+import json+import os+import platform+import shutil+from pathlib import Path+from test.helpers import qwen_3_vl_9b_name_or_path, qwen_name_or_path++import pytest+import torch++from auto_round import AutoRound++MLX_AVAILABLE = importlib.util.find_spec("mlx") is not None+MLX_LM_AVAILABLE = importlib.util.find_spec("mlx_lm") is not None+MLX_VLM_AVAILABLE = importlib.util.find_spec("mlx_vlm") is not None+IS_DARWIN = platform.system() == "Darwin"++# Native mlx checkpoint needs Darwin + mlx_lm to actually load & run.+requires_mlx_lm_runtime = pytest.mark.skipif(+ not (IS_DARWIN and MLX_LM_AVAILABLE),+ reason="Native MLX inference requires macOS (Darwin) with 'mlx_lm' installed.",+)+# auto_round GPTQ -> QuantLinearMLX post-init needs Darwin + mlx.+requires_mlx_runtime = pytest.mark.skipif(+ not (IS_DARWIN and MLX_AVAILABLE),+ reason="QuantLinearMLX post-init requires macOS (Darwin) with 'mlx' installed.",+)+# Native mlx VLM checkpoint needs Darwin + mlx_vlm to actually load & run.+requires_mlx_vlm_runtime = pytest.mark.skipif(+ not (IS_DARWIN and MLX_VLM_AVAILABLE),+ reason="Native MLX VLM inference requires macOS (Darwin) with 'mlx_vlm' installed.",+)+# Skip large 9B VLM tests when the model isn't materialized locally — we don't+# want CI to silently kick off a ~18 GB HuggingFace download.+_qwen3_vl_9b_local = os.path.isdir(qwen_3_vl_9b_name_or_path)+requires_qwen3_vl_9b_local = pytest.mark.skipif(+ not _qwen3_vl_9b_local,+ reason=(+ f"Qwen3-VL-9B model not found locally at '{qwen_3_vl_9b_name_or_path}'. "+ "Place the weights under one of the test model paths to enable this test."+ ),+)+++# --------------------------------------------------------------------------- #+# Helpers+# --------------------------------------------------------------------------- #+def _quantize_and_save(+ output_dir: str,+ scheme: str,+ bits: int,+ fmt: str,+ sym: bool = True,+ group_size: int = 128,+ layer_config: dict = None,+):+ """Run AutoRound RTN quantization on Qwen3-0.6B and export to ``fmt``."""+ ar = AutoRound(+ model=qwen_name_or_path,+ scheme=scheme,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=0, # RTN+ nsamples=1,+ seqlen=32,+ disable_opt_rtn=True,+ layer_config=layer_config,+ )+ ar.quantize_and_save(output_dir=output_dir, format=fmt)+++def _read_quant_config(output_dir: str) -> dict:+ """Return the quantization_config dict from the exported model."""+ config_path = Path(output_dir) / "config.json"+ if config_path.exists():+ with open(config_path, "r", encoding="utf-8") as f:+ cfg = json.load(f)+ if "quantization_config" in cfg:+ return cfg["quantization_config"]+ standalone = Path(output_dir) / "quantization_config.json"+ if standalone.exists():+ with open(standalone, "r", encoding="utf-8") as f:+ return json.load(f)+ return {}+++def _count_mlx_layers(model: torch.nn.Module) -> int:+ """Count QuantLinearMLX instances in a loaded HF model."""+ from auto_round_extension.mlx.qlinear_mlx import QuantLinearMLX++ return sum(1 for m in model.modules() if isinstance(m, QuantLinearMLX))+++def _mlx_lm_generate(output_dir: str, prompt: str = "The capital of France is", max_tokens: int = 8) -> str:+ """Load a native MLX checkpoint through mlx_lm and generate text."""+ from mlx_lm import generate, load++ model, tokenizer = load(output_dir)+ return generate(model, tokenizer, prompt=prompt, max_tokens=max_tokens)+++def _hf_generate(output_dir: str, prompt: str = "The capital of France is", max_new_tokens: int = 5):+ """Load an auto_round checkpoint through HF transformers and generate text.++ Returns:+ Tuple of (model, decoded_text).+ """+ from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizer++ model = AutoModelForCausalLM.from_pretrained(+ output_dir,+ device_map="cpu",+ torch_dtype="auto",+ trust_remote_code=True,+ quantization_config=AutoRoundConfig(),+ )+ tokenizer = AutoTokenizer.from_pretrained(output_dir, trust_remote_code=True)+ inputs = tokenizer(prompt, return_tensors="pt")+ with torch.no_grad():+ out = model.generate(**inputs, max_new_tokens=max_new_tokens)+ return model, tokenizer.decode(out[0], skip_special_tokens=True)+++# --------------------------------------------------------------------------- #+# Tests+# --------------------------------------------------------------------------- #+class TestMLXFormat:+ @pytest.fixture(autouse=True)+ def _save_dir(self, tmp_path):+ self.save_dir = str(tmp_path / "mlx_saved")+ yield+ shutil.rmtree(self.save_dir, ignore_errors=True)++ @classmethod+ def teardown_class(cls):+ shutil.rmtree("runs", ignore_errors=True)++ # ---- 1) Native MLX export: sweep bits ------------------------------- #+ @pytest.mark.parametrize(+ "bits, scheme",+ [+ (2, "W2A16"),+ (3, "W3A16"),+ (4, "W4A16"),+ (5, "W5A16"),+ (6, "W6A16"),+ (8, "W8A16"),+ ],+ )+ def test_mlx_native_export_bits(self, bits, scheme):+ """Export Qwen3-0.6B as a native MLX checkpoint at 2/3/4/8 bits."""+ _quantize_and_save(+ output_dir=self.save_dir,+ scheme=scheme,+ bits=bits,+ fmt="mlx",+ sym=True,+ group_size=128,+ )+ assert os.path.exists(os.path.join(self.save_dir, "config.json"))+ qcfg = _read_quant_config(self.save_dir)+ assert qcfg, f"No quantization_config found for scheme={scheme}"+ assert qcfg.get("bits") == bits, f"expected bits={bits}, got {qcfg.get('bits')}"++ @requires_mlx_lm_runtime+ @pytest.mark.parametrize(+ "bits, scheme",+ [+ (2, "W2A16"),+ (3, "W3A16"),+ (4, "W4A16"),+ (5, "W5A16"),+ (6, "W6A16"),+ (8, "W8A16"),+ ],+ )+ def test_mlx_native_inference_with_mlx_lm(self, bits, scheme):+ """Native MLX checkpoint loads and runs end-to-end through mlx_lm."""+ _quantize_and_save(+ output_dir=self.save_dir,+ scheme=scheme,+ bits=bits,+ fmt="mlx",+ sym=True,+ group_size=128,+ )+ text = _mlx_lm_generate(self.save_dir)+ assert isinstance(text, str) and len(text) > 0, f"mlx_lm generated empty output for bits={bits}"++ # ---- 2) auto_round export: sweep bits + sym/asym -------------------- #+ @pytest.mark.parametrize(+ "bits, scheme, sym",+ [+ (2, "W2A16", True),+ (3, "W3A16", True),+ (4, "W4A16", True),+ (4, "W4A16", False), # asymmetric -> exercises qzeros path+ (8, "W8A16", True),+ ],+ )+ def test_auto_round_export_bits(self, bits, scheme, sym):+ """Export Qwen3-0.6B as auto_round (GPTQ-style) across bits/sym."""+ _quantize_and_save(+ output_dir=self.save_dir,+ scheme=scheme,+ bits=bits,+ fmt="auto_round",+ sym=sym,+ group_size=128,+ )+ assert os.path.exists(os.path.join(self.save_dir, "config.json"))+ qcfg = _read_quant_config(self.save_dir)+ assert qcfg.get("bits") == bits+ assert qcfg.get("sym") == sym++ @requires_mlx_runtime+ @pytest.mark.parametrize(+ "bits, scheme, sym",+ [+ (4, "W4A16", True),+ (4, "W4A16", False),+ (8, "W8A16", True),+ ],+ )+ def test_auto_round_post_init_to_mlx(self, bits, scheme, sym):+ """On Darwin, auto_round layers must be post-init converted to QuantLinearMLX."""+ _quantize_and_save(+ output_dir=self.save_dir,+ scheme=scheme,+ bits=bits,+ fmt="auto_round:auto_gptq",+ sym=sym,+ group_size=128,+ )+ model, text = _hf_generate(self.save_dir)+ assert _count_mlx_layers(model) > 0, (+ f"Expected auto_round->MLX post-init conversion to produce QuantLinearMLX "+ f"layers on Darwin, but found none (bits={bits}, sym={sym})."+ )+ assert isinstance(text, str) and len(text) > 0++ # ---- 3) Mixed-bit via layer_config ---------------------------------- #+ def test_mixed_bits_mlx_export(self):+ """Mixed-bit layer_config exported in native MLX format.++ Verifies the resulting ``config.json`` follows the mlx-community+ mixed-bit convention: per-layer ``{group_size, bits}`` overrides for+ layers quantized with a non-default scheme, and ``false`` for layers+ we explicitly kept in fp16 (e.g. ``lm_head``).+ """+ layer_config = {+ "lm_head": {"bits": 16},+ "model.layers.0.self_attn.q_proj": {"bits": 8, "group_size": 128, "sym": True},+ "model.layers.0.self_attn.k_proj": {"bits": 8, "group_size": 128, "sym": True},+ "model.layers.0.mlp.down_proj": {"bits": 2, "group_size": 128, "sym": True},+ }+ _quantize_and_save(+ output_dir=self.save_dir,+ scheme="W4A16",+ bits=4,+ fmt="mlx",+ sym=True,+ group_size=128,+ layer_config=layer_config,+ )+ assert os.path.exists(os.path.join(self.save_dir, "config.json"))++ with open(os.path.join(self.save_dir, "config.json"), "r", encoding="utf-8") as f:+ cfg = json.load(f)+ quant = cfg.get("quantization", {})+ # global defaults+ assert quant.get("bits") == 4+ assert quant.get("group_size") == 128+ # per-layer overrides+ assert quant.get("model.layers.0.self_attn.q_proj") == {"group_size": 128, "bits": 8}+ assert quant.get("model.layers.0.self_attn.k_proj") == {"group_size": 128, "bits": 8}+ assert quant.get("model.layers.0.mlp.down_proj") == {"group_size": 128, "bits": 2}+ # lm_head kept at fp16 must be marked False so mlx-lm does not auto-quantize it+ assert quant.get("lm_head") is False, f"expected lm_head=False, got {quant.get('lm_head')!r}"++ def test_mixed_bits_auto_round_export(self):+ """Mixed-bit layer_config exported as auto_round."""+ layer_config = {+ "lm_head": {"bits": 16},+ "model.layers.0.self_attn.v_proj": {"bits": 8, "group_size": 128, "sym": True},+ "model.layers.0.mlp.gate_proj": {"bits": 2, "group_size": 128, "sym": True},+ }+ _quantize_and_save(+ output_dir=self.save_dir,+ scheme="W4A16",+ bits=4,+ fmt="auto_round",+ sym=True,+ group_size=128,+ layer_config=layer_config,+ )+ qcfg = _read_quant_config(self.save_dir)+ assert qcfg.get("bits") == 4++ @requires_mlx_lm_runtime+ def test_mixed_bits_mlx_inference_with_mlx_lm(self):+ """Mixed-bit native MLX checkpoint runs end-to-end under mlx_lm."""+ layer_config = {+ "lm_head": {"bits": 16},+ "model.layers.0.self_attn.q_proj": {"bits": 8, "group_size": 128, "sym": True},+ "model.layers.0.mlp.down_proj": {"bits": 2, "group_size": 128, "sym": True},+ }+ _quantize_and_save(+ output_dir=self.save_dir,+ scheme="W4A16",+ bits=4,+ fmt="mlx",+ sym=True,+ group_size=128,+ layer_config=layer_config,+ )+ text = _mlx_lm_generate(self.save_dir)+ assert isinstance(text, str) and len(text) > 0++ @requires_mlx_runtime+ def test_mixed_bits_auto_round_inference_on_darwin(self):+ """Mixed-bit auto_round export, loaded via HF + post-init MLX repack."""+ layer_config = {+ "lm_head": {"bits": 16},+ "model.layers.0.self_attn.q_proj": {"bits": 8, "group_size": 128, "sym": True},+ }+ _quantize_and_save(+ output_dir=self.save_dir,+ scheme="W4A16",+ bits=4,+ fmt="auto_round",+ sym=True,+ group_size=128,+ layer_config=layer_config,+ )+ model, text = _hf_generate(self.save_dir)+ assert _count_mlx_layers(model) > 0+ assert isinstance(text, str) and len(text) > 0++ @requires_qwen3_vl_9b_local+ def test_qwen3_vl_9b_mlx_export_config(self, tmp_path):+ """Quantize Qwen3-VL-9B (RTN W4A16) and validate the VLM-aware MLX config.++ Verifies the four VLM-specific guarantees of the MLX exporter:+ 1. Top-level ``quantization`` exists with global defaults.+ 2. Language-tower quantized layers are recorded with their full path.+ 3. ``text_config.quantization`` exists and uses paths *relative* to+ the language sub-module (i.e. without the ``language_model.``+ prefix), matching mlx-vlm's loader expectations.+ 4. ``vision_tower``/``visual`` un-quantized layers do NOT bloat the+ config with ``false`` entries.+ 5. ``rope_parameters`` has been flattened (``rope_theta`` surfaced).+ """+ from auto_round import AutoRoundMLLM++ save_dir = str(tmp_path / "qwen3_vl_9b_mlx")+ try:+ ar = AutoRoundMLLM(+ qwen_3_vl_9b_name_or_path,+ scheme="W4A16",+ bits=4,+ group_size=128,+ sym=True,+ iters=0,+ nsamples=1,+ seqlen=32,+ disable_opt_rtn=True,+ )+ ar.quantize_and_save(output_dir=save_dir, format="mlx")++ cfg_path = os.path.join(save_dir, "config.json")+ assert os.path.exists(cfg_path), "config.json missing after VLM mlx export"+ with open(cfg_path, "r", encoding="utf-8") as f:+ cfg = json.load(f)++ # 1) top-level defaults+ quant = cfg.get("quantization", {})+ assert quant.get("bits") == 4+ assert quant.get("group_size") == 128++ # 2) at least some language-tower layers are recorded with their full path+ lang_keys = [+ k+ for k in quant+ if isinstance(k, str)+ and k.startswith(("language_model.", "model.language_model."))+ and isinstance(quant[k], dict)+ and quant[k].get("bits") == 4+ ]+ assert lang_keys, "No language_model.* quantized entries in top-level quantization dict"++ # 3) text_config.quantization mirrors them with the prefix stripped+ sub = None+ for sub_key in ("text_config", "language_config", "thinker_config"):+ cand = cfg.get(sub_key)+ if isinstance(cand, dict) and "quantization" in cand:+ sub = cand["quantization"]+ break+ assert sub is not None, "expected text_config.quantization to be injected for VLM"+ assert sub.get("bits") == 4 and sub.get("group_size") == 128++ # Pick one full-path key and verify its prefix-stripped form is in sub+ sample = lang_keys[0]+ for prefix in ("language_model.", "model.language_model."):+ if sample.startswith(prefix):+ stripped = sample[len(prefix) :]+ break+ assert stripped in sub, (+ f"text_config.quantization missing prefix-stripped key '{stripped}' "+ f"(derived from top-level '{sample}')"+ )++ # 4) vision tower must NOT pollute the config with ``false`` entries+ vision_false_keys = [+ k+ for k, v in quant.items()+ if isinstance(k, str)+ and any(k.startswith(p + ".") for p in ("vision_tower", "vision_model", "visual"))+ and v is False+ ]+ assert not vision_false_keys, (+ f"Expected vision_tower un-quantized layers to be skipped, "+ f"but found {len(vision_false_keys)} ``false`` entries (e.g. {vision_false_keys[:3]})"+ )++ # 5) rope_parameters flattened — either at top level or inside text_config+ assert "rope_parameters" not in cfg, "top-level rope_parameters should be flattened"+ for sub_key in ("text_config", "language_config", "thinker_config"):+ sc = cfg.get(sub_key)+ if isinstance(sc, dict):+ assert "rope_parameters" not in sc, f"{sub_key}.rope_parameters should have been flattened"+ finally:+ shutil.rmtree(save_dir, ignore_errors=True)++ @requires_qwen3_vl_9b_local+ @requires_mlx_vlm_runtime+ def test_qwen3_vl_9b_mlx_inference_with_mlx_vlm(self, tmp_path):+ """End-to-end: quantize Qwen3-VL-9B → mlx → load & generate via mlx_vlm."""+ from mlx_vlm import generate, load+ from mlx_vlm.prompt_utils import apply_chat_template+ from mlx_vlm.utils import load_config++ from auto_round import AutoRoundMLLM++ save_dir = str(tmp_path / "qwen3_vl_9b_mlx_run")+ try:+ ar = AutoRoundMLLM(+ qwen_3_vl_9b_name_or_path,+ scheme="W4A16",+ bits=4,+ group_size=128,+ sym=True,+ iters=0,+ nsamples=1,+ seqlen=32,+ disable_opt_rtn=True,+ )+ ar.quantize_and_save(output_dir=save_dir, format="mlx")++ model, processor = load(save_dir)+ mlx_cfg = load_config(save_dir)+ prompt_text = "Describe this image in one sentence."+ formatted = apply_chat_template(processor, mlx_cfg, prompt_text, num_images=1)+ # Use a public example image so the test does not need local assets.+ image_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"+ output = generate(model, processor, formatted, image=[image_url], max_tokens=32).text+ assert isinstance(output, str) and len(output) > 0+ finally:+ shutil.rmtree(save_dir, ignore_errors=True)diff --git a/test_mlx_export.py b/test_mlx_export.pynew file mode 100644index 000000000..c7920a80b--- /dev/null+++ b/test_mlx_export.py@@ -0,0 +1,192 @@+#!/usr/bin/env python3+# -*- coding: utf-8 -*-+"""+Test script for MLX format export with AutoRound.+Tests quantization with W4A16 scheme on Qwen3-0.6B model.+"""++import os+import sys+from pathlib import Path++import torch++# Add project root to path+project_root = Path(__file__).parent.parent+sys.path.insert(0, str(project_root))++from auto_round import AutoRound+++def test_mlx_export():+ """Test MLX format export with W4A16 quantization."""+ print("=" * 80)+ print("AutoRound MLX Format Export Test")+ print("=" * 80)++ # Model configuration+ model_name = "Qwen/Qwen3-0.6B"+ output_dir = "./mlx_model_w4a16"++ print(f"\n[1/4] Loading model: {model_name}")+ try:+ ar = AutoRound(+ model_name,+ scheme="W4A16",+ bits=4,+ group_size=128,+ sym=True,+ iters=0, # Fast RTN mode+ disable_opt_rtn=True, # Disable optimization for faster quantization+ nsamples=32, # Use fewer samples for testing+ )+ print("✓ Model loaded successfully")+ except Exception as e:+ print(f"✗ Failed to load model: {e}")+ return False++ print("\n[2/4] Quantizing model to MLX format...")+ try:+ ar.quantize_and_save(output_dir=output_dir, format="mlx")+ print(f"✓ Model quantized and saved to {output_dir}")+ except Exception as e:+ print(f"✗ Failed to quantize and save model: {e}")+ import traceback++ traceback.print_exc()+ return False++ print("\n[3/4] Verifying output files...")+ try:+ # Check if required files exist+ required_files = [+ "config.json",+ "quantization_config.json",+ "mlx_metadata.json",+ ]++ output_path = Path(output_dir)+ for file_name in required_files:+ file_path = output_path / file_name+ if file_path.exists():+ print(f" ✓ {file_name} exists")+ else:+ print(f" ✗ {file_name} NOT found")+ return False++ # Print quantization config+ import json++ quantization_config_path = output_path / "quantization_config.json"+ with open(quantization_config_path, "r") as f:+ config = json.load(f)+ print("\n Quantization Config:")+ print(f" - Format: {config.get('format')}")+ print(f" - Quant Method: {config.get('quant_method')}")+ print(f" - Bits: {config.get('bits')}")+ print(f" - Group Size: {config.get('group_size')}")+ print(f" - Symmetric: {config.get('sym')}")++ except Exception as e:+ print(f"✗ Failed to verify output files: {e}")+ return False++ print("\n[4/4] Testing model loading from MLX format...")+ try:+ from transformers import AutoModelForCausalLM, AutoTokenizer++ # Load the quantized model+ model = AutoModelForCausalLM.from_pretrained(output_dir, device_map="auto", torch_dtype="auto")+ tokenizer = AutoTokenizer.from_pretrained(output_dir)+ print("✓ Model loaded successfully from MLX format")++ # Simple inference test+ prompt = "Hello, my name is"+ inputs = tokenizer(prompt, return_tensors="pt").to(model.device)+ with torch.no_grad():+ outputs = model.generate(**inputs, max_new_tokens=20)+ result = tokenizer.decode(outputs[0], skip_special_tokens=True)+ print(f" Generated text: {result}")+ print("✓ Inference test passed")++ except Exception as e:+ print(f"⚠ Warning: Model loading or inference test failed (this is expected if MLX not installed): {e}")++ print("\n" + "=" * 80)+ print("✓ MLX format export test completed successfully!")+ print("=" * 80)+ return True+++def test_mlx_export_w3a16():+ """Test MLX format export with W3A16 quantization."""+ print("\n" + "=" * 80)+ print("AutoRound MLX Format Export Test (W3A16)")+ print("=" * 80)++ # Model configuration+ model_name = "Qwen/Qwen3-0.6B"+ output_dir = "./mlx_model_w3a16"++ print("\n[1/3] Loading and quantizing model with W3A16...")+ try:+ ar = AutoRound(+ model_name,+ scheme="W3A16",+ bits=3,+ group_size=128,+ sym=True,+ iters=0,+ disable_opt_rtn=True,+ nsamples=32,+ )+ print("✓ Model loaded successfully")+ except Exception as e:+ print(f"✗ Failed to load model: {e}")+ return False++ print("\n[2/3] Saving model in MLX format...")+ try:+ ar.quantize_and_save(output_dir=output_dir, format="mlx")+ print(f"✓ Model saved to {output_dir}")+ except Exception as e:+ print(f"✗ Failed to save model: {e}")+ return False++ print("\n[3/3] Verifying MLX format files...")+ try:+ output_path = Path(output_dir)+ quantization_config_path = output_path / "quantization_config.json"++ import json++ with open(quantization_config_path, "r") as f:+ config = json.load(f)++ assert config.get("bits") == 3, f"Expected bits=3, got {config.get('bits')}"+ print(" ✓ Bits set correctly to 3")+ print(" ✓ All verifications passed")++ except Exception as e:+ print(f"✗ Verification failed: {e}")+ return False++ print("\n✓ W3A16 export test completed successfully!")+ return True+++if __name__ == "__main__":+ print("\n🚀 Starting AutoRound MLX Format Tests\n")++ # Run W4A16 test+ success1 = test_mlx_export()++ # Run W3A16 test+ success2 = test_mlx_export_w3a16()++ if success1 and success2:+ print("\n✅ All tests passed!")+ sys.exit(0)+ else:+ print("\n❌ Some tests failed!")+ sys.exit(1)