Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

disable torch compile in rtn/opt-rtn/iters<10 and fix glm-5.3 tuning exception (#2246)

Wenhua Cheng4dbbd15d6271
diff --git a/README.md b/README.md
index 1e3f4e1a7..4be947889 100644
--- a/README.md
+++ b/README.md
@@ -40,7 +40,7 @@ See our papers [SignRoundV1](https://arxiv.org/pdf/2309.05516) and [SignRoundV2]
* [2026/08] AutoScheme WOQ deployment on vLLM is experimentally restored, thanks to Humming Kernel: [*vLLM PR*](https://github.com/vllm-project/vllm/pull/52890), [*Example Model*](https://huggingface.co/Intel/Qwen3.8-27B-bpw2.8-AutoRound). Note: shared layers must be configured per vLLM's fusion patterns.
-* [2026/07] `torch.compile` is enabled by default except on Windows to accelerate quantization. Minor numerical differences compared with the non-compiled path are expected due to compiler optimizations. To disable it, pass `enable_torch_compile=False` to the Python API or use `--disable_torch_compile` on the CLI.
+* [2026/07] `torch.compile` is enabled in most scenarios to accelerate quantization, at the cost of ~10G extra RAM usage. Minor numerical differences compared with the non-compiled path are expected due to compiler optimizations. To disable it, pass `enable_torch_compile=False` to the Python API or use `--disable_torch_compile` on the CLI.
* [2026/06] AutoScheme has been refined to improve accuracy for gguf format. See [AutoScheme Accuracy](./docs/auto_scheme_acc.md) for details. This enhancement incurs additional tuning cost.
diff --git a/README_CN.md b/README_CN.md
index ba364abea..563644a56 100644
--- a/README_CN.md
+++ b/README_CN.md
@@ -39,7 +39,7 @@ AutoRound 是专为大语言模型(LLMs)和视觉-语言模型(VLMs)设
* [2026/08] 感谢 Humming Kernel,AutoScheme WOQ 在 vLLM 上的部署已实验性恢复:[*vLLM PR*](https://github.com/vllm-project/vllm/pull/52890),[*示例模型*](https://huggingface.co/Intel/Qwen3.8-27B-bpw2.8-AutoRound)。注意:共享层需按 vLLM 的融合模式进行配置。
-* [2026/07] 除 Windows 外默认启用 `torch.compile`,以加速量化。由于编译器优化,与未编译路径相比可能会出现细微的数值差异,这是符合预期的。如需关闭,可在 Python API 中传入 `enable_torch_compile=False`,或在命令行中使用 `--disable_torch_compile`。
+* [2026/07] 在大多数场景默认启用 `torch.compile`,以加速量化,代价是~10G的额外内存使用。由于编译器优化,与未编译路径相比可能会出现细微的数值差异,这是符合预期的。如需关闭,可在 Python API 中传入 `enable_torch_compile=False`,或在命令行中使用 `--disable_torch_compile`。
* [2026/06] AutoScheme对gguf格式优化,精度有所提升。具体结果请参考[文档](docs/auto_scheme_acc.md)。不过这些精度提升是以更高的调优时间和计算开销为代价的。
diff --git a/auto_round/calibration/inputs.py b/auto_round/calibration/inputs.py
index 25306213a..ee7ede386 100644
--- a/auto_round/calibration/inputs.py
+++ b/auto_round/calibration/inputs.py
@@ -102,7 +102,10 @@ def preprocess_block_inputs(
elif isinstance(input_others[key], list):
for i in range(len(input_others[key])):
v = input_others[key][i]
- if isinstance(v, torch.Tensor) and v.dtype in (torch.int32, torch.int64):
+ # Only floating point tensors may be recast. Boolean padding masks and
+ # integer ids must keep their dtype: e.g. GLM-5.3-Flash's DSA indexer
+ # does ``~attention_mask``, which fails on a float tensor.
+ if isinstance(v, torch.Tensor) and not v.is_floating_point():
continue
input_others[key][i] = to_dtype(v, tmp_dtype)
return input_ids, input_others
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 3b005d969..d935bac2c 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -33,7 +33,7 @@ from auto_round.compressors.config_resolution import (
resolve_quantization_config,
thaw_mapping,
)
-from auto_round.compressors.layer_config import (
+from auto_round.compressors.layer_config_resolver import (
apply_plan_to_model,
extract_regex_config,
has_quantized_layer_outside_blocks,
@@ -81,6 +81,11 @@ from auto_round.utils.device import (
from auto_round.utils.device_manager import default_enable_torch_compile, device_manager
from auto_round.utils.offload import OffloadManager
+# ``torch.compile`` only pays for itself when the compiled quant function is
+# replayed many times. Below this many SignRound iterations the one-off
+# compilation cost dominates, so compiling is pure overhead.
+MIN_ITERS_FOR_TORCH_COMPILE = 10
+
@dataclass
class SerializedCompressorConfig:
@@ -362,24 +367,20 @@ class BaseOrchestrator(object):
self.nblocks = nblocks
+ # ``None`` means "not set by the user", which is the only case where the
+ # algorithm-driven auto-disabling below may override the value.
+ self._torch_compile_user_specified = enable_torch_compile is not None
+ # Fallback explanation used by ``_log_torch_compile_state``.
+ self._torch_compile_default_off_reason = None if enable_torch_compile is None else "the user disabled it"
if enable_torch_compile is None:
enable_torch_compile = default_enable_torch_compile(self.device, platform_name=sys.platform)
if not enable_torch_compile:
- # Device values may include an ordinal (for example,
- # ``xpu:0``), so normalize before selecting the backend
- # specific warning message.
- device_type = str(self.device).split(":", 1)[0]
- if device_type == "xpu":
- logger.warning_once(
- "`torch.compile` is disabled by default on XPU for compatibility. "
- "Pass `enable_torch_compile=True` or use `--enable_torch_compile` to force enable it."
- )
- else:
- logger.warning_once(
- "`torch.compile` is disabled by default on Windows because TorchInductor requires the MSVC "
- "`cl.exe` compiler, which may not be available. Pass `enable_torch_compile=True` or use "
- "`--enable_torch_compile` to force enable it."
- )
+ self._torch_compile_default_off_reason = "it is off by default on Windows"
+ logger.warning_once(
+ "`torch.compile` is disabled by default on Windows because TorchInductor requires the MSVC "
+ "`cl.exe` compiler, which may not be available. Pass `enable_torch_compile=True` or use "
+ "`--enable_torch_compile` to force enable it."
+ )
elif enable_torch_compile and sys.platform == "win32":
logger.warning_once(
"Forcing `torch.compile` on Windows. TorchInductor may fail if the MSVC `cl.exe` compiler "
@@ -1177,6 +1178,8 @@ class BaseOrchestrator(object):
and not is_debug_mode()
and not is_raw_nv_fp
and not is_valid_act_static
+ and self._torch_compile_disabled_reason(ignore_user_override=True) is None
+ and self._torch_compile_unsupported_arch_reason() is None
and self.need_calib
):
logger.info(
@@ -1184,16 +1187,99 @@ class BaseOrchestrator(object):
"'enable_torch_compile' is disabled. Enabling it can reduce tuning cost by about 20%.",
)
+ def _torch_compile_disabled_reason(self, ignore_user_override: bool = False) -> Optional[str]:
+ """Return why torch.compile must stay off for the current algorithm, else None.
+
+ RTN and optimized RTN quantize each layer in a single pass, and very short
+ SignRound runs (``iters < MIN_ITERS_FOR_TORCH_COMPILE``) finish before the
+ compilation cost is amortized, so ``torch.compile`` only adds overhead there.
+
+ This only adjusts the *default*: when the user explicitly passed
+ ``enable_torch_compile``, their choice is always honored. Pass
+ ``ignore_user_override=True`` to query the algorithm rules alone (used to
+ suppress the "you should enable torch.compile" hint).
+ """
+ if not ignore_user_override and getattr(self, "_torch_compile_user_specified", False):
+ return None
+
+ quantize_config = getattr(self, "quantize_config", None)
+ if quantize_config is None:
+ return None
+
+ # AutoScheme runs its own delta-loss pass on top of the block quantizer and
+ # relies on torch.compile to keep VRAM down, so the rules below don't apply.
+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
+
+ if getattr(self, "is_auto_scheme", False) or isinstance(getattr(self, "scheme", None), AutoScheme):
+ return None
+
+ # OptimizedRTNConfig subclasses RTNConfig, so this covers rtn and opt-rtn.
+ from auto_round.algorithms.quantization.rtn.config import RTNConfig
+
+ if isinstance(quantize_config, RTNConfig):
+ return "RTN/OPT-RTN quantizes each layer in a single pass"
+
+ iters = getattr(quantize_config, "iters", None)
+ if iters is not None and iters < MIN_ITERS_FOR_TORCH_COMPILE:
+ return f"`iters`={iters} is below {MIN_ITERS_FOR_TORCH_COMPILE}"
+
+ return None
+
+ def _torch_compile_unsupported_arch_reason(self) -> Optional[str]:
+ """Return why the model *architecture* forbids ``torch.compile``, else ``None``.
+
+ Rules live in :mod:`auto_round.special_model_handler` so a new architecture can
+ be registered in one place.
+ """
+ from auto_round.special_model_handler import get_torch_compile_off_reason
+
+ model = getattr(getattr(self, "model_context", None), "model", None)
+ if model is None:
+ model = getattr(self, "model", None)
+ return get_torch_compile_off_reason(model)
+
def _apply_torch_compile_constraints(self, enable_torch_compile: bool) -> None:
"""Apply torch.compile disabling rules for the current compressor state."""
self.enable_torch_compile = enable_torch_compile
+ # Why compilation ended up off, used by ``_log_torch_compile_state``. When the
+ # incoming value is already False, keep the reason recorded by the earlier
+ # precheck pass instead of dropping it.
+ self._torch_compile_off_reason = (
+ None
+ if enable_torch_compile
+ else (
+ getattr(self, "_torch_compile_off_reason", None)
+ or getattr(self, "_torch_compile_default_off_reason", None)
+ )
+ )
_, is_valid_act_static = self._get_torch_compile_guard_state()
# On HPU, we rely on torch.compile to speed up the model execution.
if self.enable_torch_compile and is_valid_act_static:
self.enable_torch_compile = False
+ self._torch_compile_off_reason = "activation is static"
logger.warning_once("reset enable_torch_compile to `False` as activation is static")
+ # Architecture-level hard block (DeepSeek / GLM-5.3-Flash DSA families). These
+ # hit dynamo's recompile_limit and cannot be overridden by an explicit
+ # ``enable_torch_compile=True``.
+ if self.enable_torch_compile:
+ arch_reason = self._torch_compile_unsupported_arch_reason()
+ if arch_reason is not None:
+ self.enable_torch_compile = False
+ self._torch_compile_off_reason = arch_reason
+ logger.warning_once("reset enable_torch_compile to `False` as %s", arch_reason)
+
+ if self.enable_torch_compile:
+ disabled_reason = self._torch_compile_disabled_reason()
+ if disabled_reason is not None:
+ self.enable_torch_compile = False
+ self._torch_compile_off_reason = disabled_reason
+ logger.warning_once(
+ "reset enable_torch_compile to `False` as %s, " "so compilation cost would outweigh its benefit",
+ disabled_reason,
+ )
+
def _precheck_torch_compile(self, enable_torch_compile: bool) -> None:
"""Apply early torch.compile adjustments before scheme resolution.
@@ -1209,6 +1295,19 @@ class BaseOrchestrator(object):
self._apply_torch_compile_constraints(requested_enable_torch_compile)
if not requested_enable_torch_compile:
self._maybe_log_torch_compile_default_hint()
+ self._log_torch_compile_state()
+
+ def _log_torch_compile_state(self) -> None:
+ """Always report the final torch.compile decision so a run is self-documenting."""
+ if self.enable_torch_compile:
+ logger.info("`torch.compile` is enabled")
+ return
+
+ reason = getattr(self, "_torch_compile_off_reason", None)
+ if reason is None:
+ logger.info("`torch.compile` is disabled")
+ else:
+ logger.info("`torch.compile` is disabled, as %s", reason)
def _get_calibration_dataset(self) -> str:
"""Resolve calibration dataset: self.dataset > AutoScheme.dataset > default."""
diff --git a/auto_round/compressors/layer_config/__init__.py b/auto_round/compressors/layer_config/__init__.py
deleted file mode 100644
index 5b0adaf4f..000000000
--- a/auto_round/compressors/layer_config/__init__.py
+++ /dev/null
@@ -1,27 +0,0 @@
-# Copyright (c) 2026 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from auto_round.compressors.layer_config.resolver import (
- apply_plan_to_model,
- extract_regex_config,
- has_quantized_layer_outside_blocks,
- resolve_layer_config,
-)
-
-__all__ = [
- "apply_plan_to_model",
- "extract_regex_config",
- "has_quantized_layer_outside_blocks",
- "resolve_layer_config",
-]
diff --git a/auto_round/compressors/layer_config/resolver.py b/auto_round/compressors/layer_config_resolver.py
similarity index 100%
rename from auto_round/compressors/layer_config/resolver.py
rename to auto_round/compressors/layer_config_resolver.py
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.py
index 544f3fc19..f024b79dd 100644
--- a/auto_round/compressors/model_free.py
+++ b/auto_round/compressors/model_free.py
@@ -103,7 +103,6 @@ import multiprocessing as mp
import os
import re
import shutil
-import sys
import time
from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, ThreadPoolExecutor, as_completed, wait
from dataclasses import asdict, fields
@@ -118,7 +117,6 @@ from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme, preset_name_to_scheme
from auto_round.utils.common import AUDIO_MM_KEYS, VISION_MM_KEYS, compress_layer_names
from auto_round.utils.device import clear_memory, memory_monitor
-from auto_round.utils.device_manager import default_enable_torch_compile
from auto_round.utils.model_free_utils import (
_LM_HEAD_PATTERNS,
_apply_scheme_overrides,
@@ -495,11 +493,15 @@ class _ModelFreeCompressorCore:
self.device = device
self.quant_lm_head = quant_lm_head
self.quant_nontext_module = quant_nontext_module
- self.enable_torch_compile = (
- default_enable_torch_compile(device, platform_name=sys.platform)
- if enable_torch_compile is None
- else enable_torch_compile
- )
+ # Model-free is always zero-shot RTN/opt-RTN (each layer is quantized in a
+ # single pass), so torch.compile only adds one-off compilation overhead.
+ # Keep it off by default, but honor an explicit user request.
+ if enable_torch_compile is None:
+ self.enable_torch_compile = False
+ logger.info("`torch.compile` is disabled, as RTN/OPT-RTN quantizes each layer in a single pass")
+ else:
+ self.enable_torch_compile = enable_torch_compile
+ logger.info("`torch.compile` is %s", "enabled" if enable_torch_compile else "disabled")
self.disable_opt_rtn = disable_opt_rtn
# --- derived state populated during run() ---
diff --git a/auto_round/compressors/orchestrator.py b/auto_round/compressors/orchestrator.py
index eb6ff5ec5..b918f8e4d 100644
--- a/auto_round/compressors/orchestrator.py
+++ b/auto_round/compressors/orchestrator.py
@@ -678,85 +678,82 @@ class CompressionOrchestrator(BaseOrchestrator):
ResumeState(os.path.join(envs.AR_RESUME_DIR, f"group_{group_idx}"), sig, block_names)
)
- try:
- for group_idx, block_names in enumerate(all_blocks):
- inputs = all_inputs[block_names[0]]
- all_inputs.pop(block_names[0])
- q_inputs = None
- if all_q_inputs is not None:
- q_inputs = all_q_inputs[block_names[0]]
- all_q_inputs.pop(block_names[0])
-
- inputs, q_inputs = _update_inputs(inputs, q_inputs)
-
- clear_memory(self.inputs)
-
- resume_state = resume_states[group_idx] if resume_states is not None else None
- resume_input_ids = None
- if resume_state is not None and resume_state.resume_index > 0:
- if self.nblocks != 1:
+ for group_idx, block_names in enumerate(all_blocks):
+ inputs = all_inputs[block_names[0]]
+ all_inputs.pop(block_names[0])
+ q_inputs = None
+ if all_q_inputs is not None:
+ q_inputs = all_q_inputs[block_names[0]]
+ all_q_inputs.pop(block_names[0])
+
+ inputs, q_inputs = _update_inputs(inputs, q_inputs)
+
+ clear_memory(self.inputs)
+
+ resume_state = resume_states[group_idx] if resume_states is not None else None
+ resume_input_ids = None
+ if resume_state is not None and resume_state.resume_index > 0:
+ if self.nblocks != 1:
+ logger.warning(
+ "AR_RESUME_DIR is set but nblocks != 1; resuming mid-group is only "
+ "supported for nblocks=1 -- restarting this group from block 0."
+ )
+ resume_state = None
+ else:
+ resume_name = block_names[resume_state.resume_index]
+ # Only used here for `input_others` (position/mask info,
+ # which is legitimately re-sourced from this same cache
+ # every iteration regardless of resuming); the actual
+ # chained `input_ids` comes from `resume_input_ids`
+ # below, not this cache -- see
+ # auto_round/utils/resume.py's module docstring for why
+ # the two aren't interchangeable.
+ if resume_name in all_inputs:
+ inputs = all_inputs.pop(resume_name)
+ q_inputs = resume_state.load_q_input()
+ resume_input_ids = resume_state.load_input_ids()
+ if resume_input_ids is None:
logger.warning(
- "AR_RESUME_DIR is set but nblocks != 1; resuming mid-group is only "
- "supported for nblocks=1 -- restarting this group from block 0."
+ "AR_RESUME_DIR manifest is missing its cached input_ids tensor; "
+ "restarting this group from block 0 instead of resuming with a "
+ "possibly-inconsistent chain value."
)
resume_state = None
else:
- resume_name = block_names[resume_state.resume_index]
- # Only used here for `input_others` (position/mask info,
- # which is legitimately re-sourced from this same cache
- # every iteration regardless of resuming); the actual
- # chained `input_ids` comes from `resume_input_ids`
- # below, not this cache -- see
- # auto_round/utils/resume.py's module docstring for why
- # the two aren't interchangeable.
- if resume_name in all_inputs:
- inputs = all_inputs.pop(resume_name)
- q_inputs = resume_state.load_q_input()
- resume_input_ids = resume_state.load_input_ids()
- if resume_input_ids is None:
- logger.warning(
- "AR_RESUME_DIR manifest is missing its cached input_ids tensor; "
- "restarting this group from block 0 instead of resuming with a "
- "possibly-inconsistent chain value."
- )
- resume_state = None
- else:
- pbar.update(resume_state.resume_index)
-
- self._quantize_blocks(
- self.model_context.model,
- inputs,
- block_names,
- q_input=q_inputs if q_inputs is not None else None,
- nblocks=self.nblocks,
- pbar=pbar,
- input_others_extra_blocks=all_inputs,
- token_ids=input_ids_cache,
- resume_state=resume_state,
- resume_input_ids=resume_input_ids,
+ pbar.update(resume_state.resume_index)
+
+ self._quantize_blocks(
+ self.model_context.model,
+ inputs,
+ block_names,
+ q_input=q_inputs if q_inputs is not None else None,
+ nblocks=self.nblocks,
+ pbar=pbar,
+ input_others_extra_blocks=all_inputs,
+ token_ids=input_ids_cache,
+ resume_state=resume_state,
+ resume_input_ids=resume_input_ids,
+ )
+ if self.compress_context.is_immediate_packing and len(self.formats) != 1:
+ raise ValueError(
+ f"Expected exactly one packing format when 'immediate_packing' is True, "
+ f"but got {len(self.formats)} formats."
)
- if self.compress_context.is_immediate_packing and len(self.formats) != 1:
- raise ValueError(
- f"Expected exactly one packing format when 'immediate_packing' is True, "
- f"but got {len(self.formats)} formats."
- )
- if resume_states is not None:
- if self.compress_context.is_immediate_saving:
- # Don't clear resume state yet when exporting to shards --
- # a crash in the save/export step that follows this method
- # returning (config writing, tokenizer copy, format-specific
- # global packing pass) would otherwise force a full
- # re-tune from block 0 on the next attempt, even though
- # every block's weights are already correctly flushed to
- # disk. quantize_and_save() clears these once
- # save_quantized() actually succeeds.
- self._resume_states = resume_states
- else:
- for rs in resume_states:
- rs.clear()
- finally:
- # ── Pipeline lifecycle: finalize_quantization (model-level teardown)
- self.alg_composer.finalize_run()
+ if resume_states is not None:
+ if self.compress_context.is_immediate_saving:
+ # Don't clear resume state yet when exporting to shards --
+ # a crash in the save/export step that follows this method
+ # returning (config writing, tokenizer copy, format-specific
+ # global packing pass) would otherwise force a full
+ # re-tune from block 0 on the next attempt, even though
+ # every block's weights are already correctly flushed to
+ # disk. quantize_and_save() clears these once
+ # save_quantized() actually succeeds.
+ self._resume_states = resume_states
+ else:
+ for rs in resume_states:
+ rs.clear()
+
pbar.set_description("Quantizing done")
pbar.close()
if self.compress_context.low_cpu_mem_usage:
@@ -815,28 +812,6 @@ class CompressionOrchestrator(BaseOrchestrator):
self.model_context.quantized = True
return self.model_context.model, self.layer_config
- # def _immediate_pack_and_save_module(self, module_name):
- # from auto_round.compressors.shard_writer import ShardWriter
- #
- # shard_writer = ShardWriter.get_shard_writer()
- # to_cpu = self.compress_context.low_gpu_mem_usage
- # module = get_module(self.model, module_name)
- # if self.compress_context.is_immediate_packing:
- # immediate_pack(module_name, self.layer_config)
- # if to_cpu:
- # module = module.to("cpu")
- # packed_module = get_module(self.model, module_name)
- # set_module(self.model, module_name, packed_module.to("cpu"))
- # else:
- # if to_cpu:
- # module = module.to("cpu")
- # set_module(self.model, module_name, module)
- # if self.compress_context.is_immediate_saving:
- # module = get_module(self.model, module_name)
- # module.to("cpu")
- # shard_writer.write(module, module_name, False)
- # module.to("meta")
-
def _quantize_layers_outside_blocks(
self,
layer_names: list,
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index d247e8959..c063597a8 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -13,14 +13,23 @@
# limitations under the License.
import os
import random
-import re
from typing import Union
import torch
from torch.amp import autocast
from auto_round.compressors.config_resolution import LayerConfigResolutionError
-from auto_round.logger import logger
+
+# Explicit compatibility exports for callers that historically imported GGUF and
+# ignore-layer helpers from compressors.utils.
+from auto_round.compressors.layer_config_resolver import get_fp_layer_names
+from auto_round.export.formats.backends.gguf import (
+ _apply_gguf_shape_fallback,
+ _infer_gguf_n_layers_from_model,
+ _resolve_gguf_n_layers,
+ get_layer_config_by_gguf_format,
+ gguf_type_fallback,
+)
from auto_round.schemes import BackendDataType # re-exported: qlinear_fp/qlinear_int import it from here
from auto_round.schemes import (
QuantizationScheme,
@@ -261,7 +270,7 @@ def set_layer_config(
ResolvedScheme,
resolve_scheme_value,
)
- from auto_round.compressors.layer_config import (
+ from auto_round.compressors.layer_config_resolver import (
apply_plan_to_model,
extract_regex_config,
has_quantized_layer_outside_blocks,
@@ -321,19 +330,6 @@ def set_layer_config(
)
-from auto_round.compressors.layer_config.resolver import get_fp_layer_names
-
-# Explicit compatibility exports for callers that historically imported GGUF and
-# ignore-layer helpers from compressors.utils.
-from auto_round.export.formats.backends.gguf import (
- _apply_gguf_shape_fallback,
- _infer_gguf_n_layers_from_model,
- _resolve_gguf_n_layers,
- get_layer_config_by_gguf_format,
- gguf_type_fallback,
-)
-
-
def get_shared_keys(model):
"""
Retrieves shared keys from the model's state dictionary.
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 6bfa56d5f..4e1212640 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -15,6 +15,7 @@ import importlib
import re
import sys
from dataclasses import dataclass, field
+from types import SimpleNamespace
from typing import Any, Callable
import torch
@@ -1061,12 +1062,32 @@ register_ignore_layers(
],
)
-# glm5
+# GLM-5.3-Flash family: ``glm_moe_dsa`` (text) and ``glm5_next`` (multimodal).
+#
+# Both keep their whole DSA indexer out of quantization:
+# * ``{GlmMoeDsa,Glm5NextText}Indexer.forward`` is decorated with ``@torch.no_grad()``
+# while owning three quantizable linears (``wq_b``, ``wk``, ``weights_proj``).
+# Wrapping them makes the same compiled quant function run under both grad-disabled
+# (indexer) and grad-enabled (rest of the block) modes. dynamo keys its cache by
+# *code object* -- shared by every WrapperLinear -- so the AOTAutograd backward then
+# fails with "cannot enter context: <Context ...> is already entered".
+# * transformers marks ``indexer.weights_proj`` in ``_keep_in_fp32_modules``, i.e. it
+# is not even supposed to leave FP32.
+# The indexer is tiny anyway (``weights_proj`` outputs ``index_n_heads``), so there is
+# no compression to gain here. ``get_glm_flash_ignore_layers`` additionally keeps the
+# leading dense MLPs in full precision to work around a vLLM loading issue.
register_ignore_layers(
matchers=[
ModelTypeMatcher(r"glm_moe_dsa", mode="full"),
],
- ignore_layers=[get_glm_flash_ignore_layers, "weights_proj"], # vllm issue
+ ignore_layers=[get_glm_flash_ignore_layers, "weights_proj", "indexer"], # get_glm_flash_ignore_layers: vllm issue
+)
+
+register_ignore_layers(
+ matchers=[
+ ModelTypeMatcher(r"glm5_next", mode="full"),
+ ],
+ ignore_layers=[get_glm_flash_ignore_layers, "weights_proj", "indexer"], # get_glm_flash_ignore_layers: vllm issue
)
# step3p5
@@ -1157,6 +1178,65 @@ def get_predefined_ignore_layers(model: torch.nn.Module) -> list[str]:
return list(dict.fromkeys(layers))
+# ---------------------------------------------------------------------------
+# Architectures for which ``torch.compile`` is force-disabled during tuning.
+# ---------------------------------------------------------------------------
+@dataclass
+class PreDefinedTorchCompileOff:
+ """A rule that force-disables ``torch.compile`` for a family of models."""
+
+ matchers: list[Callable[[Any], bool]]
+ reason: str
+
+
+_PRE_DEFINED_TORCH_COMPILE_OFF: list[PreDefinedTorchCompileOff] = []
+
+
+def register_torch_compile_off(matchers: list[Callable[[Any], bool]], reason: str) -> None:
+ """Register an architecture family for which ``torch.compile`` must stay off.
+
+ Any matcher hitting is enough (``any``, not ``all``) so one rule can cover several
+ equivalent identifiers (model_type and architecture name).
+ """
+ _PRE_DEFINED_TORCH_COMPILE_OFF.append(PreDefinedTorchCompileOff(matchers, reason))
+
+
+def get_torch_compile_off_reason(model) -> str | None:
+ """Return why ``torch.compile`` must stay off for this model, else ``None``.
+
+ Accepts a model or anything exposing ``.config``.
+ """
+ if model is None:
+ return None
+ proxy = model if hasattr(model, "config") else SimpleNamespace(config=model)
+ for rule in _PRE_DEFINED_TORCH_COMPILE_OFF:
+ if any(matcher(proxy) for matcher in rule.matchers):
+ return rule.reason
+ return None
+
+
+# DeepSeek V2/V3/V3.2/V4 and the GLM-5.3-Flash family (``glm_moe_dsa`` text /
+# ``glm5_next`` multimodal) share a DSA indexer + fused MoE with shape-dependent
+# control flow. Compiling the tuning graph there hits dynamo's recompile_limit and
+# provides little benefit, so keep those runs in eager mode.
+register_torch_compile_off(
+ matchers=[
+ ModelTypeMatcher("deepseek", mode="in"),
+ ArchitectureMatcher("Deepseek", mode="in"),
+ ArchitectureMatcher("DeepSeek", mode="in"),
+ ],
+ reason="the DeepSeek architecture is incompatible with torch.compile during tuning",
+)
+register_torch_compile_off(
+ matchers=[
+ ModelTypeMatcher(r"glm_moe_dsa", mode="full"),
+ ModelTypeMatcher(r"glm5_next", mode="full"),
+ ArchitectureMatcher(r"Glm5Next", mode="in"),
+ ],
+ reason="the GLM-5 architecture is incompatible with torch.compile during tuning",
+)
+
+
def _attach_gemma4_rotary_emb(model):
"""Attach ``_rotary_emb`` to each Gemma4 decoder layer.
diff --git a/auto_round/utils/device_manager.py b/auto_round/utils/device_manager.py
index 2c8f1dddd..45b1e4dd7 100644
--- a/auto_round/utils/device_manager.py
+++ b/auto_round/utils/device_manager.py
@@ -898,8 +898,7 @@ def default_enable_torch_compile(
device: Union[None, str, int, torch.device] = None, platform_name: str | None = None
) -> bool:
"""Return the safe torch.compile default for a backend."""
- device_type = _normalize_device_type(device) or get_current_device_type()
- return device_type != "xpu" and (platform_name or sys.platform) != "win32"
+ return (platform_name or sys.platform) != "win32"
def get_current_device_manager() -> ARDevice:
diff --git a/test/unit/common/core/test_init.py b/test/unit/common/core/test_init.py
index 7c9ff5984..5c8598a6f 100644
--- a/test/unit/common/core/test_init.py
+++ b/test/unit/common/core/test_init.py
@@ -4,7 +4,7 @@ import logging
from auto_round import AutoRound
from auto_round.auto_scheme import AutoScheme
from auto_round.cli.parser import build_quantize_parser
-from auto_round.compressors.base import BaseOrchestrator
+from auto_round.compressors.base import MIN_ITERS_FOR_TORCH_COMPILE, BaseOrchestrator
from auto_round.logger import logger
from auto_round.utils.device_manager import default_enable_torch_compile
@@ -39,28 +39,118 @@ def test_auto_scheme_inherits_torch_compile_setting():
assert auto_scheme.enable_torch_compile is None
-def test_xpu_torch_compile_is_disabled_by_default():
- assert not default_enable_torch_compile("xpu", platform_name="linux")
+def test_xpu_torch_compile_is_enabled_by_default():
+ assert default_enable_torch_compile("xpu", platform_name="linux")
assert default_enable_torch_compile("cuda", platform_name="linux")
def test_torch_compile_runtime_defaults(tiny_opt_model_path):
- ar = AutoRound(model=tiny_opt_model_path, scheme="W4A16", iters=0, nsamples=1)
+ ar = AutoRound(model=tiny_opt_model_path, scheme="W4A16", iters=200, nsamples=1)
assert ar.enable_torch_compile == default_enable_torch_compile(ar.device)
ar = AutoRound(
model=tiny_opt_model_path,
scheme="W4A16",
- iters=0,
+ iters=200,
nsamples=1,
enable_torch_compile=False,
)
assert not ar.enable_torch_compile
- ar = AutoRound(model=tiny_opt_model_path, scheme="NVFP4", iters=0, nsamples=1)
+ ar = AutoRound(model=tiny_opt_model_path, scheme="NVFP4", iters=200, nsamples=1)
assert ar.enable_torch_compile == default_enable_torch_compile(ar.device)
+def _assert_compile(ar, expected: bool):
+ """Assert the effective torch.compile flag, tolerating the model-free path.
+
+ ``ModelFreeCompressor`` has no ``compress_context``; the regular compressor
+ mirrors ``enable_torch_compile`` onto it, so check both when present.
+ """
+ assert ar.enable_torch_compile is expected
+ compress_context = getattr(ar, "compress_context", None)
+ if compress_context is not None:
+ assert compress_context.enable_torch_compile is expected
+
+
+def test_torch_compile_disabled_for_rtn_and_short_signround(tiny_opt_model_path, monkeypatch):
+ """RTN / opt-RTN and `iters` < 10 never amortize the torch.compile cost."""
+ # Pin the platform default to "enabled" so the algorithm heuristic is observable
+ # regardless of the host device / OS.
+ monkeypatch.setattr("auto_round.compressors.base.default_enable_torch_compile", lambda *a, **k: True)
+
+ # Plain RTN (routes to the model-free compressor) and opt-RTN both stay off.
+ for disable_opt_rtn in (True, False):
+ ar = AutoRound(
+ model=tiny_opt_model_path,
+ scheme="W4A16",
+ iters=0,
+ nsamples=1,
+ disable_opt_rtn=disable_opt_rtn,
+ )
+ _assert_compile(ar, False)
+
+ # SignRound with too few iterations.
+ for iters in (1, MIN_ITERS_FOR_TORCH_COMPILE - 1):
+ ar = AutoRound(model=tiny_opt_model_path, scheme="W4A16", iters=iters, nsamples=1)
+ _assert_compile(ar, False)
+
+ # Enough iterations to pay back the compilation cost.
+ ar = AutoRound(model=tiny_opt_model_path, scheme="W4A16", iters=MIN_ITERS_FOR_TORCH_COMPILE, nsamples=1)
+ _assert_compile(ar, True)
+
+
+def test_explicit_torch_compile_overrides_algorithm_heuristic(tiny_opt_model_path, monkeypatch):
+ """An explicit `enable_torch_compile` is always honored, even for RTN / tiny iters."""
+ monkeypatch.setattr("auto_round.compressors.base.default_enable_torch_compile", lambda *a, **k: False)
+
+ for kwargs in ({"iters": 0}, {"iters": 1}, {"iters": 0, "disable_opt_rtn": True}):
+ ar = AutoRound(
+ model=tiny_opt_model_path,
+ scheme="W4A16",
+ nsamples=1,
+ enable_torch_compile=True,
+ **kwargs,
+ )
+ _assert_compile(ar, True)
+
+
+def test_torch_compile_state_is_always_logged(tiny_opt_model_path, monkeypatch, caplog):
+ """`post_init` reports the final torch.compile decision on every run."""
+ monkeypatch.setattr("auto_round.compressors.base.default_enable_torch_compile", lambda *a, **k: True)
+ # AutoRound's logger uses a private handler with propagation disabled.
+ monkeypatch.setattr(logger, "propagate", True)
+
+ # RTN turns compilation off, and says why.
+ with caplog.at_level(logging.INFO):
+ ar = AutoRound(model=tiny_opt_model_path, scheme="W4A16", iters=0, nsamples=1)
+ ar.post_init()
+ assert "`torch.compile` is disabled" in caplog.text
+ assert "single pass" in caplog.text
+
+ # An explicit opt-in is honored and reported as enabled.
+ caplog.clear()
+ with caplog.at_level(logging.INFO):
+ ar = AutoRound(
+ model=tiny_opt_model_path,
+ scheme="W4A16",
+ iters=0,
+ nsamples=1,
+ enable_torch_compile=True,
+ )
+ ar.post_init()
+ assert "`torch.compile` is enabled" in caplog.text
+
+
+def test_torch_compile_kept_for_auto_scheme_with_rtn(tiny_opt_model_path, monkeypatch):
+ """AutoScheme's delta-loss pass still relies on torch.compile to save VRAM."""
+ monkeypatch.setattr("auto_round.compressors.base.default_enable_torch_compile", lambda *a, **k: True)
+
+ auto_scheme = AutoScheme(avg_bits=4.0, options=["W4A16"])
+ ar = AutoRound(model=tiny_opt_model_path, scheme=auto_scheme, iters=0, nsamples=1)
+ assert ar.enable_torch_compile
+
+
def test_torch_compile_windows_defaults(monkeypatch, caplog, tiny_opt_model_path):
monkeypatch.setattr("auto_round.compressors.base.sys.platform", "win32")
# AutoRound's logger normally uses a private handler with propagation
@@ -75,7 +165,7 @@ def test_torch_compile_windows_defaults(monkeypatch, caplog, tiny_opt_model_path
ar = AutoRound(model=tiny_opt_model_path, scheme="W4A16", iters=0, nsamples=1)
assert not ar.enable_torch_compile
if str(ar.device).split(":", 1)[0] == "xpu":
- assert "disabled by default on XPU" in caplog.text
+ assert "disabled by default on XPU" not in caplog.text
else:
assert "disabled by default on Windows" in caplog.text
assert "cl.exe" in caplog.text
diff --git a/test/unit/common/core/test_pipeline_fail_fast.py b/test/unit/common/core/test_pipeline_fast.py
similarity index 100%
rename from test/unit/common/core/test_pipeline_fail_fast.py
rename to test/unit/common/core/test_pipeline_fast.py
diff --git a/test/unit/common/models/test_special_model_handler.py b/test/unit/common/models/test_special_model_handler.py
index fa3ead983..606a365ff 100644
--- a/test/unit/common/models/test_special_model_handler.py
+++ b/test/unit/common/models/test_special_model_handler.py
@@ -912,6 +912,30 @@ class TestGetPredefinedIgnoreLayers:
layers = get_predefined_ignore_layers(mock_model)
assert "layers.0.mlp" in layers
assert "layers.1.mlp" in layers
+ # The whole DSA indexer stays FP: its forward is @torch.no_grad() and
+ # transformers keeps indexer.weights_proj in _keep_in_fp32_modules.
+ assert "weights_proj" in layers
+ assert "indexer" in layers
+
+ def test_glm5_next_matcher(self):
+ """GLM-5.3-Flash keeps its whole DSA indexer in full precision.
+
+ ``Glm5NextTextIndexer.forward`` is ``@torch.no_grad()``, so quantizing its
+ ``wq_b`` / ``wk`` / ``weights_proj`` linears makes the same compiled quant
+ function run under both grad modes and breaks the AOTAutograd backward.
+ The leading dense MLPs are excluded for the same vLLM reason as glm_moe_dsa.
+ """
+ from auto_round.special_model_handler import get_predefined_ignore_layers
+
+ mock_model = MagicMock()
+ mock_model.config.model_type = "glm5_next"
+ mock_model.config.first_k_dense_replace = 3
+ layers = get_predefined_ignore_layers(mock_model)
+ assert "weights_proj" in layers
+ assert "indexer" in layers
+ assert "layers.0.mlp" in layers
+ assert "layers.1.mlp" in layers
+ assert "layers.2.mlp" in layers
def test_step3p5_matcher(self):
from auto_round.special_model_handler import get_predefined_ignore_layers
@@ -958,6 +982,36 @@ class TestGetPredefinedIgnoreLayers:
# Should not add any layers without matching rules
+class TestTorchCompileOff:
+ """DeepSeek / GLM-5.3-Flash DSA families must not run under torch.compile."""
+
+ def _cfg(self, model_type="", architectures=None):
+ from types import SimpleNamespace
+
+ return SimpleNamespace(config=SimpleNamespace(model_type=model_type, architectures=architectures or []))
+
+ def test_deepseek_disabled(self):
+ from auto_round.special_model_handler import get_torch_compile_off_reason
+
+ for mt in ("deepseek_v3", "deepseek_v32"):
+ assert get_torch_compile_off_reason(self._cfg(model_type=mt)) is not None
+
+ def test_glm5_family_disabled(self):
+ from auto_round.special_model_handler import get_torch_compile_off_reason
+
+ assert get_torch_compile_off_reason(self._cfg(model_type="glm5_next")) is not None
+ assert get_torch_compile_off_reason(self._cfg(model_type="glm_moe_dsa")) is not None
+ # Architecture name alone is enough (config before weights are materialized).
+ assert get_torch_compile_off_reason(self._cfg(architectures=["Glm5NextForConditionalGeneration"])) is not None
+
+ def test_other_models_unaffected(self):
+ from auto_round.special_model_handler import get_torch_compile_off_reason
+
+ assert get_torch_compile_off_reason(self._cfg(model_type="qwen3", architectures=["Qwen3ForCausalLM"])) is None
+ assert get_torch_compile_off_reason(self._cfg(model_type="glm4", architectures=["Glm4ForCausalLM"])) is None
+ assert get_torch_compile_off_reason(None) is None
+
+
class TestGetBagelIgnoreLayers:
"""Test get_bagel_ignore_layers function."""
diff --git a/test/unit/common/utils/test_apply.py b/test/unit/common/utils/test_apply.py
index 976cabdbc..6e8c1c9d0 100644
--- a/test/unit/common/utils/test_apply.py
+++ b/test/unit/common/utils/test_apply.py
@@ -15,7 +15,7 @@
import torch.nn as nn
from auto_round.compressors.config_resolution import ResolvedQuantizationConfig, ResolvedScheme
-from auto_round.compressors.layer_config import apply_plan_to_model
+from auto_round.compressors.layer_config_resolver import apply_plan_to_model
from auto_round.schemes import QuantizationScheme
diff --git a/test/unit/common/utils/test_layer_config_resolver.py b/test/unit/common/utils/test_layer_config_resolver.py
index ad369cd48..48c507e7f 100644
--- a/test/unit/common/utils/test_layer_config_resolver.py
+++ b/test/unit/common/utils/test_layer_config_resolver.py
@@ -15,7 +15,7 @@
import torch.nn as nn
from auto_round.compressors.config_resolution import ResolvedScheme
-from auto_round.compressors.layer_config import (
+from auto_round.compressors.layer_config_resolver import (
extract_regex_config,
has_quantized_layer_outside_blocks,
resolve_layer_config,