PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
chore: enable ruff default rules and fix violations (#2434)
Sun, Xuehaoc915fdcffaea
diff --git a/.azure-pipelines/scripts/ai_analysis/ai_analyze.py b/.azure-pipelines/scripts/ai_analysis/ai_analyze.pyindex a261f4a89..693361a1c 100644--- a/.azure-pipelines/scripts/ai_analysis/ai_analyze.py+++ b/.azure-pipelines/scripts/ai_analysis/ai_analyze.py@@ -155,7 +155,7 @@ def _call_copilot_cli(prompt: str, timeout: int, model: str, reasoning_effort: sif ai_token:env.setdefault("GITHUB_TOKEN", ai_token)env.setdefault("GH_TOKEN", ai_token)- started = datetime.datetime.now(datetime.timezone.utc)+ started = datetime.datetime.now(datetime.UTC)returncode = Nonestderr = ""stdout = ""@@ -169,7 +169,7 @@ def _call_copilot_cli(prompt: str, timeout: int, model: str, reasoning_effort: sexcept (OSError, subprocess.TimeoutExpired) as e:print(f"Warning: Copilot CLI call failed: {e}", file=sys.stderr)stderr = str(e)- ended = datetime.datetime.now(datetime.timezone.utc)+ ended = datetime.datetime.now(datetime.UTC)parsed = _parse_copilot_json(stdout)raw = parsed["answer"]_append_trace(diff --git a/.azure-pipelines/scripts/compat_smoke_test.py b/.azure-pipelines/scripts/compat_smoke_test.pyindex 3c48e701a..5419526d4 100644--- a/.azure-pipelines/scripts/compat_smoke_test.py+++ b/.azure-pipelines/scripts/compat_smoke_test.py@@ -21,7 +21,7 @@ print(f"auto_round imported from: {auto_round.__file__}")print(f"auto_round {auto_round.__version__} imported successfully (AutoRound={AutoRound.__name__})")# Verify the console_scripts entry point was installed and is runnable.-result = subprocess.run(["auto-round", "--help"], capture_output=True, text=True)+result = subprocess.run(["auto-round", "--help"], capture_output=True, text=True, check=False)if result.returncode != 0:sys.stderr.write(result.stdout)sys.stderr.write(result.stderr)diff --git a/.azure-pipelines/scripts/performance/check_performance.py b/.azure-pipelines/scripts/performance/check_performance.pyindex bdb74f6e8..ce4940f9d 100644--- a/.azure-pipelines/scripts/performance/check_performance.py+++ b/.azure-pipelines/scripts/performance/check_performance.py@@ -3,9 +3,9 @@ import reimport sysfrom dataclasses import dataclassfrom pathlib import Path-from typing import Dict, Optionallogging.basicConfig(level=logging.INFO, format="%(message)s")+logger = logging.getLogger(__name__)LOG_DIR = Path("/auto-round/log_dir")OUTPUT_BASE_DIR = Path("/auto-round/.azure-pipelines/scripts/performance")@@ -13,10 +13,10 @@ OUTPUT_BASE_DIR = Path("/auto-round/.azure-pipelines/scripts/performance")@dataclassclass QuantMetrics:- tuning_time_s: Optional[float] = None- peak_ram_gb: Optional[float] = None- peak_vram_gb: Optional[float] = None- output_size_gb: Optional[float] = None+ tuning_time_s: float | None = None+ peak_ram_gb: float | None = None+ peak_vram_gb: float | None = None+ output_size_gb: float | None = Nonedef get_dir_size_gb(path: Path) -> float:@@ -31,7 +31,7 @@ def parse_log_file(log_file: Path) -> QuantMetrics:metrics = QuantMetrics()if not log_file.exists():- logging.warning(f"Log file not found: {log_file}")+ logger.warning(f"Log file not found: {log_file}")return metricscontent = log_file.read_text(encoding="utf-8")@@ -50,7 +50,7 @@ def parse_log_file(log_file: Path) -> QuantMetrics:return metrics-def get_tuning_info() -> Dict[str, Dict[str, QuantMetrics]]:+def get_tuning_info() -> dict[str, dict[str, QuantMetrics]]:summary = {}model_list = ["Qwen/Qwen3-0.6B"]@@ -60,7 +60,7 @@ def get_tuning_info() -> Dict[str, Dict[str, QuantMetrics]]:log_file = LOG_DIR / f"perf_test_{test_mode}.log"output_dir = OUTPUT_BASE_DIR / test_mode- logging.info(f"Processing {log_file}...")+ logger.info(f"Processing {log_file}...")metrics = parse_log_file(log_file)metrics.output_size_gb = get_dir_size_gb(output_dir)@@ -70,15 +70,13 @@ def get_tuning_info() -> Dict[str, Dict[str, QuantMetrics]]:return summary-def compare_metric(- metric_name: str, current: Optional[float], baseline: Optional[float], tolerance: float = 0.1-) -> bool:+def compare_metric(metric_name: str, current: float | None, baseline: float | None, tolerance: float = 0.1) -> bool:if current is None or baseline is None:- logging.error(f" [-] {metric_name}: Incomplete data (Current: {current}, Baseline: {baseline})")+ logger.error(f" [-] {metric_name}: Incomplete data (Current: {current}, Baseline: {baseline})")return Falseif baseline == 0:- logging.warning(f" [!] {metric_name}: Baseline is 0, cannot calculate ratio.")+ logger.warning(f" [!] {metric_name}: Baseline is 0, cannot calculate ratio.")return Falseratio = current / baseline@@ -87,10 +85,10 @@ def compare_metric(msg = f" [*] {metric_name:<20}: Current = {current:<8} | Baseline = {baseline:<8} (Diff: {diff_percent:+.2f}%)"if 1.0 - tolerance <= ratio <= 1.0 + tolerance:- logging.info(f"{msg} -> PASS")+ logger.info(f"{msg} -> PASS")return Trueelse:- logging.error(f"{msg} -> FAIL")+ logger.error(f"{msg} -> FAIL")return False@@ -99,8 +97,8 @@ def check_performance():all_passed = Truefor model, modes in summary.items():- logging.info(f"\nEvaluating Model: {model}")- logging.info("-" * 60)+ logger.info(f"\nEvaluating Model: {model}")+ logger.info("-" * 60)current: QuantMetrics = modes.get("current", QuantMetrics())baseline: QuantMetrics = modes.get("baseline", QuantMetrics())@@ -117,11 +115,11 @@ def check_performance():if not compare_metric("Output Size (GB)", current.output_size_gb, baseline.output_size_gb, tolerance=0.01):all_passed = False- logging.info("=" * 60)+ logger.info("=" * 60)if all_passed:- logging.info("✅ Performance check passed: All metrics are within acceptable limits.")+ logger.info("✅ Performance check passed: All metrics are within acceptable limits.")else:- logging.error("❌ Performance check failed: Current metrics exceed acceptable limits compared to baseline.")+ logger.error("❌ Performance check failed: Current metrics exceed acceptable limits compared to baseline.")sys.exit(1)diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.pyindex 0b23ad9e2..ebc20529d 100644--- a/.azure-pipelines/scripts/ut/collect_result.py+++ b/.azure-pipelines/scripts/ut/collect_result.py@@ -181,8 +181,10 @@ class ReportGenerator:self._format_subheader(),*(self._format_row(r) for r in results),self.SEPARATOR,- f"Total: {stats['total']}, Passed: {stats['passed']}, "- f"Failed: {stats['failed']}, Skipped: {stats['skipped']}",+ (+ f"Total: {stats['total']}, Passed: {stats['passed']}, "+ f"Failed: {stats['failed']}, Skipped: {stats['skipped']}"+ ),self.SEPARATOR,"",]diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex 6b4e7d12a..278174a6e 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -73,7 +73,7 @@ repos:exclude: ^auto_round/export/export_to_gguf/conversion/.*\.py$- repo: https://github.com/astral-sh/ruff-pre-commit- rev: v0.15.20+ rev: v0.16.9hooks:- id: ruffargs: [--fix, --exit-non-zero-on-fix, --no-cache]diff --git a/auto_round/__init__.py b/auto_round/__init__.pyindex d3f87c2c4..fce0c0b4e 100644--- a/auto_round/__init__.py+++ b/auto_round/__init__.py@@ -31,20 +31,20 @@ monkey_patch()from .version import __version____all__ = [- "__version__",+ "AWQConfig",+ "AdamRoundConfig","AutoRound",- "AutoRoundLLM",- "AutoRoundMLLM","AutoRoundAdam","AutoRoundDiffusion",+ "AutoRoundLLM",+ "AutoRoundMLLM","AutoScheme",+ "OptimizedRTNConfig","QuantizationScheme","RTNConfig",- "OptimizedRTNConfig",+ "RotationConfig","SignRoundConfig",- "AdamRoundConfig","SignRoundV2Config",- "AWQConfig",- "RotationConfig","SpinQuantConfig",+ "__version__",]diff --git a/auto_round/algorithms/base.py b/auto_round/algorithms/base.pyindex b3fd72535..9940a47ff 100644--- a/auto_round/algorithms/base.py+++ b/auto_round/algorithms/base.py@@ -72,7 +72,7 @@ class BaseAlgorithm:self.config = config# Name-mangled so subclasses cannot accidentally overwrite the run context.self.__run_ctx: QuantizationRunContext | None = None- self.__block_forward_runner: "BlockForwardRunner | None" = None+ self.__block_forward_runner: BlockForwardRunner | None = None@classmethoddef from_config(cls, config: Any) -> "BaseAlgorithm":diff --git a/auto_round/algorithms/block_runner.py b/auto_round/algorithms/block_runner.pyindex e5b5462fe..90de327b3 100644--- a/auto_round/algorithms/block_runner.py+++ b/auto_round/algorithms/block_runner.py@@ -23,7 +23,7 @@ This module owns:from __future__ import annotations-from typing import TYPE_CHECKING, Any, Union+from typing import TYPE_CHECKING, Anyimport torch@@ -115,8 +115,8 @@ class BlockForwardRunner:self,batch_dim: int = 0,batch_size: int = 8,- device: Union[str, "torch.device"] = "cpu",- cache_device: Union[str, "torch.device"] = "cpu",+ device: str | torch.device = "cpu",+ cache_device: str | torch.device = "cpu",amp: bool = True,amp_dtype: torch.dtype | None = None,is_diffusion: bool = False,@@ -144,7 +144,7 @@ class BlockForwardRunner:# ── Factory ──────────────────────────────────────────────────────────────@classmethod- def from_orchestrator(cls, orchestrator: "BaseOrchestrator", enable_torch_compile=True) -> "BlockForwardRunner":+ def from_orchestrator(cls, orchestrator: BaseOrchestrator, enable_torch_compile=True) -> BlockForwardRunner:"""Create from an orchestrator instance (called once at orchestrator init)."""model_ctx = getattr(orchestrator, "model_context", None)is_diffusion = getattr(model_ctx, "is_diffusion", False) if model_ctx else False@@ -170,7 +170,7 @@ class BlockForwardRunner:def forward(self,- block: "torch.nn.Module",+ block: torch.nn.Module,inputs: list[torch.Tensor] | dict,input_others: dict,indices: torch.Tensor | None = None,@@ -310,7 +310,7 @@ class BlockForwardRunner:else:return inputs.shape[self.batch_dim]- def _normalize_output(self, output: Any, block: "torch.nn.Module" = None) -> torch.Tensor:+ def _normalize_output(self, output: Any, block: torch.nn.Module = None) -> torch.Tensor:"""Normalize block output to a single tensor."""if isinstance(output, torch.Tensor):return output@@ -343,7 +343,7 @@ class BlockForwardRunner:return firstraise TypeError(f"Block output[0] must be tensor, got {type(first).__name__}.")- def _get_output_dict(self, output: Any, block: "torch.nn.Module" = None) -> dict[str, torch.Tensor] | None:+ def _get_output_dict(self, output: Any, block: torch.nn.Module = None) -> dict[str, torch.Tensor] | None:if isinstance(output, torch.Tensor) or not isinstance(output, (tuple, list)):return Noneblock_cls_name = block.__class__.__name__ if block is not None else Nonediff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.pyindex 725ead4fc..67f1aeb2d 100644--- a/auto_round/algorithms/composer.py+++ b/auto_round/algorithms/composer.py@@ -70,7 +70,7 @@ class BlockContext:``ValueError`` with a user-readable message."""- model: "torch.nn.Module"+ model: torch.nn.Moduleblock_names: list[str] # scheduling group; len > 1 when nblocks > 1block_name: str # = block_names[0] for single-block; descriptive label for multiblock_index: int # 0-based index within the current all_blocks group@@ -98,7 +98,7 @@ class AlgorithmComposer:composer = AlgorithmComposer(configs, compressor=self)"""- def __init__(self, configs: list, orchestrator: "BaseOrchestrator" = None) -> None:+ def __init__(self, configs: list, orchestrator: BaseOrchestrator = None) -> None:"""Build the pipeline from a list of algorithm config instances.Resolution rules:@@ -240,7 +240,7 @@ class AlgorithmComposer:# ── Internal hook helpers (act_max calibration) ───────────────────────────- def _register_act_max_hooks(self, block: "torch.nn.Module") -> list:+ def _register_act_max_hooks(self, block: torch.nn.Module) -> list:"""Register per-module act_max tracking hooks for static activation quantization.Returns a list of hook handles that the caller must remove when done.@@ -301,7 +301,7 @@ class AlgorithmComposer:handles.append(module.register_forward_hook(collect_act_max))return handles- def _get_fp_act_hooks(self, block: "torch.nn.Module") -> list:+ def _get_fp_act_hooks(self, block: torch.nn.Module) -> list:"""Register FP-input act_max + quantizer forward hooks."""if not self.need_quanted_input():# If having q_input, act_max will be collected in q_input forward hook,@@ -312,13 +312,13 @@ class AlgorithmComposer:handles.extend(self.block_quantizer.register_fp_input_forward_hooks(block))return handles- def _get_q_act_hooks(self, block: "torch.nn.Module") -> list:+ def _get_q_act_hooks(self, block: torch.nn.Module) -> list:"""Register Q-input act_max + quantizer forward hooks."""handles = self._register_act_max_hooks(block)handles.extend(self.block_quantizer.register_qinput_forward_hooks(block))return handles- def _attach_act_max_for_outside_layer(self, layer: "torch.nn.Module", fp_inputs, q_inputs) -> None:+ def _attach_act_max_for_outside_layer(self, layer: torch.nn.Module, fp_inputs, q_inputs) -> None:"""Compute and attach act_max for an outside-block layer from cached inputs.Mirrors the hook-based act_max collection done for in-block layers, but@@ -334,10 +334,10 @@ class AlgorithmComposer:from auto_round.data_type.utils import reshape_pad_tensor_by_group_sizetarget_input = q_inputs or fp_inputs- act_group_size = getattr(layer, "act_group_size")+ act_group_size = layer.act_group_sizeif act_group_size is None:act_group_size = layer.group_size- act_data_type = getattr(layer, "act_data_type")+ act_data_type = layer.act_data_typeif act_data_type is None:act_data_type = layer.data_typeis_act_nv_fp_flag = is_nv_fp(act_data_type) if act_data_type else False@@ -368,9 +368,7 @@ class AlgorithmComposer:for preprocessor in self.preprocessors:if getattr(preprocessor, "enable_quanted_input", False):return True- if getattr(self.block_quantizer, "enable_quanted_input", False):- return True- return False+ return bool(getattr(self.block_quantizer, "enable_quanted_input", False))def compress_embedding_layer(self):return self.block_quantizer.quantize_embedding_layer()@@ -512,7 +510,7 @@ class AlgorithmComposer:def compress_layer_outside_block(self,- layer: "torch.nn.Module",+ layer: torch.nn.Module,fp_inputs=None,q_inputs=None,disable_opt_rtn=None, # TODO wenhuach rename this to search_init_scale@@ -537,7 +535,7 @@ class AlgorithmComposer:if fp_inputs is not None:from auto_round.compressors.utils import is_nv_fp- act_data_type = getattr(layer, "act_data_type")+ act_data_type = layer.act_data_typeif act_data_type is None:act_data_type = "fp"act_dynamic = getattr(layer, "act_dynamic", True)@@ -569,7 +567,7 @@ class AlgorithmComposer:"""return list(self.preprocessors) + list(self._rotation_members) + [self.block_quantizer]- def dispatch_block(self, block: "torch.nn.Module", input_ids, input_others: dict):+ def dispatch_block(self, block: torch.nn.Module, input_ids, input_others: dict):"""Dispatch block to device(s) via the pipeline's algorithms.Iterates all members; if exactly one overrides the default dispatch_block,@@ -597,7 +595,7 @@ class AlgorithmComposer:return overriders[0].dispatch_block(block, input_ids, input_others)return self.block_quantizer.dispatch_block(block, input_ids, input_others)- def prepare_run(self, composer: "AlgorithmComposer" = None):+ def prepare_run(self, composer: AlgorithmComposer = None):for alg in self.members():alg.prepare_run(composer=self)@@ -626,7 +624,7 @@ class AlgorithmComposer:return getattr(self.block_quantizer.config, "data_type", "mx_fp")return "mx_fp"- def apply_model_transforms(self, model: "torch.nn.Module") -> "torch.nn.Module":+ def apply_model_transforms(self, model: torch.nn.Module) -> torch.nn.Module:"""Apply model-level pre-quantisation transforms (rotation) to *model*.Generic entry point invoked once by the orchestrator before calibrationdiff --git a/auto_round/algorithms/config.py b/auto_round/algorithms/config.pyindex 3202a79a5..d533b162e 100644--- a/auto_round/algorithms/config.py+++ b/auto_round/algorithms/config.py@@ -34,7 +34,7 @@ class AlgorithmParameter:class _MutuallyExclusiveParameterRegistry:- def __init__(self, registry: "AlgorithmParameterRegistry", group_id: int) -> None:+ def __init__(self, registry: AlgorithmParameterRegistry, group_id: int) -> None:self._registry = registryself._group_id = group_iddiff --git a/auto_round/algorithms/quantization/adam_round/adam.py b/auto_round/algorithms/quantization/adam_round/adam.pyindex e30e1ab4b..bb67e6c1b 100644--- a/auto_round/algorithms/quantization/adam_round/adam.py+++ b/auto_round/algorithms/quantization/adam_round/adam.py@@ -33,8 +33,6 @@ class AdamRoundQuantizer(SignRoundQuantizer):optimizer = torch.optim.AdamWelif isinstance(optimizer, str):optimizer = getattr(torch.optim, optimizer)- else:- optimizer = optimizerreturn optimizerdef _get_scaler(self):diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.pyindex f9cfaebd4..6aa6593b0 100644--- a/auto_round/algorithms/quantization/base.py+++ b/auto_round/algorithms/quantization/base.py@@ -125,12 +125,9 @@ class BaseQuantizer(BaseAlgorithm):)except torch.OutOfMemoryError:cuda_error_msg = traceback.format_exc()- try:- logger.error(cuda_error_msg)- logger.warning("falling back to CPU")- weight, scale, zp = quant_func(module.weight.to("cpu"), **quant_kwargs)- except Exception:- raise+ logger.error(cuda_error_msg)+ logger.warning("falling back to CPU")+ weight, scale, zp = quant_func(module.weight.to("cpu"), **quant_kwargs)module.weight.data.copy_(weight.cpu())for param_name, val in zip(["scale", "zp"], [scale, zp]):if isinstance(val, dict):@@ -238,23 +235,20 @@ class BaseQuantizer(BaseAlgorithm):except torch.OutOfMemoryError:cuda_error_msg = traceback.format_exc()layer = layer.orig_layer if hasattr(layer, "orig_layer") else layer- try:- logger.error(cuda_error_msg)- logger.warning("falling back to CPU.")- layer.to("cpu")- layer = WrapperLinear(- layer,- enable_minmax_tuning=False,- enable_norm_bias_tuning=False,- enable_round_tuning=False,- enable_torch_compile=self.compress_context.enable_torch_compile,- disable_opt_rtn=disable_opt_rtn,- enable_neuqi=getattr(self.config, "enable_neuqi", False),- iters=0,- )- layer = layer.unwrapper({})- except Exception:- raise+ logger.error(cuda_error_msg)+ logger.warning("falling back to CPU.")+ layer.to("cpu")+ layer = WrapperLinear(+ layer,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ enable_round_tuning=False,+ enable_torch_compile=self.compress_context.enable_torch_compile,+ disable_opt_rtn=disable_opt_rtn,+ enable_neuqi=getattr(self.config, "enable_neuqi", False),+ iters=0,+ )+ layer = layer.unwrapper({})set_module(self.model, layer_name, layer)def _compute_valid_token_mask(self, input_ids: list) -> "list | None":diff --git a/auto_round/algorithms/quantization/registry.py b/auto_round/algorithms/quantization/registry.pyindex 82a77742c..6c4beb090 100644--- a/auto_round/algorithms/quantization/registry.py+++ b/auto_round/algorithms/quantization/registry.py@@ -8,4 +8,4 @@ def register_alg(alias, factory):register_algorithm(alias, aliases=(alias,), config_factory=factory)-__all__ = ["register_alg", "resolve_alg_config", "list_registered_algorithms"]+__all__ = ["list_registered_algorithms", "register_alg", "resolve_alg_config"]diff --git a/auto_round/algorithms/quantization/rtn/config.py b/auto_round/algorithms/quantization/rtn/config.pyindex ac4fc376c..ad0c2dc1d 100644--- a/auto_round/algorithms/quantization/rtn/config.py+++ b/auto_round/algorithms/quantization/rtn/config.py@@ -56,8 +56,8 @@ class RTNConfig(QuantizationConfig):def __init__(self,*,- disable_opt_rtn: bool = None,- enable_opt_rtn: bool = None,+ disable_opt_rtn: bool | None = None,+ enable_opt_rtn: bool | None = None,enable_neuqi: bool = False,**kwargs,) -> None:diff --git a/auto_round/algorithms/quantization/sign_round/config.py b/auto_round/algorithms/quantization/sign_round/config.pyindex a362a34e1..32be62e94 100644--- a/auto_round/algorithms/quantization/sign_round/config.py+++ b/auto_round/algorithms/quantization/sign_round/config.py@@ -11,7 +11,7 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-from typing import Callable+from collections.abc import Callablefrom auto_round.algorithms.config import AlgorithmParameterRegistryfrom auto_round.algorithms.quantization.config import QuantizationConfigdiff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.pyindex e5a7e0257..7bf06c853 100644--- a/auto_round/algorithms/quantization/sign_round/quantizer.py+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py@@ -12,8 +12,9 @@# See the License for the specific language governing permissions and# limitations under the License.import copy+from collections.abc import Callablefrom contextlib import nullcontext-from typing import TYPE_CHECKING, Any, Callable, Optional, Union+from typing import TYPE_CHECKING, Anyimport torchfrom torch import autocast@@ -135,8 +136,8 @@ class SignRoundQuantizer(BaseQuantizer):ref_output: torch.Tensor,indices: torch.Tensor,loss_func: Callable,- device: Union[str, torch.device] = "cpu",- valid_token_mask: Optional[torch.Tensor] = None,+ device: str | torch.device = "cpu",+ valid_token_mask: torch.Tensor | None = None,input_ids=None,):autocast_ctx = (@@ -608,10 +609,10 @@ class SignRoundQuantizer(BaseQuantizer):def quantize_layer_outside_block(self,layer: "torch.nn.Module",- fp_inputs: Optional[list[torch.Tensor]] = None,- q_inputs: Optional[list[torch.Tensor]] = None,- disable_opt_rtn: Optional[bool] = None,- input_ids: Optional[list[torch.Tensor]] = None,+ fp_inputs: list[torch.Tensor] | None = None,+ q_inputs: list[torch.Tensor] | None = None,+ disable_opt_rtn: bool | None = None,+ input_ids: list[torch.Tensor] | None = None,):"""Quantize a single layer that lives outside a transformer block.@@ -841,7 +842,7 @@ class SignRoundQuantizer(BaseQuantizer):def _get_scaler(self):"""Returns scaler, in SignRound, no need to use scaler."""- return None+ returndef _scale_loss_and_backward(self, scaler: Any, loss: torch.Tensor) -> torch.Tensor:"""Scales the loss and performs backward pass.diff --git a/auto_round/algorithms/quantization/sign_round/sign_sgd.py b/auto_round/algorithms/quantization/sign_round/sign_sgd.pyindex b1fe06025..671c178f5 100644--- a/auto_round/algorithms/quantization/sign_round/sign_sgd.py+++ b/auto_round/algorithms/quantization/sign_round/sign_sgd.py@@ -1,5 +1,4 @@#-# -*- coding: utf-8 -*-## Copyright (c) 2023 Intel Corporation#@@ -16,7 +15,7 @@# limitations under the License.-from typing import Iterable, List, Optional+from collections.abc import Iterable# From PyTorch:#@@ -102,7 +101,7 @@ from torch.optim.optimizer import Optimizer__all__ = ["SignSGD", "sgd"]-class _RequiredParameter(object):+class _RequiredParameter:"""Singleton class representing a required parameter for an Optimizer."""def __repr__(self):@@ -220,15 +219,15 @@ class SignSGD(Optimizer):nesterov: bool = False,*,maximize: bool = False,- foreach: Optional[bool] = None,+ foreach: bool | None = None,differentiable: bool = False,) -> None:if lr is not required and lr < 0.0:- raise ValueError("Invalid learning rate: {}".format(lr))+ raise ValueError(f"Invalid learning rate: {lr}")if momentum < 0.0:- raise ValueError("Invalid momentum value: {}".format(momentum))+ raise ValueError(f"Invalid momentum value: {momentum}")if weight_decay < 0.0:- raise ValueError("Invalid weight_decay value: {}".format(weight_decay))+ raise ValueError(f"Invalid weight_decay value: {weight_decay}")defaults = dict(lr=lr,@@ -242,7 +241,7 @@ class SignSGD(Optimizer):)if nesterov and (momentum <= 0 or dampening != 0):raise ValueError("Nesterov momentum requires a momentum and zero dampening")- super(SignSGD, self).__init__(params, defaults)+ super().__init__(params, defaults)def __setstate__(self, state):super().__setstate__(state)@@ -307,13 +306,13 @@ class SignSGD(Optimizer):def sgd(- params: List[Tensor],- d_p_list: List[Tensor],- momentum_buffer_list: List[Optional[Tensor]],+ params: list[Tensor],+ d_p_list: list[Tensor],+ momentum_buffer_list: list[Tensor | None],# kwonly args with defaults are not supported by functions compiled with torchscript issue #70627# setting this as kwarg for now as functional API is compiled by torch/distributed/optim- has_sparse_grad: bool = None,- foreach: bool = None,+ has_sparse_grad: bool | None = None,+ foreach: bool | None = None,*,weight_decay: float,momentum: float,@@ -354,9 +353,9 @@ def sgd(def _single_tensor_sgd(- params: List[Tensor],- d_p_list: List[Tensor],- momentum_buffer_list: List[Optional[Tensor]],+ params: list[Tensor],+ d_p_list: list[Tensor],+ momentum_buffer_list: list[Tensor | None],*,weight_decay: float,momentum: float,diff --git a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py b/auto_round/algorithms/quantization/sign_roundv2/quantizer.pyindex eb9c977c6..0d4b44fc4 100644--- a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py+++ b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py@@ -12,9 +12,10 @@# See the License for the specific language governing permissions and# limitations under the License.+from collections.abc import Callablefrom contextlib import nullcontextfrom functools import partial-from typing import TYPE_CHECKING, Callable, Union+from typing import TYPE_CHECKINGimport torchimport transformers@@ -25,6 +26,7 @@ from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuaif TYPE_CHECKING:from auto_round.algorithms.composer import AlgorithmComposer+from auto_round.algorithms.registry import register_pipeline_memberfrom auto_round.data_type.gguf import (double_quant_tensor_sym_rtn,@@ -154,9 +156,8 @@ class SignRoundOptimizedWrapperLinear(WrapperLinear):weight_reshape = torch.clamp(weight_reshape, clip_min, clip_max)else:logger.warning_once(- "SignRoundV2: ignoring AWQ clip range with shapes %s/%s incompatible with "- "grouped weight shape %s."- % (tuple(clip_min.shape), tuple(clip_max.shape), tuple(weight_reshape.shape))+ f"SignRoundV2: ignoring AWQ clip range with shapes {tuple(clip_min.shape)}/{tuple(clip_max.shape)} "+ f"incompatible with grouped weight shape {tuple(weight_reshape.shape)}.")return weight_reshape@@ -365,7 +366,7 @@ class SignRoundV2Quantizer(SignRoundQuantizer):ref_output: torch.Tensor,indices: torch.Tensor,mse_loss: Callable,- device: Union[str, torch.device] = "cpu",+ device: str | torch.device = "cpu",valid_token_mask: list[torch.Tensor] | None = None,):if self._use_outlier_suppressed_loss:diff --git a/auto_round/algorithms/registry.py b/auto_round/algorithms/registry.pyindex 3dfffca67..0e96bd1e4 100644--- a/auto_round/algorithms/registry.py+++ b/auto_round/algorithms/registry.py@@ -5,8 +5,9 @@ from __future__ import annotationsimport copyimport importlib+from collections.abc import Callablefrom dataclasses import dataclass, field-from typing import TYPE_CHECKING, Any, Callable+from typing import TYPE_CHECKINGif TYPE_CHECKING:from auto_round.algorithms.base import BaseAlgorithm@@ -23,7 +24,7 @@ class AlgRegistryEntry:_ALG_REGISTRY: dict[str, AlgRegistryEntry] = {}_ALIAS_TO_NAME: dict[str, str] = {}-_CONFIG_IMPL_REGISTRY: dict[type, type["BaseAlgorithm"]] = {}+_CONFIG_IMPL_REGISTRY: dict[type, type[BaseAlgorithm]] = {}_builtin_algorithms_registered = False_pipeline_members_registered = False_BUILTIN_ALGORITHM_ORDER = ("rtn", "auto_round", "awq", "svdquant", "hadamard", "quarot", "spinquant")@@ -171,14 +172,14 @@ def list_registered_algorithms() -> list[str]:def register_pipeline_member(config_cls: type):- def _decorator(member_cls: type["BaseAlgorithm"]) -> type["BaseAlgorithm"]:+ def _decorator(member_cls: type[BaseAlgorithm]) -> type[BaseAlgorithm]:_CONFIG_IMPL_REGISTRY[config_cls] = member_clsreturn member_clsreturn _decorator-def resolve_pipeline_member(config: object) -> type["BaseAlgorithm"]:+def resolve_pipeline_member(config: object) -> type[BaseAlgorithm]:_ensure_pipeline_members_registered()config_cls = type(config)for cls in config_cls.__mro__:diff --git a/auto_round/algorithms/transforms/__init__.py b/auto_round/algorithms/transforms/__init__.pyindex 13b597e71..b1ac6289b 100644--- a/auto_round/algorithms/transforms/__init__.py+++ b/auto_round/algorithms/transforms/__init__.py@@ -61,7 +61,7 @@ from auto_round.algorithms.transforms.hadamard import (RotationConfig,)-__all__ = [+__all__ = [ # noqa: RUF022# Base interfaces"BasePreprocessor","BaseWeightTransformer", # backward-compat aliasdiff --git a/auto_round/algorithms/transforms/awq/base.py b/auto_round/algorithms/transforms/awq/base.pyindex 2c8332e5e..d2e399992 100644--- a/auto_round/algorithms/transforms/awq/base.py+++ b/auto_round/algorithms/transforms/awq/base.py@@ -31,6 +31,7 @@ from __future__ import annotationsimport inspectimport re+import sysfrom typing import TYPE_CHECKING, Anyimport torch@@ -278,13 +279,13 @@ class AWQTransform(BasePreprocessor):"AWQ does not support nblocks > 1 (got nblocks=%s). ",nblocks,)- exit(-1)+ sys.exit(-1)def can_compile_block_forward(self) -> bool:"""AWQ installs per-block calibration hooks that trigger Dynamo recompiles."""return False- def prepare_run(self, composer: "AlgorithmComposer" = None) -> None:+ def prepare_run(self, composer: AlgorithmComposer = None) -> None:"""Resolve model-wide mappings and group them by transformer block."""model = self.model@@ -347,7 +348,7 @@ class AWQTransform(BasePreprocessor):return self._register_awq_hooks(self.model_context.model, block, block_name)return []- def pre_quantize_block(self, ctx: "BlockContext") -> None:+ def pre_quantize_block(self, ctx: BlockContext) -> None:"""Apply AWQ smoothing for this block and mark modified params.Called after the reference forward (activation stats collected) and@@ -382,7 +383,7 @@ class AWQTransform(BasePreprocessor):modified.extend(mapping.balance_names)modified.append(mapping.smooth_name)- def post_quantize_block(self, ctx: "BlockContext") -> None:+ def post_quantize_block(self, ctx: BlockContext) -> None:"""Release per-block AWQ caches to free memory."""block_mappings = self._block_mappings.get(ctx.block_name, [])if not block_mappings:@@ -608,9 +609,7 @@ class AWQTransform(BasePreprocessor):"""AWQ smoothing is all-or-nothing for layers sharing one smooth scale."""if self._mapping_has_ignored_layer(mapping):return False- if self._mapping_has_mixed_quant_params(mapping):- return False- return True+ return not self._mapping_has_mixed_quant_params(mapping)def _smooth_block(self, block_prefix: str, block_mappings: list) -> None:"""Run grid search and apply AWQ scales for one block.diff --git a/auto_round/algorithms/transforms/awq/config.py b/auto_round/algorithms/transforms/awq/config.pyindex affc1bad0..92018236d 100644--- a/auto_round/algorithms/transforms/awq/config.py+++ b/auto_round/algorithms/transforms/awq/config.py@@ -310,7 +310,7 @@ class AWQConfig(QuantizationConfig):def finalize_scheme(self) -> None:"""Adjust AWQ state that depends on the resolved run scheme."""data_type = self.data_type- is_gguf_double_quant = bool(data_type) and (data_type.endswith("_dq") or data_type.endswith("float_zp"))+ is_gguf_double_quant = bool(data_type) and data_type.endswith(("_dq", "float_zp"))if self.apply_clip and is_gguf_double_quant:logger.warning("AWQ weight clipping (apply_clip=True) is not supported for GGUF "diff --git a/auto_round/algorithms/transforms/base.py b/auto_round/algorithms/transforms/base.pyindex d83935c55..d904584ae 100644--- a/auto_round/algorithms/transforms/base.py+++ b/auto_round/algorithms/transforms/base.py@@ -22,7 +22,7 @@ from __future__ import annotationsfrom abc import ABC, abstractmethodfrom dataclasses import dataclass-from typing import Any, Optional+from typing import Anyimport torchimport torch.nn as nn@@ -97,7 +97,7 @@ class BaseRotation(ABC):"""# Registry populated by subclasses via ``BaseRotation.register``.- _REGISTRY: dict[str, type["BaseRotation"]] = {}+ _REGISTRY: dict[str, type[BaseRotation]] = {}def __init__(self, config: BaseRotationConfig) -> None:self.config = config@@ -144,7 +144,7 @@ class BaseRotation(ABC):model: torch.nn.Module,data_type: str = "mx_fp",**kwargs: Any,- ) -> "BaseRotation":+ ) -> BaseRotation:"""Prepare for layer-wise rotation without modifying model weights.Called once when the rotation config has ``layerwise=True``.@@ -205,7 +205,6 @@ class BaseRotation(ABC):Args:model: The fully-rotated model."""- pass# ------------------------------------------------------------------# Factory@@ -229,7 +228,7 @@ class BaseRotation(ABC):return _decorator@classmethod- def from_config(cls, config: BaseRotationConfig) -> "BaseRotation":+ def from_config(cls, config: BaseRotationConfig) -> BaseRotation:"""Instantiate the correct ``BaseRotation`` subclass for *config*.The algorithm is looked up by ``config.algorithm`` in the registry.diff --git a/auto_round/algorithms/transforms/hadamard/__init__.py b/auto_round/algorithms/transforms/hadamard/__init__.pyindex 54b749f98..64738e6f2 100644--- a/auto_round/algorithms/transforms/hadamard/__init__.py+++ b/auto_round/algorithms/transforms/hadamard/__init__.py@@ -28,7 +28,7 @@ from auto_round.algorithms.transforms.hadamard.transforms import (build_hadamard_transform,)-__all__ = [+__all__ = [ # noqa: RUF022# Algorithm class"HadamardRotation",# Configdiff --git a/auto_round/algorithms/transforms/hadamard/apply.py b/auto_round/algorithms/transforms/hadamard/apply.pyindex 9e7d03570..c599844cc 100644--- a/auto_round/algorithms/transforms/hadamard/apply.py+++ b/auto_round/algorithms/transforms/hadamard/apply.py@@ -82,7 +82,7 @@ class HadamardRotation(BaseRotation, SerializerMixin):super().__init__(config)@classmethod- def from_config(cls, config: dict | RotationConfig) -> "HadamardRotation":+ def from_config(cls, config: dict | RotationConfig) -> HadamardRotation:"""Build a :class:`HadamardRotation` from a raw dict or :class:`RotationConfig`."""if isinstance(config, dict):config = RotationConfig.model_validate(config)@@ -143,7 +143,7 @@ class HadamardRotation(BaseRotation, SerializerMixin):fuse_online_to_weight=fuse_online_to_weight,compute_device=compute_device,)- setattr(model, "_rotation_config", cfg)+ model._rotation_config = cfgreturn model# backend == "transform": original per-Linear triton-fused path.@@ -159,7 +159,7 @@ class HadamardRotation(BaseRotation, SerializerMixin):_apply_to_module(model, module, cfg, location, data_type)# Store config on model for serialisation / downstream inspection.- setattr(model, "_rotation_config", cfg)+ model._rotation_config = cfgreturn model# ------------------------------------------------------------------@@ -188,9 +188,7 @@ class HadamardRotation(BaseRotation, SerializerMixin):cfg = self.configbackend = getattr(cfg, "backend", "auto")hadamard_type = getattr(cfg, "hadamard_type", "") or ""- if backend == "inplace" or "inplace" in hadamard_type:- return False- return True+ return not (backend == "inplace" or "inplace" in hadamard_type)def prepare_layerwise(self,@@ -198,7 +196,7 @@ class HadamardRotation(BaseRotation, SerializerMixin):data_type: str = "mx_fp",location: str = "weight",**kwargs: Any,- ) -> "HadamardRotation":+ ) -> HadamardRotation:"""Prepare for per-block Hadamard rotation without touching weights.The per-Linear Hadamard needs no global pre-computation — matrices are@@ -219,7 +217,7 @@ class HadamardRotation(BaseRotation, SerializerMixin):)self._layerwise_location = locationself._layerwise_data_type = data_type- setattr(model, "_rotation_config", self.config)+ model._rotation_config = self.configreturn selfdef rotate_layer(diff --git a/auto_round/algorithms/transforms/hadamard/config.py b/auto_round/algorithms/transforms/hadamard/config.pyindex f2f2bf85a..e1893711b 100644--- a/auto_round/algorithms/transforms/hadamard/config.py+++ b/auto_round/algorithms/transforms/hadamard/config.py@@ -34,7 +34,7 @@ Two implementation backends share this one schema (method B):from __future__ import annotations-from typing import Any, ClassVar, Optional+from typing import Any, ClassVarfrom pydantic import BaseModel, Field, field_validator@@ -46,9 +46,9 @@ from auto_round.utils import logger__all__ = ["RotationConfig",+ "dump_group_size_to_rotation_config","normalize_rotation_config","to_dict_rotation_config",- "dump_group_size_to_rotation_config",]# Supported Hadamard transform types (also used by HadamardTransform registry).@@ -75,7 +75,7 @@ class RotationConfig(AlgorithmConfig, BaseModel, BaseRotationConfig):# ---- shared ----backend: str = Field(default="auto")- block_size: Optional[int] = Field(default=None)+ block_size: int | None = Field(default=None)hadamard_type: str = Field(default="hadamard")# Apply the Hadamard rotation per decoder block, in lock-step with block-wise# quantization, instead of rotating the whole model up-front. Honoured by the@@ -84,7 +84,7 @@ class RotationConfig(AlgorithmConfig, BaseModel, BaseRotationConfig):layerwise: bool = Field(default=False)# ---- inplace-only ----- fuse_online_to_weight: Optional[bool] = Field(default=None)+ fuse_online_to_weight: bool | None = Field(default=None)allow_online_rotation: bool = Field(default=True)# for random hadamard (transform path)diff --git a/auto_round/algorithms/transforms/hadamard/dispatcher.py b/auto_round/algorithms/transforms/hadamard/dispatcher.pyindex 6ae4fdbad..52e96aa44 100644--- a/auto_round/algorithms/transforms/hadamard/dispatcher.py+++ b/auto_round/algorithms/transforms/hadamard/dispatcher.py@@ -28,7 +28,7 @@ Routing is controlled by :class:`RotationConfig.backend`:from __future__ import annotations-from typing import Any, Union+from typing import Anyimport torch@@ -41,7 +41,7 @@ __all__ = ["apply_hadamard_rotation", "resolve_hadamard_backend"]def _to_config(- rotation_config: Union[str, dict, RotationConfig, None],+ rotation_config: str | dict | RotationConfig | None,data_type: str,) -> RotationConfig:"""Normalise *rotation_config* and return a :class:`RotationConfig` instance."""@@ -88,7 +88,7 @@ def resolve_hadamard_backend(config: RotationConfig, data_type: str) -> str:def apply_hadamard_rotation(model: torch.nn.Module,- rotation_config: Union[str, dict, RotationConfig, None],+ rotation_config: str | dict | RotationConfig | None,data_type: str,compute_device: torch.device | str = None,) -> (torch.nn.Module, Any):@@ -138,7 +138,7 @@ def apply_hadamard_rotation(compute_device=compute_device,)# Stash config object for downstream (export / serialization).- setattr(model, "_rotation_config", config)+ model._rotation_config = configreturn model, hookselif backend == "transform":diff --git a/auto_round/algorithms/transforms/hadamard/inplace/apply.py b/auto_round/algorithms/transforms/hadamard/inplace/apply.pyindex d2adb7508..b809830f8 100644--- a/auto_round/algorithms/transforms/hadamard/inplace/apply.py+++ b/auto_round/algorithms/transforms/hadamard/inplace/apply.py@@ -9,7 +9,6 @@ The entry point is :func:`apply_hadamard_rotation`.import gcimport typing-from typing import Dict, Unionimport torchimport tqdm@@ -280,10 +279,10 @@ def _rotate_weights(model,mapping: RotationMapping,use_fast_had: bool = True,- group_size: int = None,+ group_size: int | None = None,compute_device: torch.device = None,- had_dict: dict = None,- preset: str = None,+ had_dict: dict | None = None,+ preset: str | None = None,fuse_online_to_weight: bool = True,) -> None:"""Apply Hadamard rotation to all weights.@@ -657,9 +656,9 @@ def _register_online_hooks(mapping: RotationMapping,fp32_had: bool = False,use_fast_had: bool = True,- group_size: int = None,- had_dict: dict = None,- preset: str = None,+ group_size: int | None = None,+ had_dict: dict | None = None,+ preset: str | None = None,fuse_online_to_weight: bool = True,):"""Register online Hadamard pre-forward hooks on ``down_proj`` and ``o_proj``.@@ -710,8 +709,8 @@ def _register_online_hooks(attn_o_suffix = mapping.attn_o.split(".")[-1]# Suffixes for Q/K/V and gate/up (for online input Had hooks)- attn_qkv_suffixes = set(attr.split(".")[-1] for attr in (mapping.attn_q, mapping.attn_k, mapping.attn_v))- mlp_in_suffixes = set(attr.split(".")[-1] for attr in mapping.mlp_in)+ attn_qkv_suffixes = {attr.split(".")[-1] for attr in (mapping.attn_q, mapping.attn_k, mapping.attn_v)}+ mlp_in_suffixes = {attr.split(".")[-1] for attr in mapping.mlp_in}# --- Build hook factories ---def _make_down_proj_hook():@@ -807,12 +806,12 @@ def _register_online_hooks(def apply_rotation_transform(model,- group_size: int = None,+ group_size: int | None = None,allow_online_rotation: bool = True,- rotation_matrix: Union[str, torch.Tensor, Dict[int, torch.Tensor], None] = None,+ rotation_matrix: str | torch.Tensor | dict[int, torch.Tensor] | None = None,compute_device: torch.device | str = None,fp32_had: bool = False,- fuse_online_to_weight: bool = None,+ fuse_online_to_weight: bool | None = None,):"""Fuse layer norms, rotate weights, and register online Hadamard hooks.diff --git a/auto_round/algorithms/transforms/hadamard/inplace/hooks.py b/auto_round/algorithms/transforms/hadamard/inplace/hooks.pyindex 057d881d2..9e907749b 100644--- a/auto_round/algorithms/transforms/hadamard/inplace/hooks.py+++ b/auto_round/algorithms/transforms/hadamard/inplace/hooks.py@@ -9,7 +9,6 @@ provides the hooks and a helper to register them on the model."""import math-from typing import Optionalimport torchimport torch.nn as nn@@ -156,11 +155,11 @@ class FullOnlineHadamardHook(nn.Module):def __init__(self,- had_K: Optional[torch.Tensor],- K: Optional[int],+ had_K: torch.Tensor | None,+ K: int | None,fp32_had: bool = False,use_fast_had: bool = True,- had_matrix: Optional[torch.Tensor] = None,+ had_matrix: torch.Tensor | None = None,) -> None:super().__init__()self.custom_had = had_matrix is not None@@ -219,12 +218,12 @@ class CrossHeadOnlineHadamardHook(nn.Module):def __init__(self,- had_K: Optional[torch.Tensor],- K: Optional[int],+ had_K: torch.Tensor | None,+ K: int | None,head_dim: int,fp32_had: bool = False,use_fast_had: bool = True,- had_matrix: Optional[torch.Tensor] = None,+ had_matrix: torch.Tensor | None = None,) -> None:"""Args:@@ -613,7 +612,7 @@ class GroupOnlineHadamardHook(nn.Module):group_size: int,fp32_had: bool = False,use_fast_had: bool = True,- had_matrix: Optional[torch.Tensor] = None,+ had_matrix: torch.Tensor | None = None,) -> None:super().__init__()self.group_size = group_size@@ -788,15 +787,7 @@ def register_online_had_hooks_grouped(model, mapping, group_size, fp32_had=Falsehandles = []for name, module in model.named_modules():- if name.endswith(mlp_out_suffix) and isinstance(module, nn.Linear):- hook = GroupOnlineHadamardHook(- group_size=group_size,- fp32_had=fp32_had,- use_fast_had=use_fast_had,- )- h = module.register_forward_pre_hook(hook)- handles.append(h)- elif name.endswith(attn_o_suffix) and isinstance(module, nn.Linear):+ if isinstance(module, nn.Linear) and name.endswith((mlp_out_suffix, attn_o_suffix)):hook = GroupOnlineHadamardHook(group_size=group_size,fp32_had=fp32_had,diff --git a/auto_round/algorithms/transforms/hadamard/inplace/model_config.py b/auto_round/algorithms/transforms/hadamard/inplace/model_config.pyindex 3ecbf9b69..78fd99534 100644--- a/auto_round/algorithms/transforms/hadamard/inplace/model_config.py+++ b/auto_round/algorithms/transforms/hadamard/inplace/model_config.py@@ -12,16 +12,15 @@ New architectures can be supported by calling :func:`register_mapping`.from __future__ import annotationsfrom dataclasses import dataclass, field-from typing import Dict, List, Optionalfrom auto_round.utils import logger__all__ = [+ "MAPPING_REGISTRY","RotationMapping",- "register_mapping","get_mapping","infer_mapping_from_model",- "MAPPING_REGISTRY",+ "register_mapping",]@@ -45,7 +44,7 @@ class RotationMapping:# -- top-level modules (dot-path from model root) --embedding: str = "model.embed_tokens"lm_head: str = "lm_head"- positional_embedding: Optional[str] = None # e.g. "model.decoder.embed_positions" for OPT+ positional_embedding: str | None = None # e.g. "model.decoder.embed_positions" for OPT# -- layers container (dot-path from model root) --layers_attr: str = "model.layers"@@ -59,14 +58,14 @@ class RotationMapping:# -- per-layer: MLP (dot-path from each layer) --mlp_input_ln: str = "post_attention_layernorm"- mlp_in: List[str] = field(default_factory=lambda: ["mlp.up_proj", "mlp.gate_proj"])+ mlp_in: list[str] = field(default_factory=lambda: ["mlp.up_proj", "mlp.gate_proj"])mlp_out: str = "mlp.down_proj"# -- final norm (dot-path from model root) --pre_head_ln: str = "model.norm"# -- head dim override (None = hidden_size // num_heads) --- attn_head_dim: Optional[int] = None+ attn_head_dim: int | None = None# -- config attr names --num_heads_attr: str = "num_attention_heads"@@ -91,7 +90,7 @@ def _resolve(root, dot_path: str):# Registry# ----------------------------------------------------------------------------MAPPING_REGISTRY: Dict[str, RotationMapping] = {}+MAPPING_REGISTRY: dict[str, RotationMapping] = {}def register_mapping(key: str, mapping: RotationMapping) -> RotationMapping:diff --git a/auto_round/algorithms/transforms/hadamard/patch.py b/auto_round/algorithms/transforms/hadamard/patch.pyindex e3f5f9f95..2e9fbe3be 100644--- a/auto_round/algorithms/transforms/hadamard/patch.py+++ b/auto_round/algorithms/transforms/hadamard/patch.py@@ -31,9 +31,9 @@ from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear, pack_ffrom auto_round.wrapper import WrapperLinear, WrapperWALayer__all__ = [+ "patch_quantlinear","patch_wrapperlinear_to_apply_transform","patch_wrapperwalayer_forward_to_apply_transform",- "patch_quantlinear",]@@ -81,7 +81,7 @@ def patch_wrapperlinear_to_apply_transform(_orig_qdq_act = WrapperLinear._qdq_act- def _qdq_act_patched(self, x, act_min_scale=torch.tensor(1.0), act_max_scale=torch.tensor(1.0), act_max=None):+ def _qdq_act_patched(self, x, act_min_scale=None, act_max_scale=None, act_max=None):x = inp_transform(x)return _orig_qdq_act(self, x, act_min_scale=act_min_scale, act_max_scale=act_max_scale, act_max=act_max)@@ -191,7 +191,6 @@ def patch_quantlinear(w_transform) -> None:# add transform weightself.register_buffer("hadamard_matrix", w_transform.weight.to(device))- returnQuantLinear.pack = _pack_patchedQuantLinear._pack_patched = Truediff --git a/auto_round/algorithms/transforms/hadamard/transforms.py b/auto_round/algorithms/transforms/hadamard/transforms.pyindex 890fc5870..5d77fea47 100644--- a/auto_round/algorithms/transforms/hadamard/transforms.py+++ b/auto_round/algorithms/transforms/hadamard/transforms.py@@ -22,7 +22,8 @@ from __future__ import annotationsimport inspectimport math-from typing import Any, Callable, Dict+from collections.abc import Callable+from typing import Anyimport torchimport torch.nn as nn@@ -34,14 +35,14 @@ from auto_round.algorithms.transforms.hadamard.utils.math import (from auto_round.algorithms.transforms.hadamard.utils.matrix import apply_transform_weight__all__ = [+ "HADAMARDS","HadamardTransform","RandomHadamardTransform",- "HADAMARDS","build_hadamard_transform",]-def _filter_kwargs(fn: Callable, kwargs: Dict[str, Any]) -> Dict[str, Any]:+def _filter_kwargs(fn: Callable, kwargs: dict[str, Any]) -> dict[str, Any]:"""Return only the keyword arguments accepted by *fn*."""accepted = inspect.signature(fn).parameters.keys()return {k: v for k, v in kwargs.items() if k in accepted}diff --git a/auto_round/algorithms/transforms/hadamard/utils/math.py b/auto_round/algorithms/transforms/hadamard/utils/math.pyindex 14b15ce0d..a2b9edeb6 100644--- a/auto_round/algorithms/transforms/hadamard/utils/math.py+++ b/auto_round/algorithms/transforms/hadamard/utils/math.py@@ -28,7 +28,7 @@ from pathlib import Pathimport torchfrom safetensors import safe_open-__all__ = ["deterministic_hadamard_matrix", "random_hadamard_matrix", "is_pow2"]+__all__ = ["deterministic_hadamard_matrix", "is_pow2", "random_hadamard_matrix"]# Precomputed Hadamard matrices for non-power-of-2 sizes._HADAMARD_MATRICES_PATH: Path = Path(__file__).parent / "hadamards.safetensors"diff --git a/auto_round/algorithms/transforms/member.py b/auto_round/algorithms/transforms/member.pyindex 448e98ff3..dac931603 100644--- a/auto_round/algorithms/transforms/member.py+++ b/auto_round/algorithms/transforms/member.py@@ -57,7 +57,7 @@ class RotationPreprocessor(BasePreprocessor):def __init__(self, config: Any) -> None:super().__init__(config)- self._rotation: "BaseRotation | None" = None+ self._rotation: BaseRotation | None = None# Set once :meth:`rotate_model` decides layer-wise preparation succeeded.self._layerwise_active: bool = False@@ -65,7 +65,7 @@ class RotationPreprocessor(BasePreprocessor):# Lazy rotation construction# ------------------------------------------------------------------@property- def rotation(self) -> "BaseRotation":+ def rotation(self) -> BaseRotation:"""The concrete :class:`BaseRotation` for this member (built lazily)."""if self._rotation is None:from auto_round.algorithms.transforms import normalize_rotation_config@@ -93,9 +93,9 @@ class RotationPreprocessor(BasePreprocessor):# ------------------------------------------------------------------def rotate_model(self,- model: "torch.nn.Module",+ model: torch.nn.Module,data_type: str = "mx_fp",- ) -> "torch.nn.Module":+ ) -> torch.nn.Module:"""Rotate *model* up-front, or prepare layer-wise rotation matrices.Whether to rotate per-block is taken solely from the rotation config's@@ -129,7 +129,7 @@ class RotationPreprocessor(BasePreprocessor):# ------------------------------------------------------------------# Per-block entry (compress_block step 0)# ------------------------------------------------------------------- def on_block_ready(self, block: "torch.nn.Module", ctx: "BlockContext") -> None:+ def on_block_ready(self, block: torch.nn.Module, ctx: BlockContext) -> None:"""Rotate the block about to be quantized (layer-wise mode only).No-op unless :meth:`rotate_model` prepared layer-wise rotation. Iterates@@ -158,7 +158,7 @@ class RotationPreprocessor(BasePreprocessor):# Helpers# ------------------------------------------------------------------@staticmethod- def _iter_layers(block: "torch.nn.Module", block_index: int):+ def _iter_layers(block: torch.nn.Module, block_index: int):"""Yield ``(decoder_layer, layer_idx)`` for every layer inside *block*.``block_index`` is the global index of the *first* layer in this block.diff --git a/auto_round/algorithms/transforms/spinquant/__init__.py b/auto_round/algorithms/transforms/spinquant/__init__.pyindex b94ef5dc1..2a9339464 100644--- a/auto_round/algorithms/transforms/spinquant/__init__.py+++ b/auto_round/algorithms/transforms/spinquant/__init__.py@@ -106,7 +106,7 @@ from auto_round.algorithms.transforms.spinquant.serialize import (save_spinquant_config,)-__all__ = [+__all__ = [ # noqa: RUF022# -- Registry algorithm (unified apply_rotation() entry) --"SpinQuantRotation",# -- Preprocessor (QuaRot, recommended) --diff --git a/auto_round/algorithms/transforms/spinquant/apply.py b/auto_round/algorithms/transforms/spinquant/apply.pyindex f3c0e0c22..c8595f429 100644--- a/auto_round/algorithms/transforms/spinquant/apply.py+++ b/auto_round/algorithms/transforms/spinquant/apply.py@@ -120,7 +120,7 @@ class SpinQuantRotation(BaseRotation, SerializerMixin):model: torch.nn.Module,data_type: str = "mx_fp",**kwargs: Any,- ) -> "SpinQuantRotation":+ ) -> SpinQuantRotation:"""Prepare for layer-wise rotation: init R matrices only.Creates a :class:`SpinQuantPreprocessor`, calls itsdiff --git a/auto_round/algorithms/transforms/spinquant/cayley_optimizer.py b/auto_round/algorithms/transforms/spinquant/cayley_optimizer.pyindex f919deaef..d00ef56e2 100644--- a/auto_round/algorithms/transforms/spinquant/cayley_optimizer.py+++ b/auto_round/algorithms/transforms/spinquant/cayley_optimizer.py@@ -11,7 +11,8 @@ Core algorithm: SGD with Cayley transform to maintain orthogonality constraints.from __future__ import annotations-from typing import Any, Callable, Iterable+from collections.abc import Callable, Iterable+from typing import Anyimport torchfrom torch.optim.optimizer import Optimizerdiff --git a/auto_round/algorithms/transforms/spinquant/inplace/apply.py b/auto_round/algorithms/transforms/spinquant/inplace/apply.pyindex 18c857434..b78334f19 100644--- a/auto_round/algorithms/transforms/spinquant/inplace/apply.py+++ b/auto_round/algorithms/transforms/spinquant/inplace/apply.py@@ -19,7 +19,7 @@ R4 rotation uses a forward_pre_hook on ``down_proj`` that applies block Hadamardfrom __future__ import annotationsimport logging-from typing import Any, Optional+from typing import Anyimport torchimport torch.nn as nn@@ -35,7 +35,7 @@ logger = logging.getLogger("autoround.spinquant")def register_spinquant_hooks(model: nn.Module,config: Any,- compute_device: Optional[torch.device] = None,+ compute_device: torch.device | None = None,head_dim: int = 0,intermediate_size: int = 0,r4_rotation_size: int = 0,@@ -262,7 +262,7 @@ def remove_spinquant_hooks(handles: list[Any]) -> None:def apply_spinquant_in_place(model: nn.Module,config: Any,- dataloader: Optional[Any] = None,+ dataloader: Any | None = None,) -> nn.Module:"""Apply SpinQuant rotations to a model **in-place**.diff --git a/auto_round/algorithms/transforms/spinquant/monkeypatch.py b/auto_round/algorithms/transforms/spinquant/monkeypatch.pyindex 7fc3d9b46..747830d3e 100644--- a/auto_round/algorithms/transforms/spinquant/monkeypatch.py+++ b/auto_round/algorithms/transforms/spinquant/monkeypatch.py@@ -19,7 +19,8 @@ from __future__ import annotationsimport copyimport functoolsimport types-from typing import Any, Callable+from collections.abc import Callable+from typing import Anyimport torchimport torch.nn as nndiff --git a/auto_round/algorithms/transforms/spinquant/preprocessor.py b/auto_round/algorithms/transforms/spinquant/preprocessor.pyindex 0c41271b6..2cbec11ea 100644--- a/auto_round/algorithms/transforms/spinquant/preprocessor.py+++ b/auto_round/algorithms/transforms/spinquant/preprocessor.py@@ -23,7 +23,7 @@ from __future__ import annotationsimport loggingimport mathfrom dataclasses import dataclass-from typing import Any, Optional+from typing import Anyimport torchimport torch.nn as nn@@ -92,7 +92,7 @@ class SpinQuantConfig(BaseRotationConfig):# and R4 uses rotation_size instead of intermediate_size.# R2 always uses head_dim, R3 does not support custom size.# This follows the same convention as Quark's rotation_size.- rotation_size: Optional[int] = None+ rotation_size: int | None = None# Rotation matrix type for R1–R4# - False (default): deterministic Hadamard (same matrix every time, no need to persist)@@ -130,7 +130,7 @@ class SpinQuantConfig(BaseRotationConfig):# Numericsdtype: torch.dtype = torch.float32- device: Optional[str] = None+ device: str | None = Nonedef __post_init__(self):if self.device is None:@@ -205,7 +205,7 @@ class SpinQuantPreprocessor:original but with weight distributions better suited for quantisation."""- def __init__(self, model: nn.Module, config: Optional[SpinQuantConfig] = None) -> None:+ def __init__(self, model: nn.Module, config: SpinQuantConfig | None = None) -> None:self.model = modelself.config = config or SpinQuantConfig()@@ -242,7 +242,7 @@ class SpinQuantPreprocessor:# ------------------------------------------------------------------# Main entry point# ------------------------------------------------------------------- def preprocess(self, dataloader: Optional[Any] = None) -> nn.Module:+ def preprocess(self, dataloader: Any | None = None) -> nn.Module:logger.info("[SpinQuant] Starting preprocessing...")logger.info(f"[SpinQuant] Model architecture info: hidden_size={self.hidden_size}, "@@ -337,7 +337,7 @@ class SpinQuantPreprocessor:# Layer-wise API (for block-lifecycle / block-wise quantization)# ------------------------------------------------------------------- def prepare(self, dataloader: Optional[Any] = None) -> None:+ def prepare(self, dataloader: Any | None = None) -> None:"""Global preparation for layer-wise (block-wise) rotation.Performs all lightweight, non-destructive steps:@@ -874,7 +874,7 @@ class SpinQuantPreprocessor:if torch.cuda.is_available():torch.cuda.empty_cache()- def _get_embed_tokens(self) -> Optional[nn.Module]:+ def _get_embed_tokens(self) -> nn.Module | None:"""Get embedding module, supporting both model.embed_tokens and model.model.embed_tokens."""for attr_path in ("embed_tokens", "model.embed_tokens"):parts = attr_path.split(".")@@ -909,7 +909,7 @@ class SpinQuantPreprocessor:yield layerreturn- def _get_lm_head(self) -> Optional[nn.Module]:+ def _get_lm_head(self) -> nn.Module | None:"""Get LM head module."""return getattr(self.model, "lm_head", None)@@ -1386,7 +1386,7 @@ class SpinQuantPreprocessor:# ------------------------------------------------------------------# Helpers# ------------------------------------------------------------------- def _get_rotation_tensor(self, name: str) -> Optional[torch.Tensor]:+ def _get_rotation_tensor(self, name: str) -> torch.Tensor | None:if hasattr(self.model, name):tensor = getattr(self.model, name)if isinstance(tensor, (nn.Parameter, torch.Tensor)):diff --git a/auto_round/algorithms/transforms/spinquant/rotation_utils.py b/auto_round/algorithms/transforms/spinquant/rotation_utils.pyindex 6f5cd91ae..921abed25 100644--- a/auto_round/algorithms/transforms/spinquant/rotation_utils.py+++ b/auto_round/algorithms/transforms/spinquant/rotation_utils.py@@ -13,7 +13,6 @@ linear-algebra helpers.from __future__ import annotationsimport math-from typing import Optional, Tupleimport torchimport torch.nn as nn@@ -58,7 +57,7 @@ def is_pow2(n: int) -> bool:return n > 0 and (n & (n - 1)) == 0-def get_hadamard_K(n: int) -> Tuple[torch.Tensor, int]:+def get_hadamard_K(n: int) -> tuple[torch.Tensor, int]:"""Get the Hadamard matrix and block dimension K for a given input size.For power-of-2 sizes, K=1 (full Walsh-Hadamard via butterfly).@@ -98,7 +97,7 @@ def get_hadamard_K(n: int) -> Tuple[torch.Tensor, int]:)-def matmul_hadU(X: torch.Tensor, hadamard_K: Optional[torch.Tensor] = None, K: Optional[int] = None) -> torch.Tensor:+def matmul_hadU(X: torch.Tensor, hadamard_K: torch.Tensor | None = None, K: int | None = None) -> torch.Tensor:"""Apply normalized Hadamard transform to the last dimension of X.Uses the efficient butterfly algorithm for power-of-2 dimensions,@@ -171,19 +170,19 @@ except ImportError:__all__ = [- "rotate_in_channels_",- "rotate_out_channels_",- "fuse_rmsnorm_in_model",- "untie_word_embeddings_if_needed",+ "InputRotationWrapperHadamard",+ "apply_hadamard_to_linear",+ "create_block_diag_from_head_matrix","deterministic_hadamard_matrix",- "random_hadamard_matrix",- "is_pow2",+ "fuse_rmsnorm_in_model","get_hadamard_K",- "matmul_hadU",- "create_block_diag_from_head_matrix",- "apply_hadamard_to_linear","get_model_arch_info",- "InputRotationWrapperHadamard",+ "is_pow2",+ "matmul_hadU",+ "random_hadamard_matrix",+ "rotate_in_channels_",+ "rotate_out_channels_",+ "untie_word_embeddings_if_needed",]@@ -235,8 +234,8 @@ class InputRotationWrapperHadamard(nn.Module):self,original_module: nn.Linear,rotation_size: int,- hadamard_K: Optional[torch.Tensor] = None,- K: Optional[int] = None,+ hadamard_K: torch.Tensor | None = None,+ K: int | None = None,) -> None:super().__init__()@@ -331,9 +330,9 @@ class InputRotationWrapperHadamard(nn.Module):def rotate_in_channels_(layer: nn.Linear,- rotation_matrix: Optional[torch.Tensor] = None,- R_in: Optional[torch.Tensor] = None,- rotated_modules: Optional[set] = None,+ rotation_matrix: torch.Tensor | None = None,+ R_in: torch.Tensor | None = None,+ rotated_modules: set | None = None,) -> None:"""Fuse an input-side rotation into a linear layer's weight.@@ -387,9 +386,9 @@ def rotate_in_channels_(def rotate_out_channels_(layer: nn.Linear,- rotation_matrix: Optional[torch.Tensor] = None,- R_out: Optional[torch.Tensor] = None,- rotated_modules: Optional[set] = None,+ rotation_matrix: torch.Tensor | None = None,+ R_out: torch.Tensor | None = None,+ rotated_modules: set | None = None,) -> None:"""Fuse an output-side rotation into a linear layer's weight.diff --git a/auto_round/algorithms/transforms/spinquant/serialize.py b/auto_round/algorithms/transforms/spinquant/serialize.pyindex 634619ad8..e88df4f09 100644--- a/auto_round/algorithms/transforms/spinquant/serialize.py+++ b/auto_round/algorithms/transforms/spinquant/serialize.py@@ -29,7 +29,7 @@ import loggingimport mathimport osfrom dataclasses import asdict-from typing import TYPE_CHECKING, Any, Optional+from typing import TYPE_CHECKINGimport torchimport torch.nn as nn@@ -65,7 +65,7 @@ ROTATION_TYPE_TRAINED = 2 # Trained orthogonal, stored as float32def inject_spinquant_buffers(model: nn.Module,- config: "SpinQuantConfig",+ config: SpinQuantConfig,) -> int:"""Inject SpinQuant rotation buffers into QuantLinear modules for serialization.@@ -132,7 +132,7 @@ def inject_spinquant_buffers(def save_spinquant_config(model: nn.Module,save_dir: str,- config: "SpinQuantConfig",+ config: SpinQuantConfig,) -> None:"""Save SpinQuant config into the model's config.json for load-time reconstruction.@@ -298,7 +298,7 @@ def _preregister_buffers_on_module(def rebuild_spinquant_online(model: nn.Module,- config: Optional["SpinQuantConfig"] = None,+ config: SpinQuantConfig | None = None,) -> nn.Module:"""Rebuild online SpinQuant rotations after loading a quantized model.@@ -385,7 +385,6 @@ def _patch_quantlinear_forward_spinquant(model: nn.Module) -> int:Returns:Number of QuantLinear modules with spinquant buffers found."""- global _QUANTLINEAR_PATCHEDn_with_buffers = 0quantlinear_classes = set()@@ -523,7 +522,7 @@ def _inject_rotation_buffers(rotation_size: int,random: bool,is_trained: bool,- rotation_matrix: Optional[torch.Tensor] = None,+ rotation_matrix: torch.Tensor | None = None,) -> None:"""Register rotation buffers on a QuantLinear module.@@ -649,7 +648,7 @@ def _get_r4_target_names(model: nn.Module) -> set:return targets-def _get_stored_rotation(model: nn.Module, param_name: str) -> Optional[torch.Tensor]:+def _get_stored_rotation(model: nn.Module, param_name: str) -> torch.Tensor | None:"""Get a stored rotation matrix/parameter from the model.During preprocessing, rotation matrices are stored as model-level@@ -687,7 +686,7 @@ def _get_intermediate_size(model: nn.Module) -> int:return 0-def _config_to_serializable(config: "SpinQuantConfig", model: nn.Module) -> dict:+def _config_to_serializable(config: SpinQuantConfig, model: nn.Module) -> dict:"""Convert SpinQuantConfig to a JSON-serializable dict with model info."""from auto_round.algorithms.transforms.spinquant.preprocessor import SpinQuantConfig@@ -714,7 +713,7 @@ def _config_to_serializable(config: "SpinQuantConfig", model: nn.Module) -> dictdef _load_config_from_model(model: nn.Module,-) -> Optional["SpinQuantConfig"]:+) -> SpinQuantConfig | None:"""Try to load SpinQuantConfig from model.config."""from auto_round.algorithms.transforms.spinquant.preprocessor import SpinQuantConfigdiff --git a/auto_round/algorithms/transforms/spinquant/training.py b/auto_round/algorithms/transforms/spinquant/training.pyindex e61793315..eac6088e6 100644--- a/auto_round/algorithms/transforms/spinquant/training.py+++ b/auto_round/algorithms/transforms/spinquant/training.py@@ -47,8 +47,9 @@ from __future__ import annotationsimport copyimport loggingimport time+from collections.abc import Callablefrom dataclasses import asdict, dataclass, field-from typing import Any, Callable, Optional+from typing import Anyimport torchimport torch.nn as nn@@ -164,7 +165,7 @@ def create_dual_optimizer(model: nn.Module,lr: float = 1e-4,smooth_lr: float = 1e-3,-) -> Optional[AdamAndSGDG]:+) -> AdamAndSGDG | None:"""Create the Adam (smooth) + SGDG (rotation) dual optimiser.Returns ``None`` if no trainable parameters are found.@@ -215,8 +216,8 @@ def run_training_loop(max_iters: int = 200,loss_type: str = "kl_top",kl_top_k: int = 1000,- compute_loss_fn: Optional[Callable] = None,- on_step_end: Optional[Callable[[int, float, float], None]] = None,+ compute_loss_fn: Callable | None = None,+ on_step_end: Callable[[int, float, float], None] | None = None,log_interval: int = 50,) -> TrainingResult:"""Run the SpinQuant rotation training loop.@@ -338,7 +339,7 @@ class SpinQuantTrainingHook:self.model = modelself.config = config or SpinQuantConfig()self.enabled = enabled- self.preprocessor: Optional[SpinQuantPreprocessor] = None+ self.preprocessor: SpinQuantPreprocessor | None = Nonedef preprocess(self, dataloader: Any) -> nn.Module:"""Execute SpinQuant preprocessing."""@@ -482,11 +483,11 @@ class RotationTrainerConfig:# ---------- Misc ----------dtype: torch.dtype = torch.float32- device: Optional[str] = None+ device: str | None = Nonelog_interval: int = 50 # print every N stepseval_interval: int = 0 # 0 = neversave_interval: int = 0 # 0 = never- checkpoint_dir: Optional[str] = None+ checkpoint_dir: str | None = Nonedef __post_init__(self):if self.device is None:@@ -604,9 +605,9 @@ class RotationTrainer:def __init__(self,model: nn.Module,- config: Optional[RotationTrainerConfig] = None,- callbacks: Optional[list[RotationTrainerCallback]] = None,- compute_loss_fn: Optional[Callable[[torch.Tensor, torch.Tensor, RotationTrainerConfig], torch.Tensor]] = None,+ config: RotationTrainerConfig | None = None,+ callbacks: list[RotationTrainerCallback] | None = None,+ compute_loss_fn: Callable[[torch.Tensor, torch.Tensor, RotationTrainerConfig], torch.Tensor] | None = None,) -> None:from auto_round.algorithms.transforms.spinquant.preprocessor import (SpinQuantConfig,@@ -640,7 +641,7 @@ class RotationTrainer:# Training components (created lazily)self.optimizer = None- self._original_model: Optional[nn.Module] = None+ self._original_model: nn.Module | None = Noneself._hook_handles: list[Any] = []self._rotated_modules: set[nn.Module] = set()self._loss_buffer: list[float] = []@@ -756,7 +757,7 @@ class RotationTrainer:self._preprocessor._cleanup()return self.model- def save_checkpoint(self, path: Optional[str] = None) -> str:+ def save_checkpoint(self, path: str | None = None) -> str:"""Save rotation + smooth params to disk."""if path is None:path = f"{self.config.checkpoint_dir or '.'}/spinquant_ckpt_step{self.state['step']}.pt"diff --git a/auto_round/algorithms/transforms/svdquant/apply.py b/auto_round/algorithms/transforms/svdquant/apply.pyindex b1a767801..fffff0beb 100644--- a/auto_round/algorithms/transforms/svdquant/apply.py+++ b/auto_round/algorithms/transforms/svdquant/apply.py@@ -685,11 +685,10 @@ class SVDQuantTransform(BasePreprocessor):pattern in name or pattern in full_name for pattern in self._target_modules):return False- if self.config.exclude_modules and any(- pattern in name or pattern in full_name for pattern in self.config.exclude_modules- ):- return False- return True+ return not (+ self.config.exclude_modules+ and any(pattern in name or pattern in full_name for pattern in self.config.exclude_modules)+ )@staticmethoddef _new_linear_like(module: torch.nn.Linear, weight: torch.Tensor, bias: torch.Tensor | None):diff --git a/auto_round/algorithms/transforms/svdquant/smooth.py b/auto_round/algorithms/transforms/svdquant/smooth.pyindex 41d6b4c0c..c3d0d4d9f 100644--- a/auto_round/algorithms/transforms/svdquant/smooth.py+++ b/auto_round/algorithms/transforms/svdquant/smooth.py@@ -15,8 +15,9 @@from __future__ import annotationsimport math+from collections.abc import Iterablefrom dataclasses import dataclass-from typing import Iterable, TypeVar+from typing import TypeVarimport torchdiff --git a/auto_round/algorithms/utils.py b/auto_round/algorithms/utils.pyindex 76ce0d304..342c50cf0 100644--- a/auto_round/algorithms/utils.py+++ b/auto_round/algorithms/utils.py@@ -29,7 +29,7 @@ def _is_nvfp4_value(value: Any) -> bool:return "nv_fp" in value or "nvfp4" in value-def _has_nvfp4_layer(orchestrator: "BaseOrchestrator") -> bool:+def _has_nvfp4_layer(orchestrator: BaseOrchestrator) -> bool:"""Whether global or per-layer config enables any NVFP4 quantization."""if _is_nvfp4_value(getattr(orchestrator, "data_type", "")):return Truediff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex 8d23ff152..c5fbfe232 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -19,9 +19,9 @@ import jsonimport mathimport osimport time+from collections.abc import Iterablefrom dataclasses import asdictfrom functools import wraps-from typing import Iterable, Optional, Unionimport torchfrom accelerate import dispatch_model@@ -194,11 +194,11 @@ class AutoSchemeWrapperLinear(WrapperLinear):"""if torch.isnan(grad).any() or torch.isnan(x_diff).any():self.act_cnt -= 1- return None+ return- self.act_score += torch.abs((grad * x_diff.to(grad.device))).sum().item()+ self.act_score += torch.abs(grad * x_diff.to(grad.device)).sum().item()self.mix_score = self.weight_score + self.act_score- return None+ returnif qdq_x.requires_grad:qdq_x.register_hook(save_grad)@@ -210,7 +210,7 @@ class AutoSchemeWrapperLinear(WrapperLinear):returndevice = self.devicewith torch.no_grad():- qdq_w, _, _ = super(AutoSchemeWrapperLinear, self)._qdq_weight(+ qdq_w, _, _ = super()._qdq_weight(torch.tensor(0, device=device), torch.tensor(1.0, device=device), torch.tensor(1.0, device=device))self._score_qdq_cpu = qdq_w.detach().to("cpu")@@ -280,7 +280,6 @@ class AutoSchemeWrapperLinear(WrapperLinear):w_diff = weight.to(grad.device) - self._score_qdq_cpu.to(grad.device)self.weight_score += torch.abs(grad.to(w_diff.device) * w_diff).sum().item()self.mix_score = self.weight_score + self.act_score- return Noneqdq_w.register_hook(save_grad)return qdq_w, 1.0, None@@ -357,9 +356,8 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):def save_grad(grad):"""Backward hook: accumulate weight score from grad * (weight - qdq_w)."""w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)- self.weight_score += torch.abs((grad.to(torch.float32) * w_diff.to(grad.device))).sum().item()+ self.weight_score += torch.abs(grad.to(torch.float32) * w_diff.to(grad.device)).sum().item()self.mix_score = self.weight_score + self.act_score- return Noneself.qdq_w.requires_grad_(True)self.orig_layer.weight.requires_grad_(False)@@ -395,11 +393,11 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):"""if torch.isnan(grad).any() or torch.isnan(x_diff).any():self.act_cnt -= 1- return None+ return- self.act_score += torch.abs((grad * x_diff.to(grad.device))).sum().item()+ self.act_score += torch.abs(grad * x_diff.to(grad.device)).sum().item()self.mix_score = self.weight_score + self.act_score- return None+ returnif qdq_x.requires_grad:qdq_x.register_hook(save_grad)@@ -450,9 +448,8 @@ class AutoSchemeWrapperLinearForGGUFK(AutoSchemeWrapperLinear):"""Backward hook: accumulate weight score from grad * (weight - qdq_w)."""w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)# TODO strange, grad could be in CPU- self.weight_score += torch.abs((grad.to(w_diff.device).to(torch.float32) * w_diff)).sum().item()+ self.weight_score += torch.abs(grad.to(w_diff.device).to(torch.float32) * w_diff).sum().item()self.mix_score = self.weight_score + self.act_score- return Noneself.qdq_w.requires_grad_(True)self.orig_layer.weight.requires_grad_(False)@@ -506,9 +503,8 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):def save_grad(grad):"""Backward hook: accumulate weight score from grad * (weight - qdq_w)."""w_diff = self.orig_layer.weight - self.qdq_w.to(self.orig_layer.weight.device)- self.weight_score += torch.abs((grad.to(torch.float32) * w_diff.to(grad.device))).sum().item()+ self.weight_score += torch.abs(grad.to(torch.float32) * w_diff.to(grad.device)).sum().item()self.mix_score = self.weight_score + self.act_score- return Noneself.qdq_w.requires_grad_(True)self.orig_layer.weight.requires_grad_(False)@@ -620,9 +616,7 @@ def cal_imatrix_low_gpu(model, dataloader, major_device):clear_memory(device_list=major_device)all_move_device_hooks = []- i = 0for block_name in block_names:- i += 1block_module = get_module(model, block_name)hook_move_gpu = block_module.register_forward_pre_hook(move_to_gpu_hook)@@ -651,7 +645,7 @@ class MyCustomError(Exception):super().__init__(message)-def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_device="cpu", disk_index=None):+def prepare_model_low_gpu(model, block_inputs: dict | None = None, pbar=None, major_device="cpu", disk_index=None):"""Wrap every block's forward so that, for one calibration batch, it (1) moves itself to``major_device`` on demand, (2) records its own inputs into ``block_inputs`` (on CPU) sothey can be replayed later, and (3) moves itself back to CPU once done.@@ -1199,12 +1193,12 @@ def get_score_for_scheme(low_gpu_mem_usage=True,major_device="cpu",batch_size=1,- offload_context: Optional[OffloadManager] = None,+ offload_context: OffloadManager | None = None,processor=None,is_vlm: bool = False,force_mllm: bool = False,- model_name: Optional[str] = None,- scheme_tag: Optional[str] = None,+ model_name: str | None = None,+ scheme_tag: str | None = None,disk_index=None,):"""Wrap every quantizable layer in ``quant_layer_names`` with a scoring wrapper, run@@ -1612,7 +1606,7 @@ def get_score_for_scheme(return scores_dict-def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = None):+def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int | None = None):"""Args:layers: A dict mapping each layer name to a list of candidate options.@@ -1632,7 +1626,7 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = None# the entire path on every transition, which becomes quadratic for large# models; linked nodes keep each transition O(1) and are expanded only once.dp: dict[int, tuple[float, tuple]] = {0: (0.0, ())}- for layer_name, opts in layers.items():+ for opts in layers.values():new_dp: dict[int, tuple[float, tuple]] = {}for cur_params, (cur_loss, cur_path) in dp.items():for opt in opts:@@ -1675,7 +1669,7 @@ def choose_bits_per_layer_with_path(layers: dict, P: int, max_states: int = Nonestep = (n - 1) / (max_states - 1)selected: dict[int, tuple[float, tuple]] = {}for i in range(max_states):- idx = int(round(i * step))+ idx = round(i * step)if idx >= n:idx = n - 1k = sorted_keys[idx]@@ -2534,7 +2528,7 @@ def _score_scheme_worker(args):def _gen_layer_config(auto_scheme: AutoScheme,- model: Union[str, torch.nn.Module],+ model: str | torch.nn.Module,quant_layer_names: Iterable[str],fixed_layer_scheme: dict[str, dict],min_avg_bit_scheme,@@ -2548,7 +2542,7 @@ def _gen_layer_config(processor=None,is_vlm: bool = False,disk_index=None,- export_format: str = None,+ export_format: str | None = None,):"""Score every candidate scheme in ``auto_scheme.options`` against ``quant_layer_names``and return per-layer per-scheme losses used by the caller to pick a final bit-width@@ -3607,7 +3601,7 @@ def _enforce_w8_symmetric_entries(layer_config: dict, allow_w8_asym: bool = Fals@register_scheme_methods(("default", "DeltaLoss"))def gen_layer_config(auto_scheme: AutoScheme,- model: Union[str, torch.nn.Module],+ model: str | torch.nn.Module,quant_layer_names: Iterable[str],fixed_layer_scheme: dict[str, dict],dataset: str = "pile-10k",@@ -3617,7 +3611,7 @@ def gen_layer_config(low_gpu_mem_usage=True,min_avg_bit_scheme=None,processor=None,- export_format: str = None,+ export_format: str | None = None,**kwargs,):"""Public AutoScheme entry.diff --git a/auto_round/auto_scheme/gen_auto_scheme.py b/auto_round/auto_scheme/gen_auto_scheme.pyindex d212aca52..7e47f30a7 100644--- a/auto_round/auto_scheme/gen_auto_scheme.py+++ b/auto_round/auto_scheme/gen_auto_scheme.py@@ -12,8 +12,9 @@# See the License for the specific language governing permissions and# limitations under the License.+from collections.abc import Iterablefrom dataclasses import dataclass-from typing import Iterable, Optional, Union+from typing import Unionimport torch@@ -27,17 +28,17 @@ from auto_round.utils import get_layer_features, get_module@dataclassclass AutoScheme:- avg_bits: Union[float, list[float]]- options: Union[str, list[Union[QuantizationScheme, str]], tuple[Union[QuantizationScheme, str], ...]]- shared_layers: Optional[Iterable[Iterable[str]]] = None+ avg_bits: float | list[float]+ options: str | list[QuantizationScheme | str] | tuple[QuantizationScheme | str, ...]+ shared_layers: Iterable[Iterable[str]] | None = Nonemethod: str = "default"ignore_scale_zp_bits: bool = False- batch_size: Optional[int] = None- nsamples: Optional[int] = None- seqlen: Optional[int] = None- dataset: Optional[str] = None # Import Notice no comma for each item- device_map: Optional[Union[str, torch.device, int, dict]] = None- enable_torch_compile: Optional[bool] = None+ batch_size: int | None = None+ nsamples: int | None = None+ seqlen: int | None = None+ dataset: str | None = None # Import Notice no comma for each item+ device_map: str | torch.device | int | dict | None = None+ enable_torch_compile: bool | None = Nonelow_gpu_mem_usage: bool = Truelow_cpu_mem_usage: bool = True@@ -94,11 +95,11 @@ class GenScheme:quant_layer_names: Iterable[str],fixed_layer_scheme: dict[str, dict],dataset: str = "pile-10k",- device_map: Union[str, torch.device, int, dict, None] = None,+ device_map: str | torch.device | int | dict | None = None,tokenizer=None,enable_torch_compile=True,processor=None,- export_format: str = None,+ export_format: str | None = None,):self.auto_scheme = auto_scheme# Export-format context for the generation-time 8-bit-asym policy: thediff --git a/auto_round/auto_scheme/utils.py b/auto_round/auto_scheme/utils.pyindex c8e4f5e2a..6ffd840d9 100644--- a/auto_round/auto_scheme/utils.py+++ b/auto_round/auto_scheme/utils.py@@ -14,8 +14,8 @@import loggingimport mathimport re+from collections.abc import Iterablefrom dataclasses import asdict, fields-from typing import Iterable, Optional, Unionimport torchfrom accelerate import dispatch_model, infer_auto_device_map@@ -44,7 +44,7 @@ def apply_quant_scheme(model: torch.nn.Module,quant_layer_names: Iterable[str],fixed_layer_scheme: dict[str, dict],- scheme: Union[str, dict], # TODO add scale_dtype+ scheme: str | dict, # TODO add scale_dtype) -> None:"""Apply a quantization scheme to each quantized layer.@@ -91,7 +91,7 @@ def compute_avg_bits_for_scheme(model: torch.nn.Module,quant_layer_names: Iterable[str],fixed_layer_scheme: dict[str, dict],- scheme: Union[str, dict, None] = None,+ scheme: str | dict | None = None,ignore_scale_zp_bits: bool = False,clean_scheme: bool = True,) -> tuple[float, float]:@@ -341,7 +341,7 @@ def parse_shared_layers(model: torch.nn.Module, shared_patterns: Iterable[Iterabreturn matched_groups-def _expert_key_from_layer_name(layer_name: str) -> Optional[str]:+def _expert_key_from_layer_name(layer_name: str) -> str | None:"""Map one MoE-related linear layer to a unique expert key.Gate/up/down projections belonging to the same expert should map to one key.@@ -489,7 +489,7 @@ def _fill_inactive_expert_scores(scores_dict: dict[str, list[float]], block_nameif not active_expert_avg_losses:continuefill_value = sum(active_expert_avg_losses) / len(active_expert_avg_losses)- for _, expert_stats in expert_stats_map.items():+ for expert_stats in expert_stats_map.values():if expert_stats["has_active"]:continuefor layer_name in expert_stats["layers"]:@@ -500,8 +500,8 @@ def _log_score_summary_by_block_and_nonblock(scores_dict: dict[str, list[float]],block_names: list[str],model=None,- scheme_tag: Optional[str] = None,- summary_stage: Optional[str] = None,+ scheme_tag: str | None = None,+ summary_stage: str | None = None,):"""Log a per-block (and non-block) breakdown of ``scores_dict`` losses at debug level."""if not scores_dict:diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 4826d1b0c..8ffcc85a6 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -16,7 +16,7 @@ from __future__ import annotationsimport functoolsimport inspect-from typing import TYPE_CHECKING, Any, Optional, Union+from typing import TYPE_CHECKING, Anyimport torch@@ -74,7 +74,7 @@ def _get_compressor_class(model_type: str, base_cls: type) -> type:return combined-def _resolve_quant_config_for_routing(alg_configs) -> tuple[list, list, "QuantizationConfig"]:+def _resolve_quant_config_for_routing(alg_configs) -> tuple[list, list, QuantizationConfig]:from auto_round.algorithms.config_resolver import split_quantization_configsfrom auto_round.algorithms.quantization.config import QuantizationConfigfrom auto_round.algorithms.quantization.rtn.config import RTNConfig@@ -151,7 +151,7 @@ def _build_model_type_ctor_kwargs(model, base_kwargs, mllm_kwargs, diffusion_kwareturn model_type, ctor_kwargs-def _select_rtn_compressor_base_cls(quant_config: "RTNConfig", scheme, format, base_kwargs) -> type:+def _select_rtn_compressor_base_cls(quant_config: RTNConfig, scheme, format, base_kwargs) -> type:from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfigfrom auto_round.auto_scheme.gen_auto_scheme import AutoSchemefrom auto_round.compressors.orchestrator import CompressionOrchestrator as Compressor@@ -193,9 +193,7 @@ def _select_rtn_compressor_base_cls(quant_config: "RTNConfig", scheme, format, b# the plain min/max initialization ignores itif getattr(quant_config, "enable_neuqi", False):enable_imatrix = True- elif data_type == "int" and (bits is None or bits < 8):- enable_imatrix = True- elif is_weight_scheme(scheme):+ elif (data_type == "int" and (bits is None or bits < 8)) or is_weight_scheme(scheme):enable_imatrix = Trueact_bits = resolved_attrs.get("act_bits")@@ -277,7 +275,7 @@ def _iter_registered_alg_configs() -> list[tuple[str, type]]:return result[email protected]_cache(maxsize=None)def _discover_alg_config_fields(config_cls: type) -> frozenset:"""Discover accepted config fields without maintaining per-algorithm lists."""from pydantic import BaseModel@@ -508,7 +506,7 @@ def _prepare_entry_kwargs(alg_configs, direct_kwargs):return configs, runtime_kwargs-class _CompressorBuilder(object):+class _CompressorBuilder:"""Algorithm-config-driven entry point (``scheme`` + ``alg_configs``).This is the internal pipeline entry: it resolves the algorithm config(s),@@ -519,7 +517,7 @@ class _CompressorBuilder(object):"""@classmethod- def _resolve_config(cls, config: Union[str, object, list]) -> Union[object, list[object]]:+ def _resolve_config(cls, config: str | object | list) -> object | list[object]:"""Convert string alias(es) to the corresponding config instance(s) with default parameters."""from auto_round.algorithms.registry import resolve_alg_config@@ -531,24 +529,24 @@ class _CompressorBuilder(object):def __new__(cls,- model: Union[torch.nn.Module, str],+ model: torch.nn.Module | str,scheme="W4A16",- alg_configs: Union[str, object, list[Union[str, object]]] = None,+ alg_configs: str | object | list[str | object] = None,tokenizer=None,platform="hf",format=None,dataset="NeelNanda/pile-10k",low_gpu_mem_usage: bool = False,- device_map: Union[str, torch.device, int, dict] = 0,- iters: int = None,+ device_map: str | torch.device | int | dict = 0,+ iters: int | None = None,enable_torch_compile: bool = False,seed: int = 42,low_cpu_mem_usage: bool = True,layer_config=None,- nsamples: int = None,- seqlen: int = None,+ nsamples: int | None = None,+ seqlen: int | None = None,**kwargs,- ) -> "BaseCompressor":+ ) -> BaseCompressor:from auto_round.algorithms.quantization.rtn.config import OptimizedRTNConfig, RTNConfigfrom auto_round.algorithms.quantization.sign_round.config import SignRoundConfigfrom auto_round.algorithms.registry import normalize_algorithm_config@@ -728,28 +726,28 @@ class AutoRound:def __new__(cls,- model: Union[torch.nn.Module, str],+ model: torch.nn.Module | str,tokenizer=None,platform: str = "hf",- scheme: Union[str, dict, QuantizationScheme, "AutoScheme"] = "W4A16",- schemes: Union[str, list, tuple, None] = None,- bits: Union[int, float, None] = None,- layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,- dataset: Optional[Union[str, list, tuple, torch.utils.data.DataLoader]] = None,+ scheme: str | dict | QuantizationScheme | AutoScheme = "W4A16",+ schemes: str | list | tuple | None = None,+ bits: float | None = None,+ layer_config: dict[str, str | dict | QuantizationScheme] | None = None,+ dataset: str | list | tuple | torch.utils.data.DataLoader | None = None,iters: int | None = None,seqlen: int = 2048,nsamples: int = 128,batch_size: int = 8,gradient_accumulate_steps: int | None = None,low_gpu_mem_usage: bool = False,- device_map: Union[str, torch.device, int, dict] = 0,- enable_torch_compile: Optional[bool] = None,+ device_map: str | torch.device | int | dict = 0,+ enable_torch_compile: bool | None = None,seed: int = 42,low_cpu_mem_usage: bool = True,alg_configs=None,algorithm: str | None = None,**kwargs,- ) -> "BaseCompressor":+ ) -> BaseCompressor:direct_kwargs = dict(kwargs)legacy_device = direct_kwargs.pop("device", None)if legacy_device is not None:diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 82373687c..d2bc379fa 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -460,7 +460,7 @@ def get_github_code_clean_dataset("💡 This dataset uses an old script-based format. To load it, please install `datasets<=3.6.0`:\n\n")else:- raise error+ raisecalib_dataset = concatenate_datasets([dataset_mit, dataset_apache])calib_dataset = calib_dataset.shuffle(seed=seed).take(10000) ##TODO concat data'shuffle may have bugscalib_dataset = calib_dataset.map(tokenizer_function, batched=True)@@ -584,7 +584,7 @@ def get_ultrachat_dataset(split = "train_sft"all_splits = ["train_sft", "test_sft", "train_gen", "test_gen"]if split not in all_splits:- raise ValueError("split must be one of {} for ultrachat_200k ".format(all_splits))+ raise ValueError(f"split must be one of {all_splits} for ultrachat_200k ")dataset = load_dataset("HuggingFaceH4/ultrachat_200k", split=split, streaming=True, trust_remote_code=True)dataset = dataset.shuffle(seed=seed).take(20000)@@ -765,8 +765,8 @@ def get_mbpp_dataset(if isinstance(splits, str):splits = splits.split("+")- for split in splits:- dataset = load_dataset(dataset_name, split=split)+ for split_name in splits:+ dataset = load_dataset(dataset_name, split=split_name)for data in dataset:samples.append({"text": data["text"] + data["code"]})random.Random(seed).shuffle(samples)@@ -1021,9 +1021,7 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seedreturn Falseinput_ids = example["input_ids"][:seqlen]input_ids_list = input_ids.tolist()- if len(input_ids_list) > 1 and seqlen > 2 and input_ids_list.count(input_ids_list[-1]) > seqlen // 2:- return False- return True+ return not (len(input_ids_list) > 1 and seqlen > 2 and input_ids_list.count(input_ids_list[-1]) > seqlen // 2)def concat_dataset_element(dataset):input_ids, concat_input_ids = [eg["input_ids"] for eg in dataset], []@@ -1085,9 +1083,9 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seedif key == "num":data_lens[name] = int(values[0])if key == "concat":- do_concat = False if (len(values) > 0 and values[0].lower() == "false") else True+ do_concat = not (len(values) > 0 and values[0].lower() == "false")if key == "apply_chat_template":- apply_chat_template = False if (len(values) > 0 and values[0].lower() == "false") else True+ apply_chat_template = not (len(values) > 0 and values[0].lower() == "false")if key == "system_prompt":system_prompt = values[0]apply_chat_template = True@@ -1095,15 +1093,15 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seedget_dataset = CALIB_DATASETS.get("local")else:calib_name = name- if name not in CALIB_DATASETS.keys():+ if name not in CALIB_DATASETS:calib_name = name.split("/")[-1]- for key in CALIB_DATASETS.keys():+ for key in CALIB_DATASETS:if calib_name in key:calib_name = keybreakget_dataset = CALIB_DATASETS.get(calib_name)if get_dataset is None:- filtered_keys = [k for k in CALIB_DATASETS.keys() if "/" not in k]+ filtered_keys = [k for k in CALIB_DATASETS if "/" not in k]raise ValueError(f"Dataset '{name}' is not found. Please choose from the supported datasets: {filtered_keys}.")diff --git a/auto_round/calibration/__init__.py b/auto_round/calibration/__init__.pyindex 37baed84e..5efa7ed2d 100644--- a/auto_round/calibration/__init__.py+++ b/auto_round/calibration/__init__.py@@ -23,9 +23,9 @@ from auto_round.calibration import mllm as _mllm # noqa: F401from auto_round.calibration import diffusion as _diffusion # noqa: F401__all__ = [- "Calibrator",- "CalibrationContext","CALIBRATORS",+ "CalibrationContext",+ "Calibrator","get_calibrator","register_calibrator",]diff --git a/auto_round/calibration/base.py b/auto_round/calibration/base.pyindex 583a385f6..539d9f418 100644--- a/auto_round/calibration/base.py+++ b/auto_round/calibration/base.py@@ -26,7 +26,8 @@ Extension points for new calibration strategies:"""from abc import ABC, abstractmethod-from typing import TYPE_CHECKING, Callable+from collections.abc import Callable+from typing import TYPE_CHECKINGimport torchdiff --git a/auto_round/calibration/diffusion.py b/auto_round/calibration/diffusion.pyindex 29fca0141..c278872bc 100644--- a/auto_round/calibration/diffusion.py+++ b/auto_round/calibration/diffusion.py@@ -23,6 +23,7 @@ and customises:import inspectimport os+import sysimport torchfrom tqdm import tqdm@@ -134,7 +135,7 @@ class DiffusionCalibrator(LLMCalibrator):"Please use model path for quantization or ""move the pipeline object to GPU/XPU before passing them into API.")- exit(-1)+ sys.exit(-1)target_device = device_manager.deviceself._cpu_offload_mode = _prepare_pipeline_for_calibration(@@ -189,7 +190,7 @@ class DiffusionCalibrator(LLMCalibrator):f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "f"dataset or decease the sequence length")- exit(-1)+ sys.exit(-1)elif total_cnt < nsamples:logger.warning(f"Insufficient number of samples collected may affect the quantization. "diff --git a/auto_round/calibration/inputs.py b/auto_round/calibration/inputs.pyindex ee7ede386..5b05c9185 100644--- a/auto_round/calibration/inputs.py+++ b/auto_round/calibration/inputs.py@@ -13,14 +13,12 @@# limitations under the License."""Pure helpers for shaping cached block inputs."""-from typing import Tuple-import torchfrom auto_round.utils import clear_memory, to_device, to_dtypefrom auto_round.utils.device_manager import device_manager-__all__ = ["split_inputs", "preprocess_block_inputs"]+__all__ = ["preprocess_block_inputs", "split_inputs"]def split_inputs(@@ -29,7 +27,7 @@ def split_inputs(*,is_diffusion: bool,shared_cache_keys: tuple = (),-) -> Tuple[object, dict]:+) -> tuple[object, dict]:"""Split a captured ``inputs`` dict into ``(input_ids, input_others)``.Mirrors the original ``Compressor._split_inputs`` exactly:@@ -71,7 +69,7 @@ def preprocess_block_inputs(model_context,compress_context,first_input_name: str = "input_ids",-) -> Tuple[object, dict]:+) -> tuple[object, dict]:"""Move/cast cached block inputs onto the calibration cache device.Mirrors the original ``Compressor._preprocess_block_inputs`` exactly.diff --git a/auto_round/calibration/llm.py b/auto_round/calibration/llm.pyindex 9bd45ae2d..287d8c769 100644--- a/auto_round/calibration/llm.py+++ b/auto_round/calibration/llm.py@@ -18,9 +18,10 @@ Implements ``try_cache_inter_data_gpucpu`` / ``cache_inter_data`` /``self.compressor.X``."""+import sysimport traceback+from collections.abc import Callablefrom functools import partial-from typing import Callableimport accelerateimport torch@@ -90,7 +91,7 @@ class LLMCalibrator(Calibrator):if "flash_attn::" in error_msg and "CPU" in error_msg:cannot_calibrate_on_cpu = Trueelse:- raise error+ raiseif not calibrate_on_cpu or cannot_calibrate_on_cpu:try:@@ -178,7 +179,7 @@ class LLMCalibrator(Calibrator):except torch.OutOfMemoryError as e:if cannot_calibrate_on_cpu:- raise e+ raisecuda_error_msg = traceback.format_exc()try:logger.info("switch to cpu to cache block inputs")@@ -319,13 +320,13 @@ class LLMCalibrator(Calibrator):elif isinstance(data, str):if self.tokenizer is None:logger.error("please provide tokenizer for string input")- exit(-1)+ sys.exit(-1)data = self.tokenizer(data, truncation=True, max_length=self.seqlen, return_tensors="pt").datadata_new = {}for key in data.keys():data_new[key] = data[key].to(self.model.device)input_ids = data_new["input_ids"]- elif isinstance(data, tuple) or isinstance(data, list):+ elif isinstance(data, (tuple, list)):data_new = to_device(data, self.model.device)input_ids = data_new[0]else:@@ -406,7 +407,7 @@ class LLMCalibrator(Calibrator):if isinstance(data_new, torch.Tensor):self.model(data_new, **kwargs)- elif isinstance(data_new, tuple) or isinstance(data_new, list):+ elif isinstance(data_new, (tuple, list)):self.model(*data_new, **kwargs)else:self.model(**data_new, **kwargs)@@ -428,9 +429,9 @@ class LLMCalibrator(Calibrator):"When quantization encounters tensor shape mismatch error, ""you can try to avoid it with batch_size=1")- raise error+ raiseexcept Exception as error:- raise error+ raisetotal_cnt += input_ids.shape[0] if len(input_ids.shape) > 1 else 1if total_cnt >= nsamples:@@ -440,7 +441,7 @@ class LLMCalibrator(Calibrator):f"no data has been cached, please provide more data with sequence length "f">={self.seqlen} in the dataset or decease the sequence length")- exit(-1)+ sys.exit(-1)elif total_cnt < nsamples:logger.warning_once(f"An insufficient number of samples likely reduces the accuracy of the quantized model. "@@ -487,17 +488,13 @@ class LLMCalibrator(Calibrator):" or try to set the `batch_size` to 1 and ""`gradient_accumulate_steps` to your current batch size.")- exit(-1)+ sys.exit(-1)if hidden_states is not None:kwargs["hidden_states"] = hidden_states- for key in kwargs.keys():- if (- isinstance(kwargs[key], torch.Tensor)- or isinstance(kwargs[key], list)- or isinstance(kwargs[key], tuple)- ):+ for key, value in kwargs.items():+ if isinstance(value, (torch.Tensor, list, tuple)):if (self.has_variable_block_shapeand name not in self.blocks_requiring_input_ids@@ -505,7 +502,7 @@ class LLMCalibrator(Calibrator):):continueif key not in self.inputs[name].keys(): # initialization- data = to_device(kwargs[key], device=torch.device("cpu"))+ data = to_device(value, device=torch.device("cpu"))if data is None or key in self.shared_cache_keys:self.inputs[name][key] = datacontinue@@ -518,7 +515,7 @@ class LLMCalibrator(Calibrator):else:self.inputs[name][key] = [data]else: # append cache inputs- new_data = post_process_cache_data(self.batch_size, kwargs[key], key)+ new_data = post_process_cache_data(self.batch_size, value, key)if new_data is None: # shareable args or NoneTypeif key in self.shared_cache_keys:# Shared keys are normally the same across samples. However@@ -526,7 +523,7 @@ class LLMCalibrator(Calibrator):# varies per image because each image has a different patch count.# Upgrade from shared (raw value) to per-sample list storage so# each sample gets its own positional embeddings.- raw_new = to_device(kwargs[key], device=torch.device("cpu"))+ raw_new = to_device(value, device=torch.device("cpu"))stored = self.inputs[name].get(key)if isinstance(stored, list):stored.append(raw_new)@@ -544,9 +541,9 @@ class LLMCalibrator(Calibrator):self.inputs[name][key].extend(list(torch.split(new_data, 1, dim=self.batch_dim)))else:self.inputs[name][key].append(new_data)- elif isinstance(kwargs[key], (str, bool, type(None))):+ elif isinstance(value, (str, bool, type(None))):if key not in self.inputs[name].keys():- self.inputs[name][key] = kwargs[key]+ self.inputs[name][key] = valueelse:# Parameters not to be cachedif check_skippable_keywords(key):@@ -561,7 +558,7 @@ class LLMCalibrator(Calibrator):if hidden_states is not None:kwargs.pop("hidden_states", None)if positional_inputs:- return m.orig_forward(hidden_states=hidden_states, *positional_inputs, **kwargs)+ return m.orig_forward(*positional_inputs, hidden_states=hidden_states, **kwargs)else:return m.orig_forward(hidden_states, **kwargs)else:diff --git a/auto_round/calibration/mllm.py b/auto_round/calibration/mllm.pyindex 4eb3ea80b..24493aa20 100644--- a/auto_round/calibration/mllm.py+++ b/auto_round/calibration/mllm.py@@ -22,6 +22,8 @@ MLLM-specific runtime state (``template`` / ``extra_data_dir`` /``self.compressor``."""+import sys+import torchfrom auto_round.calibration.llm import LLMCalibrator@@ -65,7 +67,7 @@ class MLLMCalibrator(LLMCalibrator):if hasattr(self.model, "name_or_path"):name = self.model.name_or_path- if any([m in name for m in MISTRAL_3_2_MODELS]):+ if any(m in name for m in MISTRAL_3_2_MODELS):self.template = "mistral3_2"template_name = self.template@@ -207,6 +209,6 @@ class MLLMCalibrator(LLMCalibrator):if total_cnt == 0:logger.error("no data has been cached, please provide more data")- exit(-1)+ sys.exit(-1)elif total_cnt < nsamples:logger.warning(f"Insufficient number of samples: required {nsamples}, but only {total_cnt} were processed.")diff --git a/auto_round/calibration/register.py b/auto_round/calibration/register.pyindex cd8dd4900..d4a0ded30 100644--- a/auto_round/calibration/register.py+++ b/auto_round/calibration/register.py@@ -18,17 +18,15 @@ Mirrors the same pattern used elsewhere in the codebase (e.g.decorating its class with ``@register_calibrator("my_kind")``."""-from typing import Type-from auto_round.calibration.base import Calibrator-CALIBRATORS: dict[str, Type[Calibrator]] = {}+CALIBRATORS: dict[str, type[Calibrator]] = {}def register_calibrator(name: str):"""Class decorator: register a ``Calibrator`` subclass under ``name``."""- def _wrap(cls: Type[Calibrator]) -> Type[Calibrator]:+ def _wrap(cls: type[Calibrator]) -> type[Calibrator]:if not issubclass(cls, Calibrator):raise TypeError(f"{cls.__name__} must subclass auto_round.calibration.base.Calibrator")cls.name = name@@ -40,7 +38,7 @@ def register_calibrator(name: str):return _wrap-def get_calibrator(name: str) -> Type[Calibrator]:+def get_calibrator(name: str) -> type[Calibrator]:"""Look up a registered calibrator class by name."""if name not in CALIBRATORS:raise KeyError(f"No calibrator registered under '{name}'. " f"Known: {sorted(CALIBRATORS.keys())}")diff --git a/auto_round/calibration/state.py b/auto_round/calibration/state.pyindex d3c027027..0984363b3 100644--- a/auto_round/calibration/state.py+++ b/auto_round/calibration/state.py@@ -29,7 +29,7 @@ The dataclass also provides a behavioural helper:"""from dataclasses import dataclass, field-from typing import TYPE_CHECKING, Any, Optional+from typing import TYPE_CHECKING, Anyif TYPE_CHECKING:from auto_round.compressors.base import BaseOrchestrator@@ -53,7 +53,7 @@ class CalibrationContext:# blocks_requiring_input_ids: list = field(default_factory=list)#- batch_dim: Optional[int] = None+ batch_dim: int | None = None# ── Calibration parameters ─────────────────────────────────────────────batch_size: int = 8diff --git a/auto_round/calibration/utils.py b/auto_round/calibration/utils.pyindex 24707564d..113e059e8 100644--- a/auto_round/calibration/utils.py+++ b/auto_round/calibration/utils.py@@ -45,7 +45,7 @@ def _update_inputs(inputs: dict, q_inputs: dict) -> tuple[dict, torch.Tensor]:model_context = ModelContext()if model_context.is_diffusion:- input_id_str = [key for key in inputs.keys() if "hidden_state" in key]+ input_id_str = [key for key in inputs if "hidden_state" in key]if input_id_str == ["hidden_states"]:if q_inputs is not None:q_inputs = q_inputs.pop("hidden_states", None)@@ -70,7 +70,7 @@ def _update_inputs(inputs: dict, q_inputs: dict) -> tuple[dict, torch.Tensor]:def _split_inputs_diffusion(inputs: dict) -> tuple[dict, dict]:"""Split inputs for diffusion models that only have hidden_states."""- input_id_str = [key for key in inputs.keys() if "hidden_state" in key]+ input_id_str = [key for key in inputs if "hidden_state" in key]if input_id_str == ["hidden_states"]:input_ids = inputs.pop("hidden_states", None)input_others = inputsdiff --git a/auto_round/cli/algorithms.py b/auto_round/cli/algorithms.pyindex 0136a2e44..23f9dccb8 100644--- a/auto_round/cli/algorithms.py+++ b/auto_round/cli/algorithms.py@@ -127,7 +127,7 @@ def _merge_parameter(merged, parameter):if not overlapping_options and merged.dest != parameter.dest:return Noneif merged.dest != parameter.dest or _argument_compatibility_key(merged) != _argument_compatibility_key(parameter):- option = sorted(overlapping_options)[0] if overlapping_options else merged.dest+ option = min(overlapping_options) if overlapping_options else merged.destraise ValueError(f"incompatible shared CLI argument {option!r}")kwargs = dict(merged.argparse_kwargs)kwargs["default"] = argparse.SUPPRESSdiff --git a/auto_round/compressors/__init__.py b/auto_round/compressors/__init__.pyindex 7965acf4b..d8849f999 100644--- a/auto_round/compressors/__init__.py+++ b/auto_round/compressors/__init__.py@@ -22,8 +22,8 @@ if TYPE_CHECKING:from auto_round.compressors.orchestrator import CompressionOrchestrator__all__ = [- "BaseOrchestrator","BaseCompressor", # backward-compat alias+ "BaseOrchestrator","CompressionOrchestrator","ModelFreeCompressor",]diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 46fd3df87..5c3d3de4d 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -16,7 +16,7 @@ import gcimport osimport sysfrom dataclasses import asdict, dataclass, fields, replace-from typing import Any, Optional, Union+from typing import Anyimport torchfrom transformers import AutoConfig, set_seed@@ -91,39 +91,39 @@ MIN_ITERS_FOR_TORCH_COMPILE = 10@dataclassclass SerializedCompressorConfig:- bits: Optional[int] = None- act_bits: Optional[int] = None- data_type: Optional[str] = None- act_data_type: Optional[str] = None- group_size: Optional[int] = None- act_group_size: Optional[int] = None- sym: Optional[bool] = None- act_sym: Optional[bool] = None- act_dynamic: Optional[bool] = None- amp: Optional[bool] = None- batch_size: Optional[int] = None- enable_minmax_tuning: Optional[bool] = True- enable_norm_bias_tuning: Optional[bool] = False- enable_quanted_input: Optional[bool] = True- gradient_accumulate_steps: Optional[int] = None- iters: Optional[int] = None- lr: Optional[float] = None- low_gpu_mem_usage: Optional[bool] = None- minmax_lr: Optional[float] = None- nsamples: Optional[int] = None- quant_block_list: Optional[list[str]] = None- regex_config: Optional[dict[str, Any]] = None- scale_dtype: Optional[str] = None- seqlen: Optional[int] = None- supported_types: Optional[list[str]] = SUPPORTED_LAYER_TYPES- static_attention_dtype: Optional[str] = None- static_kv_dtype: Optional[str] = None- static_attention_granularity: Optional[str] = "tensor"- static_kv_granularity: Optional[str] = "tensor"- super_bits: Optional[int] = None- super_group_size: Optional[int] = None- to_quant_block_names: Optional[list[str]] = None- rotation_configs: Optional[list[dict[str, Any]]] = None+ bits: int | None = None+ act_bits: int | None = None+ data_type: str | None = None+ act_data_type: str | None = None+ group_size: int | None = None+ act_group_size: int | None = None+ sym: bool | None = None+ act_sym: bool | None = None+ act_dynamic: bool | None = None+ amp: bool | None = None+ batch_size: int | None = None+ enable_minmax_tuning: bool | None = True+ enable_norm_bias_tuning: bool | None = False+ enable_quanted_input: bool | None = True+ gradient_accumulate_steps: int | None = None+ iters: int | None = None+ lr: float | None = None+ low_gpu_mem_usage: bool | None = None+ minmax_lr: float | None = None+ nsamples: int | None = None+ quant_block_list: list[str] | None = None+ regex_config: dict[str, Any] | None = None+ scale_dtype: str | None = None+ seqlen: int | None = None+ supported_types: list[str] | None = SUPPORTED_LAYER_TYPES+ static_attention_dtype: str | None = None+ static_kv_dtype: str | None = None+ static_attention_granularity: str | None = "tensor"+ static_kv_granularity: str | None = "tensor"+ super_bits: int | None = None+ super_group_size: int | None = None+ to_quant_block_names: list[str] | None = None+ rotation_configs: list[dict[str, Any]] | None = NoneSERIALIZATION_KEYS = tuple(field.name for field in fields(SerializedCompressorConfig))@@ -190,7 +190,7 @@ def _formats_policy_string_of(formats) -> str:return ",".join(names)-class BaseOrchestrator(object):+class BaseOrchestrator:need_calib: bool = Truecompress_context: CompressContext = Nonemodel_context: ModelContext = None@@ -212,13 +212,13 @@ class BaseOrchestrator(object):locals()[_scheme_field] = _make_compressor_scheme_property(_scheme_field)@staticmethod- def _preload_model_config(model: Union[torch.nn.Module, str], trust_remote_code: bool) -> Optional[AutoConfig]:+ def _preload_model_config(model: torch.nn.Module | str, trust_remote_code: bool) -> AutoConfig | None:if not isinstance(model, str):return Nonetry:return AutoConfig.from_pretrained(model, trust_remote_code=trust_remote_code)- except (OSError, EnvironmentError, ValueError) as e:+ except (OSError, ValueError) as e:logger.debug("Failed to load config via AutoConfig.from_pretrained for %s: %s. ""Proceeding without config-based checks.",@@ -229,25 +229,25 @@ class BaseOrchestrator(object):def __init__(self,- config: Union[object, list[object]],- model: Union[torch.nn.Module, str],+ config: object | list[object],+ model: torch.nn.Module | str,tokenizer: Any = None,platform: str = "hf",- format: Union[str, list, None] = None,- scheme: Union[str, dict, QuantizationScheme, AutoScheme] = "W4A16",+ format: str | list | None = None,+ scheme: str | dict | QuantizationScheme | AutoScheme = "W4A16",low_gpu_mem_usage: bool = False,- device_map: Union[str, torch.device, int, dict] = 0,- enable_torch_compile: Optional[bool] = None,+ device_map: str | torch.device | int | dict = 0,+ enable_torch_compile: bool | None = None,seed: int = 42,low_cpu_mem_usage: bool = True,- layer_config: Optional[dict] = None,- nsamples: int = None,- seqlen: int = None,- scale_dtype: Optional[Union[str, torch.dtype]] = None,+ layer_config: dict | None = None,+ nsamples: int | None = None,+ seqlen: int | None = None,+ scale_dtype: str | torch.dtype | None = None,ignore_layers: str = "",quant_lm_head: bool = False,- to_quant_block_names: Optional[Union[str, list[str]]] = None,- dataset: Optional[Union[str, list, tuple, torch.utils.data.DataLoader]] = None,+ to_quant_block_names: str | list[str] | None = None,+ dataset: str | list | tuple | torch.utils.data.DataLoader | None = None,**kwargs,) -> None:# ``CalibrationContext`` is the single source of truth for calibration@@ -584,10 +584,7 @@ class BaseOrchestrator(object):# Layer-level scheme overrides can request static-activation paths# (e.g., global MXFP8 + local NVFP4 experts). Those still need# calibration data even when top-level scheme looks dynamic.- if self._layer_config_needs_calibration(check_need_act_calibration):- return True-- return False+ return self._layer_config_needs_calibration(check_need_act_calibration)def _layer_config_needs_calibration(self, check_need_act_calibration) -> bool:"""Return True if any raw layer_config entry implies activation calibration."""@@ -647,12 +644,12 @@ class BaseOrchestrator(object):self.__dict__["compression_plan"] = replace(plan, **changes)@property- def scheme_context(self) -> Optional[QuantizationScheme]:+ def scheme_context(self) -> QuantizationScheme | None:plan = self.__dict__.get("compression_plan")return plan.scheme.value if plan is not None else self.__dict__.get("_scheme_context")@scheme_context.setter- def scheme_context(self, value: Optional[QuantizationScheme]) -> None:+ def scheme_context(self, value: QuantizationScheme | None) -> None:self.__dict__["_scheme_context"] = valueplan = self.__dict__.get("compression_plan")if plan is not None and value is not None:@@ -675,7 +672,7 @@ class BaseOrchestrator(object):self._replace_compression_plan(formats=tuple(value))@property- def layer_config(self) -> Optional[dict]:+ def layer_config(self) -> dict | None:plan = self.__dict__.get("compression_plan")if plan is None:return self.__dict__.get("_layer_config")@@ -688,7 +685,7 @@ class BaseOrchestrator(object):self._replace_compression_plan(layer_config=value)@property- def regex_config(self) -> Optional[dict]:+ def regex_config(self) -> dict | None:plan = self.__dict__.get("compression_plan")if plan is None:return self.__dict__.get("_regex_config")@@ -738,8 +735,8 @@ class BaseOrchestrator(object):def resolve_scheme(self,- model_context: Optional[ModelContext] = None,- compress_context: Optional[CompressContext] = None,+ model_context: ModelContext | None = None,+ compress_context: CompressContext | None = None,) -> None:"""Phase-1 init: resolve scheme and bind config attrs (no model structure needed).@@ -1238,7 +1235,7 @@ class BaseOrchestrator(object):"'enable_torch_compile' is disabled. Enabling it can reduce tuning cost by about 20%.",)- def _torch_compile_disabled_reason(self, ignore_user_override: bool = False) -> Optional[str]:+ def _torch_compile_disabled_reason(self, ignore_user_override: bool = False) -> str | None:"""Return why torch.compile must stay off for the current algorithm, else None.RTN and optimized RTN quantize each layer in a single pass, and very short@@ -1287,7 +1284,7 @@ class BaseOrchestrator(object):return None- def _torch_compile_unsupported_arch_reason(self) -> Optional[str]:+ def _torch_compile_unsupported_arch_reason(self) -> str | None:"""Return why the model *architecture* forbids ``torch.compile``, else ``None``.Rules live in :mod:`auto_round.special_model_handler` so a new architecture can@@ -1530,7 +1527,7 @@ class BaseOrchestrator(object):return self._alg_composer@staticmethod- def _resolve_gguf_preset_string(formats: list["OutputFormat"]) -> Optional[str]:+ def _resolve_gguf_preset_string(formats: list["OutputFormat"]) -> str | None:"""Return the precise GGUF preset string (e.g. ``"gguf:q4_k_m"``) for thesingle resolved GGUF format, or ``None`` if no GGUF format is present.@@ -1775,7 +1772,7 @@ class BaseOrchestrator(object):return device_manager.device@device.setter- def device(self, value: Union[str, torch.device]) -> None:+ def device(self, value: str | torch.device) -> None:device_manager.device = value@property@@ -1840,7 +1837,7 @@ class BaseOrchestrator(object):logger.warning("reset low_cpu_mem_usage to False due to tied weights")returnif len(tied_weight_keys) == 1:- key = list(tied_weight_keys.keys())[0]+ key = next(iter(tied_weight_keys))if "lm_head" not in key:self.compress_context.is_immediate_saving = Falseif self.compress_context.low_cpu_mem_usage:@@ -1907,11 +1904,11 @@ class BaseOrchestrator(object):def save_quantized(self,- output_dir: str = None,- format: Union[str, list[OutputFormat]] = None,+ output_dir: str | None = None,+ format: str | list[OutputFormat] | None = None,inplace: bool = True,return_folders: bool = False,- max_shard_size: Union[int, str] = None,+ max_shard_size: int | str | None = None,**kwargs,) -> torch.nn.Module:"""Save the quantized model to the specified output directory in the specified format.@@ -1946,9 +1943,9 @@ class BaseOrchestrator(object):if isinstance(self.formats, str):self.formats = self._resolve_format_string(self.formats)self.compress_context.formats = self.formats- for format in self.formats:- save_folder = _get_save_folder_name(format)- if self.act_bits <= 8 and format.is_fake():+ for output_format in self.formats:+ save_folder = _get_save_folder_name(output_format)+ if self.act_bits <= 8 and output_format.is_fake():logger.warning("Support for exporting activation quantization is limited. ""Please ensure that your configuration is supported."@@ -1997,7 +1994,7 @@ class BaseOrchestrator(object):original_block_name, reverted_block_name)- compressed_model = format.save_quantized(+ compressed_model = output_format.save_quantized(save_folder,model=self.model_context.model,layer_config=self.layer_config,@@ -2100,9 +2097,9 @@ class BaseOrchestrator(object):def quantize_and_save(self,output_dir: str = "tmp_autoround",- format: str = None,+ format: str | None = None,inplace: bool = True,- max_shard_size: Union[int, str] = None,+ max_shard_size: int | str | None = None,**kwargs,) -> tuple[torch.nn.Module, dict[str, Any]]:"""Quantizes the model and saves it in the specified format(s).diff --git a/auto_round/compressors/config_resolution/__init__.py b/auto_round/compressors/config_resolution/__init__.pyindex dcdbb3b74..44332aef0 100644--- a/auto_round/compressors/config_resolution/__init__.py+++ b/auto_round/compressors/config_resolution/__init__.py@@ -27,11 +27,11 @@ from auto_round.compressors.config_resolution.errors import (from auto_round.compressors.config_resolution.resolve import resolve_quantization_config, resolve_scheme_value__all__ = [- "ResolvedQuantizationConfig",+ "ConfigResolutionError","FormatCompatibilityError","FormatResolution","LayerConfigResolutionError",- "ConfigResolutionError",+ "ResolvedQuantizationConfig","ResolvedScheme","SchemeResolutionError","resolve_quantization_config",diff --git a/auto_round/compressors/config_resolution/contracts.py b/auto_round/compressors/config_resolution/contracts.pyindex f095dc936..e4cd2b74c 100644--- a/auto_round/compressors/config_resolution/contracts.py+++ b/auto_round/compressors/config_resolution/contracts.py@@ -15,14 +15,15 @@from __future__ import annotationsimport copy+from collections.abc import Mappingfrom dataclasses import dataclass, fieldfrom types import MappingProxyType-from typing import Any, Mapping, Optional, Tuple+from typing import Anyfrom auto_round.schemes import QuantizationSchemeLayerConfig = Mapping[str, Mapping[str, Any]]-BlockGroups = Tuple[Tuple[str, ...], ...]+BlockGroups = tuple[tuple[str, ...], ...]def _deepcopy_mapping_proxy(value: MappingProxyType, memo: dict) -> MappingProxyType:@@ -40,7 +41,7 @@ def _deepcopy_mapping_proxy(value: MappingProxyType, memo: dict) -> MappingProxycopy._deepcopy_dispatch[MappingProxyType] = _deepcopy_mapping_proxy-def freeze_mapping(value: Optional[LayerConfig]) -> LayerConfig:+def freeze_mapping(value: LayerConfig | None) -> LayerConfig:"""Return an isolated, read-only snapshot of a layer configuration mapping.Per-layer configuration values are usually dicts (e.g. ``{"bits": 4}``), but some@@ -57,7 +58,7 @@ def freeze_mapping(value: Optional[LayerConfig]) -> LayerConfig:return MappingProxyType(frozen)-def thaw_mapping(value: Optional[LayerConfig]) -> dict:+def thaw_mapping(value: LayerConfig | None) -> dict:"""Return a fully mutable, deep-copyable plain-dict snapshot of a frozen mapping.This is the inverse of :func:`freeze_mapping` and should be used instead of@@ -71,7 +72,7 @@ def thaw_mapping(value: Optional[LayerConfig]) -> dict:return result-def freeze_block_groups(value: Optional[Tuple[Tuple[str, ...], ...]]) -> Optional[BlockGroups]:+def freeze_block_groups(value: tuple[tuple[str, ...], ...] | None) -> BlockGroups | None:"""Freeze the two-dimensional block grouping used by model traversal."""if value is None:return None@@ -81,7 +82,7 @@ def freeze_block_groups(value: Optional[Tuple[Tuple[str, ...], ...]]) -> Optiona@dataclass(frozen=True)class ResolvedScheme:_value: QuantizationScheme- preset_name: Optional[str] = None+ preset_name: str | None = Nonedef __post_init__(self) -> None:object.__setattr__(self, "_value", copy.deepcopy(self._value))@@ -92,17 +93,17 @@ class ResolvedScheme:return copy.deepcopy(self._value)@classmethod- def from_scheme(cls, value: QuantizationScheme, preset_name: Optional[str] = None) -> "ResolvedScheme":+ def from_scheme(cls, value: QuantizationScheme, preset_name: str | None = None) -> ResolvedScheme:return cls(_value=value, preset_name=preset_name)@dataclass(frozen=True)class FormatResolution:- formats: Tuple[Any, ...]+ formats: tuple[Any, ...]scheme: ResolvedSchemelayer_config_patch: LayerConfig = field(default_factory=lambda: MappingProxyType({}))scale_dtype: Any = None- quant_block_list: Optional[BlockGroups] = None+ quant_block_list: BlockGroups | None = Nonedef __post_init__(self) -> None:object.__setattr__(self, "formats", tuple(self.formats))@@ -113,12 +114,12 @@ class FormatResolution:@dataclass(frozen=True)class ResolvedQuantizationConfig:scheme: ResolvedScheme- formats: Tuple[Any, ...]+ formats: tuple[Any, ...]layer_config: LayerConfigregex_config: LayerConfig = field(default_factory=lambda: MappingProxyType({}))has_qlayer_outside_block: bool = Falsescale_dtype: Any = None- quant_block_list: Optional[BlockGroups] = None+ quant_block_list: BlockGroups | None = Nonedef __post_init__(self) -> None:object.__setattr__(self, "formats", tuple(self.formats))diff --git a/auto_round/compressors/config_resolution/resolve.py b/auto_round/compressors/config_resolution/resolve.pyindex 4d501ae0c..6e1d9882e 100644--- a/auto_round/compressors/config_resolution/resolve.py+++ b/auto_round/compressors/config_resolution/resolve.py@@ -15,7 +15,8 @@from __future__ import annotationsimport copy-from typing import Any, Mapping+from collections.abc import Mapping+from typing import Anyfrom auto_round.compressors.config_resolution.contracts import (FormatResolution,diff --git a/auto_round/compressors/diffusion/dataset.py b/auto_round/compressors/diffusion/dataset.pyindex 0b73b177b..16348eb53 100644--- a/auto_round/compressors/diffusion/dataset.py+++ b/auto_round/compressors/diffusion/dataset.py@@ -18,7 +18,6 @@ import zipfilefrom concurrent.futures import ThreadPoolExecutorfrom io import StringIOfrom pathlib import Path-from typing import Dict, Optionalimport pandas as pdimport torch@@ -27,7 +26,7 @@ from transformers import set_seedfrom auto_round.utils import download_audiocaps_csv, logger-DIFFUSION_DATASET: Dict[str, Dataset] = {}+DIFFUSION_DATASET: dict[str, Dataset] = {}COCO_URL = {@@ -228,7 +227,7 @@ class Text2ImgDataset(Dataset):self,dataset_path: str,nsamples: int = 128,- dataframe: Optional[pd.DataFrame] = None,+ dataframe: pd.DataFrame | None = None,) -> None:super().__init__()self.captions = []@@ -272,7 +271,7 @@ class Text2ImgDataset(Dataset):def __len__(self):return len(self.captions)- def __getitem__(self, i) -> Dict[str, torch.Tensor]:+ def __getitem__(self, i) -> dict[str, torch.Tensor]:if self.image_paths is not None:return self.caption_ids[i], self.captions[i], self.image_paths[i]return self.caption_ids[i], self.captions[i]@@ -312,7 +311,7 @@ class AudioCapsDataset(Dataset):def __len__(self):return len(self.captions)- def __getitem__(self, i) -> Dict[str, torch.Tensor]:+ def __getitem__(self, i) -> dict[str, torch.Tensor]:return self.caption_ids[i], self.captions[i]diff --git a/auto_round/compressors/diffusion_mixin.py b/auto_round/compressors/diffusion_mixin.pyindex 415bfcbc7..7716bf1eb 100644--- a/auto_round/compressors/diffusion_mixin.py+++ b/auto_round/compressors/diffusion_mixin.py@@ -14,7 +14,7 @@import jsonimport mathimport os-from typing import Any, Optional, Union+from typing import Anyimport torch@@ -63,8 +63,8 @@ class DiffusionMixin:guidance_scale: float = 7.5,num_inference_steps: int = 50,calib_num_inference_steps: int = 8,- generator_seed: Optional[int] = None,- diffusion_tuning_cache_size: Union[float, str] = 0,+ generator_seed: int | None = None,+ diffusion_tuning_cache_size: float | str = 0,**kwargs,) -> None:if num_inference_steps < 1:@@ -422,8 +422,8 @@ class DiffusionMixin:def save_quantized(self,- output_dir: Optional[str] = None,- format: Optional[Union[str, list]] = None,+ output_dir: str | None = None,+ format: str | list | None = None,inplace: bool = True,return_folders: bool = False,**kwargs,diff --git a/auto_round/compressors/layer_config_resolver.py b/auto_round/compressors/layer_config_resolver.pyindex f98129b80..4a70576e2 100644--- a/auto_round/compressors/layer_config_resolver.py+++ b/auto_round/compressors/layer_config_resolver.py@@ -417,7 +417,7 @@ def resolve_layer_config(enable_gguf_official_mixed: bool = True,is_mllm: bool = False,fill_default_value: bool = True,- format: str = None,+ format: str | None = None,) -> LayerConfig:"""Resolve final per-layer configuration without writing model attributes."""layer_config = expand_layer_config_for_weight_renames(layer_config, model=model, to_model_names=True)@@ -488,7 +488,7 @@ def extract_regex_config(inner_supported_types=None,ignore_layers: str = "",fill_default_value: bool = True,- format: str = None,+ format: str | None = None,) -> LayerConfig:"""Resolve only the regex entries retained for export metadata."""layer_config = expand_layer_config_for_weight_renames(layer_config, model=model, to_model_names=True)@@ -545,8 +545,7 @@ def apply_plan_to_model(model, plan: ResolvedQuantizationConfig) -> None:# (``AttributeError`` on the norm's next forward). Scope the reset to the# same modules that receive the plan below.is_quant_target = (- isinstance(module, SUPPORTED_LAYER_TYPES)- or isinstance(module, torch.nn.Embedding)+ isinstance(module, (SUPPORTED_LAYER_TYPES, torch.nn.Embedding))or module.__class__.__name__ in INNER_SUPPORTED_LAYER_TYPES)if module_name != "" and not is_quant_target:diff --git a/auto_round/compressors/mllm/__init__.py b/auto_round/compressors/mllm/__init__.pyindex 1abc19f37..90fdab74f 100644--- a/auto_round/compressors/mllm/__init__.py+++ b/auto_round/compressors/mllm/__init__.py@@ -17,10 +17,10 @@ from auto_round.compressors.mllm.processor import PROCESSORS, BasicProcessor, refrom auto_round.compressors.mllm.template import TEMPLATES, Template, get_template__all__ = [- "BasicProcessor","MLLM_DATASET","PROCESSORS","TEMPLATES",+ "BasicProcessor","Template","get_mllm_dataloader","get_template",diff --git a/auto_round/compressors/mllm/dataset.py b/auto_round/compressors/mllm/dataset.pyindex fec57f44a..e5bd4b178 100644--- a/auto_round/compressors/mllm/dataset.py+++ b/auto_round/compressors/mllm/dataset.py@@ -14,9 +14,10 @@import jsonimport os+import sysfrom collections import OrderedDictfrom inspect import signature-from typing import Any, Dict, Optional+from typing import Anyimport torchfrom torch.utils.data import DataLoader, Dataset@@ -28,7 +29,7 @@ from auto_round.special_model_handler import check_mllm_only_support_bs1from .template import Templatefrom .utils import _extract_data_dir-MLLM_DATASET: Dict[str, Dataset] = {}+MLLM_DATASET: dict[str, Dataset] = {}def register_dataset(name_list):@@ -81,7 +82,7 @@ class LlavaDataset(Dataset):model: torch.nn.Module,tokenizer: Any,dataset_path: str,- extra_data_dir: Optional[str] = None,+ extra_data_dir: str | None = None,seqlen: int = 512,padding: bool = True,truncation: bool = True,@@ -95,7 +96,8 @@ class LlavaDataset(Dataset):self.tokenizer = tokenizerif os.path.exists(dataset_path):logger.info(f"use dataset {dataset_path}, loading from disk...")- self.questions = json.load(open(dataset_path, "r"))+ with open(dataset_path, "r") as f:+ self.questions = json.load(f)else:if dataset_path == "liuhaotian/llava":dataset_path = "llava_conv_58k"@@ -186,8 +188,7 @@ class LlavaDataset(Dataset):if self.IMAGE_TOKEN in text["value"]:text["value"] = self.IMAGE_TOKEN + text["value"].replace(self.IMAGE_TOKEN, "")str_len += len(text["value"].split(" "))- if str_len > max_len:- max_len = str_len+ max_len = max(max_len, str_len)if min_word_len <= str_len < max_word_len:new_questions.append(source)if len(new_questions) >= nsamples:@@ -211,7 +212,7 @@ class LlavaDataset(Dataset):def __len__(self):return len(self.questions)- def __getitem__(self, i) -> Dict[str, torch.Tensor]:+ def __getitem__(self, i) -> dict[str, torch.Tensor]:if self.cached_data_dict is not None and i in self.cached_data_dict:self.cached_data_dict.move_to_end(i)return self.cached_data_dict[i]@@ -304,7 +305,7 @@ def get_mllm_dataloader(template, model=model, tokenizer=tokenizer, processor=processor, image_processor=image_processor)- if os.path.isfile(dataset) or dataset in MLLM_DATASET.keys():+ if os.path.isfile(dataset) or dataset in MLLM_DATASET:if seqlen > MLLM_DATASET[dataset].MAX_SUPPORT_SEQLEN:logger.warning(f"seqlen({seqlen}) is greater than the maximum length supported by the {dataset},"@@ -341,5 +342,5 @@ def get_mllm_dataloader("Text only dataset cannot be used for calibrating non-text modules,"" switching to liuhaotian/llava_conv_58k")- exit(-1)+ sys.exit(-1)return dataloader, bs, seqlendiff --git a/auto_round/compressors/mllm/processor.py b/auto_round/compressors/mllm/processor.pyindex 00ea1da25..7f91af57b 100644--- a/auto_round/compressors/mllm/processor.py+++ b/auto_round/compressors/mllm/processor.py@@ -565,8 +565,9 @@ class Mistral3Processor(BasicProcessor):file_path = hf_hub_download(repo_id=repo_id_or_path, filename=filename)with open(file_path, "r") as file:system_prompt = file.read()- today = datetime.today().strftime("%Y-%m-%d")- yesterday = (datetime.today() - timedelta(days=1)).strftime("%Y-%m-%d")+ now = datetime.now().astimezone()+ today = now.strftime("%Y-%m-%d")+ yesterday = (now - timedelta(days=1)).strftime("%Y-%m-%d")model_name = repo_id_or_path.split("/")[-1]return system_prompt.format(name=model_name, today=today, yesterday=yesterday)diff --git a/auto_round/compressors/mllm/template.py b/auto_round/compressors/mllm/template.pyindex 31a75612b..e3ece7b14 100644--- a/auto_round/compressors/mllm/template.py+++ b/auto_round/compressors/mllm/template.py@@ -16,13 +16,12 @@ import jsonimport osfrom dataclasses import dataclassfrom enum import Enum, unique-from typing import Dict, List, Optionalfrom auto_round.logger import loggerfrom .processor import PROCESSORS, BasicProcessor-TEMPLATES: Dict[str, "Template"] = {}+TEMPLATES: dict[str, "Template"] = {}def fill_content(target, **kwargs):@@ -50,7 +49,7 @@ class Template:format_observation: strformat_separator: strdefault_system: str- replace_tokens: List[tuple]+ replace_tokens: list[tuple]extra_encode: booldefault_dataset: strprocessor: "BasicProcessor"@@ -80,16 +79,16 @@ class Template:def _register_template(model_type: str,- format_user: Optional[str] = None,- format_assistant: Optional[str] = None,- format_system: Optional[str] = None,- format_function: Optional[str] = None,- format_observation: Optional[str] = None,- format_separator: Optional[str] = None,+ format_user: str | None = None,+ format_assistant: str | None = None,+ format_system: str | None = None,+ format_function: str | None = None,+ format_observation: str | None = None,+ format_separator: str | None = None,default_system: str = "",- replace_tokens: List[tuple] = None,- extra_encode: Optional[bool] = False,- default_dataset: Optional[bool] = "NeelNanda/pile-10k",+ replace_tokens: list[tuple] | None = None,+ extra_encode: bool | None = False,+ default_dataset: bool | None = "NeelNanda/pile-10k",processor: "BasicProcessor" = PROCESSORS["basic"],):"""Registers a chat template."""diff --git a/auto_round/compressors/mllm/utils.py b/auto_round/compressors/mllm/utils.pyindex eda37e92b..2729a4fa1 100644--- a/auto_round/compressors/mllm/utils.py+++ b/auto_round/compressors/mllm/utils.py@@ -43,7 +43,7 @@ def _extract_data_dir(dir_path: str):def fetch_image(path_or_url):if os.path.isfile(path_or_url):image_obj = Image.open(path_or_url)- elif path_or_url.startswith("http://") or path_or_url.startswith("https://"):+ elif path_or_url.startswith(("http://", "https://")):try:response = requests.get(path_or_url, stream=True, timeout=(3, 10))response.raise_for_status()diff --git a/auto_round/compressors/mllm_mixin.py b/auto_round/compressors/mllm_mixin.pyindex 4f61f83d6..59581dd03 100644--- a/auto_round/compressors/mllm_mixin.py+++ b/auto_round/compressors/mllm_mixin.py@@ -12,7 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.-from typing import Any, Optional, Union+from typing import Anyfrom auto_round.logger import logger@@ -49,8 +49,8 @@ class MLLMMixin:*args,processor: Any = None,image_processor: Any = None,- template: Optional[str] = None,- extra_data_dir: Optional[str] = None,+ template: str | None = None,+ extra_data_dir: str | None = None,quant_nontext_module: bool = False,**kwargs,) -> None:@@ -103,8 +103,8 @@ class MLLMMixin:def save_quantized(self,- output_dir: Optional[str] = None,- format: Union[str, list] = "auto_round",+ output_dir: str | None = None,+ format: str | list = "auto_round",inplace: bool = True,**kwargs,) -> Any:diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyindex 89ff392e7..9d5c54f4b 100644--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -110,7 +110,7 @@ import tempfileimport timefrom concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, ThreadPoolExecutor, as_completed, waitfrom dataclasses import asdict, fields-from typing import Any, Optional, Union+from typing import Anyimport torchfrom safetensors import safe_open@@ -545,14 +545,14 @@ class _ModelFreeCompressorCore:self,model_name_or_path: str,output_dir: str,- scheme: Union[str, QuantizationScheme] = "W4A16",- layer_config: Optional[dict] = None,+ scheme: str | QuantizationScheme = "W4A16",+ layer_config: dict | None = None,ignore_layers: str = "",- format: Optional[str] = None,+ format: str | None = None,device: str = "cpu",quant_lm_head: bool = False,quant_nontext_module: bool = False,- enable_torch_compile: Optional[bool] = None,+ enable_torch_compile: bool | None = None,disable_opt_rtn: bool = False,) -> None:# --- raw inputs ---@@ -790,7 +790,7 @@ class _ModelFreeCompressorCore:if incompatible:# Group by class for a cleaner warning messageincompatible_layers = []- for cls, layers in incompatible.items():+ for layers in incompatible.values():incompatible_layers.extend(layers)summary = ", ".join(f"{cls}({len(layers)})" for cls, layers in sorted(incompatible.items()))self.ignore_patterns.extend(incompatible_layers)@@ -1758,11 +1758,11 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):def __init__(self,model_name_or_path: str,- output_dir: Optional[str] = None,- scheme: Union[str, QuantizationScheme] = "W4A16",- layer_config: Optional[dict] = None,+ output_dir: str | None = None,+ scheme: str | QuantizationScheme = "W4A16",+ layer_config: dict | None = None,ignore_layers: str = "",- format: Optional[str] = None,+ format: str | None = None,device: str = "cpu",quant_lm_head: bool = False,quant_nontext_module: bool = False,@@ -1770,7 +1770,7 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):tokenizer: Any = None,device_map: Any = None,low_cpu_mem_usage: bool = True,- enable_torch_compile: Optional[bool] = None,+ enable_torch_compile: bool | None = None,disable_opt_rtn: bool = False,**kwargs,) -> None:@@ -1811,7 +1811,7 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):# Compressor-role state (mirrors BaseCompressor attributes used by# AutoRound's post-processing code)- self._output_dir_override: Optional[str] = None # set by quantize_and_save+ self._output_dir_override: str | None = None # set by quantize_and_saveself.model = Noneself.tokenizer = tokenizerself.model_free = True@@ -1857,9 +1857,9 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):# AutoScheme (two-phase delta-loss selection) state.self._auto_scheme_resolved = False- self._auto_scheme_family: Optional[str] = None+ self._auto_scheme_family: str | None = None- def _fallback_to_base_compressor(self, save_format: Optional[str] = None):+ def _fallback_to_base_compressor(self, save_format: str | None = None):from auto_round.autoround import AutoRoundlogger.info(diff --git a/auto_round/compressors/orchestrator.py b/auto_round/compressors/orchestrator.pyindex 350c7b486..96d1a262a 100644--- a/auto_round/compressors/orchestrator.py+++ b/auto_round/compressors/orchestrator.py@@ -16,7 +16,7 @@ import gcimport osimport timefrom functools import partial-from typing import TYPE_CHECKING, Any, Optional, Union+from typing import TYPE_CHECKING, Any, Optionalimport accelerateimport torch@@ -69,15 +69,15 @@ class CompressionOrchestrator(BaseOrchestrator):def __init__(self,- config: Union[object, list[object]], # TODO rename this to alg_config wenhuach- model: Union[torch.nn.Module, str],+ config: object | list[object], # TODO rename this to alg_config wenhuach+ model: torch.nn.Module | str,tokenizer: Any = None,platform: str = "hf",- format: Union[str, list, None] = None,- dataset: Optional[Union[str, list, tuple, torch.utils.data.DataLoader]] = None,+ format: str | list | None = None,+ dataset: str | list | tuple | torch.utils.data.DataLoader | None = None,low_gpu_mem_usage: bool = False,- device_map: Union[str, torch.device, int, dict] = 0,- enable_torch_compile: Optional[bool] = None,+ device_map: str | torch.device | int | dict = 0,+ enable_torch_compile: bool | None = None,seed: int = 42,low_cpu_mem_usage: bool = True,**kwargs,@@ -126,8 +126,8 @@ class CompressionOrchestrator(BaseOrchestrator):self,block_names: list,nsamples: int,- layer_names: Optional[list] = None,- last_cache_name: Optional[str] = None,+ layer_names: list | None = None,+ last_cache_name: str | None = None,) -> Any:"""Thin wrapper around ``self.calibration.collect``.@@ -520,7 +520,7 @@ class CompressionOrchestrator(BaseOrchestrator):self.alg_composer.finalize_run()remain_layer_names = []- block_name_set = set(name for block in all_blocks for name in block)+ block_name_set = {name for block in all_blocks for name in block}for n, m in self.model.named_modules():if not check_to_quantized(m):continue@@ -605,7 +605,7 @@ class CompressionOrchestrator(BaseOrchestrator):all_q_inputs = None# Leave it to gguf itself to handleif has_gguf and self.alg_composer.need_quanted_input(): # pylint: disable=E1101- is_quantized_embedding = self.alg_composer.compress_embedding_layer() #+ is_quantized_embedding = self.alg_composer.compress_embedding_layer()clear_memory()if is_quantized_embedding:all_inputs = copy.deepcopy(self.inputs)@@ -985,8 +985,8 @@ class CompressionOrchestrator(BaseOrchestrator):self,block: torch.nn.Module,inputs: Any,- q_input: Union[torch.Tensor, dict, None] = None,- device: Union[str, torch.device] = "cpu",+ q_input: torch.Tensor | dict | None = None,+ device: str | torch.device = "cpu",auto_offload: bool = True,reference_output=None,) -> Any:diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.pyindex 6232b3d88..5e26844d1 100644--- a/auto_round/compressors/shard_writer.py+++ b/auto_round/compressors/shard_writer.py@@ -16,7 +16,7 @@ import jsonimport osimport refrom collections import OrderedDict-from typing import Optional, Union+from typing import Optionalimport torch@@ -59,7 +59,7 @@ class ShardWriter:self,model: torch.nn.Module,bits: int,- max_shard_size: Optional[Union[int, str]] = None,+ max_shard_size: int | str | None = None,safe_serialization: bool = True,) -> None:if ShardWriter._initialized:@@ -218,7 +218,7 @@ class ShardWriter:logger.warning("safetensors not installed; falling back to torch.save.")return False- def save_module(self, m: torch.nn.Module, name: str = None) -> None:+ def save_module(self, m: torch.nn.Module, name: str | None = None) -> None:"""Extracts and accumulates tensors from a module."""prefix = name if name is not None else getattr(m, "global_name", "model")sd = m.state_dict()@@ -483,7 +483,7 @@ class ShardWriter:logger.info(f"model has been saved to {self.output_dir}")@torch.no_grad()- def write(self, m: torch.nn.Module = None, name: str = None, is_finalize: bool = False) -> None:+ def write(self, m: torch.nn.Module = None, name: str | None = None, is_finalize: bool = False) -> None:if m is None and name is None and not is_finalize and not is_finalize:raise ValueError("Must specify either name or m")if m is None and name is not None:diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex a0593bab0..eb149e6e3 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -123,7 +123,7 @@ def block_forward(amp_dtype: torch.dtype = torch.float16,device: torch.device = torch.device("cpu"),output_return_id: int = 0,-) -> Union[torch.Tensor, dict]:+) -> torch.Tensor | dict:"""Performs a forward pass through a block with the given inputs.Args:@@ -178,7 +178,7 @@ def block_forward(output = block(**input_others)else:output = block(**input_others)- if isinstance(output_return_id, int) and (isinstance(output, list) or isinstance(output, tuple)):+ if isinstance(output_return_id, int) and isinstance(output, (list, tuple)):output = output[output_return_id]return output@@ -195,11 +195,11 @@ def check_skippable_keywords(key):def check_need_act_calibration(- is_act_dynamic: Union[bool, None],- act_data_type: Union[str, None] = None,- act_bits: Union[int, None] = 16,- static_kv_dtype: Union[str, None] = None,- static_attention_dtype: Union[str, None] = None,+ is_act_dynamic: bool | None,+ act_data_type: str | None = None,+ act_bits: int | None = 16,+ static_kv_dtype: str | None = None,+ static_attention_dtype: str | None = None,) -> bool:if static_kv_dtype is not None or static_attention_dtype is not None:return True@@ -208,9 +208,7 @@ def check_need_act_calibration(# None is dynamicif is_act_dynamic is not None and not is_act_dynamic:return True- if act_data_type is not None and "static" in act_data_type:- return True- return False+ return act_data_type is not None and "static" in act_data_typedef collect_best_params(block, cache_device="cpu"):@@ -485,11 +483,7 @@ def _get_diffusion_save_folder_name(format) -> str:formats = compress_context.formats# Use a subfolder only if there are multiple formatsif len(formats) > 1:- return (- os.path.join(compress_context.output_dir, sanitized_format, "transformer")- if compress_context.is_immediate_saving- else os.path.join(compress_context.output_dir, sanitized_format, "transformer")- )+ return os.path.join(compress_context.output_dir, sanitized_format, "transformer")# if use is_immediate_saving, we need to save model in self.output_dir/transformer folderreturn (diff --git a/auto_round/context/compress.py b/auto_round/context/compress.pyindex 6cb0bfbca..d90036c48 100644--- a/auto_round/context/compress.py+++ b/auto_round/context/compress.py@@ -11,7 +11,7 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-from typing import Callable, Optional, Union+from collections.abc import Callableimport torch@@ -36,10 +36,10 @@ class CompressContext(BaseContext):enable_torch_compile: bool = True,is_immediate_packing: bool = False,is_immediate_saving: bool = False,- formats: Union[list, str] = None,+ formats: list | str | None = None,output_dir: str = "./compressed_models",- static_kv_dtype: Optional[torch.dtype] = None, # TODO later this should be scheme wenhuach- static_attention_dtype: Optional[torch.dtype] = None,+ static_kv_dtype: torch.dtype | None = None, # TODO later this should be scheme wenhuach+ static_attention_dtype: torch.dtype | None = None,static_kv_granularity: str = "tensor",static_attention_granularity: str = "tensor",**kwargs,diff --git a/auto_round/context/model.py b/auto_round/context/model.pyindex 7fb5c2338..a4c6c03a4 100644--- a/auto_round/context/model.py+++ b/auto_round/context/model.py@@ -15,7 +15,8 @@import gcimport importlibimport os-from typing import Any, Callable, Optional, Union+from collections.abc import Callable+from typing import Anyimport torchimport transformers@@ -61,12 +62,12 @@ class ModelContext(BaseContext):def __init__(self,- model: Union[torch.nn.Module, str, None] = None,+ model: torch.nn.Module | str | None = None,tokenizer: Any = None,platform: str = "hf",- model_dtype: Optional[Union[str, torch.dtype]] = None,+ model_dtype: str | torch.dtype | None = None,trust_remote_code: bool = True,- config: Optional[AutoConfig] = None,+ config: AutoConfig | None = None,amp: bool = True,need_calib: bool = True,is_act_quantize: bool = False,@@ -219,7 +220,7 @@ class ModelContext(BaseContext):if config is None:config = AutoConfig.from_pretrained(self.model, trust_remote_code=self.trust_remote_code)self._import_custom_moe_replacements(config)- except (OSError, EnvironmentError, ValueError) as e:+ except (OSError, ValueError) as e:logger.debug("Failed to load config via AutoConfig.from_pretrained for %s: %s. ""Proceeding without config-based checks.",@@ -353,7 +354,7 @@ class ModelContext(BaseContext):)return True- def _resolve_local_checkpoint_dir(self) -> Optional[str]:+ def _resolve_local_checkpoint_dir(self) -> str | None:"""Return a local directory holding the checkpoint shards, or ``None``.``SafetensorsIndex`` reads shards by path, so a hub repo id has to be resolved to@@ -449,7 +450,7 @@ class ModelContext(BaseContext):gate = getattr(module, "gate", None)top_k = getattr(gate, "top_k", None)if top_k is not None:- setattr(module, "top_k", top_k)+ module.top_k = top_kdef _set_amp_dtype(self) -> None:"""Sets the automatic mixed precision (AMP) data type for the model based on the device and configuration.diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex 2b4de2b7e..f111ef430 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -11,7 +11,7 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-from typing import Any, Callable, Union+from collections.abc import Callableimport torch@@ -435,10 +435,10 @@ def _gather_row_pattern(row_pattern: torch.Tensor, start: int, end: int) -> torcdef _imatrix_handle_zero(- imatrix: Union[torch.Tensor, float],+ imatrix: torch.Tensor | float,weight: torch.Tensor,bits: int,- group_size: Union[int, None] = None,+ group_size: int | None = None,raw_imatrix: torch.Tensor = None,):if not isinstance(imatrix, torch.Tensor):diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.pyindex d301f0b2e..857c83fb1 100644--- a/auto_round/data_type/int.py+++ b/auto_round/data_type/int.py@@ -11,7 +11,6 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-from typing import Unionimport torch@@ -21,7 +20,7 @@ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_from auto_round.utils import get_reciprocal-def search_scales(data: torch.Tensor, bits: int, qw: Union[None, torch.Tensor, float] = None) -> torch.Tensor:+def search_scales(data: torch.Tensor, bits: int, qw: None | torch.Tensor | float = None) -> torch.Tensor:# Maximum absolute value for symmetric quantizationnmax = int(2.0 ** (bits - 1))diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.pyindex fd0de4758..c9ff8b22d 100644--- a/auto_round/data_type/mxfp.py+++ b/auto_round/data_type/mxfp.py@@ -125,9 +125,7 @@ def search_mx_scale(tensor, bits, qw=None, data_type=None):def compute_loss(qdq_tensor, out):torch.sub(qdq_tensor, tensor, out=buf)buf.pow_(2)- if qw is not None and not isinstance(qw, (int, float)):- buf.mul_(qw)- elif qw != 1.0:+ if (qw is not None and not isinstance(qw, (int, float))) or qw != 1.0:buf.mul_(qw)torch.sum(buf, dim=-1, out=out)@@ -468,7 +466,7 @@ def quant_mx_rceil_v2(return tensor.to(orig_dtype), shared_exp.to(orig_dtype), None-for key in MXFP_FORMAT_CACHE.keys():+for key in MXFP_FORMAT_CACHE:QUANT_FUNC_WITH_DTYPE[key] = quant_mxQUANT_FUNC_WITH_DTYPE[key + "_rceil"] = quant_mx_rceilQUANT_FUNC_WITH_DTYPE["opt_rtn_" + key] = quant_mx_opt_rtndiff --git a/auto_round/data_type/neuqi.py b/auto_round/data_type/neuqi.pyindex d83afeb6d..4171e3ad2 100644--- a/auto_round/data_type/neuqi.py+++ b/auto_round/data_type/neuqi.py@@ -613,11 +613,11 @@ def neuqi_search_scale_zero(data, bits, qw=None, q_scale_thresh=1e-5, coarse_n=Nif coarse_n is None:coarse_n = envs.AR_NEUQI_COARSE or d_coarseif coarse_n < 2:- raise ValueError("AR_NEUQI_COARSE=%d is too small; need at least 2 candidates." % coarse_n)+ raise ValueError(f"AR_NEUQI_COARSE={coarse_n} is too small; need at least 2 candidates.")if fine_n is None:fine_n = envs.AR_NEUQI_FINE or d_fineif fine_n < 2:- raise ValueError("AR_NEUQI_FINE=%d is too small; need at least 2 candidates." % fine_n)+ raise ValueError(f"AR_NEUQI_FINE={fine_n} is too small; need at least 2 candidates.")_log_search_engaged(coarse_n, fine_n)if torch.is_tensor(data):ensure_sweep_warmup(data.device)@@ -1197,11 +1197,11 @@ def neuqi_search_scale_sym(data, bits, qw=None, q_scale_thresh=1e-5, coarse_n=No# default is backend-aware (wide only on the accelerated lanes)coarse_n = envs.AR_NEUQI_COARSE or d_coarseif coarse_n < 2:- raise ValueError("AR_NEUQI_COARSE=%d is too small; need at least 2 candidates." % coarse_n)+ raise ValueError(f"AR_NEUQI_COARSE={coarse_n} is too small; need at least 2 candidates.")if fine_n is None:fine_n = envs.AR_NEUQI_FINE or d_fineif fine_n < 2:- raise ValueError("AR_NEUQI_FINE=%d is too small; need at least 2 candidates." % fine_n)+ raise ValueError(f"AR_NEUQI_FINE={fine_n} is too small; need at least 2 candidates.")_log_sym_search_engaged(coarse_n, fine_n)ensure_sym_search_warmup(data.device)diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex 676b2822e..99f027c72 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -15,7 +15,6 @@import mathfrom functools import lru_cachefrom math import ceil-from typing import List, Unionimport torchfrom torch.nn import Linear, Module@@ -26,7 +25,7 @@ from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPEfrom auto_round.utils import check_to_quantized, logger-def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: Union[int, list], val: float = 0.0):+def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int | list, val: float = 0.0):"""Reshapes and pads the tensor to ensure that it can be quantized in groups of `group_size`.This function adjusts the@@ -71,7 +70,7 @@ def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: Union[int,return data_new, orig_shape, pad_len-def revert_tensor_by_pad(data: torch.Tensor, orig_shape: tuple, pad_len: Union[int, list]):+def revert_tensor_by_pad(data: torch.Tensor, orig_shape: tuple, pad_len: int | list):"""Reverts the tensor to its original shape by removing padding.This function removes the padding added during reshaping and returns the tensor to@@ -467,7 +466,7 @@ def update_fused_layer_global_scales(global_scale_name = f"{base_name}_global_scale"- def _collect_scales(mods: List[Module]) -> List[torch.Tensor]:+ def _collect_scales(mods: list[Module]) -> list[torch.Tensor]:"""Collect valid global_scale tensors from modules."""scales = []for m in mods:@@ -488,7 +487,7 @@ def update_fused_layer_global_scales("""Check for MoE expert naming: w1 (gate) and w3 (up)."""return all(hasattr(module, projection) for projection in ("w1", "w3"))- def _update_global_scales(modules: List[Module]):+ def _update_global_scales(modules: list[Module]):"""Update global scales for a list of modules."""scales = _collect_scales(modules)if not scales:@@ -544,7 +543,7 @@ def update_block_global_scale_if_needed(block, data_type, group_size):has_nvfp = Trueif not hasattr(m, "weight_global_scale"):weight_global_scale = calculate_gparam(m.weight, module_group_size)- setattr(m, "weight_global_scale", weight_global_scale)+ m.weight_global_scale = weight_global_scaleif not has_nvfp:returndiff --git a/auto_round/envs.py b/auto_round/envs.pyindex 368351c09..c0337fb75 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -15,32 +15,33 @@# For detailed usage and configuration guide, see: docs/environments.mdimport os-from typing import TYPE_CHECKING, Any, Callable, Optional+from collections.abc import Callable+from typing import TYPE_CHECKING, Anyif TYPE_CHECKING:AR_LOG_LEVEL: str = "INFO"AR_USE_MODELSCOPE: bool = "False"- AR_MODEL_FREE_SHARD_PARALLELISM: Optional[int] = None- AUTO_ROUND_CACHE: Optional[str] = None+ AR_MODEL_FREE_SHARD_PARALLELISM: int | None = None+ AUTO_ROUND_CACHE: str | None = NoneAUTO_ROUND_GGUF_AUTO_UPDATE: bool = FalseAR_DISABLE_GGUF_MTP_EXPORT: bool = False- LLAMA_CPP_ROOT: Optional[str] = None- AR_AUTO_SCHEME_NSAMPLES: Optional[int] = None- AR_AUTO_SCHEME_BATCH_SIZE: Optional[int] = None- AR_AUTO_SCHEME_CACHE: Optional[str] = None+ LLAMA_CPP_ROOT: str | None = None+ AR_AUTO_SCHEME_NSAMPLES: int | None = None+ AR_AUTO_SCHEME_BATCH_SIZE: int | None = None+ AR_AUTO_SCHEME_CACHE: str | None = NoneAR_AUTO_SCHEME_NO_SERIAL_FALLBACK: bool = FalseAR_ENABLE_AUTO_SCHEME_PARALLEL: bool = TrueAR_NVFP4_E5M3_CACHE_HP_WEIGHT: bool = FalseAR_DISK_STREAM_MODEL: bool = FalseAR_DISABLE_META_LOAD: bool = False- AR_RESUME_DIR: Optional[str] = None+ AR_RESUME_DIR: str | None = NoneAR_FORCE_MOE_ROUTING_ALL_EXPERTS: bool = FalseAR_QUANTIZE_BAGEL_MOE_GEN: bool = FalseAR_NVFP4_FUSED_LAYER_GLOBAL_SCALE: bool = TrueAR_ALLOW_W8_ASYM: bool = False-def _get_optional_positive_int_env(name: str) -> Optional[int]:+def _get_optional_positive_int_env(name: str) -> int | None:"""Read an optional env var that must be a positive integer when set."""raw = os.getenv(name)if raw is None:diff --git a/auto_round/eval/eval_cli.py b/auto_round/eval/eval_cli.pyindex 27d28994f..4433e6c29 100644--- a/auto_round/eval/eval_cli.py+++ b/auto_round/eval/eval_cli.py@@ -241,7 +241,7 @@ def eval(args):add_bos_token=args.add_bos_token,)print(make_table(res))- print("evaluation running time=%ds" % (time.time() - st))+ print(f"evaluation running time={int(time.time() - st)}s")else:st = time.time()if "auto" in str(batch_size) and args.mllm:@@ -262,7 +262,7 @@ def eval(args):from lm_eval.utils import make_table # pylint: disable=E0401print(make_table(res))- print("evaluation running time=%ds" % (time.time() - st))+ print(f"evaluation running time={int(time.time() - st)}s")def eval_with_vllm(args):@@ -349,7 +349,7 @@ def eval_with_vllm(args):)print(make_table(res))- print("evaluation running time=%ds" % (time.time() - st))+ print(f"evaluation running time={int(time.time() - st)}s")def eval_task_by_task(diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex ab4638ede..48ff1cfd5 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -13,7 +13,7 @@# limitations under the License.import os-from typing import Optional, Union+import sysfrom auto_round.logger import loggerfrom auto_round.utils import dispatch_model_block_wise@@ -57,9 +57,9 @@ def _normalize_model_eval_dtype(model, eval_model_dtype):def simple_evaluate_user_model(user_model,tokenizer,- batch_size: Optional[int] = 1,- limit: Optional[Union[int, float]] = None,- max_batch_size: Optional[int] = 64,+ batch_size: int | None = 1,+ limit: float | None = None,+ max_batch_size: int | None = 64,eval_model_dtype="auto",add_bos_token: bool = False,mllm: bool = False,@@ -98,11 +98,11 @@ def simple_evaluate_user_model(def simple_evaluate(model,- model_args: Optional[Union[str, dict]] = None,- batch_size: Optional[int] = None,- limit: Optional[Union[int, float]] = None,- max_batch_size: Optional[int] = None,- device: Optional[str] = None,+ model_args: str | dict | None = None,+ batch_size: int | None = None,+ limit: float | None = None,+ max_batch_size: int | None = None,+ device: str | None = None,**kwargs,):import lm_eval # pylint: disable=E0401@@ -141,7 +141,7 @@ def evaluate_diffusion_model(args, autoround=None, model=None, pipe=None):logger.error("Quantized model is meta and diffusers doesn't support loading auto-round quantized model now. Exit.")- exit(0)+ sys.exit(0)pipe = autoround.pipepipe.to(model.dtype)pipe.transformer = model@@ -336,7 +336,7 @@ def evaluate_with_model_instance(model, tokenizer, device_str, args):fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),)print(make_table(res))- print("evaluation running time=%ds" % (time.time() - st))+ print(f"evaluation running time={int(time.time() - st)}s")def evaluate_with_model_path(eval_folder, device_str, autoround, args):@@ -412,7 +412,7 @@ def evaluate_with_model_path(eval_folder, device_str, autoround, args):fewshot_as_multiturn=getattr(args, "fewshot_as_multiturn", False),)print(make_table(res))- print("evaluation running time=%ds" % (time.time() - st))+ print(f"evaluation running time={int(time.time() - st)}s")def run_model_evaluation(model, tokenizer, autoround, folders, formats, args):diff --git a/auto_round/experimental/attention.py b/auto_round/experimental/attention.pyindex 476159505..546e2329d 100644--- a/auto_round/experimental/attention.py+++ b/auto_round/experimental/attention.py@@ -18,8 +18,8 @@import contextlibimport inspect+from collections.abc import Callablefrom functools import partial-from typing import Callable, Optionalfrom weakref import refimport torch@@ -41,9 +41,9 @@ from auto_round.utils import logger__all__ = ["QuantizedAttentionImpl",+ "attention_quant_ctx","init_hooked_attention","is_attention_calibration_tensor_name",- "attention_quant_ctx",]diff --git a/auto_round/experimental/kv_cache.py b/auto_round/experimental/kv_cache.pyindex 948a48f91..83bef24f3 100644--- a/auto_round/experimental/kv_cache.py+++ b/auto_round/experimental/kv_cache.py@@ -19,7 +19,7 @@import contextlibfrom enum import Enumfrom functools import partial-from typing import Any, Dict, List, Optional, Tuple, Union+from typing import Anyimport torchfrom transformers.cache_utils import DynamicCache@@ -37,10 +37,10 @@ from auto_round.experimental.utils import (from auto_round.utils import logger__all__ = [- "initialize_quantized_kv_cache",- "prep_attention_module_for_calibration","freeze_module_quantization_",+ "initialize_quantized_kv_cache","kvcache_quant_context",+ "prep_attention_module_for_calibration",]@@ -71,7 +71,7 @@ class KVCacheScaleType(Enum):# NOTE: Using _ suffix to denote l is modified in place-def _pad_and_append_at_idx_(lst: List, idx: int, val: Any) -> list:+def _pad_and_append_at_idx_(lst: list, idx: int, val: Any) -> list:"""Append value val to list lst at index idx, right padding if necessaryNeeded because user may ignore some layers in configuration, meaning@@ -107,7 +107,7 @@ class QuantizedKVParameterCache(DynamicCache):def __new__(cls, *args, **kwargs):"""Singleton"""if cls._instance is None:- cls._instance = super(QuantizedKVParameterCache, cls).__new__(cls)+ cls._instance = super().__new__(cls)return cls._instancedef __init__(self, dtype: torch.dtype | str = torch.float8_e4m3fn, granularity: str = "tensor"):@@ -125,10 +125,10 @@ class QuantizedKVParameterCache(DynamicCache):super().__init__()# each index corresponds to layer_idx of the attention layer- self.k_scales: List[torch.Tensor] = []- self.v_scales: List[torch.Tensor] = []- self.k_amax: List[float] = []- self.v_amax: List[float] = []+ self.k_scales: list[torch.Tensor] = []+ self.v_scales: list[torch.Tensor] = []+ self.k_amax: list[float] = []+ self.v_amax: list[float] = []self._initialized = Truedef update(@@ -136,8 +136,8 @@ class QuantizedKVParameterCache(DynamicCache):key_states: torch.Tensor,value_states: torch.Tensor,layer_idx: int,- cache_kwargs: Optional[Dict[str, Any]] = None,- ) -> Tuple[torch.Tensor, torch.Tensor]:+ cache_kwargs: dict[str, Any] | None = None,+ ) -> tuple[torch.Tensor, torch.Tensor]:"""Get the k_scale and v_scale and output the quant-dequant key_states and value_states"""@@ -155,7 +155,7 @@ class QuantizedKVParameterCache(DynamicCache):return keys_to_return, values_to_return- def get_seq_length(self, layer_idx: Optional[int] = 0) -> int:+ def get_seq_length(self, layer_idx: int | None = 0) -> int:"""Returns the sequence length of the cached states.A layer index can be optionally passed.@@ -173,12 +173,12 @@ class QuantizedKVParameterCache(DynamicCache):def reset_states(self):"""reset the kv states (used in calibration)"""- self.key_cache: List[torch.Tensor] = []- self.value_cache: List[torch.Tensor] = []+ self.key_cache: list[torch.Tensor] = []+ self.value_cache: list[torch.Tensor] = []# Used in `generate` to keep tally of how many tokens the cache has seenself._seen_tokens = 0- self._quantized_key_cache: List[torch.Tensor] = []- self._quantized_value_cache: List[torch.Tensor] = []+ self._quantized_key_cache: list[torch.Tensor] = []+ self._quantized_value_cache: list[torch.Tensor] = []def reset(self):"""@@ -253,7 +253,7 @@ def initialize_quantized_kv_cache(module: torch.nn.Module, dtype=torch.float8_e4)quantized_kv_cache = QuantizedKVParameterCache(dtype=dtype, granularity=granularity)- setattr(module, "kv_cache", quantized_kv_cache)+ module.kv_cache = quantized_kv_cachelogger.debug(f"Initialized quantized kv_cache for {module.__class__.__name__} {getattr(module, 'layer_idx', None)}")if quantized_kv_cache.is_nvfp4:# Global scales are only registered once calibration has observed KV@@ -266,14 +266,14 @@ def initialize_quantized_kv_cache(module: torch.nn.Module, dtype=torch.float8_e4def calibrate_kv_cache_input_hook(- module: torch.nn.Module, args: Any, kwargs: Dict[str, Any]-) -> Tuple[Tuple[Any, ...], Dict[str, Any]]:+ module: torch.nn.Module, args: Any, kwargs: dict[str, Any]+) -> tuple[tuple[Any, ...], dict[str, Any]]:"""Hook to update inputs to attention layers when runningkv_cache quantization. Will update the passed inkv_cache to singleton QuantizedKVParameterCache."""- kv_cache = getattr(module, "kv_cache")+ kv_cache = module.kv_cache# Start from transformers 4.55.2, the `past_key_value` was renamed to `past_key_values`.# https://github.com/huggingface/transformers/blob/52c6c1bb6e27ca87c4faede34a4c2a7404c17c4d/src/transformers/models/llama/modeling_llama.py#L279-L280if "past_key_values" in kwargs:@@ -316,7 +316,7 @@ def calibrate_kv_cache_output_hook(module: torch.nn.Module, _args: Any, _output:"""Hook to update k_scale and v_scale parameters when running kv_cache quantization."""- kv_cache = getattr(module, "kv_cache")+ kv_cache = module.kv_cacheif kv_cache.is_nvfp4:layer_idx = module.layer_idxk_amax = kv_cache.k_amax[layer_idx] if layer_idx < len(kv_cache.k_amax) else 0.0diff --git a/auto_round/experimental/qmodules/base.py b/auto_round/experimental/qmodules/base.pyindex 8b7a9c138..be148110d 100644--- a/auto_round/experimental/qmodules/base.py+++ b/auto_round/experimental/qmodules/base.py@@ -13,7 +13,6 @@# limitations under the License.from abc import ABC, abstractmethod-from typing import Optional, Unionimport torchdiff --git a/auto_round/experimental/qmodules/fake.py b/auto_round/experimental/qmodules/fake.pyindex e7419e91d..322ab9aeb 100644--- a/auto_round/experimental/qmodules/fake.py+++ b/auto_round/experimental/qmodules/fake.py@@ -12,7 +12,6 @@# See the License for the specific language governing permissions and# limitations under the License.-from typing import Optionalimport torch@@ -31,8 +30,8 @@ class FakeActQuantLinear(QModuleBase):in_features: int,out_features: int,config: QuantizationScheme,- weight: Optional[torch.Tensor] = None,- bias: Optional[torch.Tensor] = None,+ weight: torch.Tensor | None = None,+ bias: torch.Tensor | None = None,dtype: torch.dtype = torch.bfloat16,):super().__init__()diff --git a/auto_round/experimental/qmodules/fp4_utils.py b/auto_round/experimental/qmodules/fp4_utils.pyindex f2dafa1c8..a8c1f7a4e 100644--- a/auto_round/experimental/qmodules/fp4_utils.py+++ b/auto_round/experimental/qmodules/fp4_utils.py@@ -27,7 +27,6 @@# See the License for the specific language governing permissions and# limitations under the License.-from typing import Optionalimport torch@@ -58,9 +57,7 @@ def pack_fp4_to_uint8(x: torch.Tensor) -> torch.Tensor:return packed.reshape(rows, columns // 2)-def unpack_fp4_from_uint8(- a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16-) -> torch.Tensor:+def unpack_fp4_from_uint8(a: torch.Tensor, m: int, n: int, dtype: torch.dtype | None = torch.bfloat16) -> torch.Tensor:"""Unpacks uint8 values into FP4. Each uint8 contains two FP4 values(low nibble first). The 4-bit indices are mapped to FP4 values using kE2M1ToFloat.@@ -73,22 +70,20 @@ def unpack_fp4_from_uint8(@torch.compiler.disable()def _unpack_fp4_from_uint8_cpu(- a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16+ a: torch.Tensor, m: int, n: int, dtype: torch.dtype | None = torch.bfloat16) -> torch.Tensor:return _unpack_fp4_from_uint8(a, m, n, dtype)# @torch.compile(fullgraph=True, dynamic=True)def _unpack_fp4_from_uint8_cuda(- a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16+ a: torch.Tensor, m: int, n: int, dtype: torch.dtype | None = torch.bfloat16) -> torch.Tensor:return _unpack_fp4_from_uint8(a, m, n, dtype)# reference: : https://github.com/vllm-project/vllm/pull/16362-def _unpack_fp4_from_uint8(- a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16-) -> torch.Tensor:+def _unpack_fp4_from_uint8(a: torch.Tensor, m: int, n: int, dtype: torch.dtype | None = torch.bfloat16) -> torch.Tensor:"""Unpacks uint8 values into fp4. Each uint8 consists of two fp4 values(i.e. first four bits correspond to one fp4 value, last four correspond to adiff --git a/auto_round/experimental/qmodules/fp8_static.py b/auto_round/experimental/qmodules/fp8_static.pyindex ca013fae2..03fc1be07 100644--- a/auto_round/experimental/qmodules/fp8_static.py+++ b/auto_round/experimental/qmodules/fp8_static.py@@ -13,8 +13,6 @@# limitations under the License.-from typing import Optional, Union-import torchfrom auto_round.experimental.qmodules.base import QModuleBase@@ -39,10 +37,10 @@ class WeightFP8ActFP8StaticQuantLinear(QModuleBase):self,in_features,out_features,- weight: Optional[torch.Tensor] = None,- weight_scale: Optional[torch.Tensor] = None,- bias: Union[torch.Tensor, bool, None] = None,- input_scale: Optional[torch.Tensor] = None,+ weight: torch.Tensor | None = None,+ weight_scale: torch.Tensor | None = None,+ bias: torch.Tensor | bool | None = None,+ input_scale: torch.Tensor | None = None,dtype=torch.bfloat16,):super().__init__()diff --git a/auto_round/experimental/qmodules/mx.py b/auto_round/experimental/qmodules/mx.pyindex dea099760..287b91937 100644--- a/auto_round/experimental/qmodules/mx.py+++ b/auto_round/experimental/qmodules/mx.py@@ -13,8 +13,6 @@# limitations under the License.-from typing import Optional, Union-import torchfrom auto_round.data_type.utils import get_quant_func@@ -57,9 +55,9 @@ class MXQuantLinearBase(QModuleBase):in_features,out_features,config: QuantizationScheme,- weight: Optional[torch.Tensor] = None,- weight_scale: Optional[torch.Tensor] = None,- bias: Union[torch.Tensor, bool, None] = None,+ weight: torch.Tensor | None = None,+ weight_scale: torch.Tensor | None = None,+ bias: torch.Tensor | bool | None = None,dtype=torch.bfloat16,):super().__init__()@@ -109,7 +107,7 @@ class MXQuantLinearBase(QModuleBase):),)- def initialize_weights(self, weight: Optional[torch.Tensor]) -> torch.Tensor:+ def initialize_weights(self, weight: torch.Tensor | None) -> torch.Tensor:"""Initialize weights. This method should be overridden by subclasses."""@@ -169,7 +167,7 @@ class MXQuantLinearBase(QModuleBase):return out@classmethod- def from_original(cls, config: Optional[QuantizationScheme], original_layer: torch.nn.Linear):+ def from_original(cls, config: QuantizationScheme | None, original_layer: torch.nn.Linear):"""Create an `MXQuantLinear` layer from an original linear layer."""@@ -193,7 +191,7 @@ class MXFP4QuantLinear(MXQuantLinearBase):self.weight_name = "weight_packed"super().__init__(*args, **kwargs)- def initialize_weights(self, weight: Optional[torch.Tensor]) -> torch.Tensor:+ def initialize_weights(self, weight: torch.Tensor | None) -> torch.Tensor:weight_dtype = torch.uint8weight_in_features = self.in_features // 2return torch.zeros((self.out_features, weight_in_features), dtype=weight_dtype) if weight is None else weight@@ -219,7 +217,7 @@ class MXINT4QuantLinear(MXQuantLinearBase):self.weight_name = "weight_packed"super().__init__(*args, **kwargs)- def initialize_weights(self, weight: Optional[torch.Tensor]) -> torch.Tensor:+ def initialize_weights(self, weight: torch.Tensor | None) -> torch.Tensor:weight_dtype = torch.uint8weight_in_features = self.in_features // 2return torch.zeros((self.out_features, weight_in_features), dtype=weight_dtype) if weight is None else weight@@ -236,7 +234,7 @@ class MXINT4QuantLinear(MXQuantLinearBase):return unpacked_data@classmethod- def from_original(cls, config: Optional[QuantizationScheme], original_layer: torch.nn.Linear):+ def from_original(cls, config: QuantizationScheme | None, original_layer: torch.nn.Linear):"""Create an `MXQuantLinear` layer from an original linear layer."""@@ -260,7 +258,7 @@ class MXFP8QuantLinear(MXQuantLinearBase):self.weight_name = "weight"super().__init__(*args, **kwargs)- def initialize_weights(self, weight: Optional[torch.Tensor]) -> torch.Tensor:+ def initialize_weights(self, weight: torch.Tensor | None) -> torch.Tensor:weight_dtype = torch.float8_e4m3fnweight_in_features = self.in_featuresreturn torch.zeros((self.out_features, weight_in_features), dtype=weight_dtype) if weight is None else weightdiff --git a/auto_round/experimental/qmodules/mxint4_utils.py b/auto_round/experimental/qmodules/mxint4_utils.pyindex b772f54c1..c8290307c 100644--- a/auto_round/experimental/qmodules/mxint4_utils.py+++ b/auto_round/experimental/qmodules/mxint4_utils.py@@ -27,7 +27,6 @@# See the License for the specific language governing permissions and# limitations under the License.-from typing import Optionalimport torch@@ -45,9 +44,7 @@ def get_e0m4_tensor(device):return _DEVICE_E0M4_TENSORS[device_str]-def unpack_int4_from_uint8(- a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16-) -> torch.Tensor:+def unpack_int4_from_uint8(a: torch.Tensor, m: int, n: int, dtype: torch.dtype | None = torch.bfloat16) -> torch.Tensor:"""Unpacks uint8 values into int4. Each uint8 contains two int4 values(low nibble first). The 4-bit indices are mapped to int4 values using kE0M4ToFloat.@@ -60,20 +57,20 @@ def unpack_int4_from_uint8(@torch.compiler.disable()def _unpack_int4_from_uint8_cpu(- a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16+ a: torch.Tensor, m: int, n: int, dtype: torch.dtype | None = torch.bfloat16) -> torch.Tensor:return _unpack_int4_from_uint8(a, m, n, dtype)# @torch.compile(fullgraph=True, dynamic=True)def _unpack_int4_from_uint8_cuda(- a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16+ a: torch.Tensor, m: int, n: int, dtype: torch.dtype | None = torch.bfloat16) -> torch.Tensor:return _unpack_int4_from_uint8(a, m, n, dtype)def _unpack_int4_from_uint8(- a: torch.Tensor, m: int, n: int, dtype: Optional[torch.dtype] = torch.bfloat16+ a: torch.Tensor, m: int, n: int, dtype: torch.dtype | None = torch.bfloat16) -> torch.Tensor:"""Unpacks uint8 values into int4. Each uint8 consists of two int4 valuesdiff --git a/auto_round/experimental/qmodules/nvfp4.py b/auto_round/experimental/qmodules/nvfp4.pyindex dafeb2567..e9dc81e78 100644--- a/auto_round/experimental/qmodules/nvfp4.py+++ b/auto_round/experimental/qmodules/nvfp4.py@@ -13,8 +13,6 @@# limitations under the License.-from typing import Optional, Union-import torchfrom auto_round.data_type.nvfp import get_reciprocal, ref_nvfp4_quant@@ -59,9 +57,9 @@ class NVFP4QuantLinear(QModuleBase):in_features: int,out_features: int,config: QuantizationScheme,- weight: Optional[torch.Tensor] = None,- weight_scale: Optional[torch.Tensor] = None,- bias: Union[torch.Tensor, bool, None] = None,+ weight: torch.Tensor | None = None,+ weight_scale: torch.Tensor | None = None,+ bias: torch.Tensor | bool | None = None,dtype=torch.bfloat16,):super().__init__()@@ -145,7 +143,7 @@ class NVFP4QuantLinear(QModuleBase):self._convert_global_scale_to_float32(state_dict, "input_global_scale")return super().load_state_dict(state_dict, strict, assign)- def initialize_weights(self, weight: Optional[torch.Tensor]) -> torch.Tensor:+ def initialize_weights(self, weight: torch.Tensor | None) -> torch.Tensor:"""Initialize weights."""@@ -199,7 +197,7 @@ class NVFP4QuantLinear(QModuleBase):return out@classmethod- def from_original(cls, config: Optional[QuantizationScheme], original_layer: torch.nn.Linear):+ def from_original(cls, config: QuantizationScheme | None, original_layer: torch.nn.Linear):"""Create an `NVFPQuantLinear` layer from an original linear layer."""diff --git a/auto_round/experimental/qmodules/nvfp4_e5m3.py b/auto_round/experimental/qmodules/nvfp4_e5m3.pyindex c8b115e7e..42f2b5528 100644--- a/auto_round/experimental/qmodules/nvfp4_e5m3.py+++ b/auto_round/experimental/qmodules/nvfp4_e5m3.py@@ -13,7 +13,6 @@# limitations under the License.import os-from typing import Optional, Unionimport torch@@ -33,7 +32,7 @@ __all__ = ["CuteNVFP4E5M3QuantLinear", "NVFP4E5M3QuantLinear"]_CACHE_WEIGHT_ENV = "AR_NVFP4_E5M3_CACHE_HP_WEIGHT"-def _resolve_cache_weight(cache_weight: Optional[bool], default: bool) -> bool:+def _resolve_cache_weight(cache_weight: bool | None, default: bool) -> bool:if cache_weight is not None:return cache_weightvalue = os.getenv(_CACHE_WEIGHT_ENV)@@ -53,11 +52,11 @@ class NVFP4E5M3QuantLinear(QModuleBase):in_features: int,out_features: int,config: QuantizationScheme,- weight: Optional[torch.Tensor] = None,- weight_scale: Optional[torch.Tensor] = None,- bias: Union[torch.Tensor, bool, None] = None,+ weight: torch.Tensor | None = None,+ weight_scale: torch.Tensor | None = None,+ bias: torch.Tensor | bool | None = None,dtype=torch.bfloat16,- cache_weight: Optional[bool] = None,+ cache_weight: bool | None = None,):super().__init__()assert dtype in self.SUPPORTED_COMPUTE_DTYPEdiff --git a/auto_round/experimental/utils.py b/auto_round/experimental/utils.pyindex 2391d1798..1f443bd53 100644--- a/auto_round/experimental/utils.py+++ b/auto_round/experimental/utils.py@@ -107,7 +107,7 @@ def update_parameter_data(module: torch.nn.Module, new_val: torch.Tensor, name:module.register_parameter(name, torch.nn.Parameter(new_val))else:logger.warning_once(- "Parameter %s not found in module %s, creating new parameter." % (name, module.__class__.__name__)+ f"Parameter {name} not found in module {module.__class__.__name__}, creating new parameter.")module.register_parameter(name, torch.nn.Parameter(new_val))diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 90be0c0d9..2199e2d09 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -16,8 +16,9 @@ import copyimport inspectimport jsonimport os+from collections.abc import Callablefrom dataclasses import fields-from typing import Any, Callable, Dict, Union+from typing import Any# MIT License#@@ -83,7 +84,7 @@ BLOCK_PATTERNS = [ ## copy from transformers optimumfrom auto_round.export.export_to_autoround.utils import check_neq_config-def convert_to_autogptq_dynamic(regex_config: Dict[str, Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:+def convert_to_autogptq_dynamic(regex_config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]:"""Convert AutoRound-style regex_config into AutoGPTQ-style QuantizerConfig.dynamic.@@ -101,7 +102,7 @@ def convert_to_autogptq_dynamic(regex_config: Dict[str, Dict[str, Any]]) -> Dictelif bits < 16:converted[f"+:{regex}"] = {"bits": bits}for key in GPTQ_REQUIRED_CONFIG_KEYS: # only save keys gptq supported- converted[f"+:{regex}"][key] = regex_config[name][key]+ converted[f"+:{regex}"][key] = cfg[key]else:# skip quantizationconverted[f"-:{regex}"] = {}@@ -190,12 +191,12 @@ def pack_layer(name, model, backend, device=None):def save_quantized_as_autogptq(output_dir: str,model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- device: Union[str, torch.device] = "cpu",+ device: str | torch.device = "cpu",backend: str = "auto_gptq:exllamav2",- serialization_dict: dict = None,+ serialization_dict: dict | None = None,**kwargs,) -> torch.nn.Module:"""Export the model to autogptq format to easily leverage cuda kernel."""diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex fd02d98d9..8dbdbff16 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -17,9 +17,9 @@ import functoolsimport inspectimport jsonimport os+from collections.abc import Callablefrom dataclasses import fieldsfrom enum import Enum-from typing import Callable, Unionimport torchimport torch.nn as nn@@ -242,12 +242,12 @@ def pack_layer(layer_name, model, backend, device=None):def save_quantized_as_autoround(output_dir: str,model: torch.nn.Module,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace=True,backend="auto_round:exllamav2",- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,+ device: str | torch.device = "cpu",+ serialization_dict: dict | None = None,**kwargs,):"""@@ -279,7 +279,7 @@ def save_quantized_as_autoround():backend = backend.replace("auto_round", "auto_round:auto_gptq")- safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]+ safe_serialization = kwargs.get("safe_serialization", True)if not inplace:model = copy.deepcopy(model.to("cpu"))diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.pyindex c0465dfcd..861b8d18f 100644--- a/auto_round/export/export_to_autoround/export_to_fp8.py+++ b/auto_round/export/export_to_autoround/export_to_fp8.py@@ -15,8 +15,8 @@import copyimport jsonimport os+from collections.abc import Callablefrom dataclasses import fields-from typing import Callable, Unionimport torchimport transformers@@ -202,16 +202,16 @@ def pack_layer(layer_name, model, data_type, device=None, unsqueeze=False):def save_quantized_as_autoround(output_dir: str,model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- backend: str = None,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,+ backend: str | None = None,+ device: str | torch.device = "cpu",+ serialization_dict: dict | None = None,quant_method: str = "auto-round",**kwargs,):- safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]+ safe_serialization = kwargs.get("safe_serialization", True)if not inplace:model = copy.deepcopy(model.to("cpu"))quantization_config = serialization_dictdiff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mx.py b/auto_round/export/export_to_autoround/export_to_nvfp_mx.pyindex e9be7d634..ef87ffab3 100644--- a/auto_round/export/export_to_autoround/export_to_nvfp_mx.py+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mx.py@@ -16,8 +16,8 @@ import copyimport inspectimport jsonimport os+from collections.abc import Callablefrom dataclasses import fields-from typing import Callable, Unionimport torchimport torch.nn as nn@@ -84,7 +84,7 @@ def pack_layer(name, model, backend, device=None):from auto_round.data_type.nvfp import calculate_gparaminput_global_scale = calculate_gparam(layer.act_max, layer.group_size, "cpu")- setattr(layer, "input_global_scale", input_global_scale)+ layer.input_global_scale = input_global_scaledelattr(layer, "act_max")if type(layer) == nn.Linear:@@ -137,12 +137,12 @@ def pack_layer(name, model, backend, device=None):def save_quantized_as_fp(output_dir: str,model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- device: Union[str, torch.device] = "cpu",+ device: str | torch.device = "cpu",backend: str = "autoround:exllamav2",- serialization_dict: dict = None,+ serialization_dict: dict | None = None,**kwargs,) -> torch.nn.Module:"""@@ -170,7 +170,7 @@ def save_quantized_as_fp(data_type = serialization_dict.get("data_type", None)act_bits = serialization_dict.get("act_bits", None)act_data_type = serialization_dict.get("act_data_type", None)- safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]+ safe_serialization = kwargs.get("safe_serialization", True)if not inplace:model = copy.deepcopy(model.to("cpu"))quantization_config = serialization_dict@@ -205,7 +205,7 @@ def save_quantized_as_fp(from auto_round.data_type.nvfp import calculate_gparaminput_global_scale = calculate_gparam(layer.act_max, layer.group_size, model.device)- setattr(layer, "input_global_scale", input_global_scale)+ layer.input_global_scale = input_global_scaledelattr(layer, "act_max")# update fused input_global_scalefrom auto_round.data_type.utils import update_fused_layer_global_scalesdiff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.pyindex 039c5e0b3..d9082e1e4 100644--- a/auto_round/export/export_to_autoround/qlinear_fp.py+++ b/auto_round/export/export_to_autoround/qlinear_fp.py@@ -190,7 +190,6 @@ class QuantLinear(nn.Module):if input_global_scale is not None:# TODO: the shape of `input_global_scale` is [] in some cases — need to investigate why.self.input_global_scale = input_global_scale.to(torch.float32).to(device).reshape([1])- returndef pack_fp4_to_uint8(scaled_tensor: torch.Tensor):diff --git a/auto_round/export/export_to_autoround/qlinear_triton_act.py b/auto_round/export/export_to_autoround/qlinear_triton_act.pyindex 7d5f9dee7..57deb924d 100644--- a/auto_round/export/export_to_autoround/qlinear_triton_act.py+++ b/auto_round/export/export_to_autoround/qlinear_triton_act.py@@ -171,7 +171,7 @@ class QuantLinear(nn.Module):zeros -= 1shape = scales_t.shapevalue = 0- for j in range(0, (32 // self.bits)):+ for j in range(32 // self.bits):value |= zeros << (self.bits * j)qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * valueqzeros = qzeros.astype(np.int32)diff --git a/auto_round/export/export_to_autoround/utils.py b/auto_round/export/export_to_autoround/utils.pyindex 5e9e081d2..526ecdc36 100644--- a/auto_round/export/export_to_autoround/utils.py+++ b/auto_round/export/export_to_autoround/utils.py@@ -13,12 +13,11 @@# limitations under the License.from dataclasses import fields-from typing import Listfrom auto_round.schemes import QuantizationScheme-def check_neq_config(config: dict, **expected) -> List[str]:+def check_neq_config(config: dict, **expected) -> list[str]:"""Compare a config dict against expected values.Ensures all required keys are present in both config and expected.diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex fc91b3817..306ed4c87 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -23,7 +23,7 @@import copyimport jsonimport os-from typing import Callable, Union+from collections.abc import Callableimport torchimport torch.nn as nn@@ -61,7 +61,7 @@ def _collect_modules_to_not_convert(model: torch.nn.Module,layer_config: dict,regex_config: dict,- to_quant_block_names: list = None,+ to_quant_block_names: list | None = None,) -> list:"""Collect all module names that should not be converted (not quantized).@@ -148,11 +148,11 @@ def pack_layer(name, model, backend, device=None):def save_quantized_as_autoawq(output_dir: str,model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,+ device: str | torch.device = "cpu",+ serialization_dict: dict | None = None,**kwargs,) -> torch.nn.Module:"""Export the model to autogptq format to easily leverage cuda kernel."""diff --git a/auto_round/export/export_to_awq/utils.py b/auto_round/export/export_to_awq/utils.pyindex 662987529..ddf59ac0d 100644--- a/auto_round/export/export_to_awq/utils.py+++ b/auto_round/export/export_to_awq/utils.py@@ -356,10 +356,4 @@ class WQLinear_GEMM(nn.Module):return out.reshape(out_shape)def extra_repr(self) -> str:- return "in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(- self.in_features,- self.out_features,- self.bias is not None,- self.w_bit,- self.group_size,- )+ return f"in_features={self.in_features}, out_features={self.out_features}, bias={self.bias is not None}, w_bit={self.w_bit}, group_size={self.group_size}"diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex c922be9a5..60bdbf3a3 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -37,9 +37,10 @@ from __future__ import annotationsimport jsonimport osimport re+from collections.abc import Iteratorfrom functools import partialfrom itertools import chain-from typing import TYPE_CHECKING, Any, Iterator+from typing import TYPE_CHECKINGimport numpy as npimport torch@@ -101,7 +102,7 @@ def wrapper_model_instance(def _need_low_cpu_mem(low_cpu_mem_usage):- if not low_cpu_mem_usage:+ if not low_cpu_mem_usage: # noqa: SIM103return False# process = psutil.Process(os.getpid())@@ -130,7 +131,7 @@ def get_moe_name(cls, name, new_name):tensor_type = cls.tensor_map.get_type(new_name_tmp).nameexperts_name = name_tmp.split(".")[-1]- for k, v in type_mapping.items():+ for v in type_mapping.values():if experts_name in v:idx = v.index(experts_name)name = name.replace(experts_name, type_mapping[tensor_type][idx])@@ -534,11 +535,7 @@ def _pack_spec_moe_output(cls, data_torch, data_qtype, moe_output, modify_name,def get_qtype_by_layer_config(layer_config, name, data_qtype, *, explicit_only=False):name = name[: -len(".weight")]- if name not in layer_config and name.endswith("embed_tokens"):- embedding_names = [key for key in layer_config if key.endswith("embed_tokens")]- if len(embedding_names) == 1:- name = embedding_names[0]- elif name == "token_embd":+ if (name not in layer_config and name.endswith("embed_tokens")) or name == "token_embd":embedding_names = [key for key in layer_config if key.endswith("embed_tokens")]if len(embedding_names) == 1:name = embedding_names[0]@@ -856,7 +853,7 @@ def prepare_tensors(cls):modify_name = _special_name_handle(cls, checkpoint_name)restored_outputs_completed = False- for new_name, data_torch in cls.modify_tensors(data_torch, modify_name, bid):+ for new_name, data_torch in cls.modify_tensors(data_torch, modify_name, bid): # noqa: B020restored_outputs_completed = Trueif _gguf_writer_has_tensor(cls.gguf_writer, new_name):logger.debug("%s already added to gguf_writer, skip", new_name)@@ -1012,10 +1009,9 @@ def prepare_tensors(cls):gguf.GGMLQuantizationType.Q2_K,gguf.GGMLQuantizationType.Q3_K,gguf.GGMLQuantizationType.Q4_K,+ gguf.GGMLQuantizationType.Q5_K,]:data_qtype = gguf.GGMLQuantizationType.Q5_0- elif data_qtype == gguf.GGMLQuantizationType.Q5_K:- data_qtype = gguf.GGMLQuantizationType.Q5_0elif data_qtype == gguf.GGMLQuantizationType.Q6_K:data_qtype = gguf.GGMLQuantizationType.Q8_0diff --git a/auto_round/export/export_to_gguf/hf_checkpoint_restorer.py b/auto_round/export/export_to_gguf/hf_checkpoint_restorer.pyindex 6121ef887..4b1922c36 100644--- a/auto_round/export/export_to_gguf/hf_checkpoint_restorer.py+++ b/auto_round/export/export_to_gguf/hf_checkpoint_restorer.py@@ -15,9 +15,9 @@from __future__ import annotationsfrom collections import defaultdict+from collections.abc import Callable, Iteratorfrom copy import deepcopyfrom dataclasses import dataclass-from typing import Callable, Iteratorimport torchdiff --git a/auto_round/export/export_to_gguf/llama_cpp_conversion.py b/auto_round/export/export_to_gguf/llama_cpp_conversion.pyindex e057dbbe7..99b305062 100644--- a/auto_round/export/export_to_gguf/llama_cpp_conversion.py+++ b/auto_round/export/export_to_gguf/llama_cpp_conversion.py@@ -48,15 +48,15 @@ class ConversionContext:@propertydef ModelBase(self):- return getattr(self.module, "ModelBase")+ return self.module.ModelBase@propertydef ModelType(self):- return getattr(self.module, "ModelType")+ return self.module.ModelType@propertydef get_model_architecture(self):- return getattr(self.module, "get_model_architecture")+ return self.module.get_model_architecturedef model_type(self, model_type: AutoRoundModelType | Any):if int(model_type) == int(AutoRoundModelType.MMPROJ):diff --git a/auto_round/export/export_to_gguf/special_handle.py b/auto_round/export/export_to_gguf/special_handle.pyindex 513db51bb..1b0d7cb18 100644--- a/auto_round/export/export_to_gguf/special_handle.py+++ b/auto_round/export/export_to_gguf/special_handle.py@@ -13,8 +13,8 @@# limitations under the License.import jsonimport os+from collections.abc import Iterablefrom functools import partial-from typing import Iterableimport torchfrom safetensors import safe_open@@ -55,8 +55,9 @@ def granite_moe_modify_tensors(cls, original_modify_tensors, data_torch, name, b(projectionfor projection in _GRANITE_AGGREGATED_EXPERT_TENSORS- if name.endswith(f"block_sparse_moe.experts.{projection}")- or name.endswith(f"block_sparse_moe.experts.{projection}.weight")+ if name.endswith(+ (f"block_sparse_moe.experts.{projection}", f"block_sparse_moe.experts.{projection}.weight")+ )),None,)@@ -142,7 +143,7 @@ def gptoss_generate_extra_tensors(cls) -> Iterable[tuple[str, Tensor]]:return blocks0, blocks1for name, data_torch in cls.get_tensors():- if GPTOSS_RELOAD and (name.endswith("mlp.experts.down_proj") or name.endswith("mlp.experts.gate_up_proj")):+ if GPTOSS_RELOAD and name.endswith(("mlp.experts.down_proj", "mlp.experts.gate_up_proj")):block_name = name + "_blocks"block_data_torch = get_tensor_from_file(cls.model.name_or_path, block_name)blocks0, blocks1 = repack(block_name, block_data_torch, blocks0, blocks1)diff --git a/auto_round/export/export_to_gguf/sync_llama_cpp_conversion.py b/auto_round/export/export_to_gguf/sync_llama_cpp_conversion.pyold mode 100644new mode 100755diff --git a/auto_round/export/export_to_llmcompressor/config.py b/auto_round/export/export_to_llmcompressor/config.pyindex 9c66c5ffa..99eb1196d 100644--- a/auto_round/export/export_to_llmcompressor/config.py+++ b/auto_round/export/export_to_llmcompressor/config.py@@ -53,17 +53,16 @@ if check_compressed_tensors_supported():# please refer to https://github.com/vllm-project/llm-compressor/blob/# 29f4d5644b48e9c8ebb7e36d5be9f7c92747ceb7/src/llmcompressor/modifiers/quantization/quantization/mixin.py#L168-def initialize_quantization(scheme, targets=["Linear"], config_groups=None, kv_cache_scheme=None, ignore=["lm_head"]):+def initialize_quantization(scheme, targets=None, config_groups=None, kv_cache_scheme=None, ignore=None):"""Attach quantization schemes to modules in the model and initialize the quantization config"""# apply scheme and status to model- scheme = scheme- targets = targets- config_groups = config_groups- kv_cache_scheme = kv_cache_scheme- ignore = ignore+ if targets is None:+ targets = ["Linear"]+ if ignore is None:+ ignore = ["lm_head"]using_mxfp4_for_mxfp8 = Falsecheck_compressed_tensors_supported(raise_error=True)if scheme is not None and config_groups is not None:diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.pyindex eb0d62398..12148253f 100644--- a/auto_round/export/export_to_llmcompressor/export.py+++ b/auto_round/export/export_to_llmcompressor/export.py@@ -15,8 +15,7 @@import copyimport mathimport os-from collections.abc import Mapping-from typing import Callable, Union+from collections.abc import Callable, Mappingimport torch@@ -234,8 +233,8 @@ def pack_layer(name, model, device=None):weight_device = layer.weight.devicescheme = construct_ct_scheme(layer)- setattr(layer, "quantization_scheme", scheme)- setattr(layer, "weight_scale", torch.nn.Parameter(layer.scale.to(weight_device)))+ layer.quantization_scheme = scheme+ layer.weight_scale = torch.nn.Parameter(layer.scale.to(weight_device))# AutoRound zp is the UNSIGNED level convention (q in [0, 2^b-1],# W = (q - zp) * s); compressed-tensors packs the SIGNED convention (q in# [-2^(b-1), 2^(b-1)-1], zp added BEFORE the clamp into that range - see@@ -304,7 +303,7 @@ def pack_layer(name, model, device=None):)zp = zp.to(torch.int8)- setattr(layer, "weight_zero_point", torch.nn.Parameter(zp.to(weight_device), requires_grad=False))+ layer.weight_zero_point = torch.nn.Parameter(zp.to(weight_device), requires_grad=False)delattr(layer, "scale")_compress_and_set_format(layer, scheme, device)@@ -314,11 +313,11 @@ def pack_layer(name, model, device=None):def save_quantized_as_llmcompressor(output_dir: str,model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,+ device: str | torch.device = "cpu",+ serialization_dict: dict | None = None,**kwargs,) -> torch.nn.Module:"""diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex 9e24a38e1..344b8f212 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -17,7 +17,7 @@ import inspectimport jsonimport osimport sys-from typing import Callable, Union+from collections.abc import Callableimport torchimport torch.nn as nn@@ -94,7 +94,7 @@ def pack_layer(name, model, device=None):from auto_round.data_type.nvfp import calculate_gparaminput_global_scale = calculate_gparam(layer.act_max, layer.group_size) # , model.device- setattr(layer, "input_global_scale", input_global_scale)+ layer.input_global_scale = input_global_scaledelattr(layer, "act_max")# QuantLinear = get_fp_qlinear(backend, bits, group_size, sym)@@ -303,12 +303,12 @@ def _resolve_kv_cache_scheme(def save_quantized_as_fp(output_dir: str,model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- device: Union[str, torch.device] = "cpu",- backend: str = None,- serialization_dict: dict = None,+ device: str | torch.device = "cpu",+ backend: str | None = None,+ serialization_dict: dict | None = None,**kwargs,) -> torch.nn.Module:"""@@ -336,7 +336,7 @@ def save_quantized_as_fp(data_type = serialization_dict.get("data_type", None)act_bits = serialization_dict.get("act_bits", None)act_data_type = serialization_dict.get("act_data_type", None)- safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]+ safe_serialization = kwargs.get("safe_serialization", True)if not inplace:model = copy.deepcopy(model.to("cpu"))processor = kwargs.get("processor", None)@@ -360,7 +360,7 @@ def save_quantized_as_fp(from auto_round.data_type.nvfp import calculate_gparaminput_global_scale = calculate_gparam(layer.act_max, layer.group_size, model.device)- setattr(layer, "input_global_scale", input_global_scale)+ layer.input_global_scale = input_global_scaledelattr(layer, "act_max")# update fused input_global_scalefrom auto_round.data_type.utils import update_fused_layer_global_scales@@ -460,7 +460,7 @@ def save_quantized_as_fp(attention_config = _get_attention_config(model, static_attention_granularity)else:attention_config = None- setattr(quantization_config, "format", format)+ quantization_config.format = formatquantization_config = quantization_config.to_dict()quantization_config["provider"] = "auto-round"_configure_gaudi2_fp8_dtype(quantization_config)diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.pyindex 4741112f5..364d533fe 100644--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py@@ -16,7 +16,7 @@ import copyimport jsonimport osimport sys-from typing import Callable, Union+from collections.abc import Callableimport torchimport transformers@@ -41,7 +41,7 @@ from auto_round.utils import ()-def pack_layer(layer_name: str, model: torch.nn.Module, data_type: str, device: str = None) -> None:+def pack_layer(layer_name: str, model: torch.nn.Module, data_type: str, device: str | None = None) -> None:"""Packs a model layer for quantization based on its type and configuration.@@ -179,21 +179,19 @@ def _configure_gaudi2_fp8_dtype(quantization_config: dict) -> None:if is_gaudi2():quantization_config["fp8_dtype_flavor"] = _GAUDI2_FP8_DTYPE_FLAVORlogger.warning_once(- (- "Running on Intel Gaudi2 hardware."- f" Setting FP8 dtype flavor to {_GAUDI2_FP8_DTYPE_FLAVOR} for compatibility."- )+ "Running on Intel Gaudi2 hardware."+ f" Setting FP8 dtype flavor to {_GAUDI2_FP8_DTYPE_FLAVOR} for compatibility.")def save_quantized_as_static_fp(output_dir: str,model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,+ device: str | torch.device = "cpu",+ serialization_dict: dict | None = None,**kwargs,) -> torch.nn.Module:"""@@ -217,7 +215,7 @@ def save_quantized_as_static_fp(Raises:ValueError: If the backend is not supported."""- safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]+ safe_serialization = kwargs.get("safe_serialization", True)if not inplace:model = copy.deepcopy(model.to("cpu"))diff --git a/auto_round/export/export_to_llmcompressor/utils.py b/auto_round/export/export_to_llmcompressor/utils.pyindex 304a720bd..a6020c8f0 100644--- a/auto_round/export/export_to_llmcompressor/utils.py+++ b/auto_round/export/export_to_llmcompressor/utils.py@@ -12,12 +12,11 @@# See the License for the specific language governing permissions and# limitations under the License.-from typing import Dict, Listfrom auto_round.utils import matches_any_regex, to_standard_regex-def generate_ignore_regex_list(regex_config: Dict[str, Dict], layer_config: Dict[str, Dict]) -> List[str]:+def generate_ignore_regex_list(regex_config: dict[str, dict], layer_config: dict[str, dict]) -> list[str]:"""Generate ignore regex list for llm_compressor based on regex_config and layer_config.@@ -34,7 +33,7 @@ def generate_ignore_regex_list(regex_config: Dict[str, Dict], layer_config: DictList[str]: List of regex patterns to ignore during quantization."""prefix = "re:"- ignore_regex: List[str] = []+ ignore_regex: list[str] = []# Step 1: Add regex_config keys with bits >= 16for key, cfg in regex_config.items():diff --git a/auto_round/export/export_to_mlx/export.py b/auto_round/export/export_to_mlx/export.pyindex 2681ca818..119b00bb8 100644--- a/auto_round/export/export_to_mlx/export.py+++ b/auto_round/export/export_to_mlx/export.py@@ -30,7 +30,7 @@ Supported schemes: W2A16, W3A16, W4A16, W5A16, W6A16, W8A16import copyimport jsonimport os-from typing import Callable, Union+from collections.abc import Callableimport torchimport torch.nn as nn@@ -600,11 +600,11 @@ def pack_layer(name, model, device=None, **kwargs):def save_quantized_as_mlx(output_dir: str,model: nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,+ device: str | torch.device = "cpu",+ serialization_dict: dict | None = None,**kwargs,) -> nn.Module:"""Save quantized model in MLX-compatible format.diff --git a/auto_round/export/formats/__init__.py b/auto_round/export/formats/__init__.pyindex d401af8c8..48038c853 100644--- a/auto_round/export/formats/__init__.py+++ b/auto_round/export/formats/__init__.py@@ -29,12 +29,12 @@ from auto_round.export.formats.base import (from auto_round.export.formats.resolver import resolve_formats__all__ = [- "BackendDataType",- "AutoRoundFormat","AutoAWQFormat","AutoGPTQFormat",- "FakeFormat",+ "AutoRoundFormat",+ "BackendDataType","FP8Format",+ "FakeFormat","GGUFFormat","LLMCompressorFormat","MLXFormat",diff --git a/auto_round/export/formats/backends/__init__.py b/auto_round/export/formats/backends/__init__.pyindex 3f77b50cb..e74dbf3e1 100644--- a/auto_round/export/formats/backends/__init__.py+++ b/auto_round/export/formats/backends/__init__.py@@ -26,8 +26,8 @@ __all__ = ["AutoAWQFormat","AutoGPTQFormat","AutoRoundFormat",- "FakeFormat","FP8Format",+ "FakeFormat","GGUFFormat","LLMCompressorFormat","MLXFormat",diff --git a/auto_round/export/formats/backends/auto_awq.py b/auto_round/export/formats/backends/auto_awq.pyindex 00905cb5b..f7f8c4388 100644--- a/auto_round/export/formats/backends/auto_awq.py+++ b/auto_round/export/formats/backends/auto_awq.py@@ -13,7 +13,7 @@# limitations under the License.import re-from typing import Callable, Union+from collections.abc import Callableimport torchimport transformers@@ -113,11 +113,11 @@ class AutoAWQFormat(OutputFormat):self,output_dir: str,model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,+ device: str | torch.device = "cpu",+ serialization_dict: dict | None = None,**kwargs,) -> torch.nn.Module:backend = self.get_backend_name()diff --git a/auto_round/export/formats/backends/auto_gptq.py b/auto_round/export/formats/backends/auto_gptq.pyindex 03a14a33d..805b8a157 100644--- a/auto_round/export/formats/backends/auto_gptq.py+++ b/auto_round/export/formats/backends/auto_gptq.py@@ -13,7 +13,7 @@# limitations under the License.import re-from typing import Callable, Union+from collections.abc import Callableimport torch@@ -91,11 +91,11 @@ class AutoGPTQFormat(OutputFormat):self,output_dir: str,model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,+ device: str | torch.device = "cpu",+ serialization_dict: dict | None = None,**kwargs,) -> torch.nn.Module:backend = self.get_backend_name()diff --git a/auto_round/export/formats/backends/autoround.py b/auto_round/export/formats/backends/autoround.pyindex 84134f6df..867aff2e5 100644--- a/auto_round/export/formats/backends/autoround.py+++ b/auto_round/export/formats/backends/autoround.py@@ -13,7 +13,8 @@# limitations under the License.import re-from typing import Any, Callable, Union+from collections.abc import Callable+from typing import Anyimport torch@@ -71,9 +72,12 @@ class AutoRoundFormat(OutputFormat):)if enable_awq:self.backend = AutoAWQFormat("auto_round:auto_awq", scheme, ctx)- elif scheme.is_nv_fp() or scheme.is_mx_fp() or scheme.data_type == BackendDataType.NVFP4_E5M3.value:- self.backend = AutoRoundFormat(scheme.data_type, scheme, ctx)- elif scheme.is_mx_int() and scheme.bits == 4: # only add mx_int4 now+ elif (+ scheme.is_nv_fp()+ or scheme.is_mx_fp()+ or scheme.data_type == BackendDataType.NVFP4_E5M3.value+ or (scheme.is_mx_int() and scheme.bits == 4) # only add mx_int4 now+ ):self.backend = AutoRoundFormat(scheme.data_type, scheme, ctx)elif scheme.is_act_static(): # static wfp8afp8self.backend = AutoRoundFormat(BackendDataType.FP8_STATIC.value, scheme, ctx)@@ -146,13 +150,10 @@ class AutoRoundFormat(OutputFormat):f"auto_round:{BackendDataType.MX_FP.value}",f"auto_round:{BackendDataType.MX_FP_RCEIL.value}",f"auto_round:{BackendDataType.NV_FP4_WITH_STATIC_GS.value}",+ f"auto_round:{BackendDataType.MX_INT.value}",]:from auto_round.export.export_to_autoround.export_to_nvfp_mx import pack_layer- pack_func = pack_layer- elif self.output_format in [f"auto_round:{BackendDataType.MX_INT.value}"]:- from auto_round.export.export_to_autoround.export_to_nvfp_mx import pack_layer-pack_func = pack_layerelif self.output_format in [f"auto_round:{BackendDataType.FP8.value}",@@ -172,11 +173,11 @@ class AutoRoundFormat(OutputFormat):self,output_dir: str,model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,+ device: str | torch.device = "cpu",+ serialization_dict: dict | None = None,**kwargs,) -> torch.nn.Module:if self.backend is not None:diff --git a/auto_round/export/formats/backends/fake.py b/auto_round/export/formats/backends/fake.pyindex 2c87053dd..b5e34caaa 100644--- a/auto_round/export/formats/backends/fake.py+++ b/auto_round/export/formats/backends/fake.py@@ -16,7 +16,8 @@ import copyimport globimport jsonimport os-from typing import Any, Callable, Union+from collections.abc import Callable+from typing import Anyimport torch@@ -127,11 +128,11 @@ class FakeFormat(OutputFormat):self,output_dir: str,model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,+ device: str | torch.device = "cpu",+ serialization_dict: dict | None = None,**kwargs,):has_fake_act_quant = Falsediff --git a/auto_round/export/formats/backends/fp8.py b/auto_round/export/formats/backends/fp8.pyindex 431e458d0..390956c28 100644--- a/auto_round/export/formats/backends/fp8.py+++ b/auto_round/export/formats/backends/fp8.py@@ -12,7 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.-from typing import Callable, Union+from collections.abc import Callableimport torch@@ -58,11 +58,11 @@ class FP8Format(OutputFormat):self,output_dir: str,model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,+ device: str | torch.device = "cpu",+ serialization_dict: dict | None = None,**kwargs,) -> torch.nn.Module:from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autorounddiff --git a/auto_round/export/formats/backends/gguf.py b/auto_round/export/formats/backends/gguf.pyindex cccb86719..8a215930c 100644--- a/auto_round/export/formats/backends/gguf.py+++ b/auto_round/export/formats/backends/gguf.py@@ -15,8 +15,9 @@import copyimport osimport re+from collections.abc import Callablefrom dataclasses import fields-from typing import Any, Callable, Union+from typing import Anyimport torchimport transformers@@ -176,11 +177,11 @@ class GGUFFormat(OutputFormat):self,output_dir: str,model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,+ device: str | torch.device = "cpu",+ serialization_dict: dict | None = None,**kwargs,) -> torch.nn.Module:from auto_round.export.export_to_gguf.export import save_quantized_as_gguf@@ -203,7 +204,7 @@ class GGUFFormat(OutputFormat):scheme: QuantizationScheme,model,platform: str,- formats: Union[str, list[str]] = None,+ formats: str | list[str] | None = None,model_type=ModelType.TEXT,) -> QuantizationScheme:import argparse@@ -285,9 +286,9 @@ class GGUFFormat(OutputFormat):name: str,model: torch.nn.Module,device: torch.device,- output_dir: str = None,+ output_dir: str | None = None,mllm: bool = False,- layer_config: dict = None,+ layer_config: dict | None = None,tokenizer=None,processor=None,image_processor=None,@@ -376,9 +377,7 @@ def _search_gguf_type(gguf_type):def gguf_type_fallback(gguf_type: str) -> str:gguf_type = gguf_type.lower()- if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):- gguf_type = "gguf:q5_0"- elif gguf_type == "gguf:q5_k":+ if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k", "gguf:q5_k"):gguf_type = "gguf:q5_0"elif gguf_type == "gguf:q6_k":gguf_type = "gguf:q8_0"@@ -473,9 +472,7 @@ def _get_digital_in_layer_name(layer_name):def _gguf_type_fallback(gguf_type: str) -> str:gguf_type = gguf_type.lower()- if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):- gguf_type = "gguf:q5_0"- elif gguf_type == "gguf:q5_k":+ if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k", "gguf:q5_k"):gguf_type = "gguf:q5_0"elif gguf_type == "gguf:q6_k":gguf_type = "gguf:q8_0"@@ -749,7 +746,7 @@ def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, modelbase_target_bits = int(inner_gguf_format[6])def _resolve_gguf_name(layer_name):- if model_type != ModelType.TEXT and any([key in layer_name for key in MM_MODULE_KEYS]):+ if model_type != ModelType.TEXT and any(key in layer_name for key in MM_MODULE_KEYS):gguf_layer_name = tensor_map_vision.get_name(layer_name)if gguf_layer_name is None:for key in MM_MODULE_KEYS:diff --git a/auto_round/export/formats/backends/llm_compressor.py b/auto_round/export/formats/backends/llm_compressor.pyindex 02493a0b9..749e41078 100644--- a/auto_round/export/formats/backends/llm_compressor.py+++ b/auto_round/export/formats/backends/llm_compressor.py@@ -13,7 +13,8 @@# limitations under the License.import re-from typing import Any, Callable, Union+from collections.abc import Callable+from typing import Anyimport torch@@ -157,9 +158,11 @@ class LLMCompressorFormat(OutputFormat):if scheme.act_bits <= 8 and (not scheme.is_act_standard_fp() or scheme.act_dynamic):if scheme.act_data_type == "nvfp4_v2":return None, scheme, layer_config, quant_block_list- if (scheme.is_act_nv_fp() and "static_gs" in scheme.act_data_type) or scheme.is_act_mx_fp():- return None, scheme, layer_config, quant_block_list- elif scheme.is_dynamic_afp8() and scheme.is_block_wfp8():+ if (+ (scheme.is_act_nv_fp() and "static_gs" in scheme.act_data_type)+ or scheme.is_act_mx_fp()+ or (scheme.is_dynamic_afp8() and scheme.is_block_wfp8())+ ):return None, scheme, layer_config, quant_block_listelse:bits, group_size, sym, act_bits = 8, -1, True, 8@@ -189,15 +192,11 @@ class LLMCompressorFormat(OutputFormat):from auto_round.export.export_to_llmcompressor.export_to_static_fp import pack_layerreturn pack_layer(layer_name, model, self.get_backend_name(), device=device)- elif re.search(f"{BackendDataType.INT8.value}", self.output_format):- from auto_round.export.export_to_llmcompressor.export import pack_layer-- return pack_layer(layer_name, model, device=device)- elif re.search(f"{BackendDataType.FP8_BLOCK.value}", self.output_format):- from auto_round.export.export_to_llmcompressor.export import pack_layer-- return pack_layer(layer_name, model, device=device)- elif re.search(f"{BackendDataType.WINT_A16.value}", self.output_format):+ elif (+ re.search(f"{BackendDataType.INT8.value}", self.output_format)+ or re.search(f"{BackendDataType.FP8_BLOCK.value}", self.output_format)+ or re.search(f"{BackendDataType.WINT_A16.value}", self.output_format)+ ):from auto_round.export.export_to_llmcompressor.export import pack_layerreturn pack_layer(layer_name, model, device=device)@@ -209,11 +208,11 @@ class LLMCompressorFormat(OutputFormat):self,output_dir: str,model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,+ device: str | torch.device = "cpu",+ serialization_dict: dict | None = None,**kwargs,) -> torch.nn.Module:backend = self.get_backend_name()diff --git a/auto_round/export/formats/backends/mlx.py b/auto_round/export/formats/backends/mlx.pyindex 37717a297..28ff2e909 100644--- a/auto_round/export/formats/backends/mlx.py+++ b/auto_round/export/formats/backends/mlx.py@@ -13,7 +13,7 @@# limitations under the License.import re-from typing import Callable, Union+from collections.abc import Callableimport torch@@ -55,11 +55,11 @@ class MLXFormat(OutputFormat):self,output_dir: str,model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,+ device: str | torch.device = "cpu",+ serialization_dict: dict | None = None,**kwargs,) -> torch.nn.Module:from auto_round.export.export_to_mlx.export import save_quantized_as_mlxdiff --git a/auto_round/export/formats/backends/svdquant_nunchaku.py b/auto_round/export/formats/backends/svdquant_nunchaku.pyindex d8c36c424..a5b3275d8 100644--- a/auto_round/export/formats/backends/svdquant_nunchaku.py+++ b/auto_round/export/formats/backends/svdquant_nunchaku.py@@ -15,7 +15,8 @@from __future__ import annotationsimport os-from typing import Any, Callable, Union+from collections.abc import Callable+from typing import Anyimport torch@@ -132,11 +133,11 @@ class SVDQuantNunchakuFormat(OutputFormat):self,output_dir: str,model: torch.nn.Module = None,- tokenizer: Callable = None,- layer_config: dict = None,+ tokenizer: Callable | None = None,+ layer_config: dict | None = None,inplace: bool = True,- device: Union[str, torch.device] = "cpu",- serialization_dict: dict = None,+ device: str | torch.device = "cpu",+ serialization_dict: dict | None = None,*,config=None,residual_provider=None,diff --git a/auto_round/export/formats/base.py b/auto_round/export/formats/base.pyindex 618e3e0fa..320e8293c 100644--- a/auto_round/export/formats/base.py+++ b/auto_round/export/formats/base.py@@ -18,9 +18,10 @@ import copyimport osimport refrom abc import ABC, abstractmethod+from collections.abc import Callablefrom dataclasses import asdictfrom enum import Enum-from typing import Any, Callable, Optional, Union+from typing import Anyimport torchimport transformers@@ -162,7 +163,7 @@ class OutputFormat(ABC):return self.backend.is_supported_immediate_saving() if self.backend is not None else True@classmethod- def is_support_scheme(cls: OutputFormat, scheme: Union[str, QuantizationScheme]) -> bool:+ def is_support_scheme(cls: OutputFormat, scheme: str | QuantizationScheme) -> bool:if isinstance(scheme, str) and scheme.upper() in cls.support_schemes:return Trueif isinstance(scheme, QuantizationScheme):@@ -177,7 +178,7 @@ class OutputFormat(ABC):self,scheme: QuantizationScheme,ctx: Any,- ) -> tuple[Optional[str], QuantizationScheme, dict, list]:+ ) -> tuple[str | None, QuantizationScheme, dict, list]:layer_config, quant_block_list = ctx.layer_config, ctx.quant_block_listif self.backend is not None:new_format, scheme, layer_config, quant_block_list = self.backend.check_and_reset_format(scheme, ctx)diff --git a/auto_round/export/formats/resolver.py b/auto_round/export/formats/resolver.pyindex 73c6739c1..46c94a73f 100644--- a/auto_round/export/formats/resolver.py+++ b/auto_round/export/formats/resolver.py@@ -14,8 +14,8 @@from __future__ import annotations+from collections.abc import Iterablefrom types import SimpleNamespace-from typing import Iterableimport torch@@ -94,15 +94,15 @@ def _precise_gguf_name(formats: list[OutputFormat]) -> str | None:def resolve_formats(scheme: ResolvedScheme,*,- format: str = None,- layer_config: dict = None,+ format: str | None = None,+ layer_config: dict | None = None,scale_dtype=None,quant_block_list=None,mllm: bool = False,iters: int = 0,enable_alg_ext: bool = False,quant_nontext_module: bool = False,- platform: str = None,+ platform: str | None = None,is_auto_scheme: bool = False,model=None,) -> FormatResolution:diff --git a/auto_round/export/svdquant_adapters/__init__.py b/auto_round/export/svdquant_adapters/__init__.pyindex 1404640b9..00782535d 100644--- a/auto_round/export/svdquant_adapters/__init__.py+++ b/auto_round/export/svdquant_adapters/__init__.py@@ -89,11 +89,11 @@ def resolve_svdquant_model_adapter(__all__ = [- "FLUX_TOP_LEVEL_TENSOR_KEYS","FLUX_SVDQUANT_TARGET_MODULES",+ "FLUX_TOP_LEVEL_TENSOR_KEYS","SDXL_SVDQUANT_TARGET_MODULES",- "SDXLSVDQuantNunchakuAdapter","FluxSVDQuantNunchakuAdapter",+ "SDXLSVDQuantNunchakuAdapter","detect_svdquant_model_adapter","flux_onefile_tensor_count","resolve_svdquant_model_adapter",diff --git a/auto_round/export/svdquant_nunchaku.py b/auto_round/export/svdquant_nunchaku.pyindex 7dfa31376..c2d695ab4 100644--- a/auto_round/export/svdquant_nunchaku.py+++ b/auto_round/export/svdquant_nunchaku.py@@ -16,8 +16,9 @@ from __future__ import annotationsimport jsonimport os+from collections.abc import Iterable, Mappingfrom dataclasses import dataclass, field-from typing import Iterable, Mapping, Protocol+from typing import Protocolimport torchdiff --git a/auto_round/export/utils.py b/auto_round/export/utils.pyindex 121f89e72..f76d30809 100644--- a/auto_round/export/utils.py+++ b/auto_round/export/utils.py@@ -92,7 +92,7 @@ def _state_dict_has_meta_tensor(model: nn.Module) -> bool:return False-def is_immediate_saving_mode(model: nn.Module, serialization_dict: dict = None) -> bool:+def is_immediate_saving_mode(model: nn.Module, serialization_dict: dict | None = None) -> bool:"""Determine if the model was saved via ShardWriter (immediate saving mode).Resolution order:@@ -106,9 +106,7 @@ def is_immediate_saving_mode(model: nn.Module, serialization_dict: dict = None)return Trueif unsupported_meta_device(model):return True- if _state_dict_has_meta_tensor(model):- return True- return False+ return _state_dict_has_meta_tensor(model)def is_local_pipeline_model_dir(model_dir: str) -> bool:@@ -507,8 +505,8 @@ def filter_quantization_config(quantization_config):default_dict["lr"] = 1.0 / iters if iters > 0 else 5e-3default_dict["minmax_lr"] = default_dict["lr"]- for key in default_dict:- if key in quantization_config and default_dict[key] == quantization_config[key]:+ for key, default_value in default_dict.items():+ if key in quantization_config and default_value == quantization_config[key]:quantization_config.pop(key)for k in list(quantization_config.keys()):if quantization_config[k] is None:@@ -536,9 +534,7 @@ def filter_quantization_config(quantization_config):if callable(key):quantization_config.pop(key)elif isinstance(quantization_config[key], (list, tuple)):- if any([callable(item) for item in quantization_config[key]]):- quantization_config.pop(key)- elif len(quantization_config[key]) == 0:+ if any(callable(item) for item in quantization_config[key]) or len(quantization_config[key]) == 0:quantization_config.pop(key)if key in clean_list and key in quantization_config:quantization_config.pop(key)diff --git a/auto_round/formats.py b/auto_round/formats.pyindex e28ed025c..c4866ade4 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -72,16 +72,16 @@ def get_formats(format: str, ar):__all__ = ["AutoAWQFormat",- "AutoRoundExportFormat","AutoGPTQFormat",+ "AutoRoundExportFormat","AutoRoundFormat","BackendDataType",- "FakeFormat","FP8Format",+ "FakeFormat","GGUFFormat","LLMCompressorFormat","MLXFormat","OutputFormat",- "resolve_formats","get_formats",+ "resolve_formats",]diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex d45549a75..b25236007 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -15,7 +15,7 @@ import functoolsimport platformfrom dataclasses import dataclass, fieldfrom importlib import import_module-from typing import TYPE_CHECKING, Any, Optional+from typing import TYPE_CHECKING, Anyimport torchfrom packaging.version import Version@@ -29,6 +29,8 @@ from auto_round.utils import get_library_versionBackendInfos = {}+import sys+import cpuinfoif TYPE_CHECKING:@@ -91,18 +93,18 @@ class BackendInfo:packing_format: list[str]bits: list[int]compute_dtype: list[str] = None- data_type: Optional[list[str]] = None- group_size: Optional[list[int]] = None- act_bits: Optional[list[int]] = None- act_group_size: Optional[list[int]] = None- act_sym: Optional[list[bool]] = None- act_data_type: Optional[list[str]] = None- act_dynamic: Optional[list[bool]] = None+ data_type: list[str] | None = None+ group_size: list[int] | None = None+ act_bits: list[int] | None = None+ act_group_size: list[int] | None = None+ act_sym: list[bool] | None = None+ act_data_type: list[str] | None = None+ act_dynamic: list[bool] | None = Nonepriority: int = 0 ##higher is bettercheckers: list[Any] = field(default_factory=list)- alias: Optional[list[str]] = None- requirements: Optional[list[str]] = None- systems: Optional[list[str]] = None+ alias: list[str] | None = None+ requirements: list[str] | None = None+ systems: list[str] | None = NoneBACKEND_ACT_ATTRS = [@@ -211,8 +213,8 @@ def fp8_static_scheme_checker(in_feature: int,out_feature: int,config: QuantizationScheme,- in_feature_multiplier: Optional[int] = None,- out_feature_multiplier: Optional[int] = None,+ in_feature_multiplier: int | None = None,+ out_feature_multiplier: int | None = None,):from auto_round.schemes import FP8_STATIC@@ -1154,7 +1156,7 @@ def get_autogptq_infer_linear(backend, bits=4, group_size=128, sym=False):return QuantLinear-def find_backend(backend: str, orig_backend: str = None):+def find_backend(backend: str, orig_backend: str | None = None):"""Finds the matching backend key based on the target backend name or its aliases.@@ -1196,7 +1198,7 @@ def get_all_compatible_backend(# Find compatible backendscompatible_backends = [key- for key in BackendInfos.keys()+ for key in BackendInfosif check_compatible(key, device, config, packing_format, in_features, out_features, check_requirements=False)]@@ -1240,8 +1242,8 @@ def get_layer_backend(if backend == "auto":backends = BackendInfos.keys()else:- for key in BackendInfos.keys():- if backend == key or (BackendInfos[key].alias and backend in BackendInfos[key].alias):+ for key, info in BackendInfos.items():+ if backend == key or (info.alias and backend in info.alias):backends.append(key)# Find and store other compatible backends@@ -1281,8 +1283,7 @@ def get_highest_priority_backend() -> str | None:current_system = platform.system().lower()supported_backends = []- for key in BackendInfos.keys():- backend = BackendInfos[key]+ for key, backend in BackendInfos.items():# Filter by operating system (e.g. MLX is Darwin-only; ark CPU# backends are non-Darwin only).if backend.systems is not None:@@ -1384,10 +1385,10 @@ def process_requirement(requirements: list, target_device="cuda", logger_level="for msg in install_instructions:log(msg)if logger_level == "error" and len(pip_cmds) == 0:- exit(-1)+ sys.exit(-1)joined_cmds = " and ".join(f"`{cmd}`" for cmd in pip_cmds)if joined_cmds:log(joined_cmds)if logger_level == "error":- exit(-1)+ sys.exit(-1)diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 7ebf1718b..eec7967c9 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -11,10 +11,10 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+import itertoolsimport osimport refrom types import SimpleNamespace-from typing import Unionimport torchimport torch.nn as nn@@ -70,7 +70,7 @@ def skip_not_convert_modules(model, quantization_config, layer_names, layer_confmodules_to_not_convert = _get_modules_to_not_convert(model, modules_to_not_convert)if modules_to_not_convert:for layer_name in layer_names:- if any([re.search(re.compile(n), layer_name) for n in modules_to_not_convert]):+ if any(re.search(re.compile(n), layer_name) for n in modules_to_not_convert):layer_configs[layer_name] = {"bits": 16, "act_bits": 16}return layer_configs@@ -101,9 +101,9 @@ def get_keys_to_not_convert(model):tied_params = find_tied_parameters(tied_model)# For compatibility with Accelerate < 0.18if isinstance(tied_params, dict):- tied_keys = sum(list(tied_params.values()), []) + list(tied_params.keys())+ tied_keys = list(itertools.chain.from_iterable(tied_params.values())) + list(tied_params.keys())else:- tied_keys = sum(tied_params, [])+ tied_keys = list(itertools.chain.from_iterable(tied_params))has_tied_params = len(tied_keys) > 0# If there is not tied weights, we want to keep the lm_head(output_embedding) in full precision@@ -369,7 +369,7 @@ def get_layer_config(model, quantization_config):quantization_config.modules_in_block_to_quantize) # Flatten the listfor layer_name in layer_names:- if not any([re.search(re.compile(n), layer_name) is not None for n in modules_in_block_to_quantize]):+ if not any(re.search(re.compile(n), layer_name) is not None for n in modules_in_block_to_quantize):extra_config[layer_name] = {"bits": 16} # Default to 16-bit for unquantized layers# Expand GPTQ 'dynamic' config (regex-based)@@ -408,7 +408,7 @@ def get_layer_config(model, quantization_config):return layer_configs-def get_device(obj: Union[torch.Tensor, nn.Module]) -> torch.device:+def get_device(obj: torch.Tensor | nn.Module) -> torch.device:if isinstance(obj, torch.Tensor):return obj.devicereturn next(obj.parameters()).device@@ -601,7 +601,7 @@ def _create_quant_layer(layer, layer_backend, config, in_features, out_features,)-def infer_target_device(device_map: Union[dict, int, str, None] = None) -> str:+def infer_target_device(device_map: dict | int | str | None = None) -> str:"""Infers the target device from a device_map.Args:diff --git a/auto_round/logger.py b/auto_round/logger.pyindex e439d2cad..c67e88528 100644--- a/auto_round/logger.py+++ b/auto_round/logger.py@@ -14,15 +14,16 @@import loggingimport warnings-from functools import lru_cache, wraps-from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional, TypeVar+from collections.abc import Callable, Mapping+from functools import cache, wraps+from typing import TypeVarimport auto_round.envs as envsT = TypeVar("T", bound="Callable") # used by `deprecated`-@lru_cache(maxsize=None)+@cachedef warning_once(self, msg, *args):"""Log a warning message only once per unique message/arguments combination.@@ -34,7 +35,7 @@ def warning_once(self, msg, *args):logger.warning(msg, *args, stacklevel=2)-@lru_cache(maxsize=None)+@cachedef info_once(self, msg, *args):"""Log an info message only once per unique message/arguments combination.@@ -115,7 +116,7 @@ fh.setFormatter(AutoRoundFormatter())logger.addHandler(fh)-def deprecated(future_name: Optional[str] = None, message: Optional[str] = None) -> Callable[[T], T]:+def deprecated(future_name: str | None = None, message: str | None = None) -> Callable[[T], T]:"""Decorator to mark functions as deprecateddiff --git a/auto_round/modeling/finegrained_fp8_patch.py b/auto_round/modeling/finegrained_fp8_patch.pyindex 06d85835b..8b0cfb978 100644--- a/auto_round/modeling/finegrained_fp8_patch.py+++ b/auto_round/modeling/finegrained_fp8_patch.py@@ -133,7 +133,7 @@ class Fp8Quantize(ConversionOps):def convert(self, input_dict: torch.Tensor, **kwargs) -> dict[str, torch.Tensor]:# Unpack single key/value (value may be wrapped in a list)- target_keys, value = tuple(input_dict.items())[0]+ target_keys, value = next(iter(input_dict.items()))value = value[0]# Resolve block size (support dict-like or attr-like quant_config)@@ -152,10 +152,8 @@ class Fp8Quantize(ConversionOps):# Enforce exact tiling like your originalif rows % block_m != 0 or cols % block_n != 0:raise ValueError(- (- f"Matrix dimensions ({rows}, {cols}) must be divisible by block sizes"- f" ({block_m}, {block_n}). for {target_keys}"- )+ f"Matrix dimensions ({rows}, {cols}) must be divisible by block sizes"+ f" ({block_m}, {block_n}). for {target_keys}")# Leading dims can be empty (2D) or include num_experts/... (3D+)diff --git a/auto_round/modeling/finegrained_fp8_patch_v4.py b/auto_round/modeling/finegrained_fp8_patch_v4.pyindex 11c2f6882..7b80edce1 100644--- a/auto_round/modeling/finegrained_fp8_patch_v4.py+++ b/auto_round/modeling/finegrained_fp8_patch_v4.py@@ -12,7 +12,6 @@# See the License for the specific language governing permissions and# limitations under the License.# Copied from https://github.com/huggingface/transformers/blob/v4.57.3/src/transformers/integrations/finegrained_fp8.py-from typing import Optionalfrom transformers.utils import is_accelerate_available, is_torch_available, logging@@ -48,7 +47,7 @@ class FP8Linear(nn.Linear):out_features: int,bias: bool = False,dtype=None,- block_size: Optional[tuple[int, int]] = None,+ block_size: tuple[int, int] | None = None,device=None,activation_scheme="dynamic",):diff --git a/auto_round/modeling/fp8_quant.py b/auto_round/modeling/fp8_quant.pyindex 3f4a25057..75dbc7841 100644--- a/auto_round/modeling/fp8_quant.py+++ b/auto_round/modeling/fp8_quant.py@@ -173,10 +173,8 @@ def apply_fp8_expert_replacement_patch():auto_round_logger.debug("Applied FP8 expert replacement patch to transformers.")OriginalFineGrainedFP8HfQuantizer.validate_environment = oot_validate_environmentauto_round_logger.debug(- (- "Patched FineGrainedFP8HfQuantizer.validate_environment to bypass device "- "capability check for loading FP8 models on unsupported GPUs."- )+ "Patched FineGrainedFP8HfQuantizer.validate_environment to bypass device "+ "capability check for loading FP8 models on unsupported GPUs.")except ImportError as e:auto_round_logger.warning(f"Could not apply FP8 expert replacement patch as {e}.")diff --git a/auto_round/modeling/fused_moe/__init__.py b/auto_round/modeling/fused_moe/__init__.pyindex d29e61a91..6e508e910 100644--- a/auto_round/modeling/fused_moe/__init__.py+++ b/auto_round/modeling/fused_moe/__init__.py@@ -31,7 +31,7 @@ from auto_round.modeling.fused_moe.moe_experts_interface import (resolve_experts_implementation,)-__all__ = [+__all__ = [ # noqa: RUF022"ReplacementModuleBase","apply_replacements","materialize_model_",diff --git a/auto_round/modeling/fused_moe/deepseek_v2.py b/auto_round/modeling/fused_moe/deepseek_v2.pyindex c7d386763..c65e7b66d 100644--- a/auto_round/modeling/fused_moe/deepseek_v2.py+++ b/auto_round/modeling/fused_moe/deepseek_v2.py@@ -14,8 +14,8 @@import inspectimport warnings+from collections.abc import Callablefrom functools import partial-from typing import Callable, Optionalimport torchfrom transformers.modeling_rope_utils import dynamic_rope_update@@ -67,12 +67,12 @@ def rotary_emb_forward(module, x, position_ids):def attn_forward(module,hidden_states: torch.Tensor,- attention_mask: Optional[torch.Tensor] = None,- cache_position: Optional[torch.LongTensor] = None,- position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None,- position_ids: Optional[torch.Tensor] = None,+ attention_mask: torch.Tensor | None = None,+ cache_position: torch.LongTensor | None = None,+ position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None,+ position_ids: torch.Tensor | None = None,**kwargs,-) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]:+) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]:if "padding_mask" in kwargs:warnings.warn("Passing `padding_mask` is deprecated and will be removed in v4.37. "diff --git a/auto_round/modeling/fused_moe/grouped_experts.py b/auto_round/modeling/fused_moe/grouped_experts.pyindex c9a755316..d2858ac09 100644--- a/auto_round/modeling/fused_moe/grouped_experts.py+++ b/auto_round/modeling/fused_moe/grouped_experts.py@@ -221,9 +221,7 @@ def _projection_is_supported(layer: nn.Module) -> bool:# forward hook on each expert Linear, and the grouped path -- which multiplies the# weights directly and never calls ``Linear.forward`` -- would silently skip them,# leaving every expert without ``act_max`` (breaking static-act export, e.g. NVFP4).- if layer._forward_pre_hooks or layer._forward_hooks:- return False- return True+ return not (layer._forward_pre_hooks or layer._forward_hooks)if not _is_wrapper_linear(layer):return Falseorig = layer.orig_layer@@ -231,9 +229,7 @@ def _projection_is_supported(layer: nn.Module) -> bool:return False # Conv1D / LinearAllreduce need their own forwardif orig._forward_pre_hooks or orig._forward_hooks:return False # e.g. online Hadamard rotation must run per layer- if getattr(layer, "enable_act_quant", False) and not _act_quant_is_row_independent(layer):- return False- return True+ return not (getattr(layer, "enable_act_quant", False) and not _act_quant_is_row_independent(layer))def _compute_device(layer: nn.Module) -> torch.device:diff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.pyindex cf0954b1b..dc9596854 100644--- a/auto_round/modeling/fused_moe/moe_experts_interface.py+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py@@ -122,8 +122,6 @@ class _ExpertContainer(nn.Module):which matches the standard checkpoint format without any hooks."""- pass-def _install_compact_expert_repr(module: nn.Module) -> None:"""Install compact __repr__ on the module's class.diff --git a/auto_round/modeling/fused_moe/qwen3_vl_moe.py b/auto_round/modeling/fused_moe/qwen3_vl_moe.pyindex eb76cfc34..2df48b721 100644--- a/auto_round/modeling/fused_moe/qwen3_vl_moe.py+++ b/auto_round/modeling/fused_moe/qwen3_vl_moe.py@@ -50,7 +50,7 @@ class LinearQwen3VLMoeTextSparseMoeBlock(ReplacementModuleBase):calibrate_all_experts: bool = False,):super().__init__(original)- text_config: "Qwen3VLMoeTextConfig" = config.get_text_config()+ text_config: Qwen3VLMoeTextConfig = config.get_text_config()self.hidden_size = text_config.hidden_sizeself.num_experts = text_config.num_expertsdiff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.pyindex f40867bb8..5048e8ab8 100644--- a/auto_round/modeling/fused_moe/replace_modules.py+++ b/auto_round/modeling/fused_moe/replace_modules.py@@ -15,7 +15,6 @@from abc import ABC, abstractmethodfrom contextlib import contextmanagerfrom dataclasses import dataclass-from typing import Dict, Typeimport torchfrom tqdm import tqdm@@ -124,7 +123,7 @@ def is_custom_model(model: torch.nn.Module) -> bool:def _find_first_moe_block(model: torch.nn.Module) -> tuple[str, torch.nn.Module] | tuple[None, None]:"""Return ``(name, module)`` of the first experts-like module, or ``(None, None)``."""for name, module in model.named_modules():- if name.endswith(".experts") or name.endswith(".moe"):+ if name.endswith((".experts", ".moe")):return name, modulereturn None, None@@ -226,7 +225,7 @@ class ReplacementModuleBase(ABC, torch.nn.Module):"""# Registry: module class name -> replacement module class- _replacement_registry: Dict[str, Type["ReplacementModuleBase"]] = {}+ _replacement_registry: dict[str, type["ReplacementModuleBase"]] = {}supports_gguf_fused_moe: bool = Falsedef __init_subclass__(cls, **kwargs):@@ -263,7 +262,7 @@ class ReplacementModuleBase(ABC, torch.nn.Module):self._materialized = False@classmethod- def get_replacement_class(cls, module_class_name: str) -> Type["ReplacementModuleBase"]:+ def get_replacement_class(cls, module_class_name: str) -> type["ReplacementModuleBase"]:"""Get replacement class for a given module class name."""return cls._replacement_registry.get(module_class_name)@@ -292,7 +291,6 @@ class ReplacementModuleBase(ABC, torch.nn.Module):@abstractmethoddef original_module_class(cls) -> str:"""Return the class name of the module this replaces."""- pass@classmethod@abstractmethod@@ -302,7 +300,6 @@ class ReplacementModuleBase(ABC, torch.nn.Module):config,) -> "ReplacementModuleBase":"""Create replacement module from original module."""- passdef materialize_weights(self):"""Materialize weights if needed."""@@ -316,7 +313,6 @@ class ReplacementModuleBase(ABC, torch.nn.Module):Subclasses should override this method to implementweight materialization logic."""- passdef release_original_module(self) -> None:"""Release reference to the original module to free memory."""@@ -467,7 +463,7 @@ class ModuleReplacementTracker:def __new__(cls):if cls._instance is None:- cls._instance = super(ModuleReplacementTracker, cls).__new__(cls)+ cls._instance = super().__new__(cls)return cls._instancedef __init__(self):@@ -476,9 +472,9 @@ class ModuleReplacementTracker:return# Map from replacement module id to original module- self._replacement_to_original: Dict[int, torch.nn.Module] = {}+ self._replacement_to_original: dict[int, torch.nn.Module] = {}# Map from module name to ReplacedModuleInfo- self._name_to_info: Dict[str, ReplacedModuleInfo] = {}+ self._name_to_info: dict[str, ReplacedModuleInfo] = {}ModuleReplacementTracker._initialized = Truediff --git a/auto_round/modeling/fused_moe/step3_5_moe.py b/auto_round/modeling/fused_moe/step3_5_moe.pyold mode 100755new mode 100644diff --git a/auto_round/modeling/unfused_moe/__init__.py b/auto_round/modeling/unfused_moe/__init__.pyindex 15012aea9..5f0b0747b 100644--- a/auto_round/modeling/unfused_moe/__init__.py+++ b/auto_round/modeling/unfused_moe/__init__.py@@ -147,7 +147,7 @@ def pre_check_config(model_name: str | torch.nn.Module, trust_remote_code: boolif isinstance(model_name, str):try:config = AutoConfig.from_pretrained(model_name, trust_remote_code=trust_remote_code)- except (OSError, EnvironmentError, ValueError):+ except (OSError, ValueError):return Falseelif isinstance(model_name, torch.nn.Module):config = getattr(model_name, "config", None)@@ -190,7 +190,7 @@ def apply_model_monkey_patches(model_name: str, trust_remote_code: bool = True)return False# patch blocksconfig = AutoConfig.from_pretrained(model_name, trust_remote_code=trust_remote_code)- model_type = getattr(config, "model_type")+ model_type = config.model_typecfg = MODEL_CONFIG[model_type]for orig_path, custom_path in cfg.get("block_patch", []):@@ -226,7 +226,7 @@ def apply_modeling_patch(model: torch.nn.Module) -> bool:res = pre_check_config(model)if not res:return False- model_type = getattr(model.config, "model_type")+ model_type = model.config.model_typecfg = MODEL_CONFIG[model_type]# patch blocksfor orig_path, custom_path in cfg.get("block_patch", []):diff --git a/auto_round/modeling/unfused_moe/glm_moe_light.py b/auto_round/modeling/unfused_moe/glm_moe_light.pyindex c16b52b56..b031ad7dd 100644--- a/auto_round/modeling/unfused_moe/glm_moe_light.py+++ b/auto_round/modeling/unfused_moe/glm_moe_light.py@@ -50,7 +50,6 @@ class LinearGlm4MoeLiteMoE(nn.Module):top_k_index: torch.Tensor,top_k_weights: torch.Tensor,) -> torch.Tensor:- """ """return sequential_moe_forward(hidden_states, top_k_index, top_k_weights, self.experts, self.num_experts)def route_tokens_to_experts(self, router_logits):diff --git a/auto_round/modeling/unfused_moe/qwen3_moe.py b/auto_round/modeling/unfused_moe/qwen3_moe.pyindex 2c3921014..d2c916139 100644--- a/auto_round/modeling/unfused_moe/qwen3_moe.py+++ b/auto_round/modeling/unfused_moe/qwen3_moe.py@@ -39,7 +39,6 @@ class LinearQwen3MoeSparseMoeBlock(nn.Module):)def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:- """ """batch_size, sequence_length, hidden_dim = hidden_states.shapehidden_states = hidden_states.view(-1, hidden_dim)# router_logits: (batch * sequence_length, n_experts)diff --git a/auto_round/modeling/unfused_moe/qwen3_next.py b/auto_round/modeling/unfused_moe/qwen3_next.pyindex 3441d66da..ad476e697 100644--- a/auto_round/modeling/unfused_moe/qwen3_next.py+++ b/auto_round/modeling/unfused_moe/qwen3_next.py@@ -42,7 +42,6 @@ class LinearQwen3NextSparseMoeBlock(nn.Module):self.shared_expert_gate = torch.nn.Linear(config.hidden_size, 1, bias=False)def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:- """ """batch_size, sequence_length, hidden_dim = hidden_states.shapehidden_states = hidden_states.view(-1, hidden_dim)# router_logits: (batch * sequence_length, n_experts)diff --git a/auto_round/scheme_entry.py b/auto_round/scheme_entry.pyindex 3326454db..9ec229439 100644--- a/auto_round/scheme_entry.py+++ b/auto_round/scheme_entry.py@@ -25,7 +25,7 @@ routing preview/validation helpers the entry uses before a compressor exists.from __future__ import annotations-from typing import TYPE_CHECKING, Union+from typing import TYPE_CHECKINGfrom auto_round.logger import loggerfrom auto_round.schemes import parse_scheme@@ -118,7 +118,7 @@ def eager_validate_scheme(config, scheme=None, format=None) -> None:temp_config.check_config() # raises ValueError / NotImplementedError if invalid-def is_weight_scheme(scheme: Union[str, dict, object]) -> bool:+def is_weight_scheme(scheme: str | dict | object) -> bool:if isinstance(scheme, str):return scheme.upper().startswith("W")if isinstance(scheme, dict):@@ -134,7 +134,7 @@ def is_weight_scheme(scheme: Union[str, dict, object]) -> bool:return False-def is_gguf_k_target(value: Union[str, "AutoScheme", object]) -> bool:+def is_gguf_k_target(value: str | AutoScheme | object) -> bool:from auto_round.auto_scheme.gen_auto_scheme import AutoSchemeif isinstance(value, str):diff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex 736c62cac..3b6de11c0 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -15,7 +15,7 @@ import copyfrom copy import deepcopyfrom dataclasses import asdict, dataclass, fieldsfrom enum import Enum-from typing import TYPE_CHECKING, Any, Optional, Union+from typing import TYPE_CHECKING, Any, Unionimport torch@@ -144,15 +144,15 @@ from auto_round.logger import loggerfrom auto_round.utils import SUPPORTED_DTYPES, contain_any_mm_keys, infer_bits_by_data_type__all__ = [- "QuantizationScheme",- "BackendDataType",- "GGUF_SCHEME_FACTS","GGUF_PRESET_ALIASES",- "is_standard_fp",+ "GGUF_SCHEME_FACTS",+ "BackendDataType",+ "QuantizationScheme",+ "get_gguf_scheme","is_mx_fp",- "is_nv_fp","is_mx_int",- "get_gguf_scheme",+ "is_nv_fp",+ "is_standard_fp","preset_name_to_scheme",]@@ -200,14 +200,14 @@ class QuantizationScheme:group_size: int = 128sym: bool = Truedata_type: str = "int"- act_bits: Optional[int] = None- act_group_size: Optional[int] = None- act_sym: Optional[bool] = None- act_data_type: Optional[str] = None- act_dynamic: Optional[bool] = None- super_bits: Optional[int] = None- super_group_size: Optional[int] = None- rotation_config: Optional[dict] = None+ act_bits: int | None = None+ act_group_size: int | None = None+ act_sym: bool | None = None+ act_data_type: str | None = None+ act_dynamic: bool | None = None+ super_bits: int | None = None+ super_group_size: int | None = None+ rotation_config: dict | None = None@classmethoddef empty(cls):@@ -375,7 +375,7 @@ def preset_name_to_scheme(name: str) -> QuantizationScheme:return scheme_args-def scheme_to_preset_name(scheme: Union[str, QuantizationScheme]) -> str:+def scheme_to_preset_name(scheme: str | QuantizationScheme) -> str:"""Get preset scheme name from a QuantizationScheme instance."""if isinstance(scheme, str):name = scheme.upper()@@ -423,8 +423,8 @@ def _reconcile_bits_and_dtype(config: dict, prefix: str = ""):def _override_scheme_with_user_specify(- scheme: Union[str, dict, QuantizationScheme], user_scheme_overrides: dict[str, Any], return_str=True-) -> Union[str, QuantizationScheme]:+ scheme: str | dict | QuantizationScheme, user_scheme_overrides: dict[str, Any], return_str=True+) -> str | QuantizationScheme:"""Updates a base quantization scheme with user-provided overrides.Handles GGUF formatting and synchronizes weight/activation parameters.@@ -518,8 +518,8 @@ def format_allows_w8_asym(format: str | None) -> bool:def parse_scheme(scheme: Union[str, dict, QuantizationScheme, "AutoScheme"],user_scheme_overrides: dict[str, Any],- format: str = None,-) -> tuple[Union[str, QuantizationScheme], bool, dict[str, Any]]:+ format: str | None = None,+) -> tuple[str | QuantizationScheme, bool, dict[str, Any]]:"""Parses the final scheme."""@@ -978,14 +978,12 @@ def _handle_special_schemes(layer_config[n] = {"bits": 4, "data_type": "int"}elif n != lm_head_name and mllm:layer_config[n] = {"bits": 16}- elif n != lm_head_name:- layer_config[n] = {"bits": 8, "data_type": "int"}- elif n == lm_head_name and quant_lm_head:+ elif n != lm_head_name or quant_lm_head:layer_config[n] = {"bits": 8, "data_type": "int"}return layer_config-def get_gguf_scheme(scheme: Union[str, QuantizationScheme]) -> str:+def get_gguf_scheme(scheme: str | QuantizationScheme) -> str:if scheme is None:return ""if isinstance(scheme, str) and scheme.upper().startswith("GGUF"):diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex dd9202c60..41c66099f 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -14,9 +14,10 @@import importlibimport reimport sys+from collections.abc import Callablefrom dataclasses import dataclass, fieldfrom types import SimpleNamespace-from typing import Any, Callable+from typing import Anyimport torch@@ -394,7 +395,7 @@ def _handle_special_model(model):def update_module(- model, formats: list[OutputFormat] = None, trust_remote_code: bool = True, cleanup_original: bool = True+ model, formats: list[OutputFormat] | None = None, trust_remote_code: bool = True, cleanup_original: bool = True):gguf_export = formats is not None and any(format_.is_gguf() for format_ in formats)model = apply_replacements(model, gguf_export=gguf_export)diff --git a/auto_round/utils/bit_packing.py b/auto_round/utils/bit_packing.pyindex f444cc4d1..a4dcf1a7c 100644--- a/auto_round/utils/bit_packing.py+++ b/auto_round/utils/bit_packing.py@@ -48,20 +48,18 @@ Only the block size (32 values) is required to divide the packed dimension;there is no ``32 % bits == 0`` constraint."""-from typing import Optional-import torch__all__ = ["AWQ_PACK_ORDER","AWQ_REVERSE_ORDER","SUPPORTED_PACKING_BITS",- "awq_reverse_reorder","awq_reorder",+ "awq_reverse_reorder","pack_bitstream",- "unpack_bitstream","packed_dim_size","requires_generic_bit_packing",+ "unpack_bitstream",]# Weight bit-widths that ``pack_bitstream`` / ``unpack_bitstream`` handle.@@ -240,7 +238,7 @@ def pack_scalar_zero(value: int, bits: int, num_words: int, shape, device=None)return row.reshape(*(1,) * (len(shape) - 1), num_words).expand(*shape).contiguous()-def infer_packed_bits(num_values: int, num_words: int) -> Optional[int]:+def infer_packed_bits(num_values: int, num_words: int) -> int | None:"""Best-effort recovery of ``bits`` from packed/unpacked dimension sizes."""if num_values <= 0 or num_words <= 0:return Nonediff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex 6f7dfbb3b..390c2cdae 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -126,7 +126,7 @@ def download_audiocaps_csv():logger.info(f"AudioCaps dataset cached at: {cache_file}")except requests.RequestException as e:raise RuntimeError(f"Failed to download AudioCaps from {url}: {e}") from e- except IOError as e:+ except OSError as e:raise RuntimeError(f"Failed to write AudioCaps cache to {cache_file}: {e}") from ereturn cache_file@@ -146,7 +146,7 @@ TORCH_VERSION_AT_LEAST_2_5 = torch_version_at_least("2.5.0")TORCH_VERSION_AT_LEAST_2_4 = torch_version_at_least("2.4.0")-class LazyImport(object):+class LazyImport:"""Lazy import python module till use."""def __init__(self, module_name):@@ -407,7 +407,7 @@ def monkey_patch_transformers():if parsed_version >= version.parse("5.0.0"):from transformers.initialization import no_init_weights- setattr(transformers.modeling_utils, "no_init_weights", no_init_weights)+ transformers.modeling_utils.no_init_weights = no_init_weightsif parsed_version >= version.parse("5.2.0"):# transformers 5.2.0 added Transpose.convert() which calls get_parameter() on# quantized buffer tensors (weight_packed, weight_scale), causing AttributeError.@@ -728,11 +728,11 @@ class SupportedFormats:self._support_list = self._support_format + self._gguf_formatdef __contains__(self, key):- return True if key in self._support_list else False+ return key in self._support_listdef __str__(self):# Return "(%s)" % ', '.join(self._support_format + ("gguf:q*_0", "gguf:q*_1", "gguf:q*_k_s"))- return "(%s)" % ", ".join(self._support_list)+ return f"({', '.join(self._support_list)})"def __getitem__(self, key):return self._support_list[key]diff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex fda2e7cf0..6ba56d090 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -19,10 +19,11 @@ import reimport shutilimport sysimport tempfile+from collections.abc import Callablefrom contextlib import ContextDecorator, contextmanagerfrom functools import lru_cachefrom threading import Lock-from typing import Any, Callable, Optional, Union+from typing import Anyimport cpuinfoimport psutil@@ -80,7 +81,7 @@ def _use_hpu_compile_mode():return TORCH_VERSION_AT_LEAST_2_4 and not is_hpu_lazy_mode()-def _bump_dynamo_cache_limit(min_size: Optional[int] = None):+def _bump_dynamo_cache_limit(min_size: int | None = None):"""Raise torch._dynamo cache/recompile limits.The same quant function (e.g. ``quant_tensor_sym``) is reused across@@ -109,9 +110,7 @@ def _bump_dynamo_cache_limit(min_size: Optional[int] = None):pass-def compile_func(- fun: Union[torch.nn.Module, Callable], device: Union[str, torch.device, int]-) -> Union[torch.nn.Module, Callable]:+def compile_func(fun: torch.nn.Module | Callable, device: str | torch.device | int) -> torch.nn.Module | Callable:"""Compile a function on the specified device.The shared dynamo cache-limit bump lives in :func:`_bump_dynamo_cache_limit`;@@ -162,11 +161,9 @@ def is_tbb_available(): # pragma: no coverreturn Falseif not _is_tbb_configured():logger.warning_once(- (- "TBB is installed but not configured correctly. \n"- "Please add the TBB library path to `LD_LIBRARY_PATH`, "- "for example: `export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/`."- )+ "TBB is installed but not configured correctly. \n"+ "Please add the TBB library path to `LD_LIBRARY_PATH`, "+ "for example: `export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/`.")return Falsereturn True@@ -180,9 +177,7 @@ def can_pack_with_numba(): # pragma: no coverif not is_numba_available():logger.warning_once("Numba is not installed, please install it with `pip install numba`.")return False- if not is_tbb_available():- return False- return True+ return is_tbb_available()## check hpex@@ -416,7 +411,7 @@ class fake_triton_for_hpu(ContextDecorator):return False-class CpuInfo(object):+class CpuInfo:"""Get CPU Info."""def __init__(self):@@ -596,7 +591,7 @@ def set_tuning_device_for_layer(model, name: str, device: str) -> None:def set_non_auto_device_map(- model: torch.nn.Module, device_map: Union[str, int, dict], quant_layer_names: Union[None, list, tuple] = None+ model: torch.nn.Module, device_map: str | int | dict, quant_layer_names: None | list | tuple = None) -> None:if not device_map or device_map == "auto" or isinstance(device_map, int):return@@ -1174,7 +1169,7 @@ def partition_dict_numbers(number_dict, n):# - Assign each item to the group with the current smallest sum# Complexity: O(m log n) which scales well for large m (layers)groups_sums = [0.0] * n- groups = [dict() for _ in range(n)]+ groups = [{} for _ in range(n)]# Sort items descending by sizeitems_sorted = sorted(items, key=lambda kv: kv[1], reverse=True)@@ -1242,7 +1237,7 @@ def set_avg_auto_device_map(model: torch.nn.Module, device_map):for device in device_list:if device.startswith("hpu") and len(device_list) > 1:logger.warning_once("Auto-scheme does not support multiple HPUs.")- if device.startswith("cpu") or device.startswith("hpu"):+ if device.startswith(("cpu", "hpu")):continuegpu_devices.append(device)num_devices = len(gpu_devices)@@ -1260,11 +1255,9 @@ def set_avg_auto_device_map(model: torch.nn.Module, device_map):params_dict[n] = in_features * out_featuresres_list = partition_dict_numbers(params_dict, num_devices)- device_index = 0- for res in res_list:+ for device_index, res in enumerate(res_list):for key in res.keys():set_tuning_device_for_layer(block_module, key, gpu_devices[device_index])- device_index += 1if __name__ == "__main__":@@ -1292,7 +1285,7 @@ if __name__ == "__main__":print(f"Group {i + 1}: {group}, Sum: {sum(group.values())}")-def parse_available_devices(device_map: Union[str, torch.device, int, dict, None] = None) -> list:+def parse_available_devices(device_map: str | torch.device | int | dict | None = None) -> list:"""Parse the device map and return a list of all available devices.@@ -1404,7 +1397,7 @@ def parse_available_devices(device_map: Union[str, torch.device, int, dict, Noneraise TypeError(f"Unsupported device_map type: {type(device_map)}")-@lru_cache(maxsize=None)def is_gaudi2():try:import habana_frameworks.torch.utils.experimental as htexp@@ -1602,7 +1595,7 @@ def dump_mem_usage(msg: str = "", log_level: str = "info"):# This function is designed for Auto Scheme and Diffusion Pipeline,# which requires dispatching the whole model on all available devices.def dispatch_model_by_all_available_devices(- model: torch.nn.Module, device_map: Union[str, int, dict, None]+ model: torch.nn.Module, device_map: str | int | dict | None) -> torch.nn.Module:# Important Notice: This dispatch does not follow dict device_map, just extract all available devices and use themdevice_type = get_major_device()diff --git a/auto_round/utils/device_manager.py b/auto_round/utils/device_manager.pyindex f2972d733..c06c3807d 100644--- a/auto_round/utils/device_manager.py+++ b/auto_round/utils/device_manager.py@@ -49,7 +49,6 @@ import functoolsimport gcimport reimport sys-from typing import Optional, Unionimport torch@@ -57,23 +56,23 @@ from auto_round.logger import logger__all__ = ["ARDevice",+ "ClearMemory","DeviceManager",+ "clear_memory",+ "default_enable_torch_compile",+ "detect_device_count","device_manager",- "normalize_default_device_map","get_ar_device",- "default_enable_torch_compile",+ "get_available_device_types","get_current_device_manager","get_current_device_type",- "is_device_available",- "get_available_device_types",- "get_major_device",- "detect_device_count","get_device_and_parallelism",+ "get_device_memory",+ "get_major_device","get_packing_device","is_auto_device_mapping",- "get_device_memory",- "ClearMemory",- "clear_memory",+ "is_device_available",+ "normalize_default_device_map",]@@ -89,7 +88,7 @@ __all__ = [_PREFERRED_ORDER = ("cuda", "xpu", "hpu") # add mps later-def normalize_default_device_map(device_map: Union[None, str, int, torch.device, dict]):+def normalize_default_device_map(device_map: None | str | int | torch.device | dict):"""Normalize default device selection across entry points.On Apple Silicon, the default ``0`` / ``"0"`` / ``None`` / ``"auto"``@@ -106,7 +105,7 @@ def normalize_default_device_map(device_map: Union[None, str, int, torch.device,return device_map-def _torch_accelerator_type() -> Optional[str]:+def _torch_accelerator_type() -> str | None:"""Return the canonical accelerator type reported by ``torch.accelerator``.A PyTorch build exposes at most one accelerator backend; this returns its@@ -167,7 +166,7 @@ def _hpu_available() -> bool:return False-def _normalize_device_type(device: Union[None, str, int, torch.device]) -> Optional[str]:+def _normalize_device_type(device: None | str | int | torch.device) -> str | None:"""Reduce any device spec to a bare backend type string (``"cuda"`` ...)."""if device is None:return get_current_device_type()@@ -240,7 +239,7 @@ def get_available_device_types() -> list[str]:class _DeviceIndexContext:"""Fallback for ``torch.accelerator.device_index`` on older PyTorch/backends."""- def __init__(self, device: "ARDevice", index: int):+ def __init__(self, device: ARDevice, index: int):self._device = deviceself._index = indexself._prev = None@@ -280,7 +279,7 @@ class ARDevice:#: PyTorch backend that lacks a dedicated subclass (e.g. a fresh ``npu``).device_type: str = ""- _registry: dict[str, type["ARDevice"]] = {}+ _registry: dict[str, type[ARDevice]] = {}def __init_subclass__(cls, **kwargs):super().__init_subclass__(**kwargs)@@ -289,7 +288,7 @@ class ARDevice:ARDevice._registry[dtype] = cls@classmethod- def create(cls, device_type: str) -> "ARDevice":+ def create(cls, device_type: str) -> ARDevice:"""Instantiate the most specific :class:`Device` for ``device_type``."""subclass = cls._registry.get(device_type)if subclass is not None:@@ -297,7 +296,7 @@ class ARDevice:return ARDevice(device_type)@staticmethod- def get_device_module(device: Union[None, str, int, torch.device] = None):+ def get_device_module(device: None | str | int | torch.device = None):"""Return the backend runtime module for ``device`` (e.g. ``torch.cuda``).This is a thin, version-tolerant wrapper around ``torch.get_device_module``@@ -320,7 +319,7 @@ class ARDevice:passreturn getattr(torch, device_type, None)- def __init__(self, device_type: Optional[str] = None):+ def __init__(self, device_type: str | None = None):self.type = device_type or self.device_type# Prefer the unified ``torch.accelerator`` API for runtime ops when this@@ -355,14 +354,14 @@ class ARDevice:passreturn 0- def set_device(self, index: Union[int, str, torch.device]) -> None:+ def set_device(self, index: int | str | torch.device) -> None:if self._module is None:returnok, _ = _module_call(self._module, ("set_device_index", "set_device_idx", "set_device"), index)if ok:return- def device(self, index: Union[int, str, torch.device, None] = None) -> torch.device:+ def device(self, index: int | str | torch.device | None = None) -> torch.device:"""Build a ``torch.device`` for this backend / card ``index``."""if index is None:return torch.device(self.type)@@ -377,7 +376,7 @@ class ARDevice:# return [self.device(i) for i in range(self.device_count())]# -- runtime ------------------------------------------------------------- def synchronize(self, index: Union[int, None] = None) -> None:+ def synchronize(self, index: int | None = None) -> None:if self._module is None:returnfn = getattr(self._module, "synchronize", None)@@ -510,7 +509,7 @@ class HpuARDevice(ARDevice):device_type = "hpu"@staticmethod- def get_device_module(device: Union[None, str, int, torch.device] = None):+ def get_device_module(device: None | str | int | torch.device = None):"""Return the backend runtime module for ``device`` (e.g. ``torch.cuda``).This is a thin, version-tolerant wrapper around ``torch.get_device_module``@@ -535,7 +534,7 @@ class HpuARDevice(ARDevice):except Exception: # pragma: no coverreturn None- def set_device(self, index: Union[int, str, torch.device]) -> None:+ def set_device(self, index: int | str | torch.device) -> None:if self._module is None:returnfn = getattr(self._module, "set_device", None)@@ -581,13 +580,13 @@ class MpsARDevice(ARDevice):device_type = "mps"- def __init__(self, device_type: Optional[str] = None):+ def __init__(self, device_type: str | None = None):# Always use torch.mps directly, never torch.accelerator.self.type = "mps"self._module = getattr(torch, "mps", None)@staticmethod- def get_device_module(device: Union[None, str, int, torch.device] = None):+ def get_device_module(device: None | str | int | torch.device = None):"""Return the backend runtime module for ``device`` (e.g. ``torch.cuda``).This is a thin, version-tolerant wrapper around ``torch.get_device_module``@@ -611,10 +610,10 @@ class MpsARDevice(ARDevice):def current_device(self) -> int:return 0- def set_device(self, index: Union[int, str, torch.device]) -> None:+ def set_device(self, index: int | str | torch.device) -> None:return None- def device(self, index: Union[int, str, torch.device, None] = None) -> torch.device:+ def device(self, index: int | str | torch.device | None = None) -> torch.device:"""Build a ``torch.device`` for this backend / card ``index``."""return torch.device("mps")@@ -678,7 +677,7 @@ class CpuARDevice(ARDevice):device_type = "cpu"@staticmethod- def get_device_module(device: Union[None, str, int, torch.device] = None):+ def get_device_module(device: None | str | int | torch.device = None):return None# -- discovery ----------------------------------------------------------@@ -691,20 +690,20 @@ class CpuARDevice(ARDevice):def current_device(self) -> int:return 0- def set_device(self, index: Union[int, str, torch.device]) -> None: # no-op+ def set_device(self, index: int | str | torch.device) -> None: # no-opreturn None- def device(self, index: Union[int, str, torch.device, None] = None) -> torch.device:+ def device(self, index: int | str | torch.device | None = None) -> torch.device:return torch.device("cpu")# -- runtime ------------------------------------------------------------- def synchronize(self, index: Union[int, None] = None) -> None: # no-op+ def synchronize(self, index: int | None = None) -> None: # no-opreturn Nonedef empty_cache(self) -> None: # no-op: CPU has no caching allocator.return gc.collect()- def get_device_capability(self, index: Union[int, None] = None):+ def get_device_capability(self, index: int | None = None):return Nonedef device_index(self, index: int): # nothing to switch on CPU.@@ -775,26 +774,26 @@ class DeviceManager:shared instance, so the active device / device_list is always single-sourced."""- _instance: Optional["DeviceManager"] = None+ _instance: DeviceManager | None = Nonedef __new__(cls, *args, **kwargs):if cls._instance is None:cls._instance = super().__new__(cls)return cls._instance- def __init__(self, device_map: Union[None, str, torch.device, int, dict] = None):+ def __init__(self, device_map: None | str | torch.device | int | dict = None):# Initialise backing state once; later constructions reuse the singleton.if not getattr(self, "_initialized", False):self._cache: dict[str, ARDevice] = {}self._device_map = None- self._device_list: Optional[list] = None- self._major_device: Optional[str] = None+ self._device_list: list | None = None+ self._major_device: str | None = Noneself._initialized = Trueif device_map is not None:self.configure(device_map)# -- device_map configuration ------------------------------------------- def configure(self, device_map: Union[None, str, torch.device, int, dict] = 0) -> "DeviceManager":+ def configure(self, device_map: None | str | torch.device | int | dict = 0) -> DeviceManager:"""Resolve a ``device_map`` into a concrete device list and major device.Centralises the device-map parsing the compressors used to perform by@@ -834,7 +833,7 @@ class DeviceManager:return self._major_device@device.setter- def device(self, value: Union[str, torch.device]) -> None:+ def device(self, value: str | torch.device) -> None:"""Override the major device (e.g. an OOM fallback to ``"cpu"``)."""self._major_device = str(value) if isinstance(value, torch.device) else value@@ -852,7 +851,7 @@ class DeviceManager:self._cache.pop(dtype, None)# -- lookup -------------------------------------------------------------- def get_ar_device(self, device_type: Union[None, str, int, torch.device] = None) -> ARDevice:+ def get_ar_device(self, device_type: None | str | int | torch.device = None) -> ARDevice:"""Return the cached :class:`Device` for ``device_type`` (default: current)."""normalized = _normalize_device_type(device_type) or "cpu"device = self._cache.get(normalized)@@ -889,13 +888,13 @@ class DeviceManager:device_manager = DeviceManager()-def get_ar_device(device_type: Union[None, str, int, torch.device] = None) -> ARDevice:+def get_ar_device(device_type: None | str | int | torch.device = None) -> ARDevice:"""Return the cached :class:`Device` handle for a specific backend type."""return device_manager.get_ar_device(device_type)def default_enable_torch_compile(- device: Union[None, str, int, torch.device] = None, platform_name: str | None = None+ device: None | str | int | torch.device = None, platform_name: str | None = None) -> bool:"""Return the safe torch.compile default for a backend."""return (platform_name or sys.platform) != "win32"@@ -914,7 +913,7 @@ def detect_device_count() -> int:return get_current_device_manager().device_count()-def get_device_and_parallelism(device: Union[str, torch.device, int, dict]) -> tuple[str, bool]:+def get_device_and_parallelism(device: str | torch.device | int | dict) -> tuple[str, bool]:"""Resolve a device spec into ``(device, parallelism)``.The multi-card *parallelism* policy itself is kept as a standalone function@@ -961,7 +960,7 @@ def get_device_and_parallelism(device: Union[str, torch.device, int, dict]) -> treturn device, parallelism-def get_packing_device(device: Union[str, torch.device, None] = "auto") -> torch.device:+def get_packing_device(device: str | torch.device | None = "auto") -> torch.device:"""Selects the packing device.- ``"auto"``: choose best available (active accelerator > CPU).@@ -987,12 +986,10 @@ def get_packing_device(device: Union[str, torch.device, None] = "auto") -> torchraise TypeError(f"Unsupported device type: {type(device)} ({device})")-def is_auto_device_mapping(device_map: Union[str, int, dict, None]) -> bool:+def is_auto_device_mapping(device_map: str | int | dict | None) -> bool:if device_map is None or isinstance(device_map, int):return False- elif device_map == "auto":- return True- elif isinstance(device_map, str) and "," in device_map:+ elif device_map == "auto" or (isinstance(device_map, str) and "," in device_map):return Trueelif isinstance(device_map, dict):return False@@ -1000,7 +997,7 @@ def is_auto_device_mapping(device_map: Union[str, int, dict, None]) -> bool:return False-def get_major_device(device_map: Union[None, str, torch.device, int, dict] = None) -> str:+def get_major_device(device_map: None | str | torch.device | int | dict = None) -> str:if device_map is None or isinstance(device_map, (str, torch.device, int)):"""Detects the appropriate computation device.@@ -1044,8 +1041,6 @@ def get_major_device(device_map: Union[None, str, torch.device, int, dict] = Nonif device == "tp": # pragma: no cover# should not specify card, e.g., cuda:0device = get_current_device_type() or "cpu"- else:- device = devicereturn deviceif isinstance(device_map, dict) and device_map:@@ -1091,8 +1086,8 @@ def get_device_memory(i: int = 0) -> int:def _clear_memory_for_cpu_and_cuda(- tensor: Union[torch.Tensor, list, None] = None,- device_list: Union[tuple, list, str, torch.device, None] = None,+ tensor: torch.Tensor | list | None = None,+ device_list: tuple | list | str | torch.device | None = None,):# ------------------------# Clear CPU-side references@@ -1152,7 +1147,7 @@ def _clear_memory_for_cpu_and_cuda(class ClearMemory:- def __init__(self, device_list: Union[list, tuple, None] = None):+ def __init__(self, device_list: list | tuple | None = None):self._device_list = device_list@property@@ -1167,8 +1162,8 @@ class ClearMemory:def __call__(self,- tensor: Union[torch.Tensor, None, list] = None,- device_list: Union[list, tuple, None] = None,+ tensor: torch.Tensor | None | list = None,+ device_list: list | tuple | None = None,):# Lazy imports: these symbols live in utils/device.py.from auto_round.utils.device import _force_trim_malloc, is_hpex_available, memory_monitordiff --git a/auto_round/utils/disk_stream_util.py b/auto_round/utils/disk_stream_util.pyindex 1ea98c06e..c8f9bb53a 100644--- a/auto_round/utils/disk_stream_util.py+++ b/auto_round/utils/disk_stream_util.py@@ -13,9 +13,8 @@ from __future__ import annotationsimport jsonimport loggingimport re-from functools import lru_cache+from functools import cache, lru_cachefrom pathlib import Path-from typing import Dictimport torchimport torch.nn as nn@@ -48,7 +47,7 @@ class SafetensorsIndex:# The shard names come from the checkpoint's own index, i.e. from the# artifact being loaded -- validate them against the directory before# anything downstream gets a chance to join and open them.- self.weight_map: Dict[str, str] = validate_weight_map(+ self.weight_map: dict[str, str] = validate_weight_map(json.load(f)["weight_map"], self.checkpoint_dir, index_path=index_path)else:@@ -70,14 +69,14 @@ class SafetensorsIndex:with safe_open(str(shard_path), framework="pt") as f:return tuple(f.get_slice(name).get_shape())- def read_tensors(self, names: list[str], device: str = "cpu") -> Dict[str, torch.Tensor]:+ def read_tensors(self, names: list[str], device: str = "cpu") -> dict[str, torch.Tensor]:"""Read several tensors, grouped by shard file so each shard is opened andclosed (unmapped) once regardless of how many tensors are pulled from it."""- by_shard: Dict[str, list[str]] = {}+ by_shard: dict[str, list[str]] = {}for name in names:by_shard.setdefault(self.weight_map[name], []).append(name)- result: Dict[str, torch.Tensor] = {}+ result: dict[str, torch.Tensor] = {}for shard_name, shard_tensor_names in by_shard.items():shard_path = resolve_within_directory(self.checkpoint_dir, shard_name)with safe_open(str(shard_path), framework="pt") as f:@@ -94,7 +93,7 @@ class SafetensorsIndex:@lru_cache(maxsize=8)-def get_safetensors_index(checkpoint_dir: str) -> "SafetensorsIndex":+def get_safetensors_index(checkpoint_dir: str) -> SafetensorsIndex:"""Shared ``SafetensorsIndex`` per checkpoint directory.Only the (cheap) name->shard map is cached; no ``safe_open`` handle is kept, so this@@ -152,7 +151,7 @@ def checkpoint_has_native_fused_moe_experts(checkpoint_dir: str) -> bool:# whose names already match the model are untouched.-@lru_cache(maxsize=None)+@cachedef _reverse_renamings_for(model_type):"""Invert the checkpoint-conversion WeightRenaming entries for one family.@@ -188,7 +187,7 @@ def _reverse_renamings_for(model_type):return tuple(reversed_transforms)-@lru_cache(maxsize=None)+@cachedef _model_types_for_dir(checkpoint_dir: str):"""Every model_type in the checkpoint's config, including nested sub-configs.@@ -360,7 +359,7 @@ _MODEL_SIDE_EXPERT_RE = re.compile()-@lru_cache(maxsize=None)+@cachedef _expert_projection_renames_for(model_type):"""Map a fused projection to the checkpoint-side per-expert projection names.@@ -407,7 +406,7 @@ def _expert_projection_renames_for(model_type):return tuple(renames.items())-@lru_cache(maxsize=None)+@cachedef _concat_converters_for(model_type):"""Model-side params assembled by concatenating several checkpoint tensors.@@ -499,7 +498,7 @@ def _dot_natural_key(name: str):return parts-@lru_cache(maxsize=None)+@cachedef _wildcard_concat_converters_for(model_type):"""Wildcard shard-concat converters registered for one family.@@ -546,7 +545,7 @@ def _wildcard_concat_converters_for(model_type):return tuple(converters)-@lru_cache(maxsize=None)+@cachedef _wildcard_split_converters_for(model_type):"""Save-side inverse of :func:`_wildcard_concat_converters_for`.@@ -610,7 +609,7 @@ def _resolve_num_shards(config, num_shards_attribute):return None-def split_merged_concat_tensor(config, full_name: str, tensor: "torch.Tensor"):+def split_merged_concat_tensor(config, full_name: str, tensor: torch.Tensor):"""Split a merged model-side concat parameter back into its checkpoint shards.Inverse of the load-time :func:`_assemble_sharded_tensor`. Qwen3-Next "Flash"@@ -1259,12 +1258,12 @@ class stream_block_forward:is for a plain inference-only forward pass (e.g. eval loss), not tuning."""- def __init__(self, model: nn.Module, index: SafetensorsIndex, device: str, block_names: list[str] = None):+ def __init__(self, model: nn.Module, index: SafetensorsIndex, device: str, block_names: list[str] | None = None):self.model = modelself.index = indexself.device = deviceself.block_names = block_names if block_names is not None else _default_block_names(model)- self._originals: Dict[str, "callable"] = {}+ self._originals: dict[str, callable] = {}def __enter__(self):for block_name in self.block_names:diff --git a/auto_round/utils/distributed.py b/auto_round/utils/distributed.pyindex d523e0e65..6a7936ce9 100644--- a/auto_round/utils/distributed.py+++ b/auto_round/utils/distributed.py@@ -13,14 +13,14 @@# limitations under the License.import os-from functools import lru_cache+from functools import cacheimport torchfrom auto_round.logger import logger-@lru_cache(maxsize=None)+@cachedef is_distributed():import torch.distributed as distdiff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.pyindex 82a912503..b894a5eff 100644--- a/auto_round/utils/missing_tensors.py+++ b/auto_round/utils/missing_tensors.py@@ -536,9 +536,9 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -# Pre-compile all valid regex patterns once to avoid repeated re.compile() calls# for every tensor lookup (O(N×M) → O(M) compile + O(N×M) match)._compiled_patterns: list = []- for pattern in extra_config:+ for pattern, pattern_cfg in extra_config.items():try:- _compiled_patterns.append((_re.compile(pattern), pattern, extra_config[pattern]))+ _compiled_patterns.append((_re.compile(pattern), pattern, pattern_cfg))except _re.error as exc:logger.warning("Invalid regex key in extra_config ignored during pre-compilation: %r (%s)",@@ -630,7 +630,7 @@ def _woq_quantize_missing_tensors(target_dir: str, missing_tensors_dict: dict) -if isinstance(block_name_to_quantize, list)else [b.strip() for b in block_name_to_quantize.split(",") if b.strip()])- if not any(k.startswith(b + ".") or k.startswith(b + "[") for b in blocks):+ if not any(k.startswith((b + ".", b + "[")) for b in blocks):return Falselayer_name = k[: -len(".weight")]layer_cfg = _resolve_layer_cfg(layer_name)diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 00ea444e2..d10196e7e 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -16,9 +16,11 @@ import inspectimport jsonimport osimport re+import sysfrom collections import UserDict+from collections.abc import Callablefrom pathlib import Path-from typing import TYPE_CHECKING, Any, Callable, Optional, Union+from typing import TYPE_CHECKING, Anyimport psutilimport torch@@ -191,7 +193,7 @@ def check_diffusers_installed(): # pragma: no coverreturn Trueexcept ImportError:logger.error("Please install diffusers via 'pip install diffusers'" " to run diffusion model")- exit(-1)+ sys.exit(-1)def check_start_with_block_name(name: str, block_name_to_quantize: list):@@ -211,7 +213,7 @@ def check_start_with_block_name(name: str, block_name_to_quantize: list):return False-def download_or_get_path(repo_id: str, platform: str = None) -> str:+def download_or_get_path(repo_id: str, platform: str | None = None) -> str:from auto_round import envsif platform is None:@@ -226,7 +228,7 @@ def download_or_get_path(repo_id: str, platform: str = None) -> str:return download_hf_model(repo_id)-def download_modelscope_model(repo_id: str, local_dir: str = None, cache_dir: str = None):+def download_modelscope_model(repo_id: str, local_dir: str | None = None, cache_dir: str | None = None):from modelscope.utils.file_utils import get_modelscope_cache_dir # pylint: disable=E0401system_cache = cache_dir if cache_dir is not None else get_modelscope_cache_dir()@@ -444,7 +446,7 @@ def llm_load_model(pretrained_model_name_or_path: str,platform: str = "hf",trust_remote_code: bool = True,- model_dtype: str = None,+ model_dtype: str | None = None,device: str = "cpu",**kwargs,):@@ -571,7 +573,7 @@ def llm_load_model(return model, tokenizer-def _find_pipeline_model_subfolder(model_dir_or_repo: str, file_list: list = None) -> tuple:+def _find_pipeline_model_subfolder(model_dir_or_repo: str, file_list: list | None = None) -> tuple:"""Find model/processor subfolders from a pipeline's model_index.json.Works for both local directories and remote HF repos.@@ -650,7 +652,7 @@ def mllm_load_model(torch_dtype: str = "auto",use_auto_mapping: bool = True,trust_remote_code: bool = True,- model_dtype: str = None,+ model_dtype: str | None = None,**kwargs,):from auto_round.special_model_handler import MISTRAL_3_2_MODELS@@ -884,7 +886,7 @@ def mllm_load_model(else:raise- if any([name in model.name_or_path for name in MISTRAL_3_2_MODELS]):+ if any(name in model.name_or_path for name in MISTRAL_3_2_MODELS):from mistral_common.tokens.tokenizers.mistral import MistralTokenizer # pylint: disable=E0401if os.path.isdir(pretrained_model_name_or_path):@@ -898,7 +900,7 @@ def mllm_load_model(tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path,trust_remote_code=trust_remote_code,- fix_mistral_regex=True if model_type in FIX_MISTRAL_REGEX_MODEL_TYPE_LIST else False,+ fix_mistral_regex=model_type in FIX_MISTRAL_REGEX_MODEL_TYPE_LIST,**processor_load_kwargs,)processor = AutoProcessor.from_pretrained(@@ -962,12 +964,12 @@ def _attach_diffusion_pipeline_fn(pipe):def diffusion_load_model(pretrained_model_name_or_path: str,platform: str = "hf",- device: Union[str, torch.device] = "cpu",- torch_dtype: Union[str, torch.dtype] = "auto",+ device: str | torch.device = "cpu",+ torch_dtype: str | torch.dtype = "auto",use_auto_mapping: bool = False,trust_remote_code: bool = True,- model_dtype: str = None,- default_torch_dtype: Union[str, torch.dtype] = "auto",+ model_dtype: str | None = None,+ default_torch_dtype: str | torch.dtype = "auto",**kwargs,):from functools import partial@@ -1088,8 +1090,8 @@ def diffusion_load_model(_attach_diffusion_pipeline_fn(pipe)# meta model uses model.config.save_pretrained for config saving- setattr(model.config, "save_pretrained", partial(config_save_pretrained, model.config, "config.json", model=model))- setattr(pipe.config, "save_pretrained", partial(config_save_pretrained, pipe.config, "model_index.json"))+ model.config.save_pretrained = partial(config_save_pretrained, model.config, "config.json", model=model)+ pipe.config.save_pretrained = partial(config_save_pretrained, pipe.config, "model_index.json")def model_save_pretrained(model, save_directory, **kwargs):super(model.__class__, model).save_pretrained(save_directory, **kwargs)@@ -1099,7 +1101,7 @@ def diffusion_load_model(writer.write(json.dumps(dict(model.config), indent=2, sort_keys=True) + "\n")# non-meta model uses model.save_pretrained for model and config saving- setattr(model, "save_pretrained", partial(model_save_pretrained, model))+ model.save_pretrained = partial(model_save_pretrained, model)for comp_name in pipe.components:comp = getattr(pipe, comp_name, None)@@ -1110,21 +1112,19 @@ def diffusion_load_model(and isinstance(comp, torch.nn.Module)):comp._autoround_checkpoint_subfolder = comp_name- setattr(- comp.config, "save_pretrained", partial(config_save_pretrained, comp.config, "config.json", model=comp)- )- setattr(comp, "save_pretrained", partial(model_save_pretrained, comp))+ comp.config.save_pretrained = partial(config_save_pretrained, comp.config, "config.json", model=comp)+ comp.save_pretrained = partial(model_save_pretrained, comp)return pipe, model.to(device)def load_model(- pretrained_model_name_or_path: Union[str, torch.nn.Module],+ pretrained_model_name_or_path: str | torch.nn.Module,platform: str = "hf",- model_dtype: str = None,+ model_dtype: str | None = None,trust_remote_code: bool = True,device: str = "cpu",- use_auto_mapping: bool = None,+ use_auto_mapping: bool | None = None,use_model_replacements: bool = False,**kwargs,) -> tuple:@@ -1255,7 +1255,7 @@ _is_mllm_model_cache: dict = {}_LLM_ONLY_MODEL_TYPES = {"bagel"}-def get_model_name_or_path(model_or_path: Union[str, torch.nn.Module]) -> Optional[str]:+def get_model_name_or_path(model_or_path: str | torch.nn.Module) -> str | None:if isinstance(model_or_path, str):return model_or_pathreturn getattr(model_or_path, "_name_or_path", None) or getattr(model_or_path, "name_or_path", None)@@ -1279,14 +1279,14 @@ _CODE_MODEL_FAMILIES = {_CODE_MODEL_TASKS = {"code-generation", "software-engineering", "text-to-code"}-def _match_code_model_name(value) -> Optional[str]:+def _match_code_model_name(value) -> str | None:if not isinstance(value, str) or not value:return Nonevalue = re.split(r"[/\\]", value.rstrip("/\\"))[-1]value = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", " ", value)token_matches = _CODE_MODEL_TOKENS.intersection(re.findall(r"[a-z]+", value.lower()))if token_matches:- return sorted(token_matches)[0]+ return min(token_matches)components = re.findall(r"[a-z0-9]+", value.lower())for family in sorted(_CODE_MODEL_FAMILIES):if any(component == family or re.fullmatch(rf"{family}\d+", component) for component in components):@@ -1332,7 +1332,7 @@ def _get_code_model_match(model_or_path, config=None):return None-def is_code_model(model_or_path: Union[str, torch.nn.Module], config=None) -> bool:+def is_code_model(model_or_path: str | torch.nn.Module, config=None) -> bool:"""Return whether a pure-text model is explicitly specialized for code."""match = _get_code_model_match(model_or_path, config)if match is None:@@ -1341,7 +1341,7 @@ def is_code_model(model_or_path: Union[str, torch.nn.Module], config=None) -> boreturn True-def is_mllm_model(model_or_path: Union[str, torch.nn.Module], platform: str = None):+def is_mllm_model(model_or_path: str | torch.nn.Module, platform: str | None = None):from auto_round.utils.common import MM_KEYSmodel_path = get_model_name_or_path(model_or_path)@@ -1368,29 +1368,27 @@ def is_mllm_model(model_or_path: Union[str, torch.nn.Module], platform: str = No# Only try to download if the path looks like a HF repo id (not a local filesystem path).# Skip download for absolute paths or relative paths that contain current/parent dir markers.# model_path is None for a model or pipeline built in-process, which has no name or path- _is_local_path = isinstance(model_path, str) and (- os.path.isabs(model_path) or model_path.startswith("./") or model_path.startswith("../")- )+ _is_local_path = isinstance(model_path, str) and (os.path.isabs(model_path) or model_path.startswith(("./", "../")))if model_path and not os.path.isdir(model_path) and not _is_local_path:model_path = download_or_get_path(model_path, platform=platform)result = Falseif isinstance(model_path, str):- if os.path.exists(os.path.join(model_path, "preprocessor_config.json")):- result = True- elif os.path.exists(os.path.join(model_path, "processor_config.json")):+ if os.path.exists(os.path.join(model_path, "preprocessor_config.json")) or os.path.exists(+ os.path.join(model_path, "processor_config.json")+ ):result = Trueelif os.path.exists(os.path.join(model_path, "config.json")):with open(os.path.join(model_path, "config.json")) as f:config = json.load(f)for key in config.keys():- if any([k in key for k in MM_KEYS]):+ if any(k in key for k in MM_KEYS):result = Truebreakif not result and isinstance(model_or_path, torch.nn.Module):for name, module in model_or_path.named_modules():- if any([k in name for k in MM_KEYS]):+ if any(k in name for k in MM_KEYS):result = Truebreak@@ -1401,7 +1399,7 @@ def is_mllm_model(model_or_path: Union[str, torch.nn.Module], platform: str = Noreturn result-def is_gguf_model(model_path: Union[str, torch.nn.Module]) -> bool:+def is_gguf_model(model_path: str | torch.nn.Module) -> bool:is_gguf_file = Falseif isinstance(model_path, str):if os.path.isfile(model_path) and model_path.endswith(".gguf"):@@ -1418,7 +1416,7 @@ def is_gguf_model(model_path: Union[str, torch.nn.Module]) -> bool:MODULAR_PIPELINE_INDEX_NAME = "modular_model_index.json"-def _find_pipeline_index_file(model_dir_or_repo: str) -> Optional[str]:+def _find_pipeline_index_file(model_dir_or_repo: str) -> str | None:"""Return the pipeline index file of a diffusers directory or repo, if it has one.Standard pipelines ship ``model_index.json``, Modular Diffusers pipelines ship@@ -1456,7 +1454,7 @@ def _get_modular_pipeline_class():return None-def is_diffusion_model(model_or_path: Union[str, object], trust_remote_code: bool = True) -> bool:+def is_diffusion_model(model_or_path: str | object, trust_remote_code: bool = True) -> bool:from auto_round.utils.common import LazyImport# Then check if model_index.json exists for diffusion pipeline,@@ -1750,8 +1748,8 @@ def get_gguf_architecture(dir_model, model_type=ModelType.TEXT):def get_layer_names_in_block(model: torch.nn.Module,supported_types=(torch.nn.Linear, transformers.pytorch_utils.Conv1D),- quant_block_list: list = None,- class_names: tuple = None,+ quant_block_list: list | None = None,+ class_names: tuple | None = None,) -> list[str]:"""Retrieves the names of layers within each block of the model.@@ -1784,7 +1782,7 @@ def set_nested_attr(module, attr_name: str, value):attrs = attr_name.split(".")for attr in attrs[:-1]:if not hasattr(module, attr):- return None # No need to set act_max for fp layers+ return # No need to set act_max for fp layersmodule = getattr(module, attr)setattr(module, attrs[-1], value)@@ -1888,7 +1886,7 @@ def _to_model_dtype(model, model_dtype):model = cast_model_dtype(model, torch.float32)except Exception:logger.error("please use more device to fit the device or just use one device")- exit()+ sys.exit()return model@@ -1986,10 +1984,7 @@ def unsupported_meta_device(model):if param.device.type == "meta" or target_device.type == "meta":return Trueif target_device.type == "meta":- if hasattr(model, "path"):- return False- else:- return True+ return not hasattr(model, "path")return False@@ -2020,11 +2015,11 @@ def to_device(input, device=torch.device("cpu")):return Noneif isinstance(input, torch.Tensor):return input.to(device)- if isinstance(input, dict) or isinstance(input, UserDict):+ if isinstance(input, (dict, UserDict)):for inp in input.keys():input[inp] = to_device(input[inp], device)- elif isinstance(input, list) or isinstance(input, tuple):+ elif isinstance(input, (list, tuple)):if len(input) == 0:return inputinput_res = []@@ -2150,9 +2145,7 @@ def is_moe_model_via_config(config) -> bool:config_str = str(config).lower()except Exception:config_str = str(config.to_dict()).lower() if hasattr(config, "to_dict") else ""- if "moe" in config_str or "expert" in config_str:- return True- return False+ return "moe" in config_str or "expert" in config_strdef to_dtype(input, dtype=torch.float32):@@ -2169,11 +2162,11 @@ def to_dtype(input, dtype=torch.float32):return Noneif isinstance(input, torch.Tensor):return input.to(dtype)- if isinstance(input, dict) or isinstance(input, UserDict):+ if isinstance(input, (dict, UserDict)):for inp in input.keys():input[inp] = to_dtype(input[inp], dtype)- elif isinstance(input, list) or isinstance(input, tuple):+ elif isinstance(input, (list, tuple)):if len(input) == 0:return inputinput_res = []@@ -2311,7 +2304,7 @@ def set_amax_for_all_moe_layers(model: torch.nn.Module, layer_name=None, attr_na)except AttributeError as e:# Provide more helpful debugging information- expert_types = list(set(type(expert).__name__ for expert in sub_module.experts))+ expert_types = list({type(expert).__name__ for expert in sub_module.experts})raise AttributeError(f"Failed to access attribute '{linear_name}' on experts. "f"MoE module type: {type(sub_module).__name__}, "@@ -2547,7 +2540,7 @@ def find_matching_blocks(model, all_blocks, to_quant_block_names):if not to_quant_block_names:return all_blocksto_quant_block_list = to_quant_block_names- if isinstance(to_quant_block_names, list) or isinstance(to_quant_block_names, tuple):+ if isinstance(to_quant_block_names, (list, tuple)):return to_quant_block_namesif isinstance(to_quant_block_names, str):to_quant_block_list = [name.strip() for name in to_quant_block_names.split(",")]@@ -2578,18 +2571,12 @@ def is_separate_lm_head(model: torch.nn.Module) -> bool:if "model.safetensors.index.json" in os.listdir(dir_path):with open(os.path.join(dir_path, "model.safetensors.index.json")) as f:index_mapping = json.load(f)- if lm_head_name in index_mapping["weight_map"]:- return True- else:- return False+ return lm_head_name in index_mapping["weight_map"]else:from safetensors import safe_openf = safe_open(os.path.join(dir_path, "model.safetensors"), framework="pt")- if lm_head_name in f.keys():- return True- else:- return False+ return lm_head_name in f.keys()def is_separate_tensor(model: torch.nn.Module, tensor_name: str) -> bool:@@ -2602,18 +2589,12 @@ def is_separate_tensor(model: torch.nn.Module, tensor_name: str) -> bool:if "model.safetensors.index.json" in os.listdir(dir_path):with open(os.path.join(dir_path, "model.safetensors.index.json")) as f:index_mapping = json.load(f)- if tensor_name in index_mapping["weight_map"]:- return True- else:- return False+ return tensor_name in index_mapping["weight_map"]else:from safetensors import safe_openf = safe_open(os.path.join(dir_path, "model.safetensors"), framework="pt")- if tensor_name in f.keys():- return True- else:- return False+ return tensor_name in f.keys()def handle_generation_config(model: torch.nn.Module):@@ -2731,10 +2712,12 @@ def rename_weights_files(path: str, prefix="diffusion_pytorch_model"):# rename index.jsonidx = os.path.join(path, "model.safetensors.index.json")if os.path.exists(idx):- d = json.load(open(idx))+ with open(idx) as f:+ d = json.load(f)d["weight_map"] = {k: v.replace("model-", prefix + "-") for k, v in d["weight_map"].items()}new_idx = os.path.join(path, f"{prefix}.safetensors.index.json")- json.dump(d, open(new_idx, "w"), indent=2)+ with open(new_idx, "w") as f:+ json.dump(d, f, indent=2)os.remove(idx)@@ -2891,7 +2874,7 @@ class _ShardedEmbedding(torch.nn.Module):def weight(self):# Exposed so callers that read ``embedding.weight.device`` keep working (the forward# re-routes ids to the correct shard regardless of which device they arrive on).- return getattr(self, "shard_0")+ return self.shard_0def forward(self, input_ids: "torch.Tensor") -> "torch.Tensor":flat = input_ids.reshape(-1)diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.pyindex 8c03a6df0..69731a074 100644--- a/auto_round/utils/model_free_utils.py+++ b/auto_round/utils/model_free_utils.py@@ -26,9 +26,10 @@ import osimport reimport shutilimport warnings+from collections.abc import Callablefrom dataclasses import fieldsfrom functools import lru_cache-from typing import Any, Callable, Optional, Union+from typing import Anyimport torch@@ -282,7 +283,7 @@ def quantize_weight_rtn(bits: int,group_size: int,sym: bool = True,- device: Optional[torch.device] = None,+ device: torch.device | None = None,disable_opt_rtn: bool = True,*,packing: str,@@ -458,13 +459,13 @@ class _PatternMatcher:"""Precompile ignore and layer-config patterns for shard processing."""__slots__ = (- "_ignore_re",- "_skip_re",- "_layer_config",- "_default_scheme","_compiled_lc",+ "_default_scheme","_ignore_cache",+ "_ignore_re",+ "_layer_config","_scheme_cache",+ "_skip_re",)def __init__(@@ -964,7 +965,7 @@ def _dequantize_with_device_fallback(return on_cpu()-def _normalize_scheme(scheme: Union[str, QuantizationScheme]) -> QuantizationScheme:+def _normalize_scheme(scheme: str | QuantizationScheme) -> QuantizationScheme:"""Convert *scheme* to a :class:`QuantizationScheme` instance.Raises ``ValueError`` for unknown preset names and ``TypeError`` for@@ -1046,7 +1047,7 @@ def _fused_expert_layer_name(tensor_name: str) -> str:def _quantize_moe_fused_expert_weight(tensor_name: str,tensor: torch.Tensor,- matcher: "_PatternMatcher",+ matcher: _PatternMatcher,device: str = "cpu",disable_opt_rtn: bool = False,) -> tuple[str, dict[str, torch.Tensor], str | None, str | None]:@@ -1339,7 +1340,7 @@ def _declared_int_packing(sym: bool) -> str:def _quantize_single_tensor(tensor_name: str,tensor: torch.Tensor,- matcher: "_PatternMatcher",+ matcher: _PatternMatcher,device: str = "cpu",quantize_func: Callable = quantize_weight_rtn,disable_opt_rtn: bool = False,@@ -1908,7 +1909,7 @@ def _dequant_mxfp_tensors(def _handle_mxfp_source_tensors(raw_tensors: dict[str, torch.Tensor],- matcher: "_PatternMatcher",+ matcher: _PatternMatcher,source_state: dict[str, int] | None = None,device: str = "cpu",shard_name: str | None = None,@@ -2050,13 +2051,13 @@ def _dequant_fp8_tensors(def _process_shard(shard_path: str,- default_scheme: dict = None,- layer_config: dict = None,- ignore_patterns: list[str] = None,+ default_scheme: dict | None = None,+ layer_config: dict | None = None,+ ignore_patterns: list[str] | None = None,device: str = "cpu",*,shard_name: str | None = None,- matcher: "_PatternMatcher | None" = None,+ matcher: _PatternMatcher | None = None,fp8_block_size: list | None = None,model_type: str | None = None,source_quantization_config: dict | None = None,@@ -2153,9 +2154,7 @@ def _process_shard(# so the saved model exports them in full precision.preserved_prefixes: set[str] = set()for tname in raw_tensors:- if (- tname.endswith(".weight") or tname.endswith(".weight_packed") or tname.endswith(".qweight")- ) and matcher.should_skip(tname):+ if tname.endswith((".weight", ".weight_packed", ".qweight")) and matcher.should_skip(tname):preserved_prefixes.add(tname.rsplit(".", 1)[0])preserved_tensors: dict[str, torch.Tensor] = {}@@ -2459,7 +2458,7 @@ def _is_weight_shard(fname: str) -> bool:"""if fname.endswith(".index.json"):return False- return fname.endswith(".safetensors") or fname.endswith(".bin")+ return fname.endswith((".safetensors", ".bin"))# Keep old name as an alias for backward compatibility.@@ -2800,7 +2799,7 @@ def _build_mxfp_autoround_quantization_config(quantized_layers: list[str],ignored_layers: list[str],layer_config: dict | None = None,- block_name_to_quantize: Optional[str] = None,+ block_name_to_quantize: str | None = None,) -> dict:"""Build an auto-round style quantization_config for MXFP4 / MXFP8.@@ -3071,7 +3070,7 @@ def _derive_dominant_int_scheme(layer_config=layer_config,default_scheme=fallback,)- counter: "Counter[tuple]" = Counter()+ counter: Counter[tuple] = Counter()for layer in quantized_layers:scheme = temp_matcher.resolve_scheme(f"{layer}.weight")if scheme is None:@@ -3109,7 +3108,7 @@ def _build_quantization_config(ignore_patterns: list[str],quantized_layers: list[str],ignored_layers: list[str],- block_name_to_quantize: Optional[str] = None,+ block_name_to_quantize: str | None = None,format: str = "auto_round",) -> dict:"""Build a quantization_config dict compatible with auto-round format."""@@ -3268,8 +3267,8 @@ def _build_quantization_config(def _apply_scheme_overrides(- scheme: Union[str, QuantizationScheme],- scheme_overrides: Optional[dict] = None,+ scheme: str | QuantizationScheme,+ scheme_overrides: dict | None = None,) -> QuantizationScheme:"""Return the effective scheme after applying non-None overrides."""scheme_obj = copy.deepcopy(_normalize_scheme(scheme))@@ -3285,7 +3284,7 @@ def _apply_scheme_overrides(def _validate_supported_scheme(scheme_obj: QuantizationScheme,- scheme_input: Union[str, QuantizationScheme],+ scheme_input: str | QuantizationScheme,) -> None:"""Raise ``ValueError`` if *scheme_obj* is not supported by model-free.@@ -3378,8 +3377,8 @@ def _validate_supported_scheme(def is_model_free_supported_scheme(- scheme: Union[str, QuantizationScheme],- scheme_overrides: Optional[dict] = None,+ scheme: str | QuantizationScheme,+ scheme_overrides: dict | None = None,) -> bool:"""Return True if *scheme* can be quantized via model-free mode.@@ -3477,7 +3476,7 @@ def _validate_auto_scheme_options(auto_scheme: Any) -> str:def _convert_auto_scheme_layer_config(generated: dict[str, dict],- preferred_base_scheme: Union[str, QuantizationScheme, None] = None,+ preferred_base_scheme: str | QuantizationScheme | None = None,) -> tuple[QuantizationScheme, dict[str, dict], list[str]]:"""Convert an AutoScheme-generated ``layer_config`` into model-free inputs.@@ -3496,7 +3495,7 @@ def _convert_auto_scheme_layer_config(scheme_keys = {f.name for f in fields(QuantizationScheme)}per_layer: dict[str, dict] = {}fp16_layers: list[str] = []- counter: "Counter[tuple]" = Counter()+ counter: Counter[tuple] = Counter()for name, cfg in generated.items():if not isinstance(cfg, dict):@@ -3524,9 +3523,9 @@ def _convert_auto_scheme_layer_config(# the quantization kernels ("mxfp8" / "MXFP4" → "mx_fp").if data_type_raw:dt_lower = data_type_raw.lower()- if dt_lower.startswith("mxfp") or dt_lower.startswith("mx_fp"):+ if dt_lower.startswith(("mxfp", "mx_fp")):clean["data_type"] = "mx_fp"- elif dt_lower.startswith("nvfp") or dt_lower.startswith("nv_fp"):+ elif dt_lower.startswith(("nvfp", "nv_fp")):clean["data_type"] = "nv_fp"if bits >= 16:diff --git a/auto_round/utils/offload.py b/auto_round/utils/offload.pyindex 35d1f9ae2..8aecc7d11 100644--- a/auto_round/utils/offload.py+++ b/auto_round/utils/offload.py@@ -51,7 +51,7 @@ import sysimport tempfilefrom collections import defaultdictfrom functools import partial-from typing import Any, Optional, Union+from typing import Anyimport torch@@ -190,7 +190,7 @@ def _clear_module_weights(# =====================================================================-def _resolve_model_dir(model_dir: str, revision: Optional[str] = None) -> str:+def _resolve_model_dir(model_dir: str, revision: str | None = None) -> str:"""Resolve a model name/path to a local directory containing weight files."""if os.path.isdir(model_dir):return model_dir@@ -371,11 +371,11 @@ class OffloadManager:self,enabled: bool = True,mode: str = "offload",- model_dir: Optional[str] = None,+ model_dir: str | None = None,offload_dir_prefix: str = "ar_offload",cache_numel: bool = False,retain_saved_entries: bool = False,- model_revision: Optional[str] = None,+ model_revision: str | None = None,):from auto_round import envs@@ -391,7 +391,7 @@ class OffloadManager:self.retain_saved_entries = retain_saved_entries# Disk state (offload mode)- self._tempdir: Optional[str] = None+ self._tempdir: str | None = Noneself._saved: dict[str, dict] = {} # name -> {"save_path": str}# Cached weight map for clean mode (avoids repeated disk I/O)@@ -399,12 +399,12 @@ class OffloadManager:# Hook state (for add_offload_hooks/remove_offload_hooks transparent offloading)self._hook_handles: list = []- self._model_ref: Optional[torch.nn.Module] = None+ self._model_ref: torch.nn.Module | None = Noneself._module_names: list[str] = []- self._last_loaded: Optional[str] = None+ self._last_loaded: str | None = None# Ensure-style state (for wrapping loops)- self._current_loaded: Optional[str] = None+ self._current_loaded: str | None = None# ------------------------------------------------------------------# Context manager@@ -426,7 +426,7 @@ class OffloadManager:def __call__(self,model: torch.nn.Module,- names: Union[str, list[str], list[list[str]]],+ names: str | list[str] | list[list[str]],*,skip_if_saved: bool = False,overwrite: bool = False,@@ -450,7 +450,7 @@ class OffloadManager:def offload(self,model: torch.nn.Module,- names: Union[str, list[str], list[list[str]]],+ names: str | list[str] | list[list[str]],*,skip_if_saved: bool = False,overwrite: bool = False,@@ -508,7 +508,7 @@ class OffloadManager:logger.info(f"offload done, freed {total_gb:.2f} GB")return total_gb- def _check_disk_space(self, model: torch.nn.Module, names: Union[str, list[str], list[list[str]]]) -> bool:+ def _check_disk_space(self, model: torch.nn.Module, names: str | list[str] | list[list[str]]) -> bool:"""Check whether there is enough disk space to offload the given modules.Args:@@ -576,7 +576,7 @@ class OffloadManager:self._save_to_disk(name, module)self._clear(module, block_name=name)- def reload(self, model: torch.nn.Module, names: Union[str, list[str], None] = None) -> None:+ def reload(self, model: torch.nn.Module, names: str | list[str] | None = None) -> None:"""Reload previously offloaded module(s).For ``"offload"`` mode: loads from the temp directory, then@@ -692,7 +692,7 @@ class OffloadManager:clear_memory()logger.info("module weights cleared")- def remove_offload_hooks(self, model: torch.nn.Module, names: Optional[list[str]] = None) -> None:+ def remove_offload_hooks(self, model: torch.nn.Module, names: list[str] | None = None) -> None:"""Remove hooks and reload all managed modules.Args:@@ -1007,7 +1007,7 @@ class OffloadManager:return False@staticmethod- def _flatten_names(names: Union[list[str], list[list[str]]]) -> list[str]:+ def _flatten_names(names: list[str] | list[list[str]]) -> list[str]:"""Flatten a potentially nested list of names."""flat = []for item in names:diff --git a/auto_round/utils/path_safety.py b/auto_round/utils/path_safety.pyindex c337aaf01..f2220f3a4 100644--- a/auto_round/utils/path_safety.py+++ b/auto_round/utils/path_safety.py@@ -38,12 +38,12 @@ from __future__ import annotationsimport osfrom pathlib import Path, PureWindowsPath-from typing import Any, Dict+from typing import Any__all__ = ["UnsafeCheckpointPathError",- "sanitize_shard_name","resolve_within_directory",+ "sanitize_shard_name","validate_weight_map",]@@ -129,7 +129,7 @@ def validate_weight_map(base_dir: str | os.PathLike,*,index_path: str | os.PathLike | None = None,-) -> Dict[str, str]:+) -> dict[str, str]:"""Validate every shard reference in an artifact-provided ``weight_map``.Returns a new dict with the same keys and the same relative shard names, so it@@ -144,7 +144,7 @@ def validate_weight_map(f"{label}: 'weight_map' must be a JSON object mapping tensor names to shards, "f"got {type(weight_map).__name__}")- validated: Dict[str, str] = {}+ validated: dict[str, str] = {}for tensor_name, shard_name in weight_map.items():resolve_within_directory(base_dir, shard_name, origin=f"{label}[{tensor_name}]")validated[tensor_name] = shard_namediff --git a/auto_round/utils/resume.py b/auto_round/utils/resume.pyindex 9299cbb6c..79872bb03 100644--- a/auto_round/utils/resume.py+++ b/auto_round/utils/resume.py@@ -28,7 +28,6 @@ import jsonimport osimport tempfilefrom pathlib import Path-from typing import Optionalimport torch@@ -172,7 +171,7 @@ class ResumeState:def compute_run_signature(- model_dir: Optional[str],+ model_dir: str | None,scheme_desc: str,dataset_desc: str,nsamples: int,diff --git a/auto_round/utils/weight_handler.py b/auto_round/utils/weight_handler.pyold mode 100755new mode 100644index abfca6ba4..c14aad463--- a/auto_round/utils/weight_handler.py+++ b/auto_round/utils/weight_handler.py@@ -59,10 +59,10 @@ Quick Start Guide:import osfrom abc import ABC, abstractmethod+from collections.abc import Callablefrom contextlib import ContextDecoratorfrom dataclasses import fieldsfrom enum import Enum, auto-from typing import Callable, Dict, Optional, Set, Typeimport psutilimport torch@@ -179,7 +179,6 @@ class WeightTypeHandler(ABC):Returns:True if the module is of this weight type, False otherwise."""- passdef attach_weight_shape(self, module: torch.nn.Module):"""Optional helper to attach weight shape information to the module for detection."""@@ -209,12 +208,11 @@ class WeightTypeHandler(ABC):Returns:A new high-precision layer with dequantized weights."""- pass# --- Handler Registry ----_WEIGHT_TYPE_HANDLERS: Dict[ModuleWeightType, WeightTypeHandler] = {}+_WEIGHT_TYPE_HANDLERS: dict[ModuleWeightType, WeightTypeHandler] = {}def register_weight_type_handler(weight_type: ModuleWeightType):@@ -232,7 +230,7 @@ def register_weight_type_handler(weight_type: ModuleWeightType):..."""- def decorator(handler_cls: Type[WeightTypeHandler]):+ def decorator(handler_cls: type[WeightTypeHandler]):if not issubclass(handler_cls, WeightTypeHandler):raise TypeError(f"Handler {handler_cls.__name__} must be a subclass of WeightTypeHandler")_WEIGHT_TYPE_HANDLERS[weight_type] = handler_cls()@@ -241,7 +239,7 @@ def register_weight_type_handler(weight_type: ModuleWeightType):return decorator-def get_handler(weight_type: ModuleWeightType) -> Optional[WeightTypeHandler]:+def get_handler(weight_type: ModuleWeightType) -> WeightTypeHandler | None:"""Get the registered handler for a weight type.Args:@@ -253,7 +251,7 @@ def get_handler(weight_type: ModuleWeightType) -> Optional[WeightTypeHandler]:return _WEIGHT_TYPE_HANDLERS.get(weight_type)-def get_all_handlers() -> Dict[ModuleWeightType, WeightTypeHandler]:+def get_all_handlers() -> dict[ModuleWeightType, WeightTypeHandler]:"""Get all registered weight type handlers.Returns:@@ -265,7 +263,7 @@ def get_all_handlers() -> Dict[ModuleWeightType, WeightTypeHandler]:# ============================================================================# Section 2: PUBLIC API - Detection and Conversion Functions# ============================================================================-def detect_weight_type(module: torch.nn.Module) -> Optional[ModuleWeightType]:+def detect_weight_type(module: torch.nn.Module) -> ModuleWeightType | None:"""Detect the weight type of a module or model.First checks if the module itself has a quantized_weight_type attribute.@@ -290,7 +288,7 @@ def detect_weight_type(module: torch.nn.Module) -> Optional[ModuleWeightType]:# --- Model Marking Functions ----def check_and_mark_quantized_module(model: torch.nn.Module) -> Set[ModuleWeightType]:+def check_and_mark_quantized_module(model: torch.nn.Module) -> set[ModuleWeightType]:"""Check if model contains quantized layers and mark them accordingly.This function scans the model (including the model itself) for quantized layers using@@ -303,7 +301,7 @@ def check_and_mark_quantized_module(model: torch.nn.Module) -> Set[ModuleWeightTReturns:A set of detected ModuleWeightType values. Empty set if no quantized layers found."""- detected_types: Set[ModuleWeightType] = set()+ detected_types: set[ModuleWeightType] = set()for weight_type, handler in _WEIGHT_TYPE_HANDLERS.items():# Check model itself firstif handler.detect_layer(model):@@ -332,7 +330,7 @@ def check_and_mark_quantized_module(model: torch.nn.Module) -> Set[ModuleWeightTreturn detected_types-def is_quantized_input_module(model: torch.nn.Module) -> Optional[ModuleWeightType]:+def is_quantized_input_module(model: torch.nn.Module) -> ModuleWeightType | None:"""Check if a model has quantized input weights and return the weight type.This traverses all submodules to check for the `quantized_weight_type` attribute@@ -485,7 +483,7 @@ def _pad_block_fp8_weight_naive(@with_thread_limits()def _dequant_fp8_linear_weight(- weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list = None, data_type: str = None+ weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list | None = None, data_type: str | None = None) -> torch.Tensor:"""Core dequantization logic for block-wise FP8 weights."""dtype = torch.bfloat16diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 66897972e..ac59c2916 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -95,7 +95,7 @@ class WrapperLinear(torch.nn.Module):enable_norm_bias_tuning (bool): Whether to enable normalization and tuning for the bias term.device (str): The computation device, such as 'cpu' or 'cuda'."""- super(WrapperLinear, self).__init__()+ super().__init__()self.orig_layer = orig_layerself.orig_layer.iters = kwargs.pop("iters", 200)self.disable_opt_rtn = disable_opt_rtn@@ -112,7 +112,7 @@ class WrapperLinear(torch.nn.Module):from auto_round.data_type.nvfp import calculate_gparamweight_global_scale = calculate_gparam(self.orig_layer.weight, self.orig_layer.group_size)- setattr(self, "weight_global_scale", weight_global_scale)+ self.weight_global_scale = weight_global_scaleself.weight_global_scale = self.weight_global_scale.to(self.orig_layer.weight.device)if hasattr(self.orig_layer, "scale_dtype") and self.orig_layer.scale_dtype == torch.float32:self.q_scale_thresh = 1e-8@@ -332,7 +332,7 @@ class WrapperLinear(torch.nn.Module):weight_q = weight_q.t()return weight_q, scale, zp- def _qdq_act(self, x, act_min_scale=torch.tensor(1.0), act_max_scale=torch.tensor(1.0), act_max=None):+ def _qdq_act(self, x, act_min_scale=None, act_max_scale=None, act_max=None):"""Quantizes and dequantizes activations.Args:@@ -343,6 +343,10 @@ class WrapperLinear(torch.nn.Module):Returns:tuple: Quantized activation, scale, and zero point."""+ if act_min_scale is None:+ act_min_scale = torch.tensor(1.0)+ if act_max_scale is None:+ act_max_scale = torch.tensor(1.0)act_max_scale.data.clamp_(0, 1.0)act_min_scale.data.clamp_(0, 1.0)env_act_scale = envs.AR_ACT_SCALE # fixed activation ratio,prioritize to use this one if set@@ -608,7 +612,7 @@ class WrapperLinear(torch.nn.Module):class WrapperWALayer(torch.nn.Module):def __init__(self, orig_layer, enable_torch_compile=True, device="cpu"):- super(WrapperWALayer, self).__init__()+ super().__init__()self.orig_layer = orig_layerself.enable_torch_compile = enable_torch_compileself.device = device@@ -692,7 +696,7 @@ class WrapperLayerNorm(torch.nn.Module):"""def __init__(self, orig_layer, bit=4, group_size=-1, device="cpu"):- super(WrapperLayerNorm, self).__init__()+ super().__init__()self.orig_layer = orig_layerself.bits = bitself.group_size = group_size@@ -743,7 +747,7 @@ class WrapperLlamaNorm(torch.nn.Module):"""def __init__(self, orig_layer, bit=4, group_size=-1, device="cpu"):- super(WrapperLlamaNorm, self).__init__()+ super().__init__()self.orig_layer = orig_layerself.bits = bitself.group_size = group_size@@ -803,7 +807,7 @@ class WrapperMultiblock(torch.nn.Module):"""def __init__(self, module_list):- super(WrapperMultiblock, self).__init__()+ super().__init__()self.layers = torch.nn.ModuleList(module_list)def forward(self, x, *args, **kwargs):@@ -811,7 +815,7 @@ class WrapperMultiblock(torch.nn.Module):for idx, decoder_layer in enumerate(self.layers):layer_outputs = decoder_layer(hidden_states, *args, **kwargs)hidden_states = layer_outputs- if isinstance(hidden_states, tuple) or isinstance(hidden_states, list):+ if isinstance(hidden_states, (tuple, list)):hidden_states = layer_outputs[0]return hidden_states@@ -857,7 +861,7 @@ def wrapper_block(elif enable_norm_bias_tuning:if "norm" in m.__class__.__name__.lower():- if m.__class__.__name__ in NORM_MAPPING.keys():+ if m.__class__.__name__ in NORM_MAPPING:wrapper_layer_class = NORM_MAPPING[m.__class__.__name__]new_m = wrapper_layer_class(m, device=device)set_module(block, n, new_m)diff --git a/auto_round_extension/ark/auto_round_kernel/__init__.py b/auto_round_extension/ark/auto_round_kernel/__init__.pyindex be679a144..7d484963d 100644--- a/auto_round_extension/ark/auto_round_kernel/__init__.py+++ b/auto_round_extension/ark/auto_round_kernel/__init__.py@@ -455,7 +455,7 @@ def get_lib(A: torch.Tensor):# A: mxk, B: nxk, bias: n or [1, n]-def matmul_sycl_tla(A: torch.Tensor, B: torch.Tensor, bias: Optional[torch.Tensor] = None):+def matmul_sycl_tla(A: torch.Tensor, B: torch.Tensor, bias: torch.Tensor | None = None):if A.device.type != "xpu" or B.device.type != "xpu":raise NotImplementedError("matmul_sycl_tla is only supported on XPU")if A.ndim != 2 or B.ndim != 2:@@ -1724,7 +1724,7 @@ class ArkCpuPackedKVHandle:num_heads_kv: int | None = None,*,is_causal: bool = False,- scale: Optional[float] = None,+ scale: float | None = None,tensor_layout: str = "HND",) -> torch.Tensor:del num_heads_kv@@ -1757,7 +1757,7 @@ def ark_cpu_packed_kv_descriptor(def ark_cpu_packed_kv_alloc_from_descriptor(descriptor,*,- dtype: Optional[torch.dtype] = None,+ dtype: torch.dtype | None = None,device: str = "cpu",) -> tuple[torch.Tensor, torch.Tensor]:if cpu_lib is None or not hasattr(cpu_lib, "ark_cpu_packed_kv_elems_desc"):@@ -1798,10 +1798,10 @@ def ark_cpu_packed_kv_alloc(def ark_cpu_packed_kv_info(- batch: Optional[int] = None,- num_heads_kv: Optional[int] = None,- capacity: Optional[int] = None,- head_dim: Optional[int] = None,+ batch: int | None = None,+ num_heads_kv: int | None = None,+ capacity: int | None = None,+ head_dim: int | None = None,*,dtype: torch.dtype = torch.float16,descriptor=None,@@ -2027,7 +2027,7 @@ def ark_cpu_bestla_sdpa_packed(num_heads_kv: int,*,is_causal: bool = False,- scale: Optional[float] = None,+ scale: float | None = None,tensor_layout: str = "HND",) -> torch.Tensor:"""Internal/experimental BestLA mixed-precision SDPA over a packed K/V cache.@@ -2081,7 +2081,7 @@ def ark_cpu_bestla_sdpa_packed_from_descriptor(seq_len_kv: int,*,is_causal: bool = False,- scale: Optional[float] = None,+ scale: float | None = None,tensor_layout: str = "HND",) -> torch.Tensor:"""Descriptor-based internal/experimental packed BestLA SDPA forward."""@@ -2122,7 +2122,7 @@ def sageattn(v: torch.Tensor,tensor_layout: str = "HND",is_causal: bool = False,- sm_scale: Optional[float] = None,+ sm_scale: float | None = None,return_lse: bool = False,kernel: str = "v1_pvhalf",**kwargs,@@ -2452,8 +2452,8 @@ def moe_gemm_decode(weights: torch.Tensor,num_tokens_per_expert: torch.Tensor,*,- scales: Optional[torch.Tensor] = None,- zeros: Optional[torch.Tensor] = None,+ scales: torch.Tensor | None = None,+ zeros: torch.Tensor | None = None,weight_bits: int = 4,group_size: int = 128,asym: bool = False,@@ -2605,8 +2605,8 @@ def _validate_moe_quant_args(weights: torch.Tensor,num_tokens_per_expert: torch.Tensor,*,- scales: Optional[torch.Tensor],- zeros: Optional[torch.Tensor],+ scales: torch.Tensor | None,+ zeros: torch.Tensor | None,weight_bits: int,group_size: int,asym: bool,@@ -2763,17 +2763,17 @@ class MoeSymmetricGemm:weights_ptr: intscales_ptr: intzeros_ptr: int = 0- decode_expert_id_per_token: Optional[torch.Tensor] = None+ decode_expert_id_per_token: torch.Tensor | None = None@classmethoddef prepare(cls,weights: torch.Tensor,- scales: Optional[torch.Tensor],+ scales: torch.Tensor | None,*,weight_bits: int = 4,group_size: int = 128,- activation_dtype: Optional[torch.dtype] = None,+ activation_dtype: torch.dtype | None = None,max_decode_tokens: int = 0,) -> "MoeSymmetricGemm":"""Prepare the symmetric quantized MoE GEMM fast path.@@ -2876,8 +2876,8 @@ class MoeSymmetricGemm:activations: torch.Tensor,num_tokens_per_expert: torch.Tensor,*,- outputs: Optional[torch.Tensor] = None,- expert_id_per_token: Optional[torch.Tensor] = None,+ outputs: torch.Tensor | None = None,+ expert_id_per_token: torch.Tensor | None = None,) -> torch.Tensor:total_tokens = int(activations.shape[0])if outputs is None:@@ -2915,8 +2915,8 @@ class MoeSymmetricGemm:activations: torch.Tensor,num_tokens_per_expert: torch.Tensor,*,- outputs: Optional[torch.Tensor] = None,- workspace: Optional[torch.Tensor] = None,+ outputs: torch.Tensor | None = None,+ workspace: torch.Tensor | None = None,) -> torch.Tensor:total_tokens = int(activations.shape[0])if outputs is None:@@ -2953,10 +2953,10 @@ class MoeSymmetricGemm:num_tokens_per_expert: torch.Tensor,*,phase: str = "auto",- decode_threshold: Optional[int] = None,- outputs: Optional[torch.Tensor] = None,- expert_id_per_token: Optional[torch.Tensor] = None,- workspace: Optional[torch.Tensor] = None,+ decode_threshold: int | None = None,+ outputs: torch.Tensor | None = None,+ expert_id_per_token: torch.Tensor | None = None,+ workspace: torch.Tensor | None = None,) -> torch.Tensor:if phase not in _MOE_VALID_PHASES:raise ValueError(f"phase must be one of {_MOE_VALID_PHASES}, got {phase!r}")@@ -2985,10 +2985,10 @@ class MoeSymmetricGemm:num_tokens_per_expert: torch.Tensor,*,phase: str = "auto",- decode_threshold: Optional[int] = None,- outputs: Optional[torch.Tensor] = None,- expert_id_per_token: Optional[torch.Tensor] = None,- workspace: Optional[torch.Tensor] = None,+ decode_threshold: int | None = None,+ outputs: torch.Tensor | None = None,+ expert_id_per_token: torch.Tensor | None = None,+ workspace: torch.Tensor | None = None,) -> torch.Tensor:return self.moe(activations,@@ -3006,7 +3006,7 @@ def moe_gemm(weights: torch.Tensor,num_tokens_per_expert: torch.Tensor,*,- scales: Optional[torch.Tensor] = None,+ scales: torch.Tensor | None = None,) -> torch.Tensor:"""MOE GEMM (Mixture of Experts Grouped GEMM).@@ -3258,12 +3258,12 @@ def moe_gemm_prefill(weights: torch.Tensor,num_tokens_per_expert: torch.Tensor,*,- scales: Optional[torch.Tensor] = None,- zeros: Optional[torch.Tensor] = None,+ scales: torch.Tensor | None = None,+ zeros: torch.Tensor | None = None,weight_bits: int = 4,group_size: int = 128,asym: bool = False,- scale_scheme: Optional[str] = None,+ scale_scheme: str | None = None,) -> torch.Tensor:"""MoE Grouped GEMM optimized for the prefill phase, supporting all weightencodings of ``moe_gemm_decode`` (FP16/BF16, INT8 sym/asym, INT4 sym/asym,@@ -3583,13 +3583,13 @@ def moe(weights: torch.Tensor,num_tokens_per_expert: torch.Tensor,*,- scales: Optional[torch.Tensor] = None,- zeros: Optional[torch.Tensor] = None,+ scales: torch.Tensor | None = None,+ zeros: torch.Tensor | None = None,weight_bits: int = 4,group_size: int = 128,asym: bool = False,phase: str = "auto",- decode_threshold: Optional[int] = None,+ decode_threshold: int | None = None,) -> torch.Tensor:"""Unified MoE GEMM entry point that dispatches to decode or prefill.diff --git a/auto_round_extension/ark/auto_round_kernel/qlinear.py b/auto_round_extension/ark/auto_round_kernel/qlinear.pyindex 749d2db1c..82beecbfe 100644--- a/auto_round_extension/ark/auto_round_kernel/qlinear.py+++ b/auto_round_extension/ark/auto_round_kernel/qlinear.py@@ -89,7 +89,7 @@ def convert_dtype_torch2str(dtype):elif isinstance(dtype, str) and dtype in ["int8", "fp32", "fp16", "bf16"]:return dtypeelse:- assert False, "Unsupported pytorch dtype {} to str dtype".format(dtype)+ assert False, f"Unsupported pytorch dtype {dtype} to str dtype"class QuantLinear(nn.Module):@@ -164,12 +164,9 @@ class QuantLinear(nn.Module):self.bias = Nonedef extra_repr(self) -> str:- return "in_features={}, out_features={}, bias={}, w_bit={}, group_size={}".format(- self.infeatures,- self.outfeatures,- self.bias is not None,- self.bits,- self.group_size,+ return (+ f"in_features={self.infeatures}, out_features={self.outfeatures}, bias={self.bias is not None}, "+ f"w_bit={self.bits}, group_size={self.group_size}")def post_init(self):@@ -563,14 +560,10 @@ class QuantLinearFP8(nn.Module):return outputs.to(raw_input_dtype).view(out_shape)def extra_repr(self) -> str:- return "in_features={}, out_features={}, bias={}, bits={}, " "group_size={}, data_type={}".format(- self.infeatures,- self.outfeatures,- self.bias is not None,- self.bits,- self.group_size,- self.data_type,+ return (+ f"in_features={self.infeatures}, out_features={self.outfeatures}, bias={self.bias is not None}, bits={self.bits}, "+ f"group_size={self.group_size}, data_type={self.data_type}")-__all__ = ["QuantLinear", "QuantLinearGPTQ", "QuantLinearAWQ", "QuantLinearFP8"]+__all__ = ["QuantLinear", "QuantLinearAWQ", "QuantLinearFP8", "QuantLinearGPTQ"]diff --git a/auto_round_extension/ark/auto_round_kernel/sparge_preprocess_triton.py b/auto_round_extension/ark/auto_round_kernel/sparge_preprocess_triton.pyindex 2fe1091f8..9672bd7ac 100644--- a/auto_round_extension/ark/auto_round_kernel/sparge_preprocess_triton.py+++ b/auto_round_extension/ark/auto_round_kernel/sparge_preprocess_triton.py@@ -4,7 +4,8 @@from __future__ import annotationsimport logging-from typing import Any, Callable+from collections.abc import Callable+from typing import Anyimport torchimport tritondiff --git a/auto_round_extension/ark/auto_round_kernel/sparse_attention.py b/auto_round_extension/ark/auto_round_kernel/sparse_attention.pyindex e45febf88..9e86faae3 100644--- a/auto_round_extension/ark/auto_round_kernel/sparse_attention.py+++ b/auto_round_extension/ark/auto_round_kernel/sparse_attention.py@@ -6,7 +6,7 @@ from __future__ import annotationsimport osimport warningsfrom dataclasses import dataclass-from typing import Any, Optional+from typing import Anyimport torch@@ -417,7 +417,7 @@ def sageattn(v: torch.Tensor,tensor_layout: str = "HND",is_causal: bool = False,- sm_scale: Optional[float] = None,+ sm_scale: float | None = None,return_lse: bool = False,kernel: str = "v1_pvhalf",**kwargs,@@ -499,7 +499,7 @@ def _normalize_sparse_mask(def _normalize_per_head_hparam(- value: float | int | torch.Tensor,+ value: float | torch.Tensor,num_heads: int,device: torch.device,name: str,@@ -825,7 +825,7 @@ def _prefix_protection_requested() -> bool:def _get_explicit_protected_prefix(- ctx: "_SpargePreprocessContext",+ ctx: _SpargePreprocessContext,) -> tuple[int, int]:protected_tokens = _get_protected_kv_tokens()protected_sparse_blocks = _get_protected_kv_blocks()@@ -847,7 +847,7 @@ def _get_explicit_protected_prefix(def _get_prefix_protection_blocks(- ctx: "_SpargePreprocessContext",+ ctx: _SpargePreprocessContext,*,raw_block_count: int,tile_block_count: int,diff --git a/auto_round_extension/ark/auto_round_kernel/utils.py b/auto_round_extension/ark/auto_round_kernel/utils.pyindex 17c464260..c88738bf0 100644--- a/auto_round_extension/ark/auto_round_kernel/utils.py+++ b/auto_round_extension/ark/auto_round_kernel/utils.py@@ -15,14 +15,14 @@import loggingimport reimport subprocess-from functools import lru_cache+from functools import cacheimport torchlogger = logging.getLogger(__name__)-@lru_cache(maxsize=None)+@cachedef is_oneapi_ge_2026() -> bool:try:output = subprocess.check_output(@@ -41,7 +41,7 @@ B70_DEVICE_ID = "0xe223"B70_IDENTIFIERS = (B70_DEVICE_ID, "b70")-@lru_cache(maxsize=None)+@cachedef is_b70(device: int = 0) -> bool:try:pro = torch.xpu.get_device_properties(device)diff --git a/auto_round_extension/ark/auto_round_kernel/xpu_loader.py b/auto_round_extension/ark/auto_round_kernel/xpu_loader.pyindex 6a2ec2cfe..1eea91286 100644--- a/auto_round_extension/ark/auto_round_kernel/xpu_loader.py+++ b/auto_round_extension/ark/auto_round_kernel/xpu_loader.py@@ -15,8 +15,8 @@import importlib.utilimport sysimport sysconfig+from collections.abc import Iterablefrom pathlib import Path-from typing import Iterable_DEFAULT_MODULE_NAME = "auto_round_kernel._local.auto_round_kernel_xpu"diff --git a/auto_round_extension/ark/benchmarks/bench_mxfp4_hadamard.py b/auto_round_extension/ark/benchmarks/bench_mxfp4_hadamard.pyold mode 100644new mode 100755index 04af7f551..b5852133c--- a/auto_round_extension/ark/benchmarks/bench_mxfp4_hadamard.py+++ b/auto_round_extension/ark/benchmarks/bench_mxfp4_hadamard.py@@ -1,5 +1,4 @@#!/usr/bin/env python-# -*- coding: utf-8 -*-# Copyright (C) 2026 Intel Corporation# SPDX-License-Identifier: Apache-2.0diff --git a/auto_round_extension/ark/benchmarks/bench_sparse_topk.py b/auto_round_extension/ark/benchmarks/bench_sparse_topk.pyold mode 100644new mode 100755index 9efe91e07..e77ed2294--- a/auto_round_extension/ark/benchmarks/bench_sparse_topk.py+++ b/auto_round_extension/ark/benchmarks/bench_sparse_topk.py@@ -1,5 +1,4 @@#!/usr/bin/env python-# -*- coding: utf-8 -*-# # Copyright (C) 2026 Intel Corporation# # SPDX-License-Identifier: Apache-2.0diff --git a/auto_round_extension/ark/examples/run_flux.py b/auto_round_extension/ark/examples/run_flux.pyindex 57fcf902d..c73cddae8 100644--- a/auto_round_extension/ark/examples/run_flux.py+++ b/auto_round_extension/ark/examples/run_flux.py@@ -420,23 +420,9 @@ def prepare_block_benchmark_state():img_ids = img_ids[0]image_rotary_emb = transformer.pos_embed(torch.cat((text_ids, img_ids), dim=0))- with torch.no_grad():- with transformer.cache_context("cond"):- if benchmark_block_kind == "single":- for block in joint_blocks:- encoder_hidden_states, hidden_states = block(- hidden_states=hidden_states,- encoder_hidden_states=encoder_hidden_states,- temb=temb,- image_rotary_emb=image_rotary_emb,- joint_attention_kwargs=joint_attention_kwargs,- )- preceding_blocks = (- joint_blocks[:benchmark_block_index]- if benchmark_block_kind == "joint"- else single_blocks[:benchmark_block_index]- )- for block in preceding_blocks:+ with torch.no_grad(), transformer.cache_context("cond"):+ if benchmark_block_kind == "single":+ for block in joint_blocks:encoder_hidden_states, hidden_states = block(hidden_states=hidden_states,encoder_hidden_states=encoder_hidden_states,@@ -444,6 +430,19 @@ def prepare_block_benchmark_state():image_rotary_emb=image_rotary_emb,joint_attention_kwargs=joint_attention_kwargs,)+ preceding_blocks = (+ joint_blocks[:benchmark_block_index]+ if benchmark_block_kind == "joint"+ else single_blocks[:benchmark_block_index]+ )+ for block in preceding_blocks:+ encoder_hidden_states, hidden_states = block(+ hidden_states=hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ temb=temb,+ image_rotary_emb=image_rotary_emb,+ joint_attention_kwargs=joint_attention_kwargs,+ )return {"block": target_block,diff --git a/auto_round_extension/ark/setup.py b/auto_round_extension/ark/setup.pyindex 13c030f47..6857ca8c8 100644--- a/auto_round_extension/ark/setup.py+++ b/auto_round_extension/ark/setup.py@@ -71,9 +71,7 @@ def detect_oneapi_version():Returns a string like '2025.3' or None if detection fails."""try:- result = subprocess.run(- ["icx", "--version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True- )+ result = subprocess.run(["icx", "--version"], capture_output=True, text=True, check=True)match = re.search(r"Compiler\s+(\d{4}\.\d+)", result.stdout)if match:return match.group(1)@@ -354,7 +352,7 @@ setup(version=get_build_version(),description="Auto Round Kernel binary package",author_email="[email protected]",- long_description=open("README.md", "r", encoding="utf-8").read(),+ long_description=Path("README.md").read_text(encoding="utf-8"),long_description_content_type="text/markdown",keywords="quantization,auto-around,LLM,kernel",license="Apache 2.0",diff --git a/auto_round_extension/ark/test/bench_ark_cpu_sdpa.py b/auto_round_extension/ark/test/bench_ark_cpu_sdpa.pyold mode 100644new mode 100755index c0c0fd612..1c68a59a8--- a/auto_round_extension/ark/test/bench_ark_cpu_sdpa.py+++ b/auto_round_extension/ark/test/bench_ark_cpu_sdpa.py@@ -201,8 +201,10 @@ def _run_case(shape: Shape, route: Route, route_names: dict[int, str], warmup: iark_ms, torch_ms = _measure_pair(ark_call, torch_call, warmup, runs)return (torch_ms / ark_ms,- f"{shape.name:<22}{route.name:<18}{shape.label:<24}{resolved_name:<12}"- f"{ark_ms:>10.3f}{torch_ms:>14.3f}{torch_ms / ark_ms:>10.2f}x",+ (+ f"{shape.name:<22}{route.name:<18}{shape.label:<24}{resolved_name:<12}"+ f"{ark_ms:>10.3f}{torch_ms:>14.3f}{torch_ms / ark_ms:>10.2f}x"+ ),)diff --git a/auto_round_extension/ark/test/conftest.py b/auto_round_extension/ark/test/conftest.pyindex 66b238775..dcb8b5a87 100644--- a/auto_round_extension/ark/test/conftest.py+++ b/auto_round_extension/ark/test/conftest.py@@ -1,5 +1,3 @@-#!/usr/bin/env python-# -*- coding: utf-8 -*-## Copyright (c) 2026 Intel Corporation#diff --git a/auto_round_extension/ark/test/test_flash_attn.py b/auto_round_extension/ark/test/test_flash_attn.pyold mode 100644new mode 100755index 27aad5426..07f0633b6--- a/auto_round_extension/ark/test/test_flash_attn.py+++ b/auto_round_extension/ark/test/test_flash_attn.py@@ -1,5 +1,4 @@#!/usr/bin/env python-# -*- coding: utf-8 -*-## Copyright (c) 2026 Intel Corporation#@@ -55,7 +54,7 @@ def reference_attention(Q, K, V, scale, is_causal=True):scale=scale,attn_mask=None,is_causal=is_causal,- enable_gqa=True if K.shape[1] != Q.shape[1] else False,+ enable_gqa=K.shape[1] != Q.shape[1],)return refdiff --git a/auto_round_extension/ark/test/test_matmul.py b/auto_round_extension/ark/test/test_matmul.pyold mode 100644new mode 100755index 86f06a8ee..bcad260b5--- a/auto_round_extension/ark/test/test_matmul.py+++ b/auto_round_extension/ark/test/test_matmul.py@@ -1,5 +1,4 @@#!/usr/bin/env python-# -*- coding: utf-8 -*-## Copyright (c) 2023 Intel Corporation#diff --git a/auto_round_extension/ark/test/test_moe.py b/auto_round_extension/ark/test/test_moe.pyold mode 100644new mode 100755index e4d7c39e7..3b1603d4d--- a/auto_round_extension/ark/test/test_moe.py+++ b/auto_round_extension/ark/test/test_moe.py@@ -1,5 +1,4 @@#!/usr/bin/env python-# -*- coding: utf-8 -*-## Copyright (c) 2026 Intel Corporation#diff --git a/auto_round_extension/ark/test/test_moe_decode_perf.py b/auto_round_extension/ark/test/test_moe_decode_perf.pyold mode 100644new mode 100755index ce32f765f..6554abf83--- a/auto_round_extension/ark/test/test_moe_decode_perf.py+++ b/auto_round_extension/ark/test/test_moe_decode_perf.py@@ -1,5 +1,4 @@#!/usr/bin/env python-# -*- coding: utf-8 -*-## Copyright (c) 2026 Intel Corporation#@@ -115,16 +114,13 @@ _DECODE_SKIP = _decode_skip_reason()# Surface diagnostics on collection so the user always sees why the suite# would skip, without having to add extra flags.+_xpu_lib_state = "loaded" if ark.xpu_lib is not None else "None"+_has_decode = hasattr(ark.xpu_lib, "moe_gemm_decode") if ark.xpu_lib is not None else Falseprint(- "[moe-decode-perf] xpu_available=%s xpu_lib=%s has_moe_gemm_decode=%s"- % (- _xpu_available(),- "loaded" if ark.xpu_lib is not None else "None",- hasattr(ark.xpu_lib, "moe_gemm_decode") if ark.xpu_lib is not None else False,- )+ f"[moe-decode-perf] xpu_available={_xpu_available()} xpu_lib={_xpu_lib_state} has_moe_gemm_decode={_has_decode}")if _DECODE_SKIP:- print("[moe-decode-perf] suite will SKIP. reason: %s" % _DECODE_SKIP)+ print(f"[moe-decode-perf] suite will SKIP. reason: {_DECODE_SKIP}")# ---------------------------------------------------------------------------diff --git a/auto_round_extension/ark/test/test_moe_prefill_accuracy.py b/auto_round_extension/ark/test/test_moe_prefill_accuracy.pyold mode 100644new mode 100755index e9f7ca859..c6a46c6a0--- a/auto_round_extension/ark/test/test_moe_prefill_accuracy.py+++ b/auto_round_extension/ark/test/test_moe_prefill_accuracy.py@@ -1,5 +1,4 @@#!/usr/bin/env python-# -*- coding: utf-8 -*-## Copyright (c) 2026 Intel Corporation#diff --git a/auto_round_extension/ark/test/test_moe_prefill_perf.py b/auto_round_extension/ark/test/test_moe_prefill_perf.pyold mode 100644new mode 100755index 5a61d816c..b9e1ba0db--- a/auto_round_extension/ark/test/test_moe_prefill_perf.py+++ b/auto_round_extension/ark/test/test_moe_prefill_perf.py@@ -1,5 +1,4 @@#!/usr/bin/env python-# -*- coding: utf-8 -*-## Copyright (c) 2026 Intel Corporation#@@ -150,16 +149,11 @@ _PREFILL_SKIP = _prefill_skip_reason()_QUANT_PREFILL_SKIP = _quantized_prefill_skip_reason()# Surface diagnostics on collection-print(- "[moe-prefill-perf] xpu_available=%s xpu_lib=%s has_moe_gemm=%s"- % (- _xpu_available(),- "loaded" if ark.xpu_lib is not None else "None",- hasattr(ark.xpu_lib, "moe_gemm") if ark.xpu_lib is not None else False,- )-)+_xpu_lib_state = "loaded" if ark.xpu_lib is not None else "None"+_has_prefill = hasattr(ark.xpu_lib, "moe_gemm") if ark.xpu_lib is not None else False+print(f"[moe-prefill-perf] xpu_available={_xpu_available()} xpu_lib={_xpu_lib_state} has_moe_gemm={_has_prefill}")if _PREFILL_SKIP:- print("[moe-prefill-perf] suite will SKIP. reason: %s" % _PREFILL_SKIP)+ print(f"[moe-prefill-perf] suite will SKIP. reason: {_PREFILL_SKIP}")# ---------------------------------------------------------------------------diff --git a/auto_round_extension/ark/test/test_moe_unified.py b/auto_round_extension/ark/test/test_moe_unified.pyold mode 100644new mode 100755index ad29f1158..1e969222c--- a/auto_round_extension/ark/test/test_moe_unified.py+++ b/auto_round_extension/ark/test/test_moe_unified.py@@ -1,5 +1,4 @@#!/usr/bin/env python-# -*- coding: utf-8 -*-## Copyright (c) 2026 Intel Corporation#diff --git a/auto_round_extension/ark/test/test_mxfp4_hadamard.py b/auto_round_extension/ark/test/test_mxfp4_hadamard.pyold mode 100644new mode 100755index 9c19f35ee..c94843cb3--- a/auto_round_extension/ark/test/test_mxfp4_hadamard.py+++ b/auto_round_extension/ark/test/test_mxfp4_hadamard.py@@ -1,5 +1,4 @@#!/usr/bin/env python-# -*- coding: utf-8 -*-## Copyright (c) 2026 Intel Corporation#@@ -131,7 +130,7 @@ class TestReferenceContract:y = hadamard_transform_reference(x.reshape(-1, HADAMARD_DIM), get_hadamard_matrix(HADAMARD_DIM))amax = y.abs().amax(dim=-1)for group, group_amax in enumerate(amax.tolist()):- expected = min(max(int(math.floor(math.log2(group_amax))) - 2 + 127, 0), 254)+ expected = min(max(math.floor(math.log2(group_amax)) - 2 + 127, 0), 254)assert scale.reshape(-1)[group].item() == expected# amax / scale must land in [4, 8): the E8M0 exponent is standard.ratio = group_amax / (2.0 ** (expected - 127))diff --git a/auto_round_extension/ark/test/test_packq.py b/auto_round_extension/ark/test/test_packq.pyold mode 100644new mode 100755index 7f73fc209..4b1eac62f--- a/auto_round_extension/ark/test/test_packq.py+++ b/auto_round_extension/ark/test/test_packq.py@@ -1,5 +1,4 @@#!/usr/bin/env python-# -*- coding: utf-8 -*-## Copyright (c) 2023 Intel Corporation#diff --git a/auto_round_extension/ark/test/test_sagev1_varlen.py b/auto_round_extension/ark/test/test_sagev1_varlen.pyold mode 100644new mode 100755index f3cf1493d..09378724d--- a/auto_round_extension/ark/test/test_sagev1_varlen.py+++ b/auto_round_extension/ark/test/test_sagev1_varlen.py@@ -1,5 +1,4 @@#!/usr/bin/env python-# -*- coding: utf-8 -*-## Copyright (c) 2026 Intel Corporation#@@ -348,8 +347,8 @@ def benchmark_sagev1_varlen_case(head_dim,dtype,device,- min_seq_q=1 if total_q > batch else 1,- min_seq_kv=1 if total_kv > batch else 1,+ min_seq_q=1,+ min_seq_kv=1,)scale = 1.0 / math.sqrt(head_dim)diff --git a/auto_round_extension/ark/test/test_sdpa_varlen.py b/auto_round_extension/ark/test/test_sdpa_varlen.pyold mode 100644new mode 100755index ec16c2d13..fcafff5f7--- a/auto_round_extension/ark/test/test_sdpa_varlen.py+++ b/auto_round_extension/ark/test/test_sdpa_varlen.py@@ -1,5 +1,4 @@#!/usr/bin/env python-# -*- coding: utf-8 -*-## Copyright (c) 2026 Intel Corporation#@@ -359,8 +358,8 @@ def benchmark_sdpa_varlen_case(head_dim,dtype,device,- min_seq_q=1 if total_q > batch else 1,- min_seq_kv=1 if total_kv > batch else 1,+ min_seq_q=1,+ min_seq_kv=1,)scale = 1.0 / math.sqrt(head_dim)diff --git a/auto_round_extension/ark/test/test_weightonly.py b/auto_round_extension/ark/test/test_weightonly.pyold mode 100644new mode 100755index 10bbb17e2..e7dd0863d--- a/auto_round_extension/ark/test/test_weightonly.py+++ b/auto_round_extension/ark/test/test_weightonly.py@@ -1,5 +1,4 @@#!/usr/bin/env python-# -*- coding: utf-8 -*-## Copyright (c) 2023 Intel Corporation#@@ -342,9 +341,8 @@ def test_xpu_woqgemm_graph_capture_uses_live_queue():graph = torch.xpu.CUDAGraph()try:- with torch.xpu.stream(capture_stream):- with torch.xpu.graph(graph):- graph_output = _woq_call(graph_input)+ with torch.xpu.stream(capture_stream), torch.xpu.graph(graph):+ graph_output = _woq_call(graph_input)except RuntimeError as err:pytest.skip(f"XPU graph capture is unavailable in this environment: {err}")diff --git a/auto_round_extension/ark/test/ut_utils.py b/auto_round_extension/ark/test/ut_utils.pyindex c1a93d247..df8d00eb5 100644--- a/auto_round_extension/ark/test/ut_utils.py+++ b/auto_round_extension/ark/test/ut_utils.py@@ -1,5 +1,3 @@-#!/usr/bin/env python-# -*- coding: utf-8 -*-## Copyright (c) 2023 Intel Corporation#diff --git a/auto_round_extension/ark/test/validate_non_int8_cpu_sdpa.py b/auto_round_extension/ark/test/validate_non_int8_cpu_sdpa.pyold mode 100644new mode 100755index 3ec708190..0c1a1827f--- a/auto_round_extension/ark/test/validate_non_int8_cpu_sdpa.py+++ b/auto_round_extension/ark/test/validate_non_int8_cpu_sdpa.py@@ -312,7 +312,7 @@ def main():real_cmd = cmd[cmd.index(item) :]break- result = subprocess.run(real_cmd, cwd=repo_root, env=env)+ result = subprocess.run(real_cmd, cwd=repo_root, env=env, check=False)if result.returncode != 0:print(f" FAILED (exit {result.returncode})\n")overall = Falsediff --git a/auto_round_extension/ark/tools/analyze_sycl_tla_templates.py b/auto_round_extension/ark/tools/analyze_sycl_tla_templates.pyold mode 100644new mode 100755diff --git a/auto_round_extension/ark/tools/lm_eval_with_ark_sdpa.py b/auto_round_extension/ark/tools/lm_eval_with_ark_sdpa.pyold mode 100644new mode 100755diff --git a/auto_round_extension/ark/tools/measure_sycl_tla_compile_memory.py b/auto_round_extension/ark/tools/measure_sycl_tla_compile_memory.pyold mode 100644new mode 100755index c2d2a75b8..70cbe7460--- a/auto_round_extension/ark/tools/measure_sycl_tla_compile_memory.py+++ b/auto_round_extension/ark/tools/measure_sycl_tla_compile_memory.py@@ -41,8 +41,7 @@ def measure_command(command, time_binary):completed = subprocess.run([time_binary, "-v", *command["argv"]],cwd=command["directory"],- stdout=subprocess.PIPE,- stderr=subprocess.PIPE,+ capture_output=True,text=True,check=False,)diff --git a/auto_round_extension/cuda/cute_nvfp4_e5m3.py b/auto_round_extension/cuda/cute_nvfp4_e5m3.pyindex 4be4769e2..4aa087d0b 100644--- a/auto_round_extension/cuda/cute_nvfp4_e5m3.py+++ b/auto_round_extension/cuda/cute_nvfp4_e5m3.py@@ -14,9 +14,8 @@"""Optional CuTe DSL dispatch for NVFP4 E5M3 activation QDQ."""-from functools import lru_cache+from functools import cache, lru_cachefrom importlib.util import find_spec-from typing import Optionalimport torch@@ -167,7 +166,7 @@ def _make_weight_dq_kernel():return launch_weight_dq-@lru_cache(maxsize=None)+@cachedef _get_compiled_qdq_kernel(device_index: int, dtype: torch.dtype):import cutlass.cute as cutefrom cutlass.cute.runtime import from_dlpack@@ -181,7 +180,7 @@ def _get_compiled_qdq_kernel(device_index: int, dtype: torch.dtype):)-@lru_cache(maxsize=None)+@cachedef _get_compiled_weight_dq_kernel(device_index: int, dtype: torch.dtype):import cutlass.cute as cutefrom cutlass.cute.runtime import from_dlpack@@ -197,7 +196,7 @@ def _get_compiled_weight_dq_kernel(device_index: int, dtype: torch.dtype):)-def try_cute_nvfp4_v2_qdq(activation: torch.Tensor, group_size: int) -> Optional[torch.Tensor]:+def try_cute_nvfp4_v2_qdq(activation: torch.Tensor, group_size: int) -> torch.Tensor | None:"""Run a CuTe DSL group-size-16 FP4 QDQ kernel when eligible."""if not can_use_cute_nvfp4_v2_qdq(activation, group_size):return None@@ -222,7 +221,7 @@ def try_cute_nvfp4_v2_qdq(activation: torch.Tensor, group_size: int) -> Optionaldef try_cute_nvfp4_e5m3_weight_dq(weight_packed: torch.Tensor, weight_scale: torch.Tensor, dtype: torch.dtype-) -> Optional[torch.Tensor]:+) -> torch.Tensor | None:"""Dequantize packed FP4 E5M3 weights with CuTe."""if (not is_cute_dsl_available()@@ -261,13 +260,13 @@ def try_cute_nvfp4_e5m3_linear(activation: torch.Tensor,weight_packed: torch.Tensor,weight_scale: torch.Tensor,- bias: Optional[torch.Tensor],-) -> Optional[torch.Tensor]:+ bias: torch.Tensor | None,+) -> torch.Tensor | None:"""Reserved second-stage dispatch point for fused QDQ, unpack, and GEMM.Returning ``None`` keeps the reference Linear path active until the packed-weightmainloop has been validated against the existing E5M3 checkpoint format."""del activation, weight_packed, weight_scale, bias- fused_output: Optional[torch.Tensor] = None+ fused_output: torch.Tensor | None = Nonereturn fused_outputdiff --git a/auto_round_extension/cuda/gptqmodel_marlin.py b/auto_round_extension/cuda/gptqmodel_marlin.pyindex a804572f5..7966c9bea 100644--- a/auto_round_extension/cuda/gptqmodel_marlin.py+++ b/auto_round_extension/cuda/gptqmodel_marlin.py@@ -17,7 +17,7 @@# Adapted from vllm# at https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/gptq_marlin.py-from typing import Any, Dict, List, Optional, Tuple+from typing import Anyimport numpy as npimport torch@@ -72,7 +72,7 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demanddef set_weight_attrs(weight: torch.Tensor,- weight_attrs: Optional[Dict[str, Any]],+ weight_attrs: dict[str, Any] | None,):"""Set attributes on a weight tensor.@@ -109,7 +109,7 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demandsms = torch.cuda.get_device_properties(device).multi_processor_countreturn torch.zeros(sms * max_blocks_per_sm, dtype=torch.int, device=device, requires_grad=False)- def marlin_sort_g_idx(g_idx: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:+ def marlin_sort_g_idx(g_idx: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:g_idx_sort_indices = torch.argsort(g_idx).to(torch.int)return g_idx[g_idx_sort_indices], g_idx_sort_indices@@ -137,10 +137,10 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demandreturn sdef get_scale_perms():- scale_perm: List[int] = []+ scale_perm: list[int] = []for i in range(8):scale_perm.extend([i + 8 * j for j in range(8)])- scale_perm_single: List[int] = []+ scale_perm_single: list[int] = []for i in range(4):scale_perm_single.extend([2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])return scale_perm, scale_perm_single@@ -316,7 +316,7 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demand)# toggle fp32 mode depending on MARLIN or MARLIN_FP16 backend- self.fp32 = True if self.backend in [BACKEND.MARLIN, BACKEND.AUTO] else False+ self.fp32 = self.backend in [BACKEND.MARLIN, BACKEND.AUTO]if not self.fp32:logger.warning_once(@@ -501,7 +501,7 @@ def get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demandsuper().post_init()- def list_buffers(self) -> List:+ def list_buffers(self) -> list:buf = super().list_buffers()if hasattr(self, "workspace") and self.workspace is not None:buf.append(self.workspace)diff --git a/auto_round_extension/humming/qlinear_humming.py b/auto_round_extension/humming/qlinear_humming.pyindex 421920f8e..ec74743b5 100644--- a/auto_round_extension/humming/qlinear_humming.py+++ b/auto_round_extension/humming/qlinear_humming.py@@ -226,9 +226,9 @@ class QuantLinearAWQ(QuantLinear):__all__ = [+ "SUPPORTED_BITS","QuantLinear","QuantLinearAWQ","QuantLinearGPTQ",- "SUPPORTED_BITS","is_humming_available",]diff --git a/auto_round_extension/mlx/__init__.py b/auto_round_extension/mlx/__init__.pyindex 4a8119fce..a1b7da04a 100644--- a/auto_round_extension/mlx/__init__.py+++ b/auto_round_extension/mlx/__init__.py@@ -14,4 +14,4 @@from auto_round_extension.mlx.qlinear_mlx import QuantLinearMLX, MLX_AVAILABLE-__all__ = ["QuantLinearMLX", "MLX_AVAILABLE"]+__all__ = ["MLX_AVAILABLE", "QuantLinearMLX"]diff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.pyindex 466057700..cc364327c 100644--- a/auto_round_extension/torch/qlinear_torch.py+++ b/auto_round_extension/torch/qlinear_torch.py@@ -234,7 +234,7 @@ class QuantLinear(GenericBitPackingMixin, nn.Module):else:shape = scales_t.shapevalue = 0- for j in range(0, (32 // self.bits)):+ for j in range(32 // self.bits):value |= zeros << (self.bits * j)qzeros = torch.ones((shape[0], shape[1] // 32 * self.bits), dtype=torch.int32) * valueself.qzeros = qzeros.cpu()diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.pyindex d6fef7beb..117997ce7 100644--- a/auto_round_extension/torch/qlinear_torch_zp.py+++ b/auto_round_extension/torch/qlinear_torch_zp.py@@ -165,7 +165,7 @@ class QuantLinear(GenericBitPackingMixin, nn.Module):zeros = int(min(max(zeros - 1, 0), self.maxq))shape = scales_t.shapevalue = 0- for j in range(0, (32 // self.bits)):+ for j in range(32 // self.bits):value |= zeros << (self.bits * j)qzeros = torch.ones((shape[0], shape[1] // 32 * self.bits), dtype=torch.int32) * valueself.qzeros = qzeros.cpu()diff --git a/auto_round_extension/triton/neuqi_sweep.py b/auto_round_extension/triton/neuqi_sweep.pyindex 251d5557b..98d639631 100644--- a/auto_round_extension/triton/neuqi_sweep.py+++ b/auto_round_extension/triton/neuqi_sweep.py@@ -124,7 +124,7 @@ if triton is not None:qwt = tl.load(qw_ptr + c_off[:, None] * G + g_off[None, :], mask=cm[:, None] & gm[None, :], other=0.0)else:qwt = tl.zeros((BC, GP), dtype=tl.float32) + 1.0- for k in range(0, K):+ for k in range(K):sc = tl.load(scales_ptr + c_off * K + k, mask=cm, other=1.0) # [BC]x = d / sc[:, None]r = _rint_f32(x)@@ -144,7 +144,7 @@ if triton is not None:best = tl.where(better, acc, best)bz = tl.where(better, z, bz)else:- for z in range(0, NZ):+ for z in range(NZ):q = tl.minimum(tl.maximum(r + z, 0.0), MAXQ)err = sc[:, None] * (q - z) - dloss2 = err * err * qwt@@ -190,7 +190,7 @@ if triton is not None:best = tl.zeros((BC,), dtype=tl.float32) + float("inf")bk = tl.zeros((BC,), dtype=tl.int32)bmir = tl.zeros((BC,), dtype=tl.int32)- for k in range(0, K):+ for k in range(K):sc = tl.load(scales_ptr + c_off * K + k, mask=cm, other=1.0) # [BC]x = d / sc[:, None]fl = tl.floor(x)@@ -264,7 +264,7 @@ if triton is not None:best = tl.zeros((BC,), dtype=tl.float32) + float("inf")bk = tl.zeros((BC,), dtype=tl.int32)bz = tl.zeros((BC,), dtype=tl.int32)- for k in range(0, K):+ for k in range(K):f = tl.load(fracs_ptr + k)invf = tl.load(invf_ptr + k)x = d * invf@@ -285,7 +285,7 @@ if triton is not None:l_best = tl.where(better, acc, l_best)l_bz = tl.where(better, z, l_bz)else:- for z in range(0, MAXQ + 1):+ for z in range(MAXQ + 1):q = tl.minimum(tl.maximum(r + z, 0.0), MAXQ)err = f * (q - z) - dloss2 = err * err * qwt@@ -469,7 +469,7 @@ if triton is not None:best = tl.zeros((BC,), dtype=tl.float32) + float("inf")bk = tl.zeros((BC,), dtype=tl.int32)bmir = tl.zeros((BC,), dtype=tl.int32)- for k in range(0, K):+ for k in range(K):f = tl.load(fracs_ptr + k)invf = tl.load(invf_ptr + k)x = d * invfdiff --git a/auto_round_extension/triton/qlinear_tritonv2.py b/auto_round_extension/triton/qlinear_tritonv2.pyindex 2fcb87f92..28798ee2c 100644--- a/auto_round_extension/triton/qlinear_tritonv2.py+++ b/auto_round_extension/triton/qlinear_tritonv2.py@@ -13,6 +13,7 @@# limitations under the License.import math+import sysfrom logging import getLoggerimport numpy as np@@ -30,7 +31,7 @@ try:except ImportError as e:if torch.xpu.is_available():logger.error("please make sure your triton version is same with `pytorch-triton-xpu` library ")- exit(-1)+ sys.exit(-1)triton_import_exception = edef error_raiser_triton(*args, **kwargs):@@ -154,7 +155,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):else:shape = scales_t.shapevalue = 0- for j in range(0, (32 // self.bits)):+ for j in range(32 // self.bits):value |= zeros << (self.bits * j)qzeros = np.ones((shape[0], shape[1] // 32 * self.bits), dtype=np.uint32) * valueqzeros = qzeros.astype(np.int32)@@ -210,7 +211,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):logger.info(f"Found {len(kn_values)} unique KN Linear values.")logger.info("Warming up autotune cache ...")with torch.no_grad():- for m in tqdm(range(0, math.ceil(math.log2(seqlen)) + 1)):+ for m in tqdm(range(math.ceil(math.log2(seqlen)) + 1)):m = 2**mfor (k, n), (qweight,diff --git a/auto_round_extension/triton/qlinear_tritonv2_zp.py b/auto_round_extension/triton/qlinear_tritonv2_zp.pyindex 58ff9de0a..e10bcb21f 100644--- a/auto_round_extension/triton/qlinear_tritonv2_zp.py+++ b/auto_round_extension/triton/qlinear_tritonv2_zp.py@@ -13,6 +13,7 @@# limitations under the License.import math+import sysfrom logging import getLoggerimport torch@@ -27,7 +28,7 @@ try:except ImportError as e:if torch.xpu.is_available():logger.error("please make sure your triton version is same with `pytorch-triton-xpu` library ")- exit(-1)+ sys.exit(-1)triton_import_exception = edef error_raiser_triton(*args, **kwargs):@@ -221,7 +222,7 @@ class QuantLinear(nn.Module, TritonModuleMixin):logger.info(f"Found {len(kn_values)} unique KN Linear values.")logger.info("Warming up autotune cache ...")with torch.no_grad():- for m in tqdm(range(0, math.ceil(math.log2(seqlen)) + 1)):+ for m in tqdm(range(math.ceil(math.log2(seqlen)) + 1)):m = 2**mfor (k, n), (qweight,diff --git a/auto_round_extension/triton/triton_utils/custom_autotune.py b/auto_round_extension/triton/triton_utils/custom_autotune.pyindex 5b5b5b14d..1ac6fe36b 100644--- a/auto_round_extension/triton/triton_utils/custom_autotune.py+++ b/auto_round_extension/triton/triton_utils/custom_autotune.py@@ -36,7 +36,6 @@import builtinsimport mathimport time-from typing import Dictimport triton@@ -54,7 +53,7 @@ class CustomizedTritonAutoTuner(triton.KernelInterface):configs,key,reset_to_zero,- prune_configs_by: Dict = None,+ prune_configs_by: dict | None = None,nearest_power_of_two: bool = False,):if not configs:@@ -207,9 +206,9 @@ def matmul248_kernel_config_pruner(configs, nargs):"""The main purpose of this function is to shrink BLOCK_SIZE_* when the corresponding dimension is smaller."""- m = max(2 ** int(math.ceil(math.log2(nargs["M"]))), 16)- n = max(2 ** int(math.ceil(math.log2(nargs["N"]))), 16)- k = max(2 ** int(math.ceil(math.log2(nargs["K"]))), 16)+ m = max(2 ** math.ceil(math.log2(nargs["M"])), 16)+ n = max(2 ** math.ceil(math.log2(nargs["N"])), 16)+ k = max(2 ** math.ceil(math.log2(nargs["K"])), 16)used = set()for config in configs:diff --git a/auto_round_extension/triton/triton_utils/kernels.py b/auto_round_extension/triton/triton_utils/kernels.pyindex efe809fb4..7ab222509 100644--- a/auto_round_extension/triton/triton_utils/kernels.py+++ b/auto_round_extension/triton/triton_utils/kernels.py@@ -186,7 +186,7 @@ def quant_matmul_248_kernel(zeros_shifter = (offs_bn % infearure_per_bits) * bitsaccumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)- for k in range(0, num_pid_k):+ for k in range(num_pid_k):g_idx = tl.load(g_ptrs)# Fetch scales and zeros; these are per-outfeature and thus reused in the inner loop@@ -348,7 +348,7 @@ def transpose_quant_matmul_248_kernel(zeros_shifter = (offs_n % infearure_per_bits) * bitsaccumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_K), dtype=tl.float32)- for k in range(0, num_pid_n):+ for k in range(num_pid_n):# Fetch scales and zeros; these are per-outfeature and thus reused in the inner loopscales = tl.load(scales_ptrs) # (BLOCK_SIZE_K, BLOCK_SIZE_N,)zeros = tl.load(zeros_ptrs) # (BLOCK_SIZE_K, BLOCK_SIZE_N,)diff --git a/auto_round_extension/triton/triton_utils_zp/custom_autotune.py b/auto_round_extension/triton/triton_utils_zp/custom_autotune.pyindex 5b5b5b14d..1ac6fe36b 100644--- a/auto_round_extension/triton/triton_utils_zp/custom_autotune.py+++ b/auto_round_extension/triton/triton_utils_zp/custom_autotune.py@@ -36,7 +36,6 @@import builtinsimport mathimport time-from typing import Dictimport triton@@ -54,7 +53,7 @@ class CustomizedTritonAutoTuner(triton.KernelInterface):configs,key,reset_to_zero,- prune_configs_by: Dict = None,+ prune_configs_by: dict | None = None,nearest_power_of_two: bool = False,):if not configs:@@ -207,9 +206,9 @@ def matmul248_kernel_config_pruner(configs, nargs):"""The main purpose of this function is to shrink BLOCK_SIZE_* when the corresponding dimension is smaller."""- m = max(2 ** int(math.ceil(math.log2(nargs["M"]))), 16)- n = max(2 ** int(math.ceil(math.log2(nargs["N"]))), 16)- k = max(2 ** int(math.ceil(math.log2(nargs["K"]))), 16)+ m = max(2 ** math.ceil(math.log2(nargs["M"])), 16)+ n = max(2 ** math.ceil(math.log2(nargs["N"])), 16)+ k = max(2 ** math.ceil(math.log2(nargs["K"])), 16)used = set()for config in configs:diff --git a/auto_round_extension/triton/triton_utils_zp/kernels.py b/auto_round_extension/triton/triton_utils_zp/kernels.pyindex b8715519b..f79e386d7 100644--- a/auto_round_extension/triton/triton_utils_zp/kernels.py+++ b/auto_round_extension/triton/triton_utils_zp/kernels.py@@ -185,7 +185,7 @@ def quant_matmul_248_kernel(zeros_shifter = (offs_bn % infearure_per_bits) * bitsaccumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)- for k in range(0, num_pid_k):+ for k in range(num_pid_k):g_idx = tl.load(g_ptrs)# Fetch scales and zeros; these are per-outfeature and thus reused in the inner loop@@ -346,7 +346,7 @@ def transpose_quant_matmul_248_kernel(zeros_shifter = (offs_n % infearure_per_bits) * bitsaccumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_K), dtype=tl.float32)- for k in range(0, num_pid_n):+ for k in range(num_pid_n):# Fetch scales and zeros; these are per-outfeature and thus reused in the inner loopscales = tl.load(scales_ptrs) # (BLOCK_SIZE_K, BLOCK_SIZE_N,)zeros = tl.load(zeros_ptrs) # (BLOCK_SIZE_K, BLOCK_SIZE_N,)diff --git a/pyproject.toml b/pyproject.tomlindex f95721f73..b13c77f97 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -74,11 +74,10 @@ indent-width = 4target-version = "py311"[tool.ruff.lint]-# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default.-# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or-# McCabe complexity (`C901`) by default.-select = ["E4", "E7", "E9", "F", "NPY", "FURB"]+# Using default rules: https://docs.astral.sh/ruff/default-rules/ignore = [+ "B023", # Function definition does not bind loop variable (closures here are invoked within the same iteration)+ "BLE001", # Do not catch blind exception: `Exception`"E402", # Module level import not at top of file"E501", # Line too long (121 > 120 characters)"E721", # Do not compare types, use isinstance()@@ -89,6 +88,16 @@ ignore = ["F403", # from {name} import * used; unable to detect undefined names"F841", # Local variable is assigned to but never used{name}"FURB171", # Membership test against single-item container+ "I001", # Import block is un-sorted or un-formatted (handled by isort hook)+ "PLR0402", # Use `from torch import nn` in lieu of alias (`import torch.nn as nn` is the PyTorch idiom)+ "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar`+ "RUF059", # Unpacked variable is never used+ "RUF100", # Unused `noqa` directive (keeps noqa for globally ignored rules)+ "S110", # `try`-`except`-`pass` detected+ "S112", # `try`-`except`-`continue` detected+ "SIM102", # Use a single `if` statement instead of nested `if` statements+ "SIM118", # Use `key in dict` instead of `key in dict.keys()` (safetensors handles only expose `.keys()`)+ "TRY004", # Prefer `TypeError` exception for invalid type (changing exception types breaks callers)]# Allow fix for all enabled rules (when `--fix`) is provided.@@ -98,6 +107,12 @@ unfixable = []# Allow unused variables when underscore-prefixed.dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"+[tool.ruff.lint.flake8-comprehensions]+allow-dict-calls-with-keyword-arguments = true++[tool.ruff.lint.flake8-bugbear]+extend-immutable-calls = ["torch.device"]+[tool.ruff.format]# Like Black, use double quotes for strings.quote-style = "double"diff --git a/setup.py b/setup.pyindex 6131dcb4b..7de42ca2b 100644--- a/setup.py+++ b/setup.py@@ -1,9 +1,9 @@+import builtinsimport osimport reimport subprocessimport sysfrom functools import lru_cache-from io import openfrom setuptools import find_packages, setup@@ -11,10 +11,10 @@ os.environ["CC"] = "g++"os.environ["CXX"] = "g++"try:filepath = "./auto_round/version.py"- with open(filepath) as version_file:+ with builtins.open(filepath) as version_file:(__version__,) = re.findall('__version__ = "(.*)"', version_file.read())except Exception as error:- assert False, "Error: Could not open '%s' due %s\n" % (filepath, error)+ assert False, f"Error: Could not open '{filepath}' due {error}\n"# All BUILD_* flags are initially set to `False` and# will be updated to `True` if the corresponding environment check passes.@@ -40,7 +40,7 @@ def get_build_version():# it avoids the sdist/wheel version mismatch that occurs because git is not# available inside the extracted sdist directory.if os.path.exists("PKG-INFO"):- with open("PKG-INFO", encoding="utf-8") as f:+ with builtins.open("PKG-INFO", encoding="utf-8") as f:for line in f:if line.startswith("Version:"):return line.split(":", 1)[1].strip()@@ -103,7 +103,7 @@ def is_cpu_env():def fetch_requirements(path):requirements = []- with open(path, "r") as fd:+ with builtins.open(path, "r") as fd:requirements = [r.strip() for r in fd]return requirements@@ -162,13 +162,16 @@ if __name__ == "__main__":install_requires = INSTALL_CFG.get("install_requires", [])extras_require = INSTALL_CFG.get("extras_require", {})+ with builtins.open("README.md", "r", encoding="utf-8") as f:+ long_description = f.read()+setup(name=package_name,author="Intel AIPT Team",version=get_build_version(),description="Repository of AutoRound: Advanced Weight-Only Quantization Algorithm for LLMs",- long_description=open("README.md", "r", encoding="utf-8").read(),+ long_description=long_description,long_description_content_type="text/markdown",keywords="quantization,auto-around,LLM,SignRound",license="Apache 2.0",diff --git a/test/conftest.py b/test/conftest.pyindex 006079a3b..7b85a985d 100644--- a/test/conftest.py+++ b/test/conftest.py@@ -1,6 +1,6 @@import osimport sys-from typing import Mapping+from collections.abc import Mappingimport pytestdiff --git a/test/e2e/test_cpu/conftest.py b/test/e2e/test_cpu/conftest.pyindex a603c55d5..de985a252 100644--- a/test/e2e/test_cpu/conftest.py+++ b/test/e2e/test_cpu/conftest.py@@ -32,7 +32,6 @@ import subprocessimport sysimport timefrom dataclasses import asdict, dataclass, field-from typing import List, Optionalimport pytest@@ -144,7 +143,7 @@ class ModelCase:# Default CPU matrix: real, small (≤1.5B) LLMs that finish quantize+eval# in a few minutes on a 32 GiB host. These are the models most likely# to actually run on CPU in production.-DEFAULT_MODEL_CASES: List[ModelCase] = [+DEFAULT_MODEL_CASES: list[ModelCase] = [# Qwen family - small + well supported across all formats.ModelCase("Qwen/Qwen3-0.6B", 4, 128, True, "auto_round", min_ram_gib=8, eval_limit=80),ModelCase("Qwen/Qwen3-0.6B", 4, 128, True, "auto_gptq", min_ram_gib=8, eval_limit=80),@@ -166,7 +165,7 @@ DEFAULT_MODEL_CASES: List[ModelCase] = []# Heavier cases - 1.5B-2B; need ~24 GiB free RAM and longer wall-clock.-LARGE_MODEL_CASES: List[ModelCase] = [+LARGE_MODEL_CASES: list[ModelCase] = [ModelCase("Qwen/Qwen2.5-1.5B-Instruct", 4, 128, True, "auto_round", min_ram_gib=14, eval_limit=80),ModelCase("Qwen/Qwen2.5-1.5B-Instruct", 4, 128, True, "auto_gptq", min_ram_gib=14, eval_limit=80),ModelCase("Qwen/Qwen2.5-1.5B-Instruct", 4, 128, True, "auto_awq", min_ram_gib=14, eval_limit=80),@@ -175,7 +174,7 @@ LARGE_MODEL_CASES: List[ModelCase] = []-def _resolve_mem_override(request) -> Optional[int]:+def _resolve_mem_override(request) -> int | None:return request.config.getoption("--e2e-cpu-mem-gib") if request else None@@ -185,7 +184,7 @@ def _resolve_mem_override(request) -> Optional[int]:@pytest.fixture(scope="session")-def model_matrix(request) -> List[ModelCase]:+def model_matrix(request) -> list[ModelCase]:preset = os.environ.get("E2E_CPU_PRESET", "default")if preset == "default":return DEFAULT_MODEL_CASES@@ -261,9 +260,9 @@ class EvalResult:bits: intgroup_size: intsym: bool- task: Optional[str] = None- metric: Optional[str] = None- value: Optional[float] = None+ task: str | None = None+ metric: str | None = None+ value: float | None = Noneextra: dict = field(default_factory=dict)wall_time_s: float = 0.0@@ -295,8 +294,8 @@ def quantize_and_save(iters: int = 200,nsamples: int = 128,seqlen: int = 2048,- extra_kwargs: Optional[dict] = None,- scheme: Optional[str] = None,+ extra_kwargs: dict | None = None,+ scheme: str | None = None,):"""Quantize a model with the Python API and save it.@@ -342,7 +341,7 @@ def run_lm_eval(limit: int = 100,batch_size: str = "auto",model_type: str = "hf",- extra_model_args: Optional[dict] = None,+ extra_model_args: dict | None = None,):"""Run ``lm-eval`` over a saved checkpoint.@@ -366,7 +365,7 @@ def run_lm_eval()-def extract_metric(results: dict, task: str, metric: str = "acc,none") -> Optional[float]:+def extract_metric(results: dict, task: str, metric: str = "acc,none") -> float | None:"""Pull a single metric out of the lm-eval results dict (may be missing)."""try:return float(results["results"][task][metric])@@ -379,7 +378,7 @@ def extract_metric(results: dict, task: str, metric: str = "acc,none") -> Option# ----------------------------------------------------------------------------def run_cli(argv: List[str], env: Optional[dict] = None, timeout: int = 60 * 60) -> int:+def run_cli(argv: list[str], env: dict | None = None, timeout: int = 60 * 60) -> int:"""Spawn ``python -m auto_round <argv>`` and return the exit code.Used by the CLI e2e tests; intentionally goes through the actualdiff --git a/test/e2e/test_cpu/test_bf16_vs_quant_quality.py b/test/e2e/test_cpu/test_bf16_vs_quant_quality.pyindex 1826caefc..1a47ee730 100644--- a/test/e2e/test_cpu/test_bf16_vs_quant_quality.py+++ b/test/e2e/test_cpu/test_bf16_vs_quant_quality.py@@ -39,7 +39,6 @@ from test.e2e.test_cpu.conftest import ( # noqa: E402quantize_and_save,record,)-from typing import Dict, Optionalimport pytest@@ -75,7 +74,7 @@ def _case_id(model_id: str, scheme: str) -> str:# ----------------------------------------------------------------------------def _evaluate_bf16(model_id: str, tasks: str, limit: int) -> Dict[str, Optional[float]]:+def _evaluate_bf16(model_id: str, tasks: str, limit: int) -> dict[str, float | None]:"""Run ``lm-eval`` on the bf16 model and return a {task: acc} dict."""from auto_round.eval.evaluation import simple_evaluate_user_modelfrom auto_round.utils import llm_load_modeldiff --git a/test/e2e/test_cpu/test_diffusion_quantize_e2e.py b/test/e2e/test_cpu/test_diffusion_quantize_e2e.pyindex 126cf3252..464a60c76 100644--- a/test/e2e/test_cpu/test_diffusion_quantize_e2e.py+++ b/test/e2e/test_cpu/test_diffusion_quantize_e2e.py@@ -36,7 +36,6 @@ from test.e2e.test_cpu.conftest import ( # noqa: E402EvalResult,record,)-from typing import Optionalimport pytestimport torchdiff --git a/test/e2e/test_cpu/test_gguf_conversion_e2e.py b/test/e2e/test_cpu/test_gguf_conversion_e2e.pyindex 91ce205ac..bc1afd1fa 100644--- a/test/e2e/test_cpu/test_gguf_conversion_e2e.py+++ b/test/e2e/test_cpu/test_gguf_conversion_e2e.py@@ -43,7 +43,6 @@ from test.e2e.test_cpu.conftest import ( # noqa: E402assert_non_garbage_output,record,)-from typing import List, Optionalimport pytest@@ -88,7 +87,7 @@ def _case_id(gguf_type: str) -> str:def _find_gguf(save_dir: str) -> str:- matches: List[str] = []+ matches: list[str] = []for root, _, files in os.walk(save_dir):for name in files:if name.endswith(".gguf"):diff --git a/test/e2e/test_cpu/test_gguf_cpu_inference.py b/test/e2e/test_cpu/test_gguf_cpu_inference.pyindex 5440dc36e..bbb992c9a 100644--- a/test/e2e/test_cpu/test_gguf_cpu_inference.py+++ b/test/e2e/test_cpu/test_gguf_cpu_inference.py@@ -42,7 +42,6 @@ from test.e2e.test_cpu.conftest import ( # noqa: E402quantize_and_save,record,)-from typing import Listimport pytest@@ -99,7 +98,7 @@ def _build_llamacpp(gguf_path: str, n_ctx: int = 512, n_threads: int = 0):def _find_gguf(save_dir: str) -> str:"""Locate the .gguf file produced by ``auto-round --format gguf:*``."""- matches: List[str] = []+ matches: list[str] = []for root, _, files in os.walk(save_dir):for name in files:if name.endswith(".gguf"):diff --git a/test/e2e/test_cpu/test_moe_e2e.py b/test/e2e/test_cpu/test_moe_e2e.pyindex 9b6e7493a..1a8417012 100644--- a/test/e2e/test_cpu/test_moe_e2e.py+++ b/test/e2e/test_cpu/test_moe_e2e.py@@ -39,7 +39,6 @@ from test.e2e.test_cpu.conftest import ( # noqa: E402quantize_and_save,record,)-from typing import Listimport pytestimport torchdiff --git a/test/e2e/test_cpu/test_save_load_roundtrip.py b/test/e2e/test_cpu/test_save_load_roundtrip.pyindex dca6742d8..dd5dfff8b 100644--- a/test/e2e/test_cpu/test_save_load_roundtrip.py+++ b/test/e2e/test_cpu/test_save_load_roundtrip.py@@ -46,7 +46,6 @@ from test.e2e.test_cpu.conftest import ( # noqa: E402assert_non_garbage_output,record,)-from typing import List, Optionalimport pytestimport torch@@ -117,7 +116,7 @@ def _reload_and_generate_gguf(saved_dir: str) -> str:"""Reload a GGUF checkpoint via llama.cpp and run a short generation."""from llama_cpp import Llama- matches: List[str] = []+ matches: list[str] = []for root, _, files in os.walk(saved_dir):for name in files:if name.endswith(".gguf"):diff --git a/test/e2e/test_cuda/conftest.py b/test/e2e/test_cuda/conftest.pyindex 7adf16464..a6d9fb926 100644--- a/test/e2e/test_cuda/conftest.py+++ b/test/e2e/test_cuda/conftest.py@@ -35,7 +35,6 @@ import shutilimport sysimport timefrom dataclasses import dataclass, field-from typing import List, Optionalimport pytestimport torch@@ -126,7 +125,7 @@ class ModelCase:# every user runs, (b) the W2A16 low-memory path, (c) the activation# quant path and (d) the GPTQ/AWQ back-compat paths. All models are# small enough to fit on a single 24 GiB GPU at W4A16 with offloading.-DEFAULT_MODEL_CASES: List[ModelCase] = [+DEFAULT_MODEL_CASES: list[ModelCase] = [ModelCase("Qwen/Qwen3-1.7B", 4, 128, True, "auto_round", min_gpu_gib=10),ModelCase("Qwen/Qwen3-1.7B", 4, 128, True, "auto_gptq", min_gpu_gib=10),ModelCase("Qwen/Qwen3-1.7B", 4, 128, True, "auto_awq", min_gpu_gib=10),@@ -136,7 +135,7 @@ DEFAULT_MODEL_CASES: List[ModelCase] = [# A more demanding matrix for nightly runs (8 GiB / 7B-class). These# require ~16 GiB free at fp16 master + W4A16 weights.-LARGE_MODEL_CASES: List[ModelCase] = [+LARGE_MODEL_CASES: list[ModelCase] = [ModelCase("Qwen/Qwen2.5-7B-Instruct", 4, 128, True, "auto_round", min_gpu_gib=18),ModelCase("Qwen/Qwen2.5-7B-Instruct", 4, 128, True, "auto_gptq", min_gpu_gib=18),ModelCase("meta-llama/Llama-3.2-3B-Instruct", 4, 128, True, "auto_round", min_gpu_gib=12),@@ -159,7 +158,7 @@ def pytest_addoption(parser):@pytest.fixture(scope="session")-def model_matrix(request) -> List[ModelCase]:+def model_matrix(request) -> list[ModelCase]:preset = request.config.getoption("--e2e-model-preset")if preset == "default":return DEFAULT_MODEL_CASES@@ -211,7 +210,7 @@ def quantize_and_save(iters: int = 200,nsamples: int = 128,seqlen: int = 2048,- extra_kwargs: Optional[dict] = None,+ extra_kwargs: dict | None = None,):"""Run the full AutoRound pipeline and return the saved checkpoint dir.@@ -264,14 +263,14 @@ class BenchResult:# End-to-end output tokens per second (prompt processing + decoding).output_tokens_per_s: float# Decoding-only tokens per second (excludes prompt eval), if measurable.- gen_tokens_per_s: Optional[float]+ gen_tokens_per_s: float | None# Time-to-first-token seconds (mean over the batch, if available).- ttft_s: Optional[float]+ ttft_s: float | None# Generated text for the first prompt; useful for sanity checks.sample_output: str-def _standard_prompts() -> List[str]:+def _standard_prompts() -> list[str]:"""A fixed prompt list so different runs are comparable."""return ["The capital of France is",@@ -285,7 +284,7 @@ def _standard_prompts() -> List[str]:]-def make_bench_prompts(tokenizer, num_prompts: int, target_input_tokens: int = 64) -> List[str]:+def make_bench_prompts(tokenizer, num_prompts: int, target_input_tokens: int = 64) -> list[str]:"""Pad each base prompt with lorem-style text to ~target_input_tokens.The result is a list of prompts whose prompt-eval cost is similar@@ -296,7 +295,7 @@ def make_bench_prompts(tokenizer, num_prompts: int, target_input_tokens: int = 6" Lorem ipsum dolor sit amet, consectetur adipiscing elit. ""Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ")- out: List[str] = []+ out: list[str] = []i = 0while len(out) < num_prompts:prompt = base[i % len(base)] + pad * 4diff --git a/test/e2e/test_cuda/test_sglang_throughput.py b/test/e2e/test_cuda/test_sglang_throughput.pyindex 3ed58a3ac..0d031b934 100644--- a/test/e2e/test_cuda/test_sglang_throughput.py+++ b/test/e2e/test_cuda/test_sglang_throughput.py@@ -41,7 +41,6 @@ from test.e2e.test_cuda.conftest import ( # noqa: E402make_bench_prompts,quantize_and_save,)-from typing import Listimport pytestimport torchdiff --git a/test/e2e/test_cuda/test_vllm_throughput.py b/test/e2e/test_cuda/test_vllm_throughput.pyindex 03d03d011..6f7cb11d7 100644--- a/test/e2e/test_cuda/test_vllm_throughput.py+++ b/test/e2e/test_cuda/test_vllm_throughput.py@@ -51,7 +51,6 @@ from test.e2e.test_cuda.conftest import ( # noqa: E402make_bench_prompts,quantize_and_save,)-from typing import Listimport pytestimport torchdiff --git a/test/helpers.py b/test/helpers.pyindex a05b0a9aa..68fc6cf97 100644--- a/test/helpers.py+++ b/test/helpers.py@@ -381,14 +381,14 @@ def get_tiny_model(config = json.load(f)_reduce_config_layers(config, num_layers, num_experts)_apply_config_overrides(config, config_overrides)- return getattr(getattr(diffusers_module, mod_name), "from_config")(config)+ return getattr(diffusers_module, mod_name).from_config(config)else:config = transformers.AutoConfig.from_pretrained(os.path.join(local_dir, folder_name, "config.json"))_reduce_config_layers(config, num_layers, num_experts)_apply_config_overrides(config, config_overrides)- return getattr(getattr(transformers_module, mod_name), "_from_config")(config)+ return getattr(transformers_module, mod_name)._from_config(config)with open(os.path.join(local_dir, "model_index.json"), "r", encoding="utf-8") as f:model_index = json.load(f)@@ -452,7 +452,7 @@ def get_tiny_model(return slicedkwargs["dtype"] = "auto" if "auto" not in kwargs else kwargs["dtype"]- kwargs["trust_remote_code"] = True if "trust_remote_code" not in kwargs else kwargs["trust_remote_code"]+ kwargs["trust_remote_code"] = kwargs.get("trust_remote_code", True)if is_mllm:model, processor, tokenizer, image_processor = mllm_load_model(model_name_or_path, **kwargs)if hasattr(model.config, "vision_config"):diff --git a/test/integration/test_cpu/test_inc_integration.py b/test/integration/test_cpu/test_inc_integration.pyindex b5d71b6ff..257d68aad 100644--- a/test/integration/test_cpu/test_inc_integration.py+++ b/test/integration/test_cpu/test_inc_integration.py@@ -31,7 +31,7 @@ target_modules = ["QuantLinear", "QuantLinearGPTQ", "QuantLinearAWQ", "WQLinear_@torch.no_grad()def run_fn(model, dataloader):for data in dataloader:- if isinstance(data, tuple) or isinstance(data, list):+ if isinstance(data, (tuple, list)):model(*data)elif isinstance(data, dict):model(**data)diff --git a/test/unit/common/algorithms/transforms/hadamard/test_dispatcher.py b/test/unit/common/algorithms/transforms/hadamard/test_dispatcher.pyindex bd490df56..8abf418c4 100644--- a/test/unit/common/algorithms/transforms/hadamard/test_dispatcher.py+++ b/test/unit/common/algorithms/transforms/hadamard/test_dispatcher.py@@ -289,16 +289,18 @@ class TestApplyHadamardRotation:fuse_online_to_weight=None,)model = nn.Linear(8, 8)- with patch(- "auto_round.algorithms.transforms.hadamard.dispatcher.resolve_hadamard_backend",- return_value="transform",- ):- with patch(+ with (+ patch(+ "auto_round.algorithms.transforms.hadamard.dispatcher.resolve_hadamard_backend",+ return_value="transform",+ ),+ patch("auto_round.algorithms.transforms.hadamard.dispatcher._to_config",return_value=fake_cfg,- ):- with pytest.raises(ValueError, match="only supports hadamard or random_hadamard"):- apply_hadamard_rotation(model, fake_cfg, "mx_fp", compute_device="cpu")+ ),+ pytest.raises(ValueError, match="only supports hadamard or random_hadamard"),+ ):+ apply_hadamard_rotation(model, fake_cfg, "mx_fp", compute_device="cpu")def test_rotation_config_stored_on_model(self):"""After apply, ``_rotation_config`` is set on the model (inplace path)."""diff --git a/test/unit/common/calibration/test_diffusion.py b/test/unit/common/calibration/test_diffusion.pyindex 3a6de3177..b578e2828 100644--- a/test/unit/common/calibration/test_diffusion.py+++ b/test/unit/common/calibration/test_diffusion.py@@ -157,10 +157,13 @@ class TestDiffusionCalibrator:calibrator.pipe = FakePipeline(fn=lambda *args, **kwargs: None)calibrator._requires_calibration_image = lambda: False- with patch(- "auto_round.compressors.diffusion.dataset.get_diffusion_dataloader",- return_value=(new_dataloader, 2),- ), patch("auto_round.calibration.diffusion.tqdm", FakeTqdm):+ with (+ patch(+ "auto_round.compressors.diffusion.dataset.get_diffusion_dataloader",+ return_value=(new_dataloader, 2),+ ),+ patch("auto_round.calibration.diffusion.tqdm", FakeTqdm),+ ):calibrator.calib(nsamples=2, bs=1)assert calibrator.dataloader is new_dataloader@@ -203,12 +206,14 @@ class TestDiffusionCalibrator:)calibrator.dataset = "mock"- with patch(- "auto_round.compressors.diffusion.dataset.get_diffusion_dataloader",- return_value=([], 2),+ with (+ patch(+ "auto_round.compressors.diffusion.dataset.get_diffusion_dataloader",+ return_value=([], 2),+ ),+ pytest.raises(SystemExit),):- with pytest.raises(SystemExit):- calibrator.calib(nsamples=1, bs=1)+ calibrator.calib(nsamples=1, bs=1)def test_calib_moves_pipeline_to_target_device(self, calibrator):seen = []@@ -224,9 +229,12 @@ class TestDiffusionCalibrator:calibrator.pipe.to = fake_to- with patch("auto_round.calibration.diffusion.tqdm", FakeTqdm), patch(- "auto_round.calibration.diffusion.device_manager",- SimpleNamespace(device="cuda:0"),+ with (+ patch("auto_round.calibration.diffusion.tqdm", FakeTqdm),+ patch(+ "auto_round.calibration.diffusion.device_manager",+ SimpleNamespace(device="cuda:0"),+ ),):calibrator.calib(nsamples=2, bs=1)@@ -294,9 +302,11 @@ class TestDiffusionCalibrator:calibrator.pipe = FakePipeline(fn=failing_pipe)calibrator._requires_calibration_image = lambda: False- with patch("auto_round.calibration.diffusion.tqdm", FakeTqdm):- with pytest.raises(NotImplementedError, match="unsupported op"):- calibrator.calib(nsamples=1, bs=1)+ with (+ patch("auto_round.calibration.diffusion.tqdm", FakeTqdm),+ pytest.raises(NotImplementedError, match="unsupported op"),+ ):+ calibrator.calib(nsamples=1, bs=1)def test_calib_other_exceptions_propagate(self, calibrator):def failing_pipe(*args, **kwargs):@@ -306,9 +316,8 @@ class TestDiffusionCalibrator:calibrator.pipe = FakePipeline(fn=failing_pipe)calibrator._requires_calibration_image = lambda: False- with patch("auto_round.calibration.diffusion.tqdm", FakeTqdm):- with pytest.raises(RuntimeError, match="unexpected"):- calibrator.calib(nsamples=1, bs=1)+ with patch("auto_round.calibration.diffusion.tqdm", FakeTqdm), pytest.raises(RuntimeError, match="unexpected"):+ calibrator.calib(nsamples=1, bs=1)def test_calib_single_sample_stops_early(self, calibrator):seen = []@@ -330,9 +339,8 @@ class TestDiffusionCalibrator:calibrator.pipe = FakePipeline(fn=lambda *args, **kwargs: None)calibrator._requires_calibration_image = lambda: False- with patch("auto_round.calibration.diffusion.tqdm", FakeTqdm):- with pytest.raises(SystemExit):- calibrator.calib(nsamples=1, bs=1)+ with patch("auto_round.calibration.diffusion.tqdm", FakeTqdm), pytest.raises(SystemExit):+ calibrator.calib(nsamples=1, bs=1)def test_calib_insufficient_samples_warns_and_truncates(self, calibrator):def fake_pipe(prompts, **kwargs):@@ -357,6 +365,8 @@ class TestDiffusionCalibrator:calibrator.pipe = FakePipeline(fn=fake_pipe)calibrator._requires_calibration_image = lambda: False- with patch("auto_round.calibration.diffusion.tqdm", FakeTqdm):- with pytest.raises(ValueError, match="valid sample count is less than batch_size"):- calibrator.calib(nsamples=3, bs=2)+ with (+ patch("auto_round.calibration.diffusion.tqdm", FakeTqdm),+ pytest.raises(ValueError, match="valid sample count is less than batch_size"),+ ):+ calibrator.calib(nsamples=3, bs=2)diff --git a/test/unit/common/compressors/test_compressors_init.py b/test/unit/common/compressors/test_compressors_init.pyindex 672c4095b..08fc46865 100644--- a/test/unit/common/compressors/test_compressors_init.py+++ b/test/unit/common/compressors/test_compressors_init.py@@ -17,7 +17,7 @@ class TestCompressorsLazyImports:def test_auto_round_lazy_import(self):with pytest.raises(AttributeError, match="has no attribute"):- getattr(compressors, "AutoRound")+ _ = compressors.AutoRounddef test_base_compressor_lazy_import(self):BaseCompressor = compressors.BaseCompressor@@ -41,7 +41,7 @@ class TestCompressorsLazyImports:def test_unknown_attribute_raises(self):with pytest.raises(AttributeError, match="has no attribute"):- getattr(compressors, "UnknownClass123")+ _ = compressors.UnknownClass123def test_all_contains_expected(self):assert "BaseOrchestrator" in compressors.__all__diff --git a/test/unit/common/eval/test_evaluation.py b/test/unit/common/eval/test_evaluation.pyindex 028345c06..d9a9da081 100644--- a/test/unit/common/eval/test_evaluation.py+++ b/test/unit/common/eval/test_evaluation.py@@ -242,20 +242,24 @@ class TestPrepareModelForEval:def test_raises_when_meta_device(self):m = nn.Linear(4, 4)m.dtype = torch.bfloat16- with patch("auto_round.eval.evaluation._normalize_model_eval_dtype", return_value=m):- with patch("auto_round.eval.evaluation.dispatch_model_block_wise") as mock_dispatch:- result = prepare_model_for_eval(m, "cpu", "auto")- assert result is m- mock_dispatch.assert_called_once()+ with (+ patch("auto_round.eval.evaluation._normalize_model_eval_dtype", return_value=m),+ patch("auto_round.eval.evaluation.dispatch_model_block_wise") as mock_dispatch,+ ):+ result = prepare_model_for_eval(m, "cpu", "auto")+ assert result is m+ mock_dispatch.assert_called_once()def test_multi_device_dispatch(self):m = nn.Linear(4, 4)m.hf_device_map = {"linear": "cpu", "linear2": "cpu"}- with patch("auto_round.eval.evaluation._normalize_model_eval_dtype", return_value=m):- with patch("accelerate.big_modeling.dispatch_model") as mock_dispatch:- result = prepare_model_for_eval(m, "cpu", "auto")- assert result is m- mock_dispatch.assert_called_once()+ with (+ patch("auto_round.eval.evaluation._normalize_model_eval_dtype", return_value=m),+ patch("accelerate.big_modeling.dispatch_model") as mock_dispatch,+ ):+ result = prepare_model_for_eval(m, "cpu", "auto")+ assert result is m+ mock_dispatch.assert_called_once()class TestSimpleEvaluate:@@ -274,16 +278,18 @@ class TestSimpleEvaluateUserModel:def test_creates_hflm(self):mock_hflm = MagicMock()- with patch.dict(- "sys.modules",- {- "lm_eval": MagicMock(),- "lm_eval.models": MagicMock(),- "lm_eval.models.huggingface": MagicMock(HFLM=mock_hflm),- },+ with (+ patch.dict(+ "sys.modules",+ {+ "lm_eval": MagicMock(),+ "lm_eval.models": MagicMock(),+ "lm_eval.models.huggingface": MagicMock(HFLM=mock_hflm),+ },+ ),+ patch("lm_eval.simple_evaluate", return_value={"results": {}}) as mock_eval,):- with patch("lm_eval.simple_evaluate", return_value={"results": {}}) as mock_eval:- model = MagicMock()- tokenizer = MagicMock()- result = simple_evaluate_user_model(model, tokenizer, batch_size=4)- assert "results" in result or mock_hflm.called+ model = MagicMock()+ tokenizer = MagicMock()+ result = simple_evaluate_user_model(model, tokenizer, batch_size=4)+ assert "results" in result or mock_hflm.calleddiff --git a/test/unit/common/export/test_gguf_conversion.py b/test/unit/common/export/test_gguf_conversion.pyindex aa3de4016..85de13ec0 100644--- a/test/unit/common/export/test_gguf_conversion.py+++ b/test/unit/common/export/test_gguf_conversion.py@@ -1,6 +1,3 @@-#!/usr/bin/env python3-# -*- coding: utf-8 -*--"""Comprehensive unit tests for GGUF conversion modules with low coverage.Tests conversion modules that have 0% coverage in the test suite, covering@@ -343,9 +340,11 @@ class TestMimoConversion:obj.tensor_map.mapping = {"tensor": ("KEY", "tensor_name")}# Mock super().prepare_tensors() to skip the base class work- with patch("auto_round.export.export_to_gguf.conversion.base.ModelBase.prepare_tensors"):- with pytest.raises(ValueError, match="Unprocessed experts"):- obj.prepare_tensors()+ with (+ patch("auto_round.export.export_to_gguf.conversion.base.ModelBase.prepare_tensors"),+ pytest.raises(ValueError, match="Unprocessed experts"),+ ):+ obj.prepare_tensors()# ==============================================================================@@ -813,9 +812,11 @@ class TestOlmoConversion:obj._experts = [{"unprocessed.tensor": None}]obj.tensor_map.mapping = {"tensor": ("KEY", "tensor_name")}- with patch("auto_round.export.export_to_gguf.conversion.base.ModelBase.prepare_tensors"):- with pytest.raises(ValueError, match="Unprocessed experts"):- obj.prepare_tensors()+ with (+ patch("auto_round.export.export_to_gguf.conversion.base.ModelBase.prepare_tensors"),+ pytest.raises(ValueError, match="Unprocessed experts"),+ ):+ obj.prepare_tensors()# ==============================================================================@@ -1387,9 +1388,11 @@ class TestSmallThinkerConversion:obj._experts = [{"unprocessed.tensor": None}]obj.tensor_map.mapping = {"tensor": ("KEY", "tensor_name")}- with patch("auto_round.export.export_to_gguf.conversion.base.ModelBase.prepare_tensors"):- with pytest.raises(ValueError, match="Unprocessed experts"):- obj.prepare_tensors()+ with (+ patch("auto_round.export.export_to_gguf.conversion.base.ModelBase.prepare_tensors"),+ pytest.raises(ValueError, match="Unprocessed experts"),+ ):+ obj.prepare_tensors()def test_smallthinker_set_gguf_parameters_expert_gating(self):"""Test expert gating function is set correctly."""@@ -3044,9 +3047,11 @@ class TestMistral3Conversion:},)obj.rope_parameters = {"rope_type": "linear", "mscale_all_dim": 1.0, "llama_4_scaling_beta": 0.5}- with patch.object(cls.__mro__[1], "set_gguf_parameters", lambda self: None):- with pytest.raises(AssertionError, match="rope_type must be 'yarn'"):- obj.set_gguf_parameters()+ with (+ patch.object(cls.__mro__[1], "set_gguf_parameters", lambda self: None),+ pytest.raises(AssertionError, match="rope_type must be 'yarn'"),+ ):+ obj.set_gguf_parameters()# ==============================================================================@@ -5686,9 +5691,11 @@ class TestBertConversion:obj = _make_mock_model(BertModel)obj.cls_out_labels = None- with patch.object(BertModel.__mro__[1], "set_gguf_parameters", lambda self: None):- with patch.object(obj, "_try_set_pooling_type"):- obj.set_gguf_parameters()+ with (+ patch.object(BertModel.__mro__[1], "set_gguf_parameters", lambda self: None),+ patch.object(obj, "_try_set_pooling_type"),+ ):+ obj.set_gguf_parameters()obj.gguf_writer.add_causal_attention.assert_called_once_with(False)def test_bert_set_gguf_parameters_with_classifier_labels(self):@@ -5697,9 +5704,11 @@ class TestBertConversion:obj = _make_mock_model(BertModel)obj.cls_out_labels = {"0": "NEGATIVE", "1": "POSITIVE"}- with patch.object(BertModel.__mro__[1], "set_gguf_parameters", lambda self: None):- with patch.object(obj, "_try_set_pooling_type"):- obj.set_gguf_parameters()+ with (+ patch.object(BertModel.__mro__[1], "set_gguf_parameters", lambda self: None),+ patch.object(obj, "_try_set_pooling_type"),+ ):+ obj.set_gguf_parameters()obj.gguf_writer.add_classifier_output_labels.assert_called_once_with(["NEGATIVE", "POSITIVE"])def test_bert_filter_tensors_strips_bert_prefix(self):diff --git a/test/unit/common/export/test_gguf_dtype_helpers.py b/test/unit/common/export/test_gguf_dtype_helpers.pyindex 71368b8c3..f8606cb24 100644--- a/test/unit/common/export/test_gguf_dtype_helpers.py+++ b/test/unit/common/export/test_gguf_dtype_helpers.py@@ -195,7 +195,7 @@ class TestUseMoreBits:from auto_round.export.export_to_gguf.gguf_dtype import _use_more_bits# 8 layers: first 8/8=1 layer uses more bits- for i in range(0, 1):+ for i in range(1):assert _use_more_bits(i, 8) is Truedef test_last_eighth(self):diff --git a/test/unit/common/export/test_mlx_export.py b/test/unit/common/export/test_mlx_export.pyindex d9cef5885..cb41d2a19 100644--- a/test/unit/common/export/test_mlx_export.py+++ b/test/unit/common/export/test_mlx_export.py@@ -552,7 +552,8 @@ class TestSaveQuantizedAsMlx:)cfg_path = os.path.join(output_dir, "config.json")assert os.path.exists(cfg_path)- cfg = json.load(open(cfg_path))+ with open(cfg_path) as f:+ cfg = json.load(f)assert "quantization" in cfgdef test_autoround_format_flag(self, tmp_path):diff --git a/test/unit/common/export/test_qlinear_fp_helpers.py b/test/unit/common/export/test_qlinear_fp_helpers.pyindex 050c56535..3effc8881 100644--- a/test/unit/common/export/test_qlinear_fp_helpers.py+++ b/test/unit/common/export/test_qlinear_fp_helpers.py@@ -67,7 +67,7 @@ class TestQuantLinearInit:)assert layer.weight_global_scale.shape == (1,)# act_bits > 8 -> input_global_scale NOT registered- assert not hasattr(layer, "input_global_scale") or layer.input_global_scale is None or True+ assert not hasattr(layer, "input_global_scale") or layer.input_global_scale is Nonedef test_construction_4bit_nv_act_global(self):from auto_round.export.export_to_autoround.qlinear_fp import QuantLineardiff --git a/test/unit/common/modeling/test_fp8_quant.py b/test/unit/common/modeling/test_fp8_quant.pyindex 167dc6eaf..52e3ba33f 100644--- a/test/unit/common/modeling/test_fp8_quant.py+++ b/test/unit/common/modeling/test_fp8_quant.py@@ -94,15 +94,17 @@ class TestOotReplaceWithFp8Linear:model = _NoLinear()config = _QuantConfigStub(dequantize=False)- with patch("transformers.integrations.finegrained_fp8.FP8Linear"):- with patch(+ with (+ patch("transformers.integrations.finegrained_fp8.FP8Linear"),+ patch("transformers.integrations.finegrained_fp8.should_convert_module",return_value=True,- ):- with patch("transformers.integrations.finegrained_fp8.logger") as mock_logger:- result = oot_replace_with_fp8_linear(model, quantization_config=config)- assert result is model- assert mock_logger.warning.called+ ),+ patch("transformers.integrations.finegrained_fp8.logger") as mock_logger,+ ):+ result = oot_replace_with_fp8_linear(model, quantization_config=config)+ assert result is model+ assert mock_logger.warning.calleddef test_replaces_linear_modules(self):"""All ``nn.Linear`` children should be replaced with the FP8 class."""@@ -113,21 +115,23 @@ class TestOotReplaceWithFp8Linear:def _make_module(*args, **kwargs):return nn.Linear(8, 8)- with patch(- "transformers.integrations.finegrained_fp8.FP8Linear",- side_effect=_make_module,- ):- with patch(+ with (+ patch(+ "transformers.integrations.finegrained_fp8.FP8Linear",+ side_effect=_make_module,+ ),+ patch("transformers.integrations.finegrained_fp8.should_convert_module",return_value=True,- ):- with patch(- "auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5_4_0",- return_value=False,- ):- result = oot_replace_with_fp8_linear(model, quantization_config=config)- # FP8Linear was called for each nn.Linear child.- assert result is model+ ),+ patch(+ "auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5_4_0",+ return_value=False,+ ),+ ):+ result = oot_replace_with_fp8_linear(model, quantization_config=config)+ # FP8Linear was called for each nn.Linear child.+ assert result is modeldef test_with_modules_to_not_convert(self):"""Names listed in ``modules_to_not_convert`` are skipped."""@@ -135,19 +139,21 @@ class TestOotReplaceWithFp8Linear:model = _TinyModel(with_bias=True)config = _QuantConfigStub(dequantize=False)- with patch("transformers.integrations.finegrained_fp8.FP8Linear") as mock_fp8:- with patch(+ with (+ patch("transformers.integrations.finegrained_fp8.FP8Linear") as mock_fp8,+ patch("transformers.integrations.finegrained_fp8.should_convert_module",return_value=False,- ):- oot_replace_with_fp8_linear(- model,- modules_to_not_convert=["fc1"],- quantization_config=config,- )- # No replacement calls should have happened- # because should_convert_module returned False everywhere.- assert not mock_fp8.called+ ),+ ):+ oot_replace_with_fp8_linear(+ model,+ modules_to_not_convert=["fc1"],+ quantization_config=config,+ )+ # No replacement calls should have happened+ # because should_convert_module returned False everywhere.+ assert not mock_fp8.calleddef test_pre_quantized(self):"""The ``pre_quantized=True`` path passes ``dtype=None`` instead of@@ -162,23 +168,25 @@ class TestOotReplaceWithFp8Linear:captured_kwargs.append(kwargs)return nn.Linear(8, 8)- with patch("transformers.integrations.finegrained_fp8.FP8Linear", side_effect=_capture):- with patch(+ with (+ patch("transformers.integrations.finegrained_fp8.FP8Linear", side_effect=_capture),+ patch("transformers.integrations.finegrained_fp8.should_convert_module",return_value=True,- ):- with patch(- "auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5_4_0",- return_value=True,- ):- oot_replace_with_fp8_linear(- model,- quantization_config=config,- pre_quantized=True,- )- # Every captured call must include ``dtype=None``.- for kw in captured_kwargs:- assert kw.get("dtype") is None+ ),+ patch(+ "auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5_4_0",+ return_value=True,+ ),+ ):+ oot_replace_with_fp8_linear(+ model,+ quantization_config=config,+ pre_quantized=True,+ )+ # Every captured call must include ``dtype=None``.+ for kw in captured_kwargs:+ assert kw.get("dtype") is Nonedef test_bias_kwarg_name_pre_v5_4(self):"""On transformers < 5.4, the bias flag is passed as ``bias``."""@@ -191,23 +199,25 @@ class TestOotReplaceWithFp8Linear:captured_kwargs.append(kwargs)return nn.Linear(8, 8)- with patch("transformers.integrations.finegrained_fp8.FP8Linear", side_effect=_capture):- with patch(+ with (+ patch("transformers.integrations.finegrained_fp8.FP8Linear", side_effect=_capture),+ patch("transformers.integrations.finegrained_fp8.should_convert_module",return_value=True,- ):- with patch(- "auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5_4_0",- return_value=False,- ):- oot_replace_with_fp8_linear(model, quantization_config=config)- # At least one replacement happened.- assert len(captured_kwargs) >= 1- for kw in captured_kwargs:- # On pre-5.4, ``bias`` is the kwarg (not ``has_bias``).- assert "bias" in kw- assert "has_bias" not in kw- assert kw["bias"] is True+ ),+ patch(+ "auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5_4_0",+ return_value=False,+ ),+ ):+ oot_replace_with_fp8_linear(model, quantization_config=config)+ # At least one replacement happened.+ assert len(captured_kwargs) >= 1+ for kw in captured_kwargs:+ # On pre-5.4, ``bias`` is the kwarg (not ``has_bias``).+ assert "bias" in kw+ assert "has_bias" not in kw+ assert kw["bias"] is Truedef test_bias_kwarg_name_v5_4_plus(self):"""On transformers >= 5.4, the bias flag is passed as ``has_bias``."""@@ -220,21 +230,23 @@ class TestOotReplaceWithFp8Linear:captured_kwargs.append(kwargs)return nn.Linear(8, 8)- with patch("transformers.integrations.finegrained_fp8.FP8Linear", side_effect=_capture):- with patch(+ with (+ patch("transformers.integrations.finegrained_fp8.FP8Linear", side_effect=_capture),+ patch("transformers.integrations.finegrained_fp8.should_convert_module",return_value=True,- ):- with patch(- "auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5_4_0",- return_value=True,- ):- oot_replace_with_fp8_linear(model, quantization_config=config)- assert len(captured_kwargs) >= 1- for kw in captured_kwargs:- assert "has_bias" in kw- assert "bias" not in kw- assert kw["has_bias"] is True+ ),+ patch(+ "auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5_4_0",+ return_value=True,+ ),+ ):+ oot_replace_with_fp8_linear(model, quantization_config=config)+ assert len(captured_kwargs) >= 1+ for kw in captured_kwargs:+ assert "has_bias" in kw+ assert "bias" not in kw+ assert kw["has_bias"] is Truedef test_no_bias_flag_passed_correctly(self):"""When the linear module has ``bias=False``, the OOT function must@@ -249,19 +261,21 @@ class TestOotReplaceWithFp8Linear:captured_kwargs.append(kwargs)return nn.Linear(8, 8)- with patch("transformers.integrations.finegrained_fp8.FP8Linear", side_effect=_capture):- with patch(+ with (+ patch("transformers.integrations.finegrained_fp8.FP8Linear", side_effect=_capture),+ patch("transformers.integrations.finegrained_fp8.should_convert_module",return_value=True,- ):- with patch(- "auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5_4_0",- return_value=False,- ):- oot_replace_with_fp8_linear(model, quantization_config=config)- assert len(captured_kwargs) >= 1- for kw in captured_kwargs:- assert kw.get("bias") is False+ ),+ patch(+ "auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5_4_0",+ return_value=False,+ ),+ ):+ oot_replace_with_fp8_linear(model, quantization_config=config)+ assert len(captured_kwargs) >= 1+ for kw in captured_kwargs:+ assert kw.get("bias") is Falsedef test_returns_self(self):"""The function returns the (mutated) model object."""@@ -272,20 +286,22 @@ class TestOotReplaceWithFp8Linear:def _make_module(*args, **kwargs):return nn.Linear(8, 8)- with patch(- "transformers.integrations.finegrained_fp8.FP8Linear",- side_effect=_make_module,- ):- with patch(+ with (+ patch(+ "transformers.integrations.finegrained_fp8.FP8Linear",+ side_effect=_make_module,+ ),+ patch("transformers.integrations.finegrained_fp8.should_convert_module",return_value=True,- ):- with patch(- "auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5_4_0",- return_value=True,- ):- result = oot_replace_with_fp8_linear(model, quantization_config=config)- assert result is model+ ),+ patch(+ "auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5_4_0",+ return_value=True,+ ),+ ):+ result = oot_replace_with_fp8_linear(model, quantization_config=config)+ assert result is model# ---------------------------------------------------------------------------@@ -333,23 +349,27 @@ class TestApplyFp8ExpertReplacementPatch:def test_no_cuda_does_nothing(self):"""On a non-CUDA host the function must be a no-op without raising."""- with patch("torch.cuda.is_available", return_value=False):- with patch(+ with (+ patch("torch.cuda.is_available", return_value=False),+ patch("auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5",return_value=True,- ):- # Should not raise.- assert apply_fp8_expert_replacement_patch() is None+ ),+ ):+ # Should not raise.+ assert apply_fp8_expert_replacement_patch() is Nonedef test_old_transformers_does_nothing(self):"""With transformers < 5 the function must be a no-op."""- with patch("torch.cuda.is_available", return_value=True):- with patch(+ with (+ patch("torch.cuda.is_available", return_value=True),+ patch("auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5",return_value=False,- ):- assert apply_fp8_expert_replacement_patch() is None+ ),+ ):+ assert apply_fp8_expert_replacement_patch() is Nonedef test_import_error_is_swallowed(self):"""If the local import of ``transformers.integrations.finegrained_fp8``@@ -365,14 +385,16 @@ class TestApplyFp8ExpertReplacementPatch:raise ImportError("boom")return real_import(name, globals, locals, fromlist, level)- with patch("torch.cuda.is_available", return_value=True):- with patch(+ with (+ patch("torch.cuda.is_available", return_value=True),+ patch("auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5",return_value=True,- ):- with patch("builtins.__import__", side_effect=fake_import):- # Should not raise despite the ImportError.- assert apply_fp8_expert_replacement_patch() is None+ ),+ patch("builtins.__import__", side_effect=fake_import),+ ):+ # Should not raise despite the ImportError.+ assert apply_fp8_expert_replacement_patch() is Nonedef test_replaces_upstream_replace_with_fp8_linear(self):"""When transformers >= 5 and CUDA is available, the upstream@@ -385,13 +407,15 @@ class TestApplyFp8ExpertReplacementPatch:original = upstream.replace_with_fp8_lineartry:- with patch("torch.cuda.is_available", return_value=True):- with patch(+ with (+ patch("torch.cuda.is_available", return_value=True),+ patch("auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5",return_value=True,- ):- apply_fp8_expert_replacement_patch()- assert upstream.replace_with_fp8_linear is fp8q.oot_replace_with_fp8_linear+ ),+ ):+ apply_fp8_expert_replacement_patch()+ assert upstream.replace_with_fp8_linear is fp8q.oot_replace_with_fp8_linearfinally:upstream.replace_with_fp8_linear = original@@ -408,13 +432,15 @@ class TestApplyFp8ExpertReplacementPatch:original = FineGrainedFP8HfQuantizer.validate_environmenttry:- with patch("torch.cuda.is_available", return_value=True):- with patch(+ with (+ patch("torch.cuda.is_available", return_value=True),+ patch("auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5",return_value=True,- ):- apply_fp8_expert_replacement_patch()- assert FineGrainedFP8HfQuantizer.validate_environment is fp8q.oot_validate_environment+ ),+ ):+ apply_fp8_expert_replacement_patch()+ assert FineGrainedFP8HfQuantizer.validate_environment is fp8q.oot_validate_environmentfinally:FineGrainedFP8HfQuantizer.validate_environment = original@@ -463,9 +489,11 @@ class TestPatchBehaviorMatrix:def test_all_combinations_no_raise(self, cuda_available, transformers_v5):"""Every combination of gating conditions must not raise."""- with patch("torch.cuda.is_available", return_value=cuda_available):- with patch(+ with (+ patch("torch.cuda.is_available", return_value=cuda_available),+ patch("auto_round.modeling.fp8_quant.is_transformers_version_greater_or_equal_5",return_value=transformers_v5,- ):- assert apply_fp8_expert_replacement_patch() is None+ ),+ ):+ assert apply_fp8_expert_replacement_patch() is Nonediff --git a/test/unit/common/models/test_bagel.py b/test/unit/common/models/test_bagel.pyindex 137005a27..d8b9a9048 100644--- a/test/unit/common/models/test_bagel.py+++ b/test/unit/common/models/test_bagel.py@@ -454,9 +454,7 @@ class TestConfigErrorHandling:# The branch diff adds ValueError to the except clause around line 294# Verify the pattern appears (line numbers may shift)- assert (- "except (OSError, EnvironmentError, ValueError)" in content- ), "BaseCompressor should catch ValueError alongside OSError/EnvironmentError"+ assert "except (OSError, ValueError)" in content, "BaseCompressor should catch ValueError alongside OSError"def test_autoconfig_valueerror_caught_in_model_context(self):"""ModelContext: ValueError should be caught in AutoConfig.from_pretrained."""@@ -465,9 +463,7 @@ class TestConfigErrorHandling:content = f.read()# The branch diff adds ValueError to the except clause around line 146- assert (- "except (OSError, EnvironmentError, ValueError)" in content- ), "ModelContext should catch ValueError alongside OSError/EnvironmentError"+ assert "except (OSError, ValueError)" in content, "ModelContext should catch ValueError alongside OSError"# ================= Test: mllm_load_model for bagel =================diff --git a/test/unit/common/models/test_block_names.py b/test/unit/common/models/test_block_names.pyindex e26461457..771ba34ff 100644--- a/test/unit/common/models/test_block_names.py+++ b/test/unit/common/models/test_block_names.py@@ -5,7 +5,7 @@ import torch.nn as nn# ================= simple multimodal model =================class TextEncoder(nn.Module):def __init__(self, input_size, hidden_size):- super(TextEncoder, self).__init__()+ super().__init__()self.fc = nn.Linear(input_size, hidden_size)def forward(self, x):@@ -14,7 +14,7 @@ class TextEncoder(nn.Module):class VisionEncoder(nn.Module):def __init__(self, input_size, hidden_size):- super(VisionEncoder, self).__init__()+ super().__init__()self.fc = nn.Linear(input_size, hidden_size)def forward(self, x):@@ -31,7 +31,7 @@ class VisionEncoderModuleList(nn.ModuleList):class SimpleMultimodalModel(nn.Module):def __init__(self, text_input_size, image_input_size, hidden_size, num_text_encoders, num_image_encoders):- super(SimpleMultimodalModel, self).__init__()+ super().__init__()self.text_encoders = TextEncoderModuleList([TextEncoder(text_input_size, hidden_size) for _ in range(num_text_encoders)])@@ -56,7 +56,7 @@ class SimpleMultimodalModel(nn.Module):# ================= simple MoE model =================class Expert(nn.Module):def __init__(self, input_size, hidden_size):- super(Expert, self).__init__()+ super().__init__()self.fc = nn.Linear(input_size, hidden_size)def forward(self, x):@@ -73,7 +73,7 @@ class ExpertModuleList(nn.ModuleList):class NestedMoEModel(nn.Module):def __init__(self, input_size, hidden_size, num_groups, experts_per_group):- super(NestedMoEModel, self).__init__()+ super().__init__()self.expert_groups = ExpertModuleList([ExpertGroupModuleList([Expert(input_size, hidden_size) for _ in range(experts_per_group)])diff --git a/test/unit/common/models/test_cosmos3.py b/test/unit/common/models/test_cosmos3.pyindex 8f0713cbf..d73862ad5 100644--- a/test/unit/common/models/test_cosmos3.py+++ b/test/unit/common/models/test_cosmos3.py@@ -38,9 +38,8 @@ class TestCosmos3ForwardMode:def test_forward_mode_cleanup_on_exception(self):from auto_round.special_model_handler import _cosmos3_forward_mode, _cosmos3_forward_state- with pytest.raises(RuntimeError):- with _cosmos3_forward_mode():- raise RuntimeError("simulated error")+ with pytest.raises(RuntimeError), _cosmos3_forward_mode():+ raise RuntimeError("simulated error")assert not getattr(_cosmos3_forward_state, "active", False)def test_forward_mode_nested_context(self):@@ -591,12 +590,14 @@ class TestCosmos3DiffusionLoadModel:tmpdir = self._write_model_index("DDPMScheduler")try:- with patch("auto_round.special_model_handler.load_cosmos3_diffusion") as mock_cosmos:- with patch("auto_round.utils.common.LazyImport", return_value=MagicMock()):- try:- diffusion_load_model(tmpdir)- except Exception:- pass- mock_cosmos.assert_not_called()+ with (+ patch("auto_round.special_model_handler.load_cosmos3_diffusion") as mock_cosmos,+ patch("auto_round.utils.common.LazyImport", return_value=MagicMock()),+ ):+ try:+ diffusion_load_model(tmpdir)+ except Exception:+ pass+ mock_cosmos.assert_not_called()finally:shutil.rmtree(tmpdir)diff --git a/test/unit/common/models/test_moe_model.py b/test/unit/common/models/test_moe_model.pyindex 91fcd7178..e2d0b87a7 100644--- a/test/unit/common/models/test_moe_model.py+++ b/test/unit/common/models/test_moe_model.py@@ -13,7 +13,7 @@ from auto_round import AutoRounddef quantize_model(model, output_dir, scheme, iters=0, ignore_layers="self_attn,router,lm_head,mlp.gate"):"""Helper function to quantize the model with the given scheme."""- disable_opt_rtn = True if iters == 0 else False+ disable_opt_rtn = iters == 0autoround = AutoRound(model,scheme=scheme,diff --git a/test/unit/common/models/test_unfused_moe_init.py b/test/unit/common/models/test_unfused_moe_init.pyindex 8dfae1fcb..03506fe5b 100644--- a/test/unit/common/models/test_unfused_moe_init.py+++ b/test/unit/common/models/test_unfused_moe_init.py@@ -106,19 +106,21 @@ def test_get_checkpoint_conversion_mapping_ar_passthrough():transformers mapping function. We mock that to verify the call."""sentinel = ["x", "y"]- with mock.patch(- "transformers.conversion_mapping.orig_get_checkpoint_conversion_mapping",- create=True,- return_value=sentinel,- new_callable=mock.MagicMock,- ) as fake:- # Some transformers versions don't expose ``orig_*`` yet; guard- # against that by also patching the public name.- with mock.patch(+ # Some transformers versions don't expose ``orig_*`` yet; guard+ # against that by also patching the public name.+ with (+ mock.patch(+ "transformers.conversion_mapping.orig_get_checkpoint_conversion_mapping",+ create=True,+ return_value=sentinel,+ new_callable=mock.MagicMock,+ ) as fake,+ mock.patch("transformers.conversion_mapping.get_checkpoint_conversion_mapping",side_effect=lambda mt: sentinel,- ):- result = get_checkpoint_conversion_mapping_ar("not_in_our_list")+ ),+ ):+ result = get_checkpoint_conversion_mapping_ar("not_in_our_list")assert result == sentineldiff --git a/test/unit/common/models/test_vlm_ram_reduction.py b/test/unit/common/models/test_vlm_ram_reduction.pyindex 2e78813d9..bd841146b 100644--- a/test/unit/common/models/test_vlm_ram_reduction.py+++ b/test/unit/common/models/test_vlm_ram_reduction.py@@ -167,11 +167,10 @@ class TestDiffusionMultiDeviceDispatch:pipe = FakePipe()pipe.components = {"transformer": pipe.transformer, "vae": pipe.vae}- with patch("torch.cuda.is_available", return_value=False):- with patch("torch.cuda.device_count", return_value=0):- # Single device path (falls back to pipe.to)- result = dispatch_model_by_all_available_devices(pipe, "cpu")- assert result is pipe+ with patch("torch.cuda.is_available", return_value=False), patch("torch.cuda.device_count", return_value=0):+ # Single device path (falls back to pipe.to)+ result = dispatch_model_by_all_available_devices(pipe, "cpu")+ assert result is pipedef test_multi_device_respects_non_main_memory(self):"""With multiple devices, memory reservation must be computed for non-main components."""diff --git a/test/unit/common/schemes/test_w8_asym_policy.py b/test/unit/common/schemes/test_w8_asym_policy.pyindex 732439deb..f9ba87564 100644--- a/test/unit/common/schemes/test_w8_asym_policy.py+++ b/test/unit/common/schemes/test_w8_asym_policy.py@@ -82,7 +82,7 @@ class TestAutoSchemeW8AsymPolicy:return AutoScheme(options=["W8A16", "W4A16"], avg_bits=6.0)def _w8_option(self, scheme):- return [o for o in scheme.options if getattr(o, "bits", None) == 8][0]+ return next(o for o in scheme.options if getattr(o, "bits", None) == 8)@pytest.mark.parametrize("fmt,opt_in,expect_sym",diff --git a/test/unit/common/utils/test_common_pure_helpers.py b/test/unit/common/utils/test_common_pure_helpers.pyindex 2d712867a..1744c3cd2 100644--- a/test/unit/common/utils/test_common_pure_helpers.py+++ b/test/unit/common/utils/test_common_pure_helpers.py@@ -381,7 +381,7 @@ class TestParseLayerConfigArg:def test_invalid_input_raises(self):from auto_round.utils.common import parse_layer_config_arg- with pytest.raises(Exception):+ with pytest.raises(ValueError):parse_layer_config_arg("")def test_multi_key_unquoted_dict(self):diff --git a/test/unit/common/utils/test_device.py b/test/unit/common/utils/test_device.pyindex 836652807..dc2962a81 100644--- a/test/unit/common/utils/test_device.py+++ b/test/unit/common/utils/test_device.py@@ -81,8 +81,9 @@ class TestUseHpuCompileMode:from auto_round.utils.device import _use_hpu_compile_mode# Mock both is_hpu_lazy_mode and TORCH_VERSION_AT_LEAST_2_4 (imported inside function)- with patch("auto_round.utils.device.is_hpu_lazy_mode", return_value=False), patch.dict(- "sys.modules", {"auto_round.utils.common": MagicMock(TORCH_VERSION_AT_LEAST_2_4=True)}+ with (+ patch("auto_round.utils.device.is_hpu_lazy_mode", return_value=False),+ patch.dict("sys.modules", {"auto_round.utils.common": MagicMock(TORCH_VERSION_AT_LEAST_2_4=True)}),):result = _use_hpu_compile_mode()assert result is True@@ -98,8 +99,9 @@ class TestUseHpuCompileMode:"""Test compile mode False when torch < 2.4."""from auto_round.utils.device import _use_hpu_compile_mode- with patch("auto_round.utils.device.is_hpu_lazy_mode", return_value=False), patch.dict(- "sys.modules", {"auto_round.utils.common": MagicMock(TORCH_VERSION_AT_LEAST_2_4=False)}+ with (+ patch("auto_round.utils.device.is_hpu_lazy_mode", return_value=False),+ patch.dict("sys.modules", {"auto_round.utils.common": MagicMock(TORCH_VERSION_AT_LEAST_2_4=False)}),):result = _use_hpu_compile_mode()assert result is False@@ -118,11 +120,13 @@ class TestBumpDynamoCacheLimit:mock_config.accumulated_cache_size_limit = 8mock_config.recompile_limit = 8- with patch.dict("sys.modules", {"torch._dynamo.config": mock_config}):- with patch("torch._dynamo.config", mock_config):- _bump_dynamo_cache_limit(min_size=32)- # Function should attempt to set values >= 32- # Best effort - it may or may not raise depending on imports+ with (+ patch.dict("sys.modules", {"torch._dynamo.config": mock_config}),+ patch("torch._dynamo.config", mock_config),+ ):+ _bump_dynamo_cache_limit(min_size=32)+ # Function should attempt to set values >= 32+ # Best effort - it may or may not raise depending on importsdef test_bump_without_value_uses_default(self):"""Test _bump_dynamo_cache_limit without min_size uses env default."""@@ -1125,17 +1129,19 @@ class TestDispatchModelBlockWise:# Multi-device path: provide 2 "cpu" entries so ``len(devices) > 1``.# After the inner loop dedupes, ``device == "cpu"`` is used to index# the mocked max_memory dict.- with patch(- "auto_round.utils.device.parse_available_devices",- return_value=["cpu", "cpu"],- ), patch(- "auto_round.utils.device.get_max_memory", return_value={"cpu": 1024}- ), patch("auto_round.utils.device.get_balanced_memory", return_value={"cpu": 512}), patch(- "auto_round.utils.device.infer_auto_device_map",- return_value={"0": "cpu"},- ) as mock_infer, patch(- "auto_round.utils.device.dispatch_model", return_value="MOCKED"- ) as mock_dispatch:+ with (+ patch(+ "auto_round.utils.device.parse_available_devices",+ return_value=["cpu", "cpu"],+ ),+ patch("auto_round.utils.device.get_max_memory", return_value={"cpu": 1024}),+ patch("auto_round.utils.device.get_balanced_memory", return_value={"cpu": 512}),+ patch(+ "auto_round.utils.device.infer_auto_device_map",+ return_value={"0": "cpu"},+ ) as mock_infer,+ patch("auto_round.utils.device.dispatch_model", return_value="MOCKED") as mock_dispatch,+ ):result = dispatch_model_block_wise(model, device_map="cpu,cpu", max_mem_ratio=0.5)assert mock_infer.calledassert mock_dispatch.called@@ -1164,9 +1170,11 @@ class TestDispatchModelByAllAvailableDevices:from auto_round.utils.device import dispatch_model_by_all_available_devicesmodel = MagicMock(spec=nn.Module)- with patch("auto_round.utils.device.get_balanced_memory", return_value={0: 1024}) as balanced, patch(- "auto_round.utils.device.infer_auto_device_map", return_value={"0": "cpu"}- ), patch("auto_round.utils.device.dispatch_model", return_value="AUTO_MODEL"):+ with (+ patch("auto_round.utils.device.get_balanced_memory", return_value={0: 1024}) as balanced,+ patch("auto_round.utils.device.infer_auto_device_map", return_value={"0": "cpu"}),+ patch("auto_round.utils.device.dispatch_model", return_value="AUTO_MODEL"),+ ):with patch("auto_round.utils.device.parse_available_devices",return_value=["cpu"],@@ -1455,9 +1463,11 @@ class TestDumpMemoryUsageCtx:def test_context_manager_with_warning_level(self):from auto_round.utils.device import dump_memory_usage_ctx- with patch("auto_round.utils.device.logger.warning") as warn:- with dump_memory_usage_ctx(msg="warn-ctx", log_level="warning"):- pass+ with (+ patch("auto_round.utils.device.logger.warning") as warn,+ dump_memory_usage_ctx(msg="warn-ctx", log_level="warning"),+ ):+ passassert warn.calleddef test_decorator_runs_function(self):@@ -1720,12 +1730,13 @@ class TestFakeCudaForHpuExtra:from auto_round.utils.device import fake_cuda_for_hpuoriginal = MagicMock(return_value=True)- with patch("auto_round.utils.device.is_hpex_available", return_value=True), patch(- "torch.cuda.is_available", original+ with (+ patch("auto_round.utils.device.is_hpex_available", return_value=True),+ patch("torch.cuda.is_available", original),+ fake_cuda_for_hpu(),):- with fake_cuda_for_hpu():- # Should be temporarily faked.- pass+ # Should be temporarily faked.+ pass# After exit, original is restored.# We can't strictly assert identity due to dynamic restoration,# but should at least have called __exit__ without raising.@@ -1738,13 +1749,15 @@ class TestFakeTritonForHpuExtra:from auto_round.utils.device import fake_triton_for_hpu# Create a fake triton module- with patch.dict(- sys.modules,- {"triton": MagicMock(), "triton.language": MagicMock()},+ with (+ patch.dict(+ sys.modules,+ {"triton": MagicMock(), "triton.language": MagicMock()},+ ),+ patch("auto_round.utils.device.is_hpex_available", return_value=True),+ fake_triton_for_hpu(),):- with patch("auto_round.utils.device.is_hpex_available", return_value=True):- with fake_triton_for_hpu():- pass+ pass# ===========================================================================@@ -1853,9 +1866,10 @@ class TestMallocTrimCounter:from auto_round.utils import device as device_moddevice_mod._malloc_trim_counter = 0- with patch.dict(os.environ, {"AR_ENABLE_MALLOC_TRIM": "1"}, clear=False), patch(- "auto_round.utils.device.ctypes.CDLL"- ) as mock_cdll:+ with (+ patch.dict(os.environ, {"AR_ENABLE_MALLOC_TRIM": "1"}, clear=False),+ patch("auto_round.utils.device.ctypes.CDLL") as mock_cdll,+ ):mock_libc = MagicMock()mock_cdll.return_value = mock_libcfrom auto_round.utils.device import _maybe_trim_malloc@@ -1870,11 +1884,14 @@ class TestMallocTrimCounter:def test_invalid_every_falls_back_to_default(self):from auto_round.utils.device import _maybe_trim_malloc- with patch.dict(- os.environ,- {"AR_ENABLE_MALLOC_TRIM": "1", "AR_MALLOC_TRIM_EVERY": "notanumber"},- clear=False,- ), patch("auto_round.utils.device.ctypes.CDLL") as mock_cdll:+ with (+ patch.dict(+ os.environ,+ {"AR_ENABLE_MALLOC_TRIM": "1", "AR_MALLOC_TRIM_EVERY": "notanumber"},+ clear=False,+ ),+ patch("auto_round.utils.device.ctypes.CDLL") as mock_cdll,+ ):mock_libc = MagicMock()mock_cdll.return_value = mock_libc_maybe_trim_malloc()@@ -1885,11 +1902,14 @@ class TestMallocTrimCounter:from auto_round.utils import device as device_modfrom auto_round.utils.device import _maybe_trim_malloc- with patch.dict(- os.environ,- {"AR_ENABLE_MALLOC_TRIM": "1", "AR_MALLOC_TRIM_EVERY": "0"},- clear=False,- ), patch("auto_round.utils.device.ctypes.CDLL") as mock_cdll:+ with (+ patch.dict(+ os.environ,+ {"AR_ENABLE_MALLOC_TRIM": "1", "AR_MALLOC_TRIM_EVERY": "0"},+ clear=False,+ ),+ patch("auto_round.utils.device.ctypes.CDLL") as mock_cdll,+ ):mock_libc = MagicMock()mock_cdll.return_value = mock_libcdiff --git a/test/unit/common/utils/test_device_manager.py b/test/unit/common/utils/test_device_manager.pyindex f6a22de25..f93445d24 100644--- a/test/unit/common/utils/test_device_manager.py+++ b/test/unit/common/utils/test_device_manager.py@@ -358,18 +358,22 @@ class TestGetCurrentDeviceType:)# Force every discovery path to report "nothing"- with patch.object(- __import__("auto_round.utils.device_manager", fromlist=["_hpu_available"]),- "_hpu_available",- return_value=False,- ), patch.object(- __import__("auto_round.utils.device_manager", fromlist=["_torch_accelerator_type"]),- "_torch_accelerator_type",- return_value=None,- ), patch.object(- __import__("auto_round.utils.device_manager", fromlist=["_PREFERRED_ORDER"]),- "_PREFERRED_ORDER",- (),+ with (+ patch.object(+ __import__("auto_round.utils.device_manager", fromlist=["_hpu_available"]),+ "_hpu_available",+ return_value=False,+ ),+ patch.object(+ __import__("auto_round.utils.device_manager", fromlist=["_torch_accelerator_type"]),+ "_torch_accelerator_type",+ return_value=None,+ ),+ patch.object(+ __import__("auto_round.utils.device_manager", fromlist=["_PREFERRED_ORDER"]),+ "_PREFERRED_ORDER",+ (),+ ),):# Need to clear the lru_cacheget_current_device_type.cache_clear()@@ -415,8 +419,9 @@ class TestAvailableHelpers:def test_get_available_device_types_cpu_only(self):from auto_round.utils.device_manager import get_available_device_types- with patch("auto_round.utils.device_manager._hpu_available", return_value=False), patch(- "auto_round.utils.device_manager._torch_accelerator_type", return_value=None+ with (+ patch("auto_round.utils.device_manager._hpu_available", return_value=False),+ patch("auto_round.utils.device_manager._torch_accelerator_type", return_value=None),):assert get_available_device_types() == []@@ -766,12 +771,14 @@ class TestGetDeviceMemory:def test_cpu_raises_runtime_error(self):from auto_round.utils.device_manager import get_device_memory- with patch(- "auto_round.utils.device_manager.get_current_device_type",- return_value="cpu",+ with (+ patch(+ "auto_round.utils.device_manager.get_current_device_type",+ return_value="cpu",+ ),+ pytest.raises(RuntimeError),):- with pytest.raises(RuntimeError):- get_device_memory()+ get_device_memory()# ---------------------------------------------------------------------------diff --git a/test/unit/common/utils/test_distributed.py b/test/unit/common/utils/test_distributed.pyindex a83c8951b..519d09455 100644--- a/test/unit/common/utils/test_distributed.py+++ b/test/unit/common/utils/test_distributed.py@@ -27,26 +27,32 @@ class TestIsDistributed:"""Test is_distributed with mocked torch.distributed."""def test_not_initialized(self):- with patch("auto_round.utils.distributed.is_distributed", return_value=False):- with patch("torch.distributed.is_initialized", return_value=False):- # Clear cache to ensure fresh evaluation- is_distributed.cache_clear()- result = is_distributed()- assert result is False+ with (+ patch("auto_round.utils.distributed.is_distributed", return_value=False),+ patch("torch.distributed.is_initialized", return_value=False),+ ):+ # Clear cache to ensure fresh evaluation+ is_distributed.cache_clear()+ result = is_distributed()+ assert result is Falsedef test_initialized_single_device(self):- with patch("torch.distributed.is_initialized", return_value=True):- with patch("torch.distributed.get_world_size", return_value=1):- is_distributed.cache_clear()- result = is_distributed()- assert result is False+ with (+ patch("torch.distributed.is_initialized", return_value=True),+ patch("torch.distributed.get_world_size", return_value=1),+ ):+ is_distributed.cache_clear()+ result = is_distributed()+ assert result is Falsedef test_initialized_multi_device(self):- with patch("torch.distributed.is_initialized", return_value=True):- with patch("torch.distributed.get_world_size", return_value=4):- is_distributed.cache_clear()- result = is_distributed()- assert result is True+ with (+ patch("torch.distributed.is_initialized", return_value=True),+ patch("torch.distributed.get_world_size", return_value=4),+ ):+ is_distributed.cache_clear()+ result = is_distributed()+ assert result is Truedef test_dist_not_initialized(self):with patch("torch.distributed.is_initialized", return_value=False):@@ -55,18 +61,22 @@ class TestIsDistributed:is_distributed.cache_clear()def test_dist_initialized_single_world(self):- with patch("torch.distributed.is_initialized", return_value=True):- with patch("torch.distributed.get_world_size", return_value=1):- is_distributed.cache_clear()- assert is_distributed() is False- is_distributed.cache_clear()+ with (+ patch("torch.distributed.is_initialized", return_value=True),+ patch("torch.distributed.get_world_size", return_value=1),+ ):+ is_distributed.cache_clear()+ assert is_distributed() is False+ is_distributed.cache_clear()def test_dist_initialized_multi_world(self):- with patch("torch.distributed.is_initialized", return_value=True):- with patch("torch.distributed.get_world_size", return_value=2):- is_distributed.cache_clear()- assert is_distributed() is True- is_distributed.cache_clear()+ with (+ patch("torch.distributed.is_initialized", return_value=True),+ patch("torch.distributed.get_world_size", return_value=2),+ ):+ is_distributed.cache_clear()+ assert is_distributed() is True+ is_distributed.cache_clear()class TestNoopSync:@@ -164,13 +174,15 @@ class TestSetupDdpIfNeeded:pytest.skip("CUDA not available")model = nn.Linear(4, 4).to("cpu")- with patch("torch.distributed.is_initialized", return_value=True):- with patch("torch.distributed.get_world_size", return_value=2):- with patch("torch.distributed.get_rank", return_value=0):- with patch("torch.nn.parallel.DistributedDataParallel"):- ar = SimpleNamespace()- block, sync_fn = setup_ddp_if_needed_(ar, model, [0])- assert sync_fn is _noop_sync+ with (+ patch("torch.distributed.is_initialized", return_value=True),+ patch("torch.distributed.get_world_size", return_value=2),+ patch("torch.distributed.get_rank", return_value=0),+ patch("torch.nn.parallel.DistributedDataParallel"),+ ):+ ar = SimpleNamespace()+ block, sync_fn = setup_ddp_if_needed_(ar, model, [0])+ assert sync_fn is _noop_syncdef test_single_device_returns_noop_when_distributed(self):"""Test the multi-GPU case which doesn't need to move to GPU device."""@@ -201,12 +213,14 @@ class TestSetupDdpIfNeeded:model = nn.Linear(4, 4)- with patch("torch.distributed.is_initialized", return_value=True):- with patch("torch.distributed.get_world_size", return_value=4):- with patch("torch.distributed.get_rank", return_value=0):- ar = SimpleNamespace()- block, sync_fn = setup_ddp_if_needed_(ar, model, [0, 1])- # Should not be the noop- assert sync_fn is not _noop_sync- # Calling it should not raise- sync_fn()+ with (+ patch("torch.distributed.is_initialized", return_value=True),+ patch("torch.distributed.get_world_size", return_value=4),+ patch("torch.distributed.get_rank", return_value=0),+ ):+ ar = SimpleNamespace()+ block, sync_fn = setup_ddp_if_needed_(ar, model, [0, 1])+ # Should not be the noop+ assert sync_fn is not _noop_sync+ # Calling it should not raise+ sync_fn()diff --git a/test/unit/common/utils/test_generation.py b/test/unit/common/utils/test_generation.pyindex b5635face..f7bf507ce 100644--- a/test/unit/common/utils/test_generation.py+++ b/test/unit/common/utils/test_generation.py@@ -63,7 +63,7 @@ class TestAutoRoundFormatGeneration:for bits in [4]:model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)- bits, group_size, sym = bits, 128, True+ group_size, sym = 128, Trueautoround = AutoRound(model,tokenizer,diff --git a/test/unit/common/utils/test_missing_tensors.py b/test/unit/common/utils/test_missing_tensors.pyindex 2867450a4..e727b4b8e 100644--- a/test/unit/common/utils/test_missing_tensors.py+++ b/test/unit/common/utils/test_missing_tensors.py@@ -134,8 +134,8 @@ class TestSplitFusedExpertTensors:}result = split_fused_expert_tensors(tensors)assert set(result.keys()) == set(tensors.keys())- for k in tensors:- assert torch.equal(result[k], tensors[k])+ for k, v in tensors.items():+ assert torch.equal(result[k], v)def test_warns_on_3d_tensor_with_unsupported_parent(self, caplog, _autoround_log_propagate):tensors = {diff --git a/test/unit/common/utils/test_model_utils.py b/test/unit/common/utils/test_model_utils.pyindex ecbc6a1c7..c1362272a 100644--- a/test/unit/common/utils/test_model_utils.py+++ b/test/unit/common/utils/test_model_utils.py@@ -993,7 +993,7 @@ class TestSetAttr:set_attr(model, "inner.new_attr", "new_value")- assert getattr(model.inner, "new_attr") == "new_value"+ assert model.inner.new_attr == "new_value"def test_set_attr_missing_parent(self):"""Test set_attr does not raise when the parent path doesn't exist."""diff --git a/test/unit/common/utils/test_weight_handler.py b/test/unit/common/utils/test_weight_handler.pyindex 418ee67d4..8a2a1eb44 100644--- a/test/unit/common/utils/test_weight_handler.py+++ b/test/unit/common/utils/test_weight_handler.py@@ -871,7 +871,6 @@ class TestGetHandler:# Actually, ModuleWeightType is an Enum, so we can't easily create a new one# Let's just verify that unknown combinations return None# The function should return None for any unregistered type- pass# ==============================================================================diff --git a/test/unit/envs.py b/test/unit/envs.pyindex 5b6b5730c..dab67083d 100644--- a/test/unit/envs.py+++ b/test/unit/envs.py@@ -14,8 +14,9 @@import importlib.utilimport unittest+from collections.abc import Callablefrom functools import wraps-from typing import Callable, Literal+from typing import Literalimport torchfrom transformers.utils.versions import require_versiondiff --git a/test/unit/test_cpu/core/test_autoround.py b/test/unit/test_cpu/core/test_autoround.pyindex d639be085..f1dff0278 100644--- a/test/unit/test_cpu/core/test_autoround.py+++ b/test/unit/test_cpu/core/test_autoround.py@@ -238,7 +238,6 @@ class TestAutoRound:)autoround.quantize()- #def test_signround(self, tiny_opt_model_path, dataloader):model_name = tiny_opt_model_pathbits, group_size, sym = 4, -1, Falsediff --git a/test/unit/test_cpu/core/test_resume_integration.py b/test/unit/test_cpu/core/test_resume_integration.pyindex 6396a65aa..222575416 100644--- a/test/unit/test_cpu/core/test_resume_integration.py+++ b/test/unit/test_cpu/core/test_resume_integration.py@@ -60,10 +60,12 @@ class TestResumeIntegration:if len(crashed_after) == 1:raise RuntimeError("simulated crash")- with mock.patch.object(ResumeState, "mark_block_done", crash_after_first_block):- with pytest.raises(RuntimeError, match="simulated crash"):- ar = AutoRound(model=tiny_opt_model_path, scheme="W4A16", iters=1, nsamples=1)- ar.quantize()+ with (+ mock.patch.object(ResumeState, "mark_block_done", crash_after_first_block),+ pytest.raises(RuntimeError, match="simulated crash"),+ ):+ ar = AutoRound(model=tiny_opt_model_path, scheme="W4A16", iters=1, nsamples=1)+ ar.quantize()assert crashed_after == ["model.decoder.layers.0"]diff --git a/test/unit/test_cpu/export/test_llmc_format.py b/test/unit/test_cpu/export/test_llmc_format.pyindex 807853047..e98155176 100644--- a/test/unit/test_cpu/export/test_llmc_format.py+++ b/test/unit/test_cpu/export/test_llmc_format.py@@ -265,7 +265,8 @@ class TestLLMC:from safetensors import safe_open- config = json.load(open(os.path.join(quantized_model_path, "config.json")))+ with open(os.path.join(quantized_model_path, "config.json")) as f:+ config = json.load(f)assert "group_0" in config["quantization_config"]["config_groups"]assert config["quantization_config"]["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8assert config["quantization_config"]["config_groups"]["group_0"]["weights"]["strategy"] == "channel"@@ -289,7 +290,8 @@ class TestLLMC:import json- config = json.load(open(os.path.join(quantized_model_path, "config.json")))+ with open(os.path.join(quantized_model_path, "config.json")) as f:+ config = json.load(f)assert "group_0" in config["quantization_config"]["config_groups"]assert config["quantization_config"]["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8assert config["quantization_config"]["config_groups"]["group_0"]["weights"]["strategy"] == "tensor"diff --git a/test/unit/test_cpu/models/test_block_names.py b/test/unit/test_cpu/models/test_block_names.pyindex c97999593..0876d6555 100644--- a/test/unit/test_cpu/models/test_block_names.py+++ b/test/unit/test_cpu/models/test_block_names.py@@ -61,7 +61,7 @@ class TestQuantizationBlocks:model = Qwen2VLForConditionalGeneration.from_pretrained(model_name, trust_remote_code=True, device_map="auto")block_name = get_block_names(model, quant_vision=True)assert len(block_name) == 2- assert all(["visual.merger.mlp" not in n for n in block_name])+ assert all("visual.merger.mlp" not in n for n in block_name)block_name = get_block_names(model, quant_vision=False)assert len(block_name) == 1assert block_name == get_block_names(model)diff --git a/test/unit/test_cpu/quantization/test_model_free_parity.py b/test/unit/test_cpu/quantization/test_model_free_parity.pyold mode 100755new mode 100644diff --git a/test/unit/test_cpu/quantization/test_neuqi.py b/test/unit/test_cpu/quantization/test_neuqi.pyindex b4e4b63c3..ab4769c4c 100644--- a/test/unit/test_cpu/quantization/test_neuqi.py+++ b/test/unit/test_cpu/quantization/test_neuqi.py@@ -795,10 +795,10 @@ class TestNeuqiSymSharedCoarse:def spy_launch(dn, q, fracs, invf, nm):spy_launch.called = True- return None # decline -> per-candidate fallback+ # implicit None: decline -> per-candidate fallbackspy_launch.called = False- monkeypatch.setattr(N, "_sym_coarse_pass_shared", lambda *a, **k: (None if not spy_launch.called else None))+ monkeypatch.setattr(N, "_sym_coarse_pass_shared", lambda *a, **k: None)# direct wiring check: the core calls _sym_coarse_pass_shared at allreal = N._sym_coarse_pass_shared@@ -996,7 +996,7 @@ class TestNeuqiAsymSharedCoarse:def wrapper(data_, qw_, s0_, coarse_, maxq_):wrapper.called = True- return None # decline -> batched fallback still correct+ # implicit None: decline -> batched fallback still correctwrapper.called = Falsemonkeypatch.setattr(N, "_zp_coarse_pass_shared", wrapper)@@ -1520,7 +1520,7 @@ class TestExpertBatching:cfg = OptimizedRTNConfig(bits=4, group_size=32, sym=False, enable_neuqi=enable_neuqi)cfg.disable_opt_rtn = disable_opt_rtn- cfg.orig_disable_opt_rtn = False if not disable_opt_rtn else True+ cfg.orig_disable_opt_rtn = bool(disable_opt_rtn)q = OptimizedRTNQuantizer(cfg)if is_moe:# model_context is a read-only property backed by the run ctx;diff --git a/test/unit/test_cuda/advanced/test_multiple_card.py b/test/unit/test_cuda/advanced/test_multiple_card.pyindex e84d95470..1e5ed76ae 100644--- a/test/unit/test_cuda/advanced/test_multiple_card.py+++ b/test/unit/test_cuda/advanced/test_multiple_card.py@@ -128,8 +128,8 @@ class TestAutoRound:model_name = "OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc"device_map = {}- for i in range(0, 32):- key = f"model.layers.{str(i)}"+ for i in range(32):+ key = f"model.layers.{i!s}"device_map[key] = "cuda:0"device_map["model.layers.1"] = "cpu"device_map["model.layers.2"] = "cpu"diff --git a/test/unit/test_cuda/backends/test_marlin_backend.py b/test/unit/test_cuda/backends/test_marlin_backend.pyindex 5b868c5ba..bf142f12e 100644--- a/test/unit/test_cuda/backends/test_marlin_backend.py+++ b/test/unit/test_cuda/backends/test_marlin_backend.py@@ -67,7 +67,7 @@ class TestAutoRoundMarlinBackend:print(f"{group_size}!!!!!!!!!!!!!!!!!")model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)- bits, group_size, sym = 4, group_size, True+ bits, sym = 4, Trueautoround = AutoRound(model,tokenizer,@@ -97,7 +97,7 @@ class TestAutoRoundMarlinBackend:print(f"{group_size}!!!!!!!!!!!!!!!!!")model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)- bits, group_size, sym = 4, group_size, True+ bits, sym = 4, Trueautoround = AutoRound(model,tokenizer,diff --git a/test/unit/test_cuda/models/test_get_block_name.py b/test/unit/test_cuda/models/test_get_block_name.pyindex 633b797c2..b3190273f 100644--- a/test/unit/test_cuda/models/test_get_block_name.py+++ b/test/unit/test_cuda/models/test_get_block_name.py@@ -29,7 +29,7 @@ class TestAutoRound:def teardown_class(self):shutil.rmtree("runs", ignore_errors=True)- def check_block_names(self, block_names, prefixs=[], n_layers=[]):+ def check_block_names(self, block_names, prefixs=(), n_layers=()):assert len(block_names) == len(prefixs) == len(n_layers)for i, block_name in enumerate(block_names):prefix = prefixs[i]@@ -213,7 +213,7 @@ class TestAutoRound:block_names = get_block_names(model)self.check_block_names(block_names, ["transformer_blocks", "single_transformer_blocks"], [19, 38])- assert any(["context_embedder" not in n for n in block_names])+ assert any("context_embedder" not in n for n in block_names)block_names = get_block_names(model, quant_vision=True)self.check_block_names(block_names, ["transformer_blocks", "single_transformer_blocks"], [19, 38])diff --git a/test/unit/test_cuda/models/test_mllm.py b/test/unit/test_cuda/models/test_mllm.pyindex c91b99325..630667d93 100644--- a/test/unit/test_cuda/models/test_mllm.py+++ b/test/unit/test_cuda/models/test_mllm.py@@ -137,8 +137,8 @@ class TestAutoRound:model = MllamaForConditionalGeneration.from_pretrained(model_name, trust_remote_code=True, device_map="auto")block_name = get_block_names(model, quant_vision=True)assert len(block_name) == 3- assert any(["vision_model.global_transformer.layers.0" not in n for n in block_name])- assert any(["vision_model.transformer.layers.0" not in n for n in block_name])+ assert any("vision_model.global_transformer.layers.0" not in n for n in block_name)+ assert any("vision_model.transformer.layers.0" not in n for n in block_name)block_name = get_block_names(model, quant_vision=False)assert len(block_name) == 1assert get_block_names(model) == block_namediff --git a/test/unit/test_mlx/test_mlx_format.py b/test/unit/test_mlx/test_mlx_format.pyindex a302e3bd8..51ccfe9e8 100644--- a/test/unit/test_mlx/test_mlx_format.py+++ b/test/unit/test_mlx/test_mlx_format.py@@ -78,7 +78,7 @@ def _quantize_and_save(fmt: str,sym: bool = True,group_size: int = 128,- layer_config: dict = None,+ layer_config: dict | None = None,):"""Run AutoRound RTN quantization on Qwen3-0.6B and export to ``fmt``."""ar = AutoRound(diff --git a/test/unit/test_xpu/test_autoround.py b/test/unit/test_xpu/test_autoround.pyindex da32e35a6..ee4264cc6 100644--- a/test/unit/test_xpu/test_autoround.py+++ b/test/unit/test_xpu/test_autoround.py@@ -16,12 +16,10 @@ class TestAutoRoundXPU:@classmethoddef setup_class(self):self.device = "xpu"- pass@classmethoddef teardown_class(self):shutil.rmtree("runs", ignore_errors=True)- pass@pytest.fixture(autouse=True)def _save_dir(self, tmp_path):