PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Add ruff and isort (#578)
Sun, Xuehao774a5ae008ac
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex 6d14162d0..b81bdeaa0 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -3,13 +3,23 @@ ci:autoupdate_schedule: quarterlyrepos:+ - repo: https://github.com/pre-commit/pre-commit-hooks+ rev: v5.0.0+ hooks:+ - id: check-json+ - id: check-yaml+ - id: debug-statements+ - id: mixed-line-ending+ args: [--fix=lf]+- repo: https://github.com/Lucas-C/pre-commit-hooksrev: v1.5.5hooks:- id: insert-licensefiles: |(?x)^(- auto_round/.*(py|yaml|yml|sh)+ auto_round/.*(py|yaml|yml|sh)|+ auto_round_extension/.*(py|yaml|yml|sh))$args:[@@ -26,7 +36,14 @@ repos:args: [-w]additional_dependencies:- tomli- exclude: |- (?x)^(- examples/.*(txt|patch)- )$++ - repo: https://github.com/pycqa/isort+ rev: 6.0.1+ hooks:+ - id: isort++ - repo: https://github.com/astral-sh/ruff-pre-commit+ rev: v0.12.4+ hooks:+ - id: ruff+ args: [--fix, --exit-non-zero-on-fix, --no-cache]diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 1a8c885d7..003035922 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -13,6 +13,7 @@# limitations under the License.import sys+def run_eval():from auto_round.script.llm import setup_eval_parserargs = setup_eval_parser()@@ -58,7 +59,7 @@ def run_fast():def run_mllm():if "--eval" in sys.argv:- from auto_round.script.llm import setup_eval_parser, eval+ from auto_round.script.llm import eval, setup_eval_parsersys.argv.remove("--eval")args = setup_eval_parser()args.mllm = True@@ -76,7 +77,7 @@ def run_mllm():def run_lmms():# from auto_round.script.lmms_eval import setup_lmms_args, eval- from auto_round.script.mllm import setup_lmms_parser, lmms_eval+ from auto_round.script.mllm import lmms_eval, setup_lmms_parserargs = setup_lmms_parser()lmms_eval(args)diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 6a11b075f..056308f41 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -12,54 +12,62 @@# See the License for the specific language governing permissions and# limitations under the License.+import copyimport osimport reimport sys+import time+from typing import Any, Union+import accelerateimport torch-import copy-import time-from typing import Union, Any-from transformers import set_seedfrom torch import autocastfrom tqdm import tqdm-import accelerate+from transformers import set_seed+from auto_round.data_type import QUANT_FUNC_WITH_DTYPEfrom auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType-from auto_round.wrapper import WrapperMultiblock, wrapper_block, unwrapper_block, WrapperLinear, unwrapper_layer+from auto_round.low_cpu_mem.utils import get_layers_before_blockfrom auto_round.utils import (+ SUPPORTED_DTYPES,+ SUPPORTED_LAYER_TYPES,+ TORCH_VERSION_AT_LEAST_2_6,CpuInfo,+ _gguf_args_check,block_forward,check_is_cpu,+ check_seqlen_compatible,+ check_skippable_keywords,check_to_quantized,+ clear_memory,collect_best_params,+ compile_func,convert_dtype_str2torch,detect_device,+ find_matching_blocks,+ flatten_list,get_block_names,+ get_layer_config_by_gguf_format,+ get_layer_features,+ get_layer_names_in_block,+ get_lm_head_name,get_module,+ get_shared_keys,htcore,+ infer_bits_by_data_type,+ init_cache,+ is_debug_mode,is_optimum_habana_available,+ llm_load_model,logger,- to_device,- to_dtype,- get_layer_names_in_block,mv_module_from_gpu,- unsupport_meta_device, clear_memory,- compile_func,- find_matching_blocks, is_debug_mode,- TORCH_VERSION_AT_LEAST_2_6,- SUPPORTED_LAYER_TYPES,- get_layer_features,- set_module,- llm_load_model,reset_params,- init_cache, check_skippable_keywords, get_shared_keys, SUPPORTED_DTYPES, infer_bits_by_data_type,- _gguf_args_check,- check_seqlen_compatible,- get_layer_config_by_gguf_format, get_lm_head_name, flatten_list+ set_module,+ to_device,+ to_dtype,+ unsupport_meta_device,)-from auto_round.data_type import QUANT_FUNC_WITH_DTYPE-from auto_round.low_cpu_mem.utils import get_layers_before_block+from auto_round.wrapper import WrapperLinear, WrapperMultiblock, unwrapper_block, unwrapper_layer, wrapper_blockclass AutoRound(object):@@ -232,9 +240,9 @@ class AutoRound(object):self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device##activation- self.act_group_size = act_group_size if not (act_group_size is None) else self.group_size- self.act_bits = act_bits if not (act_bits is None) else self.bits- self.act_sym = act_sym if not (act_sym is None) else self.sym+ self.act_group_size = act_group_size if act_group_size is not None else self.group_size+ self.act_bits = act_bits if act_bits is not None else self.bits+ self.act_sym = act_sym if act_sym is not None else self.symself.act_dynamic = act_dynamicself.act_data_type = act_data_typeif self.act_data_type is None:@@ -272,7 +280,7 @@ class AutoRound(object):self.enable_torch_compile = enable_torch_compileif not self.enable_torch_compile and TORCH_VERSION_AT_LEAST_2_6 and self.act_bits > 8 and not is_debug_mode() \- and self.low_cpu_mem_usage != True and "fp8" not in self.data_type and "fp8" not in self.act_data_type:+ and not self.low_cpu_mem_usage and "fp8" not in self.data_type and "fp8" not in self.act_data_type:logger.info("'enable_torch_compile' is set to `False` by default. " \"Enabling it can reduce tuning cost by 20%, but it might throw an exception.")@@ -280,7 +288,7 @@ class AutoRound(object):self.enable_torch_compile = Falselogger.warning("reset enable_torch_compile to `False` as activation quantization is enabled")- if self.low_cpu_mem_usage == True and self.enable_torch_compile:+ if self.low_cpu_mem_usage and self.enable_torch_compile:self.enable_torch_compile = Falselogger.warning("reset enable_torch_compile to `False` as low_cpu_mem_usage is enabled")@@ -535,7 +543,7 @@ class AutoRound(object):only_gguf = Falseif only_gguf:self.scale_dtype = torch.float32- logger.info(f"change `scale_dtype` to `torch.float32`")+ logger.info("change `scale_dtype` to `torch.float32`")# Adjust format settings based on compatibilityfor index in range(len(formats)):@@ -592,7 +600,7 @@ class AutoRound(object):f" group_size={self.group_size}, sym={self.sym}, act_bits={self.act_bits}"elif format != "fake":logger.warning(- f"Currently only support to export auto_round format quantized model"+ "Currently only support to export auto_round format quantized model"" with fp8 dtype activation for activation quantization."" Change format to fake and save.")@@ -1320,7 +1328,7 @@ class AutoRound(object):keys = inputs.keys()input_id_str = [key for key in keys if key.startswith('hidden_state')]if len(input_id_str) != 1:- raise RuntimeError(f"hidden_states arg mismatch error,"+ raise RuntimeError("hidden_states arg mismatch error,""please raise an issue in https://github.com/intel/auto-round/issues")inputs["input_ids"] = inputs.pop(input_id_str[0], None)if q_inputs is not None:@@ -1853,10 +1861,10 @@ class AutoRound(object):self.batch_dim = 1if len(hidden_states.shape) > 1 and hidden_states.shape[1] > self.batch_size:logger.error(- f"this model has not been supported, "- f"please raise an issue in https://github.com/intel/auto-round/issues"- f" or try to set the `batch_size` to 1 and "- f"`gradient_accumulate_steps` to your current batch size.")+ "this model has not been supported, "+ "please raise an issue in https://github.com/intel/auto-round/issues"+ " or try to set the `batch_size` to 1 and "+ "`gradient_accumulate_steps` to your current batch size.")exit(-1)if hidden_states is not None:@@ -2094,8 +2102,7 @@ class AutoRound(object):hook_handles = []for n, m in model.named_modules():- # for block- if hasattr(m, "act_dynamic") and m.act_dynamic == False and check_to_quantized(m):+ if hasattr(m, "act_dynamic") and not m.act_dynamic and check_to_quantized(m):hook = m.register_forward_hook(get_act_max_hook)hook_handles.append(hook)continue@@ -2405,7 +2412,7 @@ class AutoRound(object):model_type=model_type)else:PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.tmp_name, self.model, self.formats[0])- pbar.set_description(f"Quantizing done")+ pbar.set_description("Quantizing done")pbar.update(1)pbar.close()diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 6fdd2b32b..de4eba825 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -14,12 +14,12 @@import jsonimport random+import sysimport torch-from datasets import Dataset, IterableDataset, load_dataset, concatenate_datasets-from datasets import Features, Sequence, Value+from datasets import Dataset, Features, IterableDataset, Sequence, Value, concatenate_datasets, load_datasetfrom torch.utils.data import DataLoader-import sys+from .utils import is_local_path, loggerCALIB_DATASETS = {}@@ -136,7 +136,7 @@ def get_pile_dataset(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", spliterror_message = str(e)# Check for proxy or SSL errorif "proxy" in error_message.lower() or isinstance(e, ssl.SSLError) or "SSL" in error_message.upper():- logger.error(f"Network error detected, please checking proxy settings." \+ logger.error("Network error detected, please checking proxy settings." \"Error: {error_message}. Or consider using a backup dataset by `pip install modelscope`" \" and set '--dataset swift/pile-val-backup' in AutoRound API.")else:diff --git a/auto_round/data_type/fp8.py b/auto_round/data_type/fp8.pyindex 70873e6ed..6253d662e 100644--- a/auto_round/data_type/fp8.py+++ b/auto_round/data_type/fp8.py@@ -13,9 +13,15 @@# limitations under the License.import torch-from auto_round.data_type.utils import get_gaudi_fp8_ste_func, float8_e4m3fn_ste, reshape_pad_tensor_by_group_size, \- revert_tensor_by_pad, float8_e5m2_ste+from auto_round.data_type.register import register_dtype+from auto_round.data_type.utils import (+ float8_e4m3fn_ste,+ float8_e5m2_ste,+ get_gaudi_fp8_ste_func,+ reshape_pad_tensor_by_group_size,+ revert_tensor_by_pad,+)@register_dtype(("fp8_sym","fp8","fp8_e4m3"))diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex eba6db810..0d094f41d 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -13,8 +13,9 @@# limitations under the License.import torch-from auto_round.data_type.utils import round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad, logger+from auto_round.data_type.register import register_dtype+from auto_round.data_type.utils import logger, reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_stefrom auto_round.utils import get_reciprocal@@ -506,7 +507,7 @@ def quant_tensor_gguf_sym_dq(Returns:Quantized and de-quantized tensor, scale, zero-point"""- from auto_round.export.export_to_gguf.config import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES+ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, K_SCALE_SIZE, QK_Kfrom auto_round.export.export_to_gguf.packing import make_q3_quants, make_qx_quantsif bits not in [3, 6]:diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.pyindex 5367b6925..be5d7cf33 100644--- a/auto_round/data_type/int.py+++ b/auto_round/data_type/int.py@@ -13,8 +13,9 @@# limitations under the License.import torch-from auto_round.data_type.utils import round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad+from auto_round.data_type.register import register_dtype+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_ste@register_dtype("int_sym")diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.pyindex af115311d..f5ad2aa08 100644--- a/auto_round/data_type/mxfp.py+++ b/auto_round/data_type/mxfp.py@@ -13,8 +13,9 @@# limitations under the License.import torch-from auto_round.data_type.utils import floor_ste, round_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad-from auto_round.data_type.register import register_dtype, QUANT_FUNC_WITH_DTYPE++from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE, register_dtype+from auto_round.data_type.utils import floor_ste, reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_steMXFP_FORMAT_CACHE = {# data type: ebits, mbits, emax, max_norm, min_normdiff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex d1293e899..147264c68 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -16,7 +16,7 @@ import torchfrom auto_round.data_type.fp8 import float8_e4m3fn_stefrom auto_round.data_type.register import register_dtype-from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, logger+from auto_round.data_type.utils import logger, reshape_pad_tensor_by_group_size, revert_tensor_by_pad# taken fromdiff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex 5265d55fc..748417038 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -12,11 +12,14 @@# See the License for the specific language governing permissions and# limitations under the License.+from functools import lru_cache+import torch+from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE-from functools import lru_cachefrom auto_round.utils import logger+def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int):"""Reshapes and pads the tensor to ensure that it can be quantized in groups of `group_size`.diff --git a/auto_round/data_type/w4fp8.py b/auto_round/data_type/w4fp8.pyindex 9b5123759..1dd550b22 100644--- a/auto_round/data_type/w4fp8.py+++ b/auto_round/data_type/w4fp8.py@@ -15,8 +15,7 @@import torchfrom auto_round.data_type.register import register_dtype-from auto_round.data_type.utils import get_gaudi_fp8_ste_func, float8_e4m3fn_ste-+from auto_round.data_type.utils import float8_e4m3fn_ste, get_gaudi_fp8_ste_func# @register_dtype("fp8_gaudi3_to_int_sym")# def progressive_quant_fp8_int4_gaudi3(diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex 5536c48c4..293379bab 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -12,10 +12,10 @@# See the License for the specific language governing permissions and# limitations under the License.+import osfrom typing import Optional, Unionfrom lm_eval import simple_evaluate as lm_simple_evaluate-import osos.environ["TOKENIZERS_PARALLELISM"] = "false"diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 34dd7aea0..d74040419 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -12,6 +12,14 @@# See the License for the specific language governing permissions and# limitations under the License.+import copy+import inspect+import json+import os+from concurrent.futures import ThreadPoolExecutor++import threadpoolctl as tctl+# MIT License## Copyright (c) 2023 潘其威(William)@@ -34,22 +42,21 @@# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE# SOFTWARE.import torch--import auto_round.export.export_to_autogptq.qlinear_triton-from auto_round.utils import check_to_quantized, get_block_names, \- get_module, logger, set_module, SUPPORTED_LAYER_TYPES, filter_quantization_config-import copy-import json-import os-import torch.nn as nnimport transformers--import threadpoolctl as tctl-import inspectfrom tqdm import tqdm-from concurrent.futures import ThreadPoolExecutor-from auto_round.utils import get_autogptq_packing_qlinear++import auto_round.export.export_to_autogptq.qlinear_triton+from auto_round.utils import (+ SUPPORTED_LAYER_TYPES,+ check_to_quantized,+ filter_quantization_config,+ get_autogptq_packing_qlinear,+ get_block_names,+ get_module,+ logger,+ set_module,+)BLOCK_PATTERNS = [ ## copy from transformers optimum"transformer.h",@@ -142,7 +149,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllflattened_list = [item for sublist in all_blocks for item in sublist]common_prefix = os.path.commonprefix(flattened_list).rstrip('.')if common_prefix not in BLOCK_PATTERNS:- logger.error(f"auto-gptq format may not support loading this quantized model")+ logger.error("auto-gptq format may not support loading this quantized model")quantization_config['block_name_to_quantize'] = common_prefixquantization_config.pop("to_quant_block_names", None)diff --git a/auto_round/export/export_to_autogptq/qlinear_triton.py b/auto_round/export/export_to_autogptq/qlinear_triton.pyindex c51920109..671737d3b 100644--- a/auto_round/export/export_to_autogptq/qlinear_triton.py+++ b/auto_round/export/export_to_autogptq/qlinear_triton.py@@ -19,6 +19,7 @@ import torchimport torch.nn as nnimport transformers+class TritonModuleMixin:@classmethoddef warmup(cls, model, transpose=False, seqlen=2048):diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 10da020a2..a0ac9ae8b 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -14,20 +14,28 @@import copy+import inspectimport jsonimport os+from concurrent.futures import ThreadPoolExecutor+import threadpoolctl as tctlimport torchimport torch.nn as nnimport transformers--from auto_round.utils import get_module, logger, set_module, SUPPORTED_LAYER_TYPES, check_to_quantized, \- filter_quantization_config, SUPPORTED_FORMATS-import threadpoolctl as tctl-import inspectfrom tqdm import tqdm-from concurrent.futures import ThreadPoolExecutor-from auto_round.utils import get_autogptq_packing_qlinear, check_start_with_block_name++from auto_round.utils import (+ SUPPORTED_FORMATS,+ SUPPORTED_LAYER_TYPES,+ check_start_with_block_name,+ check_to_quantized,+ filter_quantization_config,+ get_autogptq_packing_qlinear,+ get_module,+ logger,+ set_module,+)def check_neq_config(config, data_type, bits, group_size, sym):@@ -83,7 +91,7 @@ def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_elif "awq" in backend:from ..export_to_awq.utils import WQLinear_GEMMreturn WQLinear_GEMM- elif "gptq" in backend and not "gptqmodel" in backend: ## have g_idx+ elif "gptq" in backend and "gptqmodel" not in backend: ## have g_idxreturn get_autogptq_packing_qlinear(backend, bits, group_size, sym)else:raise ValueError(@@ -261,7 +269,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exreturn save_quantized_as_autoround(output_dir, inplace=inplace, backend="auto_round", **kwargs)##if using sym, we change to gptq sym kernel to avoid compiling from auto_round source- if (kwargs.get("sym") is None or kwargs.get("sym") == True) and ("gptq" not in backend and "awq" not in backend):+ if (kwargs.get("sym") is None or kwargs.get("sym")) and ("gptq" not in backend and "awq" not in backend):backend = backend.replace('auto_round', 'auto_round:auto_gptq')model = kwargs["model"]@@ -279,8 +287,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exprocessor = kwargs.get("processor", None)extra_config = {}block_name_to_quantize = quantization_config["block_name_to_quantize"]- if isinstance(block_name_to_quantize, str): \- block_name_to_quantize = block_name_to_quantize.split(",")+ if isinstance(block_name_to_quantize, str):+ block_name_to_quantize = block_name_to_quantize.split(",")elif isinstance(block_name_to_quantize, list):for i in range(len(block_name_to_quantize)):block_name_to_quantize[i] = os.path.commonprefix(block_name_to_quantize[i]).rstrip('.')diff --git a/auto_round/export/export_to_autoround/export_to_fp8_woq.py b/auto_round/export/export_to_autoround/export_to_fp8_woq.pyindex d29e021c2..02ef4d755 100644--- a/auto_round/export/export_to_autoround/export_to_fp8_woq.py+++ b/auto_round/export/export_to_autoround/export_to_fp8_woq.py@@ -13,15 +13,24 @@# limitations under the License.import copy-import os-import torchimport json-from auto_round.utils import logger, set_module, SUPPORTED_LAYER_TYPES, check_to_quantized, \- filter_quantization_config, get_module, check_start_with_block_name+import os+from concurrent.futures import ThreadPoolExecutor+import threadpoolctl as tctl+import torchimport transformersfrom tqdm import tqdm-from concurrent.futures import ThreadPoolExecutor++from auto_round.utils import (+ SUPPORTED_LAYER_TYPES,+ check_start_with_block_name,+ check_to_quantized,+ filter_quantization_config,+ get_module,+ logger,+ set_module,+)def check_neq_config(config, data_type, bits, group_size, sym):@@ -169,8 +178,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",processor = kwargs.get("processor", None)extra_config = {}block_name_to_quantize = quantization_config["block_name_to_quantize"]- if isinstance(block_name_to_quantize, str): \- block_name_to_quantize = block_name_to_quantize.split(",")+ if isinstance(block_name_to_quantize, str):+ block_name_to_quantize = block_name_to_quantize.split(",")elif isinstance(block_name_to_quantize, list):for i in range(len(block_name_to_quantize)):block_name_to_quantize[i] = os.path.commonprefix(block_name_to_quantize[i]).rstrip('.')diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex 4ef02662e..22c13048e 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -20,22 +20,28 @@# SOFTWARE.-import os-import torch-import torch.nn as nn--from auto_round.utils import (logger, get_module,- set_module,- check_to_quantized,- get_block_names,- extract_block_names_to_str, SUPPORTED_LAYER_TYPES, filter_quantization_config)import copyimport json-from auto_round.export.export_to_awq.utils import WQLinear_GEMM+import osfrom concurrent.futures import ThreadPoolExecutor+import threadpoolctl as tctl+import torch+import torch.nn as nnfrom tqdm import tqdm+from auto_round.export.export_to_awq.utils import WQLinear_GEMM+from auto_round.utils import (+ SUPPORTED_LAYER_TYPES,+ check_to_quantized,+ extract_block_names_to_str,+ filter_quantization_config,+ get_block_names,+ get_module,+ logger,+ set_module,+)+def pack_layer(name, model, backend):if name == "lm_head": ##dese not support lm-headdiff --git a/auto_round/export/export_to_awq/utils.py b/auto_round/export/export_to_awq/utils.pyindex 64e826e9e..72e218df5 100644--- a/auto_round/export/export_to_awq/utils.py+++ b/auto_round/export/export_to_awq/utils.py@@ -33,8 +33,9 @@# SOFTWARE.import gc-import torchimport warnings++import torchimport torch.nn as nnfrom torch.autograd import Functiondiff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.pyindex d2b6a821e..e6b71586b 100644--- a/auto_round/export/export_to_gguf/config.py+++ b/auto_round/export/export_to_gguf/config.py@@ -13,6 +13,7 @@# limitations under the License.from enum import IntEnum+class ModelType(IntEnum):TEXT = 1MMPROJ = 2diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 45b99d2f0..ea69b0bbd 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -34,29 +34,29 @@from __future__ import annotations-import ast-import gc-import loggingimport argparse+import astimport contextlib+import gcimport json+import logging+import mathimport osimport reimport sys-import psutilfrom enum import IntEnum-from pathlib import Pathfrom hashlib import sha256-from typing import TYPE_CHECKING, Any, Callable, ContextManager, Iterable, Iterator, Literal, Sequence, TypeVar, castfrom itertools import chain+from pathlib import Path+from typing import TYPE_CHECKING, Any, Callable, ContextManager, Iterable, Iterator, Literal, Sequence, TypeVar, cast-import mathimport numpy as np+import psutilimport torchfrom transformers import AutoConfig-from auto_round.utils import logger, LazyImport, get_module, clean_module_parameterfrom auto_round.export.export_to_gguf.packing import ggml_quant+from auto_round.utils import LazyImport, clean_module_parameter, get_module, loggergguf = LazyImport("gguf")@@ -4741,8 +4741,7 @@ class BertModel(TextModel):def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:del bid # unused- if name.startswith("bert."):- name = name[5:]+ name = name.removeprefix("bert.")if name.endswith(".gamma"):name = name[:-6] + ".weight"@@ -4798,6 +4797,7 @@ class BertModel(TextModel):raise FileNotFoundError(f"File not found: {tokenizer_path}")from base64 import b64decode+from transformers import AutoTokenizertokenizer = AutoTokenizer.from_pretrained(self.dir_model)@@ -4918,8 +4918,7 @@ class DistilBertModel(BertModel):super().set_gguf_parameters()def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:- if name.startswith("distilbert."):- name = name[11:]+ name = name.removeprefix("distilbert.")# These layers act as MLM head, so we don't need themif name.startswith("vocab_"):@@ -4960,8 +4959,7 @@ class RobertaModel(BertModel):def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:# if name starts with "roberta.", remove the prefix# e.g. https://huggingface.co/BAAI/bge-reranker-v2-m3/tree/main- if name.startswith("roberta."):- name = name[8:]+ name = name.removeprefix("roberta.")# position embeddings start at pad_token_id + 1, so just chop down the weight tensorif name == "embeddings.position_embeddings.weight":@@ -5076,8 +5074,7 @@ class NeoBert(BertModel):if name.startswith("decoder."):return []- if name.startswith("model."):- name = name[6:]+ name = name.removeprefix("model.")return super().modify_tensors(data_torch, name, bid)@@ -5096,8 +5093,7 @@ class XLMRobertaModel(BertModel):def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:# if name starts with "roberta.", remove the prefix# e.g. https://huggingface.co/BAAI/bge-reranker-v2-m3/tree/main- if name.startswith("roberta."):- name = name[8:]+ name = name.removeprefix("roberta.")# position embeddings start at pad_token_id + 1, so just chop down the weight tensorif name == "embeddings.position_embeddings.weight":diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex 14b8d0993..e42b49f95 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -13,14 +13,23 @@# limitations under the License.import os-import sysimport shutil-import torch-from pathlib import Path+import sysimport time+from pathlib import Path++import torch+from auto_round.export.export_to_gguf.convert import ModelBase, ModelType, get_model_architecture-from auto_round.utils import logger, LazyImport, get_block_names, flatten_list, check_to_quantized, get_module, \- clear_memory+from auto_round.utils import (+ LazyImport,+ check_to_quantized,+ clear_memory,+ flatten_list,+ get_block_names,+ get_module,+ logger,+)TMP_DIR_NAME = "tmp_dir"diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex b377fdf28..2dfe3b82b 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -12,10 +12,10 @@# See the License for the specific language governing permissions and# limitations under the License.-import torchimport numpy as np+import torch-from auto_round.export.export_to_gguf.config import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES+from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, K_SCALE_SIZE, QK_Kfrom auto_round.utils import get_reciprocalGGML_QUANT_TYPE = {}diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.pyindex 435eed66d..34aa1aed7 100644--- a/auto_round/export/export_to_itrex/export.py+++ b/auto_round/export/export_to_itrex/export.py@@ -19,14 +19,14 @@ from typing import Dict, List, Optional, Unionimport torchimport transformers+from auto_round.export.register import register_format-from auto_round.utils import get_module, logger, set_module, detect_device, check_to_quantized+from auto_round.utils import check_to_quantized, detect_device, get_module, logger, set_modulefrom .config import QuantConfigfrom .model_wrapper import WeightOnlyLinear-def quant_weight_w_scale(weight, scale, zp, group_size=-1, device="cpu"):"""Quant and dequant tensor with group size.diff --git a/auto_round/export/export_to_itrex/model_wrapper.py b/auto_round/export/export_to_itrex/model_wrapper.pyindex c8e985939..05d0ce45e 100644--- a/auto_round/export/export_to_itrex/model_wrapper.py+++ b/auto_round/export/export_to_itrex/model_wrapper.py@@ -19,12 +19,13 @@# since the model classes inherit torch.nn.Module.import math+import numpy as npimport torchfrom packaging.version import Versionfrom torch.autograd import Functionfrom torch.nn import functional as F-import numpy as np-from auto_round.utils import logger, can_pack_with_numba++from auto_round.utils import can_pack_with_numba, loggerNF4 = [-1.0,diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.pyindex a60338752..d9e5dc50d 100644--- a/auto_round/export/export_to_llmcompressor/export.py+++ b/auto_round/export/export_to_llmcompressor/export.py@@ -13,10 +13,12 @@# limitations under the License.import os+import torch-from auto_round.utils import get_module, logger, set_module, detect_device-from auto_round.wrapper import WrapperWALayer+from auto_round.export.export_to_llmcompressor.config import quantization_config+from auto_round.utils import detect_device, get_module, logger, set_module+from auto_round.wrapper import WrapperWALayer@torch.no_grad()diff --git a/auto_round/inference/auto_quantizer.py b/auto_round/inference/auto_quantizer.pyindex 3aea03e04..765a7e150 100644--- a/auto_round/inference/auto_quantizer.py+++ b/auto_round/inference/auto_quantizer.py@@ -29,8 +29,9 @@import importlib.utilimport warningsfrom dataclasses import dataclass+from enum import Enumfrom logging import getLogger-from typing import Any, Dict, Optional, Tuple, Union, List+from typing import Any, Dict, List, Optional, Tuple, Unionimport torchimport torch.nn as nn@@ -39,9 +40,9 @@ from transformers.modeling_utils import PreTrainedModelfrom transformers.quantizers import AutoQuantizationConfig, HfQuantizerfrom transformers.quantizers.auto import AUTO_QUANTIZER_MAPPINGfrom transformers.utils.quantization_config import AwqConfig, GPTQConfig, QuantizationConfigMixin, QuantizationMethod-from auto_round.utils import (is_hpu_supported)+from auto_round.inference.convert_model import convert_hf_model, infer_target_device, post_init-from enum import Enum+from auto_round.utils import is_hpu_supportedlogger = getLogger(__name__)import sysdiff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex d1ad65f97..47bffe6dd 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -14,7 +14,7 @@import functoolsfrom dataclasses import dataclass, field-from typing import List, Any, Optional+from typing import Any, List, Optionalfrom transformers.utils.versions import require_version@@ -308,7 +308,7 @@ def check_compatible(backend_name, device, bits, group_size, sym, packing_formatbackend = BackendInfos[backend_name]# Check if device is supported by the backend- if not device in backend.device:+ if device not in backend.device:return False# Check if bit-width is supporteddiff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 1e6a565b5..6004e8acc 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -15,19 +15,31 @@ import osimport refrom logging import getLoggerfrom typing import Union-from tqdm import tqdm+import torchimport torch.nn as nn-+from tqdm import tqdmfrom transformers.pytorch_utils import Conv1D-from auto_round.utils import (- get_module, set_module, is_hpu_supported, get_block_names, find_matching_blocks,- get_layer_names_in_block, check_to_quantized, check_start_with_block_name, SUPPORTED_LAYER_TYPES)-from auto_round.inference.backend import (- get_layer_backend, dynamic_import_inference_linear, find_backend, BackendInfos, get_highest_priority_backend,- process_requirement)+ BackendInfos,+ dynamic_import_inference_linear,+ find_backend,+ get_highest_priority_backend,+ get_layer_backend,+ process_requirement,+)+from auto_round.utils import (+ SUPPORTED_LAYER_TYPES,+ check_start_with_block_name,+ check_to_quantized,+ find_matching_blocks,+ get_block_names,+ get_layer_names_in_block,+ get_module,+ is_hpu_supported,+ set_module,+)logger = getLogger(__name__)@@ -69,7 +81,9 @@ def get_keys_to_not_convert(model):Input model"""from copy import deepcopy+from accelerate.utils import find_tied_parameters+# Create a copy of the model and tie the weights, then# check if it contains tied weightstied_model = deepcopy(model) # this has 0 cost since it is done inside `init_empty_weights` context manager`@@ -508,7 +522,7 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):quantization_config = model.config.quantization_config- if hasattr(quantization_config, "desc_act") and quantization_config.desc_act == True:+ if hasattr(quantization_config, "desc_act") and quantization_config.desc_act:##check static_groupif (hasattr(quantization_config, "static_groups") and not quantization_config.static_groups) or (not hasattr(quantization_config, "static_groups")):diff --git a/auto_round/low_cpu_mem/load.py b/auto_round/low_cpu_mem/load.pyindex 1ff291b16..44a8f2b0b 100644--- a/auto_round/low_cpu_mem/load.py+++ b/auto_round/low_cpu_mem/load.py@@ -33,7 +33,6 @@ from torch.serialization import ()from ..low_cpu_mem import modified_pickle as pickle-from .utils import torchtorch_version = torch.__version__.split("+")[0]diff --git a/auto_round/low_cpu_mem/utils.py b/auto_round/low_cpu_mem/utils.pyindex f4165b4eb..715556f0f 100644--- a/auto_round/low_cpu_mem/utils.py+++ b/auto_round/low_cpu_mem/utils.py@@ -16,13 +16,13 @@# limitations under the License."""Utils for layer wise quantization."""-import osimport gcimport json-import pickle-from functools import partialimport logging+import os+import picklefrom collections import OrderedDict+from functools import partialimport torchfrom accelerate import init_empty_weights@@ -30,9 +30,10 @@ from accelerate.utils import set_module_tensor_to_devicefrom transformers import AutoConfig, AutoModelForCausalLMfrom transformers.models.auto.auto_factory import _BaseAutoModelClass-from .load import loadfrom auto_round.utils import detect_device+from .load import load+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(filename)s L%(lineno)d: %(message)s")logger = logging.getLogger("low_cpu_mem_tools")diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex 5ab25e92d..ed6d06611 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -12,29 +12,29 @@# See the License for the specific language governing permissions and# limitations under the License.-from typing import Union-from tqdm import tqdmfrom copy import deepcopy+from typing import Unionimport torch+from tqdm import tqdm++from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS, _handle_special_model+from ..autoround import AutoRound+from ..low_cpu_mem.utils import get_layers_before_blockfrom ..utils import (- logger,+ clear_memory,detect_device,+ extract_block_names_to_str,+ find_matching_blocks,+ get_block_names,+ logger,+ mllm_load_model,to_device,to_dtype,- get_block_names,- find_matching_blocks,- extract_block_names_to_str,- clear_memory,- mllm_load_model)-from ..autoround import AutoRound-from .template import get_template, Template-from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS, _handle_special_modelfrom .mllm_dataset import get_mllm_dataloader-from ..low_cpu_mem.utils import get_layers_before_block-+from .template import Template, get_templatedef _only_text_test(model, tokenizer, device, model_type):@@ -198,7 +198,7 @@ class AutoRoundMLLM(AutoRound):(dataset in CALIB_DATASETS.keys() and not \_only_text_test(model, tokenizer, device, self.template.model_type)):if quant_nontext_module:- logger.warning(f"Text only dataset cannot be used for calibrating non-text modules,"+ logger.warning("Text only dataset cannot be used for calibrating non-text modules,""switching to liuhaotian/llava_conv_58k")else:logger.warning(f"{model.config.model_type} not support for {dataset},"diff --git a/auto_round/mllm/eval.py b/auto_round/mllm/eval.pyindex 997dea72d..dfe4aaa52 100644--- a/auto_round/mllm/eval.py+++ b/auto_round/mllm/eval.py@@ -46,14 +46,15 @@# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE# SOFTWARE.+import jsonimport osimport time-import jsonfrom functools import partial-import pandas as pd-from ..utils import logger, LazyImportimport numpy as np+import pandas as pd++from ..utils import LazyImport, loggervlmeval = LazyImport("vlmeval")diff --git a/auto_round/mllm/mllm_dataset.py b/auto_round/mllm/mllm_dataset.pyindex 18b7d0fd5..953c3e0c2 100644--- a/auto_round/mllm/mllm_dataset.py+++ b/auto_round/mllm/mllm_dataset.py@@ -12,18 +12,18 @@# See the License for the specific language governing permissions and# limitations under the License.-import osimport json+import osfrom typing import Dictimport torch-from torch.utils.data import Dataset, DataLoader+from torch.utils.data import DataLoader, Datasetfrom transformers import set_seed-from .utils import _extract_data_dir-from .template import Template-from ..utils import loggerfrom ..special_model_handler import check_mllm_model_batch+from ..utils import logger+from .template import Template+from .utils import _extract_data_dirMLLM_DATASET: Dict[str, Dataset] = {}@@ -250,7 +250,7 @@ def get_mllm_dataloader(tokenizer, seqlen, dataset, seed, bs, nsamples)if quant_nontext_module:logger.error(- f"Text only dataset cannot be used for calibrating non-text modules,"+ "Text only dataset cannot be used for calibrating non-text modules,"" switching to liuhaotian/llava_conv_58k")exit(-1)return dataloader, bs, gradient_accumulate_stepsdiff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.pyindex 0782fd09c..1794cca06 100644--- a/auto_round/mllm/template.py+++ b/auto_round/mllm/template.py@@ -12,14 +12,14 @@# See the License for the specific language governing permissions and# limitations under the License.-import osimport json+import osfrom dataclasses import dataclass-from typing import Dict, Optional, Listfrom enum import Enum, unique+from typing import Dict, List, Optionalfrom ..utils import logger-from .processor import BasicProcessor, PROCESSORS+from .processor import PROCESSORS, BasicProcessorTEMPLATES: Dict[str, "Template"] = {}diff --git a/auto_round/mllm/utils.py b/auto_round/mllm/utils.pyindex eebb2a189..90e67e855 100644--- a/auto_round/mllm/utils.py+++ b/auto_round/mllm/utils.py@@ -13,6 +13,7 @@# limitations under the License.import os+import requestsfrom ..utils import LazyImportdiff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 8faea4e8f..7463dbcc4 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -12,6 +12,9 @@# See the License for the specific language governing permissions and# limitations under the License.+import argparse+import logging+# Copyright (c) 2024 Intel Corporation## Licensed under the Apache License, Version 2.0 (the "License");@@ -27,16 +30,16 @@# limitations under the License.import osimport re-import argparseimport sys-import logging+from auto_round.utils import (- get_fp_layer_names,clear_memory,get_device_and_parallelism,+ get_fp_layer_names,get_model_dtype,+ set_cuda_visible_devices,str2bool,- set_cuda_visible_devices)+)os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"@@ -341,11 +344,9 @@ def tune(args):args.eval_bs = "auto"import transformers+ from transformers import AutoConfig, AutoModel, AutoModelForCausalLM, AutoTokenizer- from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel, AutoConfig-- from auto_round.utils import detect_device, get_library_version- from auto_round.utils import logger+ from auto_round.utils import detect_device, get_library_version, loggerif args.format is None:args.format = "auto_round"@@ -427,8 +428,8 @@ def tune(args):if lm_head_layer_name in item: ##TODO extend to encoder-decoder layer, seq classification modelargs.quant_lm_head = Falselogger.warning(- f"reset `quant_lm_head` to `False` as quantizing lm_head with tied weights has not been "- f"supported currently")+ "reset `quant_lm_head` to `False` as quantizing lm_head with tied weights has not been "+ "supported currently")breakif args.quant_lm_head:@@ -670,8 +671,8 @@ def eval(args):eval_model_dtype = get_model_dtype(args.eval_model_dtype)if is_gguf_file:import torch- from transformers import AutoTokenizer, AutoModelForCausalLMfrom lm_eval.utils import make_table # pylint: disable=E0401+ from transformers import AutoModelForCausalLM, AutoTokenizertokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)logger.warning("evaluating gguf model is an experimental feature, the accuracy may be not correct.")@@ -719,12 +720,13 @@ def eval_task_by_task(# load after _eval_int in order to make sure import torch after set CUDA_VISBILE_DEVICESimport traceback- from auto_round.utils import logger+from lm_eval import simple_evaluate as lm_simple_evaluatefrom lm_eval.models.huggingface import HFLMfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundConfig # pylint: disable=E0611+ from auto_round.utils import loggerif batch_size is None:batch_size = "auto:8"is_gguf_file = Falsediff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 2e913fd92..dabbc4014 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -12,17 +12,17 @@# See the License for the specific language governing permissions and# limitations under the License.+import argparseimport osimport sys-import argparsefrom auto_round.utils import (- get_fp_layer_names,clear_memory,- is_debug_mode,get_device_and_parallelism,- set_cuda_visible_devices,+ get_fp_layer_names,+ is_debug_mode,logger,+ set_cuda_visible_devices,)os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"@@ -284,8 +284,7 @@ def setup_lmeval_parser():def tune(args):import transformers-- from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig, AutoProcessor+ from transformers import AutoConfig, AutoModelForCausalLM, AutoProcessor, AutoTokenizerif args.format is None:args.format = "auto_round"@@ -377,8 +376,8 @@ def tune(args):if lm_head_layer_name in item: ##TODO extend to encoder-decoder layer, seq classification modelargs.quant_lm_head = Falseprint(- f"warning, disable quant_lm_head as quantizing lm_head with tied weights has not been "- f"supported currently")+ "warning, disable quant_lm_head as quantizing lm_head with tied weights has not been "+ "supported currently")breakif args.quant_lm_head:layer_config[lm_head_layer_name] = {"bits": args.bits}@@ -390,8 +389,8 @@ def tune(args):if args.quant_lm_head and args.low_gpu_mem_usage:print(- f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "- f"gpu")+ "warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "+ "gpu")if "--truncation" not in sys.argv:args.truncation = Nonediff --git a/auto_round/testing_utils.py b/auto_round/testing_utils.pyindex be856083c..c859c6188 100644--- a/auto_round/testing_utils.py+++ b/auto_round/testing_utils.py@@ -12,13 +12,13 @@# See the License for the specific language governing permissions and# limitations under the License.-import unittestimport importlib.util+import unittestimport torch-from transformers.utils.versions import require_version+def is_gguf_available():return importlib.util.find_spec("gguf") is not Nonediff --git a/auto_round/utils.py b/auto_round/utils.pyindex f64c8fb41..988af3db8 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -13,23 +13,24 @@# limitations under the License.import copy+import gcimport loggingimport os-import sys+import reimport subprocess+import sysfrom collections import UserDict-import re+from functools import lru_cache+import cpuinfoimport psutilimport torch+import transformers+from packaging import versionfrom torch.amp import autocast-from functools import lru_cache-from packaging import version-import gc+from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelTypefrom auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK, SPECIAL_SHARED_CACHE_KEYS-import transformers-from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGML_QUANT_SIZES, GGUF_INNER_CONFIG, QK_K, ModelTypeSHARED_CACHE_KEYS = ("position_ids", "cache_position", "position_embeddings")@@ -108,6 +109,7 @@ fh.setFormatter(AutoRoundFormatter())logger.addHandler(fh)import importlib+import transformers@@ -1094,7 +1096,7 @@ def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=Non- str: An error message describing why the model is incompatible, or an empty string if compatible."""if bits != 4:- return False, f"AutoAWQ GEMM kernel only supports 4 bits"+ return False, "AutoAWQ GEMM kernel only supports 4 bits"for n, m in model.named_modules():if isinstance(m, transformers.pytorch_utils.Conv1D):return False, "AutoAWQ GEMM kernel does not support conv1d"@@ -1174,9 +1176,10 @@ def get_layer_features(layer):def _gguf_args_check(args_or_ar, format_str=None, model_type=ModelType.TEXT):- from auto_round.utils import loggerimport argparse+ from auto_round.utils import logger+if format_str is None:args_or_ar.format = args_or_ar.format.replace("q*_", f"q{args_or_ar.bits}_")format_str = args_or_ar.format@@ -1209,6 +1212,7 @@ def _gguf_args_check(args_or_ar, format_str=None, model_type=ModelType.TEXT):if isinstance(args_or_ar.model, str) and os.path.isdir(args_or_ar.model):from pathlib import Path+from auto_round.export.export_to_gguf.convert import ModelBasehparams = ModelBase.load_hparams(Path(args_or_ar.model))model_architecture = hparams["architectures"][0]@@ -1275,7 +1279,7 @@ def llm_load_model(low_cpu_mem_mode=0,low_cpu_mem_tmp_dir=None,**kwargs):- from transformers import AutoTokenizer, AutoModel, AutoModelForCausalLM+ from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizeris_glm = bool(re.search("chatglm", pretrained_model_name_or_path.lower()))low_cpu_mem_usage = False@@ -1342,9 +1346,10 @@ def mllm_load_model(model_dtype=None,**kwargs):import json+import transformers- from transformers import AutoProcessor, AutoTokenizer, AutoModelForCausalLM, AutoModel- from huggingface_hub import HfApi, hf_hub_download, HfFileSystem+ from huggingface_hub import HfApi, HfFileSystem, hf_hub_download+ from transformers import AutoModel, AutoModelForCausalLM, AutoProcessor, AutoTokenizerif os.path.isdir(pretrained_model_name_or_path):config = json.load(open(os.path.join(pretrained_model_name_or_path, "config.json")))@@ -1372,7 +1377,7 @@ def mllm_load_model(processor, image_processor = None, Noneif "deepseek_vl_v2" == model_type:- from deepseek_vl2.models import DeepseekVLV2Processor, DeepseekVLV2ForCausalLM # pylint: disable=E0401+ from deepseek_vl2.models import DeepseekVLV2ForCausalLM, DeepseekVLV2Processor # pylint: disable=E0401processor = DeepseekVLV2Processor.from_pretrained(pretrained_model_name_or_path)tokenizer = processor.tokenizermodel: DeepseekVLV2ForCausalLM = AutoModelForCausalLM.from_pretrained(@@ -1628,6 +1633,7 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model, model_typetarget_gguf_format = next((fmt for fmt in gguf_format if fmt != "fake"), None)import gguf # pylint: disable=E0401+from auto_round.export.export_to_gguf.convert import ModelBase, get_model_architecturemodel_architecture = get_model_architecture(hparams=model.config.to_dict(), model_type=model_type)try:@@ -1912,7 +1918,7 @@ def get_gguf_qtype_by_layer_config(layer_config):return gguf.GGMLQuantizationType.Q6_Kif bits == 8 and sym and group_size == 32:return gguf.GGMLQuantizationType.Q8_0- raise ValueError(f"Unknown layer config")+ raise ValueError("Unknown layer config")def flatten_list(nested_list):flattened = []diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex a226daeed..12eb573fc 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -13,15 +13,12 @@# limitations under the License.import torch-from torch.functional import Fimport transformers+from torch.functional import F+from auto_round.data_type import get_quant_func-from .utils import (- check_to_quantized,- get_scale_shape,- set_module,- logger, SUPPORTED_LAYER_TYPES-)++from .utils import SUPPORTED_LAYER_TYPES, check_to_quantized, get_scale_shape, logger, set_moduledef reshape_and_pad_tensor(v, group_size=-1):diff --git a/auto_round_extension/__init__.py b/auto_round_extension/__init__.pyindex e69de29bb..2b414eb72 100644--- a/auto_round_extension/__init__.py+++ b/auto_round_extension/__init__.py@@ -0,0 +1,14 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+diff --git a/auto_round_extension/cuda/__init__.py b/auto_round_extension/cuda/__init__.pyindex e69de29bb..2b414eb72 100644--- a/auto_round_extension/cuda/__init__.py+++ b/auto_round_extension/cuda/__init__.py@@ -0,0 +1,14 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+diff --git a/auto_round_extension/cuda/gptqmodel_marlin.py b/auto_round_extension/cuda/gptqmodel_marlin.pyindex 242037d9b..42bc2c75a 100644--- a/auto_round_extension/cuda/gptqmodel_marlin.py+++ b/auto_round_extension/cuda/gptqmodel_marlin.py@@ -24,13 +24,13 @@ import torchdef get_marlin_layer(): ##use an ugly wrapper to import gptqmodel on demand- from gptqmodel.models._const import DEVICE, PLATFORM # pylint: disable=E0401- from gptqmodel.nn_modules.qlinear import BaseQuantLinear # pylint: disable=E0401- from gptqmodel.utils.backend import BACKEND # pylint: disable=E0401+ from gptqmodel.models._const import DEVICE, PLATFORM # pylint: disable=E0401+ from gptqmodel.nn_modules.qlinear import BaseQuantLinear # pylint: disable=E0401+ from gptqmodel.utils.backend import BACKEND # pylint: disable=E0401marlin_import_exception = Nonetry:- import gptqmodel_marlin_kernels # pylint: disable=E0401+ import gptqmodel_marlin_kernels # pylint: disable=E0401except ImportError as e:marlin_import_exception = ediff --git a/auto_round_extension/hpu/__init__.py b/auto_round_extension/hpu/__init__.pyindex e69de29bb..2b414eb72 100644--- a/auto_round_extension/hpu/__init__.py+++ b/auto_round_extension/hpu/__init__.py@@ -0,0 +1,14 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+diff --git a/auto_round_extension/hpu/qlinear_hpu.py b/auto_round_extension/hpu/qlinear_hpu.pyindex 9a6eb1490..5bcd06927 100644--- a/auto_round_extension/hpu/qlinear_hpu.py+++ b/auto_round_extension/hpu/qlinear_hpu.py@@ -40,6 +40,7 @@ import numpy as npimport torchimport torch.nn as nnimport transformers+try:import habana_frameworks.torch.core as htcoreconvert_from_uint4 = torch.ops.hpu.convert_from_uint4diff --git a/auto_round_extension/hpu/qlinear_hpu_gptq.py b/auto_round_extension/hpu/qlinear_hpu_gptq.pyindex c2f3a884e..0016ff032 100644--- a/auto_round_extension/hpu/qlinear_hpu_gptq.py+++ b/auto_round_extension/hpu/qlinear_hpu_gptq.py@@ -40,6 +40,7 @@ import numpy as npimport torchimport torch.nn as nnimport transformers+try:import habana_frameworks.torch.core as htcoreconvert_from_uint4 = torch.ops.hpu.convert_from_uint4diff --git a/auto_round_extension/ipex/__init__.py b/auto_round_extension/ipex/__init__.pyindex 2d32570d3..2929160d8 100644--- a/auto_round_extension/ipex/__init__.py+++ b/auto_round_extension/ipex/__init__.py@@ -1,3 +1,17 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+from auto_round_extension.ipex.qlinear_ipex_awq import QuantLinear as IpexAWQQuantLinearfrom auto_round_extension.ipex.qlinear_ipex_gptq import (QuantLinear as IpexGPTQQuantLinear,diff --git a/auto_round_extension/ipex/qlinear_ipex_awq.py b/auto_round_extension/ipex/qlinear_ipex_awq.pyindex b03d2104f..83afdb2ff 100644--- a/auto_round_extension/ipex/qlinear_ipex_awq.py+++ b/auto_round_extension/ipex/qlinear_ipex_awq.py@@ -1,6 +1,21 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import torchimport torch.nn as nn+class QuantLinear(nn.Module):QUANT_TYPE = "ipex_awq"def __init__(self, w_bit, group_size, in_features, out_features, bias, dev):diff --git a/auto_round_extension/ipex/qlinear_ipex_gptq.py b/auto_round_extension/ipex/qlinear_ipex_gptq.pyindex 042f0226f..e308ec5b9 100644--- a/auto_round_extension/ipex/qlinear_ipex_gptq.py+++ b/auto_round_extension/ipex/qlinear_ipex_gptq.py@@ -1,12 +1,25 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import math+from logging import getLoggerimport numpy as npimport torchimport torch.nn as nnimport transformers-from logging import getLogger-logger = getLogger(__name__)BITS_DTYPE_MAPPING = {@@ -100,9 +113,13 @@ class QuantLinear(nn.Module):# if not self.training and IPEX_AVAILABLE:if not self.training:import intel_extension_for_pytorch as ipex- from intel_extension_for_pytorch.nn.modules.weight_only_quantization import WeightOnlyQuantizedLinear, \- QuantDtype, QuantMethod+ from intel_extension_for_pytorch.nn.modules.weight_only_quantization import (+ QuantDtype,+ QuantMethod,+ WeightOnlyQuantizedLinear,+ )from packaging import version+from auto_round.utils import get_library_versionipex_version = get_library_version("intel_extension_for_pytorch")if version.parse(ipex_version) >= version.parse("2.5"):diff --git a/auto_round_extension/qbits/__init__.py b/auto_round_extension/qbits/__init__.pyindex b4594b642..04cea3305 100644--- a/auto_round_extension/qbits/__init__.py+++ b/auto_round_extension/qbits/__init__.py@@ -1,3 +1,17 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+from auto_round_extension.qbits.qlinear_qbits import QuantLinear as QBitsQuantLinearfrom auto_round_extension.qbits.qlinear_qbits_gptq import (QuantLinear as QBitsGPTQQuantLinear,diff --git a/auto_round_extension/qbits/qbits_awq.py b/auto_round_extension/qbits/qbits_awq.pyindex 0b3adefb0..07b3012ba 100644--- a/auto_round_extension/qbits/qbits_awq.py+++ b/auto_round_extension/qbits/qbits_awq.py@@ -1,5 +1,20 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import torchimport torch.nn as nn+AWQ_REVERSE_ORDER = [0, 4, 1, 5, 2, 6, 3, 7]def unpack_awq(qweight: torch.Tensor, qzeros: torch.Tensor, bits: int):shifts = torch.arange(0, 32, bits, device="cpu")diff --git a/auto_round_extension/qbits/qlinear_qbits.py b/auto_round_extension/qbits/qlinear_qbits.pyindex aee384f32..2dee2d506 100644--- a/auto_round_extension/qbits/qlinear_qbits.py+++ b/auto_round_extension/qbits/qlinear_qbits.py@@ -18,7 +18,9 @@ import mathimport numpy as npimport torchimport torch.nn as nn+from auto_round.utils import convert_dtype_torch2str, logger+QBITS_AVAILABLE = TrueBITS_DTYPE_MAPPING = {diff --git a/auto_round_extension/qbits/qlinear_qbits_gptq.py b/auto_round_extension/qbits/qlinear_qbits_gptq.pyindex 8a1211955..e67bf6ef5 100644--- a/auto_round_extension/qbits/qlinear_qbits_gptq.py+++ b/auto_round_extension/qbits/qlinear_qbits_gptq.py@@ -18,7 +18,9 @@ import mathimport numpy as npimport torchimport torch.nn as nn+from auto_round.utils import convert_dtype_torch2str, logger+QBITS_AVAILABLE = TrueBITS_DTYPE_MAPPING = {diff --git a/auto_round_extension/torch/__init__.py b/auto_round_extension/torch/__init__.pyindex e69de29bb..2b414eb72 100644--- a/auto_round_extension/torch/__init__.py+++ b/auto_round_extension/torch/__init__.py@@ -0,0 +1,14 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+diff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.pyindex 066ba5beb..401ce149e 100644--- a/auto_round_extension/torch/qlinear_torch.py+++ b/auto_round_extension/torch/qlinear_torch.py@@ -20,7 +20,6 @@ import torchimport torch.nn as nnimport transformers-logger = getLogger(__name__)diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.pyindex a274f57be..503e40e49 100644--- a/auto_round_extension/torch/qlinear_torch_zp.py+++ b/auto_round_extension/torch/qlinear_torch_zp.py@@ -21,7 +21,6 @@ import torchimport torch.nn as nnimport transformers-logger = getLogger(__name__)diff --git a/auto_round_extension/triton/__init__.py b/auto_round_extension/triton/__init__.pyindex e69de29bb..2b414eb72 100644--- a/auto_round_extension/triton/__init__.py+++ b/auto_round_extension/triton/__init__.py@@ -0,0 +1,14 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+diff --git a/auto_round_extension/triton/qlinear_tritonv2.py b/auto_round_extension/triton/qlinear_tritonv2.pyindex b9d8931b9..7320ba03f 100644--- a/auto_round_extension/triton/qlinear_tritonv2.py+++ b/auto_round_extension/triton/qlinear_tritonv2.py@@ -1,3 +1,17 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import mathfrom logging import getLogger@@ -6,17 +20,15 @@ import torchimport torch.nn as nnimport transformers-from auto_round_extension.triton.triton_utils.mixin import TritonModuleMixin-logger = getLogger(__name__)try:from auto_round_extension.triton.triton_utils.dequant import QuantLinearFunction, quant_matmul_248except ImportError as e:if torch.xpu.is_available():- logger.error(f"please make sure your triton version is same with `pytorch-triton-xpu` library ")+ logger.error("please make sure your triton version is same with `pytorch-triton-xpu` library ")exit(-1)triton_import_exception = ediff --git a/auto_round_extension/triton/qlinear_tritonv2_zp.py b/auto_round_extension/triton/qlinear_tritonv2_zp.pyindex 530e14d8d..af18f27b0 100644--- a/auto_round_extension/triton/qlinear_tritonv2_zp.py+++ b/auto_round_extension/triton/qlinear_tritonv2_zp.py@@ -1,3 +1,17 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+import mathfrom logging import getLogger@@ -6,17 +20,15 @@ import torchimport torch.nn as nnimport transformers-from auto_round_extension.triton.triton_utils_zp.mixin import TritonModuleMixin-logger = getLogger(__name__)try:from auto_round_extension.triton.triton_utils_zp.dequant import QuantLinearFunction, quant_matmul_248except ImportError as e:if torch.xpu.is_available():- logger.error(f"please make sure your triton version is same with `pytorch-triton-xpu` library ")+ logger.error("please make sure your triton version is same with `pytorch-triton-xpu` library ")exit(-1)triton_import_exception = ediff --git a/auto_round_extension/triton/triton_utils/custom_autotune.py b/auto_round_extension/triton/triton_utils/custom_autotune.pyindex b511579cc..5b5b5b14d 100644--- a/auto_round_extension/triton/triton_utils/custom_autotune.py+++ b/auto_round_extension/triton/triton_utils/custom_autotune.py@@ -40,7 +40,6 @@ from typing import Dictimport triton-# code based https://github.com/fpgaminer/GPTQ-triton"""Mostly the same as the autotuner in Triton, but with a few changes like using 40 runs instead of 100.diff --git a/auto_round_extension/triton/triton_utils/kernels.py b/auto_round_extension/triton/triton_utils/kernels.pyindex eebd29cdb..0fb42ac45 100644--- a/auto_round_extension/triton/triton_utils/kernels.py+++ b/auto_round_extension/triton/triton_utils/kernels.py@@ -43,7 +43,6 @@ from torch.cuda.amp import custom_bwd, custom_fwdfrom . import custom_autotune-logger = getLogger(__name__)diff --git a/auto_round_extension/triton/triton_utils_zp/custom_autotune.py b/auto_round_extension/triton/triton_utils_zp/custom_autotune.pyindex b511579cc..5b5b5b14d 100644--- a/auto_round_extension/triton/triton_utils_zp/custom_autotune.py+++ b/auto_round_extension/triton/triton_utils_zp/custom_autotune.py@@ -40,7 +40,6 @@ from typing import Dictimport triton-# code based https://github.com/fpgaminer/GPTQ-triton"""Mostly the same as the autotuner in Triton, but with a few changes like using 40 runs instead of 100.diff --git a/auto_round_extension/triton/triton_utils_zp/kernels.py b/auto_round_extension/triton/triton_utils_zp/kernels.pyindex f361a3dde..2d4a358aa 100644--- a/auto_round_extension/triton/triton_utils_zp/kernels.py+++ b/auto_round_extension/triton/triton_utils_zp/kernels.py@@ -42,7 +42,6 @@ import triton.language as tlfrom . import custom_autotune-logger = getLogger(__name__)diff --git a/pyproject.toml b/pyproject.tomlindex 60bdae880..5fe81084a 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,2 +1,97 @@[tool.codespell]-ignore-words = ".azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt"\ No newline at end of file+ignore-words = ".azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt"++[tool.isort]+profile = "black"+line_length = 120+known_first_party = ["auto_round"]+extend_skip_glob = ["**/__init__.py"]++[tool.ruff]+# Exclude a variety of commonly ignored directories.+exclude = [+ ".bzr",+ ".direnv",+ ".eggs",+ ".git",+ ".git-rewrite",+ ".hg",+ ".ipynb_checkpoints",+ ".mypy_cache",+ ".nox",+ ".pants.d",+ ".pyenv",+ ".pytest_cache",+ ".pytype",+ ".ruff_cache",+ ".svn",+ ".tox",+ ".venv",+ ".vscode",+ "__pypackages__",+ "_build",+ "buck-out",+ "build",+ "dist",+ "node_modules",+ "site-packages",+ "venv",+]++# Same as Black.+line-length = 120+indent-width = 4++# Assume Python 3.10+target-version = "py310"++[tool.ruff.lint]+# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default.+# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or+# McCabe complexity (`C901`) by default.+select = ["E4", "E7", "E9", "F", "NPY", "FURB"]+ignore = [+ "E402", # Module level import not at top of file+ "E501", # Line too long (121 > 120 characters)+ "E721", # Do not compare types, use isinstance()+ "E722", # Do not use bare except+ "E731", # Do not assign a lambda expression, use a def+ "E741", # Do not use variables named ‘l’, ‘O’, or ‘I’+ "F401", # {name} imported but unused+ "F403", # from {name} import * used; unable to detect undefined names+ "F841", # Local variable is assigned to but never used{name}+]++# Allow fix for all enabled rules (when `--fix`) is provided.+fixable = ["ALL"]+unfixable = []++# Allow unused variables when underscore-prefixed.+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"++[tool.ruff.format]+# Like Black, use double quotes for strings.+quote-style = "double"++# Like Black, indent with spaces, rather than tabs.+indent-style = "space"++# Like Black, respect magic trailing commas.+skip-magic-trailing-comma = false++# Like Black, automatically detect the appropriate line ending.+line-ending = "auto"++# Enable auto-formatting of code examples in docstrings. Markdown,+# reStructuredText code/literal blocks and doctests are all supported.+#+# This is currently disabled by default, but it is planned for this+# to be opt-out in the future.+docstring-code-format = false++# Set the line length limit used when formatting code snippets in+# docstrings.+#+# This only has an effect when the `docstring-code-format` setting is+# enabled.+docstring-code-line-length = "dynamic"diff --git a/setup.py b/setup.pyindex a725e32e6..1d21ef8d5 100644--- a/setup.py+++ b/setup.py@@ -1,9 +1,10 @@-import re-from io import openimport os-from setuptools import find_packages, setup+import reimport sysfrom functools import lru_cache+from io import open++from setuptools import find_packages, setupos.environ["CC"] = "g++"os.environ["CXX"] = "g++"@@ -67,7 +68,7 @@ def is_cpu_env():def fetch_requirements(path):requirements = []with open(path, "r") as fd:- requirements = [r.strip() for r in fd.readlines()]+ requirements = [r.strip() for r in fd]return requirementsdiff --git a/test/test_cpu/test_auto_round_hpu_only.py b/test/test_cpu/test_auto_round_hpu_only.pyindex 006e44f63..a6fdb433a 100644--- a/test/test_cpu/test_auto_round_hpu_only.py+++ b/test/test_cpu/test_auto_round_hpu_only.py@@ -1,14 +1,15 @@import pytestimport torch-from auto_round.utils import is_hpu_supported-from _test_helpers import is_pytest_mode_compile, is_pytest_mode_lazy+from auto_round.utils import is_hpu_supported+def run_opt_125m_on_hpu():- from auto_round import AutoRoundfrom transformers import AutoModelForCausalLM, AutoTokenizer+ from auto_round import AutoRound+model_name = "facebook/opt-125m"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)@@ -24,7 +25,7 @@ def run_opt_125m_on_hpu():seqlen=2,)q_model, qconfig = autoround.quantize()- assert q_model is not None, f"Expected q_model to be not None"+ assert q_model is not None, "Expected q_model to be not None"@pytest.mark.skipif(not is_hpu_supported(), reason="HPU is not supported")@@ -42,8 +43,7 @@ def test_opt_125m_compile_mode():def test_import():from auto_round import AutoRound- from auto_round.export.export_to_itrex.export import (- WeightOnlyLinear, save_quantized_as_itrex)+ from auto_round.export.export_to_itrex.export import WeightOnlyLinear, save_quantized_as_itrex@pytest.mark.parametrize(@@ -51,9 +51,10 @@ def test_import():["fp8_to_int_sym"],)def test_w4a8(data_type):- from auto_round import AutoRoundfrom transformers import AutoModelForCausalLM, AutoTokenizer+ from auto_round import AutoRound+model_name = "facebook/opt-125m"model = AutoModelForCausalLM.from_pretrained(model_name,@@ -76,4 +77,4 @@ def test_w4a8(data_type):act_dynamic=False,)q_model, qconfig = autoround.quantize()- assert q_model is not None, f"Expected q_model to be not None"+ assert q_model is not None, "Expected q_model to be not None"diff --git a/test/test_cpu/test_autoopt.py b/test/test_cpu/test_autoopt.pyindex 6a986706d..536906e8f 100644--- a/test/test_cpu/test_autoopt.py+++ b/test/test_cpu/test_autoopt.py@@ -8,7 +8,7 @@ import torchimport transformersfrom transformers import AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoRoundOPT, AutoRoundAdam+from auto_round import AutoRoundAdam, AutoRoundOPTclass LLMDataLoader:diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex 25b5afe14..fc577e319 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -7,10 +7,10 @@ from auto_round.eval.evaluation import simple_evaluate_user_modelsys.path.insert(0, "../..")import torch-from transformers import AutoModelForCausalLM, AutoTokenizer, AutoRoundConfig+from _test_helpers import model_infer+from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizerfrom auto_round import AutoRound-from _test_helpers import model_inferclass LLMDataLoader:@@ -423,7 +423,8 @@ class TestAutoRound(unittest.TestCase):def test_not_convert_modules(self):import requestsfrom PIL import Image- from transformers import Qwen2VLForConditionalGeneration, AutoProcessor+ from transformers import AutoProcessor, Qwen2VLForConditionalGeneration+from auto_round_extension.ipex.qlinear_ipex_awq import QuantLinearmodel_name = "Qwen/Qwen2-VL-2B-Instruct-AWQ"quantization_config = AutoRoundConfig()diff --git a/test/test_cpu/test_autoround_acc.py b/test/test_cpu/test_autoround_acc.pyindex 1158e9be3..656064907 100644--- a/test/test_cpu/test_autoround_acc.py+++ b/test/test_cpu/test_autoround_acc.py@@ -3,14 +3,18 @@ import copyimport shutilimport sysimport unittest+sys.path.insert(0, "../..")+from math import isclose+import torchimport transformers-from math import isclosefrom transformers import AutoModelForCausalLM, AutoTokenizer+from auto_round import AutoRound # pylint: disable=E0401from auto_round.export.export_to_itrex.export import pack_model # pylint: disable=E0401+class LLMDataLoader:def __init__(self):self.batch_size = 1diff --git a/test/test_cpu/test_block_names.py b/test/test_cpu/test_block_names.pyindex 6f1436315..f7555a944 100644--- a/test/test_cpu/test_block_names.py+++ b/test/test_cpu/test_block_names.py@@ -2,13 +2,16 @@ import osimport shutilimport sysimport unittest+sys.path.insert(0, ".")sys.path.insert(0, "../..")import torchimport torch.nn as nn-from transformers import AutoModelForCausalLM, AutoTokenizer, AutoConfig+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer+from auto_round import AutoRound+class LLMDataLoader:def __init__(self, input_size=10):self.batch_size = 1@@ -175,8 +178,9 @@ class TestQuantizationBlocks(unittest.TestCase):def test_mm_block_name(self):- from auto_round.utils import get_block_namesfrom transformers import Qwen2VLForConditionalGeneration++ from auto_round.utils import get_block_namesmodel_name = "Qwen/Qwen2-VL-2B-Instruct"model = Qwen2VLForConditionalGeneration.from_pretrained(model_name, trust_remote_code=True, device_map="auto")diff --git a/test/test_cpu/test_conv1d.py b/test/test_cpu/test_conv1d.pyindex 8e87170a2..0009bcedf 100644--- a/test/test_cpu/test_conv1d.py+++ b/test/test_cpu/test_conv1d.py@@ -5,10 +5,12 @@ import unittestsys.path.insert(0, "../..")import torch+from _test_helpers import model_inferfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRound-from _test_helpers import model_infer++class LLMDataLoader:def __init__(self):self.batch_size = 1diff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.pyindex 8915c548e..bfd54ba31 100644--- a/test/test_cpu/test_export.py+++ b/test/test_cpu/test_export.py@@ -4,7 +4,7 @@ import unittestsys.path.insert(0, "../..")import torch-from transformers import AutoModelForCausalLM, AutoTokenizer, AutoRoundConfig+from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizerfrom auto_round import AutoRound@@ -206,6 +206,7 @@ class TestAutoRound(unittest.TestCase):def test_static_afp8_export(self):import os+from safetensors import safe_openmodel_name = "facebook/opt-125m"diff --git a/test/test_cpu/test_generation.py b/test/test_cpu/test_generation.pyindex cec33e991..11c9ca1bb 100644--- a/test/test_cpu/test_generation.py+++ b/test/test_cpu/test_generation.py@@ -5,7 +5,7 @@ import unittestsys.path.insert(0, "../..")import torch-from transformers import AutoModelForCausalLM, AutoTokenizer, AutoRoundConfig+from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizerfrom auto_round import AutoRounddiff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.pyindex ae5cfba5a..cb7691a1b 100644--- a/test/test_cpu/test_gguf_format.py+++ b/test/test_cpu/test_gguf_format.py@@ -1,7 +1,8 @@import os+import shutilimport sysimport unittest-import shutil+sys.path.insert(0, "../..")import torchdiff --git a/test/test_cpu/test_hpu.py b/test/test_cpu/test_hpu.pyindex 629a93212..035060817 100644--- a/test/test_cpu/test_hpu.py+++ b/test/test_cpu/test_hpu.py@@ -9,7 +9,6 @@ from transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRound-class LLMDataLoader:def __init__(self):self.batch_size = 1@@ -20,7 +19,7 @@ class LLMDataLoader:def is_hpu_supported():try:- import habana_frameworks.torch.core as htcore # pylint: disable=E0401+ import habana_frameworks.torch.core as htcore # pylint: disable=E0401except ImportError as e:return Falsereturn Truediff --git a/test/test_cpu/test_llmcompressor_w8a8.py b/test/test_cpu/test_llmcompressor_w8a8.pyindex 7156b6a07..978a8ed5c 100644--- a/test/test_cpu/test_llmcompressor_w8a8.py+++ b/test/test_cpu/test_llmcompressor_w8a8.py@@ -1,7 +1,8 @@import os+import shutilimport sysimport unittest-import shutil+sys.path.insert(0, "../..")import torchdiff --git a/test/test_cpu/test_load_awq_gptq.py b/test/test_cpu/test_load_awq_gptq.pyindex 202605491..88e9730bc 100644--- a/test/test_cpu/test_load_awq_gptq.py+++ b/test/test_cpu/test_load_awq_gptq.py@@ -4,9 +4,7 @@ import unittestsys.path.insert(0, "../..")-from transformers import AutoModelForCausalLM, AutoTokenizer--from transformers import AutoRoundConfig+from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizerclass TestAutoRound(unittest.TestCase):diff --git a/test/test_cpu/test_low_cpu_mem.py b/test/test_cpu/test_low_cpu_mem.pyindex 2c4378f07..5ad8f5659 100644--- a/test/test_cpu/test_low_cpu_mem.py+++ b/test/test_cpu/test_low_cpu_mem.py@@ -1,20 +1,21 @@+import osimport shutilimport sys-import osimport unittest+sys.path.insert(0, "../..")import torchfrom transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRoundfrom auto_round.low_cpu_mem.utils import (- load_model_with_hooks,- load_empty_model,get_layers_before_block,layer_wise_load,layer_wise_save,- )--from auto_round import AutoRound+ load_empty_model,+ load_model_with_hooks,+)class LLMDataLoader:diff --git a/test/test_cpu/test_mllm.py b/test/test_cpu/test_mllm.pyindex 5d5426b9f..4206847f0 100644--- a/test/test_cpu/test_mllm.py+++ b/test/test_cpu/test_mllm.py@@ -3,11 +3,11 @@ import unittestsys.path.insert(0, "../..")-from auto_round import AutoRoundMLLM+import shutilfrom transformers import AutoProcessor, AutoTokenizer, Qwen2VLForConditionalGeneration-import shutil+from auto_round import AutoRoundMLLMclass FakeDataLoader:@@ -73,7 +73,7 @@ class TestAutoRoundMLLM(unittest.TestCase):autoround.save_quantized("./saved/", format="auto_round", inplace=True)def test_quant_block_names(self):- from auto_round.utils import get_block_names,find_matching_blocks+ from auto_round.utils import find_matching_blocks, get_block_namestokenizer = AutoTokenizer.from_pretrained(self.model_name)processor = AutoProcessor.from_pretrained(self.model_name, trust_remote_code=True)model = Qwen2VLForConditionalGeneration.from_pretrained(@@ -109,6 +109,7 @@ class TestAutoRoundMLLM(unittest.TestCase):def test_pure_text_model_check(self):from transformers import AutoModelForCausalLM+from auto_round.utils import is_pure_text_modelmodel = Qwen2VLForConditionalGeneration.from_pretrained(self.model_name, trust_remote_code=True, device_map="auto")diff --git a/test/test_cpu/test_utils.py b/test/test_cpu/test_utils.pyindex eff324e43..846af0036 100644--- a/test/test_cpu/test_utils.py+++ b/test/test_cpu/test_utils.py@@ -1,8 +1,10 @@-from unittest.mock import patchimport sys+from unittest.mock import patch+sys.path.insert(0, "../..")import auto_round.utils as auto_round_utils+class TestPackingWithNumba:@patch.object(auto_round_utils, "_is_tbb_installed", lambda: False)diff --git a/test/test_cpu/test_woq_linear.py b/test/test_cpu/test_woq_linear.pyindex 1f48e2309..673b5d06d 100644--- a/test/test_cpu/test_woq_linear.py+++ b/test/test_cpu/test_woq_linear.py@@ -1,6 +1,8 @@+import sys+import pytestimport torch-import sys+sys.path.insert(0, "../..")from auto_round.export.export_to_itrex.model_wrapper import WeightOnlyLineardiff --git a/test/test_cuda/test_2_3bits.py b/test/test_cuda/test_2_3bits.pyindex 73f73d8f2..519d4e6de 100644--- a/test/test_cuda/test_2_3bits.py+++ b/test/test_cuda/test_2_3bits.py@@ -1,17 +1,17 @@import copy+import reimport shutilimport sysimport unittest-import resys.path.insert(0, "../..")import torchimport transformers+from lm_eval.utils import make_table # pylint: disable=E0401from transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round.eval.evaluation import simple_evaluate-from lm_eval.utils import make_table # pylint: disable=E0401from auto_round.testing_utils import require_autogptq, require_greater_than_050, require_greater_than_051diff --git a/test/test_cuda/test_auto_round_format.py b/test/test_cuda/test_auto_round_format.pyindex 817b5087d..22c963a48 100644--- a/test/test_cuda/test_auto_round_format.py+++ b/test/test_cuda/test_auto_round_format.py@@ -4,15 +4,13 @@ import sysimport unittestsys.path.insert(0, "../..")-from auto_round.eval.evaluation import simple_evaluate_user_model-from auto_round.testing_utils import require_greater_than_050, require_autogptq, require_awq, require_ipex-import torchimport transformersfrom transformers import AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoRound-from auto_round import AutoRoundConfig+from auto_round import AutoRound, AutoRoundConfig+from auto_round.eval.evaluation import simple_evaluate_user_model+from auto_round.testing_utils import require_autogptq, require_awq, require_greater_than_050, require_ipexclass LLMDataLoader:diff --git a/test/test_cuda/test_conv1d.py b/test/test_cuda/test_conv1d.pyindex bf0daba34..b8fa0aa43 100644--- a/test/test_cuda/test_conv1d.py+++ b/test/test_cuda/test_conv1d.py@@ -5,11 +5,13 @@ import unittestsys.path.insert(0, "../..")import torch+from _test_helpers import model_inferfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round.testing_utils import require_gptqmodel-from _test_helpers import model_infer++class LLMDataLoader:def __init__(self):self.batch_size = 1@@ -35,7 +37,7 @@ class TestQuantizationConv1d(unittest.TestCase):def test_quant(self):self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)bits, group_size, sym = 4, 128, True- from auto_round import AutoRoundConfig+ from auto_round import AutoRoundConfigautoround = AutoRound(self.model,self.tokenizer,diff --git a/test/test_cuda/test_exllamav2_backend.py b/test/test_cuda/test_exllamav2_backend.pyindex 2a6539306..4279d6e57 100644--- a/test/test_cuda/test_exllamav2_backend.py+++ b/test/test_cuda/test_exllamav2_backend.py@@ -1,15 +1,16 @@import shutilimport sysimport unittest+import pytest+sys.path.insert(0, "../..")import torchfrom transformers import AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoRound-from auto_round import AutoRoundConfig+from auto_round import AutoRound, AutoRoundConfigfrom auto_round.eval.evaluation import simple_evaluate_user_modelfrom auto_round.testing_utils import require_autogptq, require_gptqmodeldiff --git a/test/test_cuda/test_get_block_name.py b/test/test_cuda/test_get_block_name.pyindex fd89e2aa1..f796970d3 100644--- a/test/test_cuda/test_get_block_name.py+++ b/test/test_cuda/test_get_block_name.py@@ -6,10 +6,17 @@ import unittestsys.path.insert(0, "../..")import torchimport transformers-from transformers import AutoModelForCausalLM, AutoTokenizer, Qwen2VLForConditionalGeneration, AutoModelForVision2Seq, \- Gemma3ForConditionalGeneration, Mistral3ForConditionalGeneration-from auto_round.utils import get_block_names, is_pure_text_model+from transformers import (+ AutoModelForCausalLM,+ AutoModelForVision2Seq,+ AutoTokenizer,+ Gemma3ForConditionalGeneration,+ Mistral3ForConditionalGeneration,+ Qwen2VLForConditionalGeneration,+)+from auto_round import AutoRound+from auto_round.utils import get_block_names, is_pure_text_modelclass TestAutoRound(unittest.TestCase):@@ -33,7 +40,7 @@ class TestAutoRound(unittest.TestCase):model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)block_names = get_block_names(model)self.check_block_names(block_names, ["transformer.encoder.layers"], [40])- assert is_pure_text_model(model) == True+ assert is_pure_text_model(model), "Expected model to be pure text model"def test_opt_125m(self):model_name = "/models/opt-125m"@@ -41,56 +48,56 @@ class TestAutoRound(unittest.TestCase):block_names = get_block_names(model)self.check_block_names(block_names, ["model.decoder.layers"], [12])- assert is_pure_text_model(model) == True+ assert is_pure_text_model(model)def test_Qwen(self):model_name = "/models/Qwen2.5-7B-Instruct"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)block_names = get_block_names(model)self.check_block_names(block_names, ["model.layers"], [28])- assert is_pure_text_model(model) == True+ assert is_pure_text_model(model)def test_phi4(self):model_name = "/models/phi-4"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)block_names = get_block_names(model)self.check_block_names(block_names, ["model.layers"], [40])- assert is_pure_text_model(model) == True+ assert is_pure_text_model(model)def test_llama3(self):model_name = "/models/Meta-Llama-3.1-8B-Instruct"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)block_names = get_block_names(model)self.check_block_names(block_names, ["model.layers"], [32])- assert is_pure_text_model(model) == True+ assert is_pure_text_model(model)def test_mixtral(self):model_name = "/models/Mixtral-8x7B-Instruct-v0.1"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)block_names = get_block_names(model)self.check_block_names(block_names, ["model.layers"], [32])- assert is_pure_text_model(model) == True+ assert is_pure_text_model(model)def test_falcon(self):model_name = "/models/Falcon3-7B-Instruct"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)block_names = get_block_names(model)self.check_block_names(block_names, ["model.layers"], [28])- assert is_pure_text_model(model) == True+ assert is_pure_text_model(model)def test_orca(self):model_name = "/models/Orca-2-7b"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)block_names = get_block_names(model)self.check_block_names(block_names, ["model.layers"], [32])- assert is_pure_text_model(model) == True+ assert is_pure_text_model(model)def test_OLMo(self):model_name = "/models/OLMo-2-1124-7B-Instruct"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)block_names = get_block_names(model)self.check_block_names(block_names, ["model.layers"], [32])- assert is_pure_text_model(model) == True+ assert is_pure_text_model(model)def test_Qwen2VL(self):model_name = "/models/Qwen2-VL-2B-Instruct"@@ -100,7 +107,7 @@ class TestAutoRound(unittest.TestCase):block_names = get_block_names(model, quant_vision=True)self.check_block_names(block_names, ["visual.blocks", "model.layers"], [32, 28])- assert is_pure_text_model(model) == False+ assert not is_pure_text_model(model)def test_Llama32(self):model_name = "/models/Llama-3.2-11B-Vision-Instruct"@@ -113,7 +120,7 @@ class TestAutoRound(unittest.TestCase):["vision_model.transformer.layers", "vision_model.global_transformer.layers","language_model.model.layers"], [32, 8, 40])- assert is_pure_text_model(model) == False+ assert not is_pure_text_model(model)def test_SmolVLM(self):model_name = "/models/SmolVLM-Instruct"@@ -123,7 +130,7 @@ class TestAutoRound(unittest.TestCase):block_names = get_block_names(model, quant_vision=True)self.check_block_names(block_names, ["model.vision_model.encoder.layers", "model.text_model.layers"], [27, 24])- assert is_pure_text_model(model) == False+ assert not is_pure_text_model(model)def test_glm_4v(self):model_name = "/models/glm-4v-9b"@@ -134,7 +141,7 @@ class TestAutoRound(unittest.TestCase):block_names = get_block_names(model, quant_vision=True)self.check_block_names(block_names, ["transformer.encoder.layers", "transformer.vision.transformer.layers"],[40, 63])- assert is_pure_text_model(model) == False+ assert not is_pure_text_model(model)def test_gemma3(self):model_name = "/models/gemma-3-12b-it"@@ -145,7 +152,7 @@ class TestAutoRound(unittest.TestCase):block_names = get_block_names(model, quant_vision=True)self.check_block_names(block_names, ["vision_tower.vision_model.encoder.layers", "language_model.model.layers"],[27, 48])- assert is_pure_text_model(model) == False+ assert not is_pure_text_model(model)def test_Mistral3(self):model_name = "/models/Mistral-Small-3.1-24B-Instruct-2503"@@ -156,7 +163,7 @@ class TestAutoRound(unittest.TestCase):block_names = get_block_names(model, quant_vision=True)self.check_block_names(block_names, ["vision_tower.transformer.layers", "language_model.model.layers"],[24, 40])- assert is_pure_text_model(model) == False+ assert not is_pure_text_model(model)def test_Molmo(self):model_name = "/models/Molmo-7B-D-0924"@@ -168,7 +175,7 @@ class TestAutoRound(unittest.TestCase):self.check_block_names(block_names,["model.transformer.blocks", "model.vision_backbone.image_vit.transformer.resblocks"],[28, 23])- assert is_pure_text_model(model) == False+ assert not is_pure_text_model(model)if __name__ == "__main__":diff --git a/test/test_cuda/test_gguf.py b/test/test_cuda/test_gguf.pyindex ece1c8c1a..6faf31eb4 100644--- a/test/test_cuda/test_gguf.py+++ b/test/test_cuda/test_gguf.py@@ -11,6 +11,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round.testing_utils import require_gguf+class LLMDataLoader:def __init__(self):self.batch_size = 1diff --git a/test/test_cuda/test_main_func.py b/test/test_cuda/test_main_func.pyindex 7f520cb38..d32faf92a 100644--- a/test/test_cuda/test_main_func.py+++ b/test/test_cuda/test_main_func.py@@ -1,18 +1,18 @@import copy+import reimport shutilimport sysimport unittest-import resys.path.insert(0, "../..")import torchimport transformers+from lm_eval.utils import make_table # pylint: disable=E0401from transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRound, AutoRoundAdamfrom auto_round.eval.evaluation import simple_evaluate-from auto_round.testing_utils import require_gptqmodel, require_optimum, require_awq-from lm_eval.utils import make_table # pylint: disable=E0401+from auto_round.testing_utils import require_awq, require_gptqmodel, require_optimumdef get_accuracy(data):diff --git a/test/test_cuda/test_marlin_backend.py b/test/test_cuda/test_marlin_backend.pyindex cb3516cad..f398029c2 100644--- a/test/test_cuda/test_marlin_backend.py+++ b/test/test_cuda/test_marlin_backend.py@@ -1,14 +1,14 @@import shutilimport sysimport unittest+import pytestsys.path.insert(0, "../..")import torchfrom transformers import AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoRound-from auto_round import AutoRoundConfig+from auto_round import AutoRound, AutoRoundConfigfrom auto_round.eval.evaluation import simple_evaluate_user_modeldiff --git a/test/test_cuda/test_multiple_card.py b/test/test_cuda/test_multiple_card.pyindex 5f8ab73d7..1710c1660 100644--- a/test/test_cuda/test_multiple_card.py+++ b/test/test_cuda/test_multiple_card.py@@ -1,16 +1,18 @@import re+import shutilimport sysimport unittest-import shutil+sys.path.insert(0, "../..")import torchfrom lm_eval.utils import make_table # pylint: disable=E0401from transformers import AutoModelForCausalLM, AutoTokenizer+from auto_round import AutoRoundfrom auto_round.eval.evaluation import simple_evaluate-from auto_round.testing_utils import multi_card, require_greater_than_050, require_gptqmodel+from auto_round.testing_utils import multi_card, require_gptqmodel, require_greater_than_050def get_accuracy(data):@@ -105,7 +107,6 @@ class TestAutoRound(unittest.TestCase):autoround = AutoRound(model, tokenizer, iters=2, device_map=device_map, nsamples=7,seqlen=32)autoround.quantize()- from transformers import AutoModelForCausalLM, AutoTokenizermodel_name = "OPEA/Meta-Llama-3.1-8B-Instruct-int4-sym-inc"device_map = {}diff --git a/test/test_cuda/test_multiple_card_calib.py b/test/test_cuda/test_multiple_card_calib.pyindex 63a494f75..8f3b584bc 100644--- a/test/test_cuda/test_multiple_card_calib.py+++ b/test/test_cuda/test_multiple_card_calib.py@@ -1,13 +1,14 @@import osimport re-import sysimport shutil+import sysimport unittestsys.path.insert(0, "../..")from auto_round.testing_utils import multi_card+def get_accuracy(data):match = re.search(r'\|acc\s+\|[↑↓]\s+\|\s+([\d.]+)\|', data)diff --git a/test/test_cuda/test_qbits.py b/test/test_cuda/test_qbits.pyindex 8b79d97b5..4af9fb358 100644--- a/test/test_cuda/test_qbits.py+++ b/test/test_cuda/test_qbits.py@@ -6,8 +6,8 @@ sys.path.insert(0, "../..")from transformers import AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoRoundConfig, AutoRound-from auto_round.testing_utils import require_itrex, require_gptqmodel+from auto_round import AutoRound, AutoRoundConfig+from auto_round.testing_utils import require_gptqmodel, require_itrexclass TestAutoRound(unittest.TestCase):@@ -78,8 +78,9 @@ class TestAutoRound(unittest.TestCase):layer_config["model.decoder.layers.0.self_attn.k_proj"] = {"bits": 8}layer_config["model.decoder.layers.6.self_attn.out_proj"] = {"bits": 2, "group_size": 32}bits, group_size, sym = 4, 128, True- from auto_round import AutoRoundimport torch++ from auto_round import AutoRoundautoround = AutoRound(model,tokenizer,diff --git a/test/test_cuda/test_support_vlms.py b/test/test_cuda/test_support_vlms.pyindex ddca458aa..8208ae255 100644--- a/test/test_cuda/test_support_vlms.py+++ b/test/test_cuda/test_support_vlms.py@@ -1,15 +1,16 @@import os-import sysimport shutil+import sysimport unittestsys.path.insert(0, '../..')-from auto_round import AutoRoundConfig ## must import for auto-round format-from auto_round.testing_utils import require_gptqmodel, require_vlm_envimport requestsfrom PIL import Image+from auto_round import AutoRoundConfig # # must import for auto-round format+from auto_round.testing_utils import require_gptqmodel, require_vlm_env+class TestSupportVLMS(unittest.TestCase):@classmethod@@ -352,7 +353,7 @@ class TestSupportVLMS(unittest.TestCase):self.assertFalse(res > 0 or res == -1, msg="deepseek vl2 tuning fail")quantized_model_path = os.path.join(self.save_dir, "deepseek-vl2-tiny-w4g32")- from deepseek_vl2.models import DeepseekVLV2Processor, DeepseekVLV2ForCausalLM+ from deepseek_vl2.models import DeepseekVLV2ForCausalLM, DeepseekVLV2Processorfrom transformers import AutoModelForCausalLMvl_chat_processor: DeepseekVLV2Processor = DeepseekVLV2Processor.from_pretrained(quantized_model_path)tokenizer = vl_chat_processor.tokenizerdiff --git a/test/test_cuda/test_transformers.py b/test/test_cuda/test_transformers.pyindex 527f5aec9..e429f14d4 100644--- a/test/test_cuda/test_transformers.py+++ b/test/test_cuda/test_transformers.py@@ -26,7 +26,6 @@ from transformers.testing_utils import ()from transformers.utils import is_torch_available-if is_torch_available():import torchdiff --git a/test/test_cuda/test_triton_backend.py b/test/test_cuda/test_triton_backend.pyindex a0d69322f..e31b659dd 100644--- a/test/test_cuda/test_triton_backend.py+++ b/test/test_cuda/test_triton_backend.py@@ -4,12 +4,11 @@ import sysimport unittestsys.path.insert(0, "../..")-from auto_round.eval.evaluation import simple_evaluate_user_model-import torchfrom transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRound, AutoRoundConfig+from auto_round.eval.evaluation import simple_evaluate_user_modelfrom auto_round.testing_utils import require_greater_than_050diff --git a/test/test_cuda/test_vlms.py b/test/test_cuda/test_vlms.pyindex 15ad6ca1c..62444d786 100644--- a/test/test_cuda/test_vlms.py+++ b/test/test_cuda/test_vlms.py@@ -1,16 +1,18 @@-import re-import os-import sysimport copy+import os+import reimport shutil+import sysimport unittest+import requestssys.path.insert(0, "../..")from PIL import Image+from auto_round import AutoRoundConfig-from auto_round.testing_utils import require_gptqmodel, require_vlm_env, require_optimum+from auto_round.testing_utils import require_gptqmodel, require_optimum, require_vlm_envclass TestAutoRound(unittest.TestCase):@@ -42,8 +44,7 @@ class TestAutoRound(unittest.TestCase):# res == """<s> There is a girl who likes adventure, and she is looking for a partner to go on a treasure hunt. She has found a map that leads to a hidden treasure, but she needs a partner to help her decipher the clues and find the treasure. You""")def qwen_inference(self, quantized_model_dir):- import requests- from transformers import Qwen2VLForConditionalGeneration, AutoProcessor, AutoTokenizer+ from transformers import AutoProcessor, AutoTokenizer, Qwen2VLForConditionalGenerationtokenizer = AutoTokenizer.from_pretrained(quantized_model_dir)processor = AutoProcessor.from_pretrained(quantized_model_dir, trust_remote_code=True)model = Qwen2VLForConditionalGeneration.from_pretrained(@@ -92,8 +93,9 @@ class TestAutoRound(unittest.TestCase):@require_gptqmodel@require_optimumdef test_vlm_tune(self):+ from transformers import AutoProcessor, AutoTokenizer, Qwen2VLForConditionalGeneration+from auto_round import AutoRoundMLLM- from transformers import Qwen2VLForConditionalGeneration, AutoProcessor, AutoTokenizer## load the modelmodel_name = "/models/Qwen2-VL-2B-Instruct"diff --git a/test/test_xpu/test_autoround.py b/test/test_xpu/test_autoround.pyindex caad16634..0c547b236 100644--- a/test/test_xpu/test_autoround.py+++ b/test/test_xpu/test_autoround.py@@ -8,8 +8,8 @@ import torchimport transformersfrom transformers import AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoRoundConfig-from auto_round import AutoRound+from auto_round import AutoRound, AutoRoundConfig+class LLMDataLoader:def __init__(self):@@ -52,7 +52,6 @@ class TestAutoRoundXPU(unittest.TestCase):quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path)- from auto_round import AutoRoundConfigquantization_config = AutoRoundConfig(backend="auto")@@ -88,7 +87,6 @@ class TestAutoRoundXPU(unittest.TestCase):autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round:auto_awq")quantized_model_path = "./saved"- from auto_round import AutoRoundConfigquantization_config = AutoRoundConfig(backend="auto")