PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support fp8 model and str as input in llm quantization (#699)
Wenhua Cheng32777f7327f0
diff --git a/README.md b/README.mdindex a2c5397d9..0e5c3539f 100644--- a/README.md+++ b/README.md@@ -29,10 +29,10 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).## 🆕 What's New[2025/07] AutoRound now offers experimental support for **GGUF** format, and recommends using optimized RTN mode (--iters 0) for- all bits other than 3 bits. **A more advanced algorithm** tailored for specific configurations may be available in- v0.6.1. Example+ all bits other than 3 bits. Examplemodels: [Intel/Qwen3-235B-A22B-q2ks-mixed-ar](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-ar)- and [Intel/DeepSeek-R1-0528-q2ks-mixed-ar](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-ar).+ and [Intel/DeepSeek-R1-0528-q2ks-mixed-ar](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-ar). **A more advanced algorithm** tailored for specific configurations may be available in+ v0.6.1.[2025/05] AutoRound provides some recipes for **DeepSeek-R1-0528**, please referto [Intel/DeepSeek-R1-0528-int2-mixed-ar](https://huggingface.co/Intel/DeepSeek-R1-0528-int2-mixed-ar), [Intel/DeepSeek-R1-0528-int4-ar](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-ar)diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex bd3237d7a..f6d6b6ea2 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -31,6 +31,7 @@ from auto_round.data_type.utils import reshape_pad_tensor_by_group_sizefrom auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelTypefrom auto_round.low_cpu_mem.utils import get_layers_before_blockfrom auto_round.utils import (+ INNER_SUPPORTED_LAYER_TYPES,SUPPORTED_DTYPES,SUPPORTED_LAYER_TYPES,TORCH_VERSION_AT_LEAST_2_6,@@ -46,6 +47,8 @@ from auto_round.utils import (collect_best_params,compile_func,convert_dtype_str2torch,+ convert_fp8_layer_to_linear,+ convert_fp8_model_to_16b_model,detect_device,find_matching_blocks,flatten_list,@@ -55,6 +58,7 @@ from auto_round.utils import (get_layer_names_in_block,get_lm_head_name,get_module,+ get_quant_keys,get_shared_keys,htcore,infer_bits_by_data_type,@@ -141,15 +145,15 @@ class AutoRound(object):def __init__(self,- model: torch.nn.Module,- tokenizer,+ model: Union[torch.nn.Module, str],+ tokenizer=None,bits: int = 4,group_size: int = 128,sym: bool = True,layer_config: dict = None,batch_size: int = 8,amp: bool = True,- device: str = None,+ device: Union[str, torch.device, int] = 0,lr_scheduler=None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",enable_quanted_input: bool = True,@@ -157,7 +161,7 @@ class AutoRound(object):lr: float = None,minmax_lr: float = None,low_gpu_mem_usage: bool = False,- low_cpu_mem_usage: bool = False,+ low_cpu_mem_usage: int = 0,iters: int = 200,seqlen: int = 2048,nsamples: int = 128,@@ -188,14 +192,28 @@ class AutoRound(object):if kwargs:logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")self.quantized = False- self.model_orig_dtype = model.dtypeself.seed = seedset_seed(self.seed)+ if device is not None and "," in str(device):+ raise ValueError(+ "API does not support explicit set multiple devices,"+ " please set CUDA_VISIBLE_DEVICES yourself and use `device=auto` instead"+ )++ if isinstance(model, str):+ model, tokenizer, low_cpu_mem_usage = llm_load_model(+ model, device=device, low_cpu_mem_mode=low_cpu_mem_usage+ )+ elif tokenizer is None and iters > 0:+ raise ValueError("A tokenizer must be set for non-str model input")+ self.low_cpu_mem_usage = bool(low_cpu_mem_usage)if unsupport_meta_device(model):raise RuntimeError("AutoRound does not support parameters on meta device. ""Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU.")+ # self.model_orig_dtype = model.dtype+ self.device = detect_device(device) ##must place after llm_load_model, because this one will convert auto## important tuning hype-parametersself.amp = amp@@ -208,7 +226,7 @@ class AutoRound(object):self.sym = symself.low_gpu_mem_usage = low_gpu_mem_usage- self.low_cpu_mem_usage = low_cpu_mem_usage+self.layer_config = {} if layer_config is None else layer_configself.seqlen = seqlenself.batch_size, self.gradient_accumulate_steps = batch_size, gradient_accumulate_steps@@ -232,9 +250,10 @@ class AutoRound(object):)self.bits = tmp_bitsself.supported_types = SUPPORTED_LAYER_TYPES+ self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPESself.model = model.eval()self.tokenizer = tokenizer- self.device = detect_device(device)+self.scale_dtype = convert_dtype_str2torch(scale_dtype)self.set_amp_dtype()self.to_quant_block_names = to_quant_block_names@@ -837,7 +856,12 @@ class AutoRound(object):# Load datasetfrom .calib_dataset import get_dataloader+ if hasattr(self.model, "is_fp8"):+ convert_fp8_model_to_16b_model(self.model, self.amp_dtype)+if isinstance(self.dataset, str):+ if self.tokenizer is None:+ raise ValueError("A tokenizer must be set for the model when using a dataset string.")dataset_name = self.dataset.replace(" ", "")self.dataloader = get_dataloader(self.tokenizer, self.seqlen, dataset_name, self.seed, self.batch_size, self.nsamples@@ -1063,6 +1087,10 @@ class AutoRound(object):"""m = get_module(self.model, name)+ if m.__class__.__name__ == "FP8Linear":+ m = convert_fp8_layer_to_linear(m, self.amp_dtype)+ set_module(self.model, name, m)+# Step 1: Use optimized RTN data type if availableif not self.disable_opt_rtn and not m.data_type.startswith("rtn_"):from auto_round.data_type import QUANT_FUNC_WITH_DTYPE@@ -1192,7 +1220,9 @@ class AutoRound(object):clear_memory()cnt = 1cnt += 1-+ ##convert remainning fp8+ if hasattr(self.model, "is_fp8"):+ convert_fp8_model_to_16b_model(self.model, self.amp_dtype)self.quantized = Truereturn self.model, self.layer_config@@ -1250,6 +1280,8 @@ class AutoRound(object):pbar.set_description(f"Quantizing {block_name}")block = get_module(self.model, block_name)block = block.to(self.device)+ if hasattr(self.model, "is_fp8"):+ convert_fp8_model_to_16b_model(block, dtype=self.amp_dtype)# Dispatch model if neededif self.device_map is not None:from accelerate import dispatch_model@@ -1401,7 +1433,7 @@ class AutoRound(object):self.batch_size = total_sampleslogger.warning(f"force the train batch size to {total_samples}")- self.quant_blocks(+ self.quantize_blocks(self.model,inputs,block_names,@@ -1419,6 +1451,12 @@ class AutoRound(object):self.quant_layers(layer_names, all_inputs) ##TODO pack layer immediately+ if hasattr(self.model, "is_fp8"):+ for n, m in self.model.named_modules():+ if m.__class__.__name__ == "FP8Linear":+ new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype).to("cpu")+ set_module(self.model, n, new_layer)+end_time = time.time()cost_time = end_time - self.start_timelogger.info(f"quantization tuning time {cost_time}")@@ -1456,17 +1494,22 @@ class AutoRound(object):"""##TODO currently we take all the layers outside blocks as post block layers which is not optimal## if there is no input for layer, we use rtn+for layer_name in copy.deepcopy(layer_names):if layer_name not in layer_inputs:logger.info(f"using rtn to quantize {layer_name}")from auto_round.data_type import QUANT_FUNC_WITH_DTYPElayer = get_module(self.model, layer_name)+ if layer.__class__.__name__ == "FP8Linear":+ new_layer = convert_fp8_layer_to_linear(layer, self.amp_dtype).to(self.device)+ set_module(self.model, layer_name, new_layer)+ layer = new_layer+if not self.disable_opt_rtn and "rtn_" + layer.data_type in QUANT_FUNC_WITH_DTYPE:layer.data_type = "rtn_" + layer.data_typelogger.info("using optimized rtn method for quantizing %s", layer_name)self.layer_config[layer_name]["data_type"] = layer.data_type- layer.to(self.device)wrapper_layer = WrapperLinear(layer,enable_round_tuning=False,@@ -1529,24 +1572,12 @@ class AutoRound(object):otherwise returns False."""# Get the names of layers in quantization blocks- layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)+ supported_types = self.supported_types + self.inner_supported_types+ layers_in_blocks = get_layer_names_in_block(self.model, supported_types, self.quant_block_list)##process regex in layer_configall_supported_layer_names = []# List of configuration keys- keys = [- "bits",- "group_size",- "sym",- "data_type",- "scale_dtype",- "act_bits",- "act_group_size",- "act_sym",- "act_dynamic",- "act_data_type",- "super_bits",- "super_group_size",- ]+ keys = get_quant_keys()for n, m in self.model.named_modules():# Delete previous configuration to avoid conflicts with prior tuning@@ -1554,9 +1585,6 @@ class AutoRound(object):if hasattr(m, key):delattr(m, key)- # Skip unsupported types- supported_types = self.supported_types-if not isinstance(m, supported_types):continueall_supported_layer_names.append(n)@@ -1794,6 +1822,8 @@ class AutoRound(object):Raises:Exception: If caching on GPU fails, switches to CPU and caches there."""+ if hasattr(self.model, "is_fp8"):+ layer_names = []if layer_names is None:layer_names = []@@ -2211,7 +2241,7 @@ class AutoRound(object):continuereturn hook_handles- def quant_block(self, block, input_ids, input_others, q_input=None, device=torch.device("cpu")):+ def quantize_block(self, block, input_ids, input_others, q_input=None, device=torch.device("cpu")):"""Quantize the weights of a given block of the model.Args:@@ -2224,6 +2254,12 @@ class AutoRound(object):Returns:Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)"""+ if hasattr(self.model, "is_fp8"):+ for n, m in block.named_modules():+ if m.__class__.__name__ == "FP8Linear":+ new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype).to(device)+ set_module(block, n, new_layer)+if self.device_map is not None:from accelerate import dispatch_model@@ -2421,7 +2457,7 @@ class AutoRound(object):clear_memory(input_ids)return None, output- def quant_blocks(+ def quantize_blocks(self, model: torch.nn.Module, inputs, block_names, q_input=None, nblocks=1, device="cpu", pbar=None):"""Quantize and dequantize the weights of the specified blocks in the model.@@ -2459,9 +2495,9 @@ class AutoRound(object):for i in range(len(input_others[key])):to_dtype(input_others[key][i], tmp_dtype)if self.enable_torch_compile:- quant_block = compile_func(self.quant_block, device)+ quant_block = compile_func(self.quantize_block, device)else:- quant_block = self.quant_block+ quant_block = self.quantize_blockif pbar is None:pbar = tqdm(range(0, len(block_names), nblocks))@@ -2806,7 +2842,7 @@ class AutoRoundOPT(AutoRound):def __init__(self,- model,+ model: Union[torch.nn.Module, str],tokenizer=None,bits: int = 4,group_size: int = 128,@@ -2814,7 +2850,7 @@ class AutoRoundOPT(AutoRound):layer_config=None,batch_size: int = 8,amp: bool = True,- device=None,+ device: Union[str, torch.device, int] = 0,lr_scheduler=None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",enable_quanted_input: bool = True,@@ -2822,7 +2858,7 @@ class AutoRoundOPT(AutoRound):lr: float = None,minmax_lr: float = None,low_gpu_mem_usage: bool = False,- low_cpu_mem_usage: bool = False,+ low_cpu_mem_usage: int = 0,iters: int = 200,seqlen: int = 2048,nsamples: int = 128,@@ -2985,7 +3021,7 @@ class AutoRoundAdam(AutoRoundOPT):def __init__(self,- model,+ model: Union[torch.nn.Module, str],tokenizer=None,bits: int = 4,group_size: int = 128,@@ -2993,7 +3029,7 @@ class AutoRoundAdam(AutoRoundOPT):layer_config=None,batch_size: int = 8,amp: bool = True,- device=None,+ device: Union[str, torch.device, int] = 0,lr_scheduler=None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",enable_quanted_input: bool = True,@@ -3001,7 +3037,7 @@ class AutoRoundAdam(AutoRoundOPT):lr: float = None,minmax_lr: float = None,low_gpu_mem_usage: bool = False,- low_cpu_mem_usage: bool = False,+ low_cpu_mem_usage: int = 0,iters: int = 200,seqlen: int = 2048,nsamples: int = 128,diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex 79fbafb23..7ba4fb8d1 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -15,7 +15,7 @@import osfrom typing import Optional, Union-from lm_eval import simple_evaluate as lm_simple_evaluate+from lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0611os.environ["TOKENIZERS_PARALLELISM"] = "false"diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 52e97f557..1b042d31e 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -418,18 +418,13 @@ def tune(args):if model_name[-1] == "/":model_name = model_name[:-1]logger.info(f"start to quantize {model_name}")- torch_dtype = "auto"- if device_str is not None and "hpu" in device_str:- torch_dtype = torch.bfloat16from auto_round.utils import llm_load_modelmodel, tokenizer, low_cpu_mem_usage = llm_load_model(model_name,- torch_dtype=torch_dtype,- use_auto_mapping=use_auto_mapping,trust_remote_code=not args.disable_trust_remote_code,- device=device_str,+ device=args.device,low_cpu_mem_mode=args.low_cpu_mem_mode,low_cpu_mem_tmp_dir=args.low_cpu_mem_tmp_dir,model_dtype=args.model_dtype,@@ -440,9 +435,7 @@ def tune(args):if "bloom" in model_name:args.low_gpu_mem_usage = False- round = AutoRound- if args.adam:- round = AutoRoundAdam+ round = AutoRound if not args.adam else AutoRoundAdamlayer_config = {}not_quantize_layer_names = get_fp_layer_names(model, args.fp_layers)@@ -516,7 +509,7 @@ def tune(args):enable_minmax_tuning=not args.disable_minmax_tuning,act_bits=args.act_bits,act_group_size=args.act_group_size,- low_cpu_mem_usage=low_cpu_mem_usage,+ low_cpu_mem_usage=args.low_cpu_mem_mode,data_type=args.data_type,enable_norm_bias_tuning=args.enable_norm_bias_tuning,not_use_best_mse=args.not_use_best_mse,@@ -773,10 +766,10 @@ def eval_task_by_task(set_cuda_visible_devices(device)device_str, parallelism = get_device_and_parallelism(device)- # load after _eval_int in order to make sure import torch after set CUDA_VISBILE_DEVICES+ # load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICESimport traceback- from lm_eval import simple_evaluate as lm_simple_evaluate+ from lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0611from lm_eval.models.huggingface import HFLMfrom transformers import AutoModelForCausalLM, AutoTokenizer@@ -827,7 +820,7 @@ def eval_task_by_task(if isinstance(tasks, str):tasks = tasks.replace(" ", "").split(",")- from lm_eval.utils import make_table # pylint: disable=E0401+ from lm_eval.utils import make_table # pylint: disable=E0611res_all = {}res_keys = ["results", "versions", "n-shot", "higher_is_better"]diff --git a/auto_round/utils.py b/auto_round/utils.pyindex f7fe856c8..f1d160757 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -67,6 +67,8 @@ SUPPORTED_FORMATS = SupportedFormats()SUPPORTED_LAYER_TYPES = (torch.nn.Linear, transformers.pytorch_utils.Conv1D)+INNER_SUPPORTED_LAYER_TYPES = (transformers.integrations.finegrained_fp8.FP8Linear,)+SUPPORTED_DTYPES = ("int", "mx_fp", "fp", "nv_fp")@@ -571,6 +573,8 @@ def detect_device(device=None):return str(device)elif isinstance(device, torch.device):device = str(device)+ elif isinstance(device, str): ## for cuda:0+ device = devicereturn device@@ -1146,7 +1150,12 @@ def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=Nondef get_device_and_parallelism(device):from auto_round.utils import detect_device- devices = device.replace(" ", "").split(",")+ if isinstance(device, str):+ devices = device.replace(" ", "").split(",")+ elif isinstance(device, int):+ devices = str(device)+ else:+ devices = deviceif all(s.isdigit() for s in devices) and len(devices) > 1 and torch.cuda.is_available():device = "cuda"parallelism = True@@ -1302,10 +1311,21 @@ def _to_model_dtype(model, model_dtype):return model+def set_fake_cuda_device_capability(func=None):+ if func is not None:+ torch.cuda.get_device_capability = func+ return func++ def fake_cuda():+ return 100, 1++ orig_func = torch.cuda.get_device_capability+ torch.cuda.get_device_capability = fake_cuda+ return orig_func++def llm_load_model(pretrained_model_name_or_path,- torch_dtype="auto",- use_auto_mapping=True,trust_remote_code=True,model_dtype=None,device="cpu",@@ -1315,6 +1335,11 @@ def llm_load_model():from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer+ device_str, use_auto_mapping = get_device_and_parallelism(device)+ torch_dtype = "auto"+ if device_str is not None and "hpu" in device_str:+ torch_dtype = torch.bfloat16+is_glm = bool(re.search("chatglm", pretrained_model_name_or_path.lower()))low_cpu_mem_usage = False@@ -1365,6 +1390,19 @@ def llm_load_model(trust_remote_code=trust_remote_code,device_map="auto" if use_auto_mapping else None,)+ except ValueError as e:+ if "FP8 quantized" in str(e):+ orig_func = set_fake_cuda_device_capability()+ model = model_cls.from_pretrained(+ pretrained_model_name_or_path,+ torch_dtype=torch_dtype,+ trust_remote_code=trust_remote_code,+ device_map="auto" if use_auto_mapping else None,+ )+ torch.cuda.get_device_capability = orig_func+ model.is_fp8 = True ##tricky setting+ logger.warning("the support for fp8 model as input is experimental, please use with caution.")+except OSError as e:logger.warning(f"fail to load {pretrained_model_name_or_path}, set trust_remote_code to False and retry."@@ -2055,6 +2093,78 @@ def check_need_act_calibration(is_act_dynamic, act_data_type=None):return False+def dequant_block_fp8_weight(weight, weight_scale, block_size):+ dtype = torch.bfloat16+ if weight_scale is None:+ return weight+ assert len(block_size) == 2++ weight_shape_len = len(weight.shape)++ block_size_m, block_size_n = block_size++ # mul scale+ if weight_shape_len == 2:+ weight_scale_m, weight_scale_n = weight_scale.shape+ weight_scale = weight_scale.view(weight_scale_m, 1, weight_scale_n, 1)+ weight = weight.view(weight_scale_m, block_size_m, weight_scale_n, block_size_n)+ dequant_weight = weight.to(dtype) * weight_scale.to(dtype)+ dequant_weight = dequant_weight.view(weight_scale_m * block_size_m, weight_scale_n * block_size_n)+ elif weight_shape_len == 3:+ fd, weight_scale_m, weight_scale_n = weight_scale.shape+ weight_scale = weight_scale.view(fd, weight_scale_m, 1, weight_scale_n, 1)+ weight = weight.view(fd, weight_scale_m, block_size_m, weight_scale_n, block_size_n)+ dequant_weight = weight.to(dtype) * weight_scale.to(dtype)+ dequant_weight = dequant_weight.view(fd, weight_scale_m * block_size_m, weight_scale_n * block_size_n)+ else:+ raise ValueError("Only support original weight shape is either 2 or 3")++ return dequant_weight+++def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16):+ """ """+ new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)+ if layer.bias:+ new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))+ keys = get_quant_keys() + ["tmp_name"]+ for key in keys:+ setattr(new_layer, key, getattr(layer, key, None))+ dq_weight = dequant_block_fp8_weight(layer.weight, layer.weight_scale_inv, layer.block_size)+ new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))+ return new_layer+++def convert_fp8_model_to_16b_model(model, dtype=torch.bfloat16):+ """+ Convert a model with FP8 quantized layers to a model with 16-bit linear layers.+ This is useful for compatibility with other frameworks or for further processing.+ """+ for n, m in model.named_modules():+ if m.__class__.__name__ == "FP8Linear":+ new_module = convert_fp8_layer_to_linear(m, dtype=dtype)+ set_module(model, n, new_module)+ return model+++def get_quant_keys():+ keys = [+ "bits",+ "group_size",+ "sym",+ "data_type",+ "scale_dtype",+ "act_bits",+ "act_group_size",+ "act_sym",+ "act_dynamic",+ "act_data_type",+ "super_bits",+ "super_group_size",+ ]+ return keys++def out_of_vram(error_msg):error_msg = str(error_msg)# CUDAdiff --git a/requirements.txt b/requirements.txtindex 13809cc16..282f3a5b6 100644--- a/requirements.txt+++ b/requirements.txt@@ -8,6 +8,7 @@ packagingpillownumbatbb+triton ## for fp8 modeltorchtransformers>=4.38threadpoolctldiff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex 94097d01b..84cf50f7e 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -596,6 +596,16 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()+ def test_fp8_model_input_rtn_generation(self):+ model_name = "Qwen/Qwen3-0.6B-FP8"+ ar = AutoRound(model=model_name, iters=0)+ ar.quantize_and_save(output_dir=self.save_folder)+ model = AutoModelForCausalLM.from_pretrained(self.save_folder, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+if __name__ == "__main__":unittest.main()diff --git a/test/test_cuda/test_fp8_input.py b/test/test_cuda/test_fp8_input.pynew file mode 100644index 000000000..00af4ee14--- /dev/null+++ b/test/test_cuda/test_fp8_input.py@@ -0,0 +1,101 @@+import os+import shutil+import sys+import unittest++from auto_round.eval.evaluation import simple_evaluate++sys.path.insert(0, "../..")+import torch+import transformers+from transformers import AutoModelForCausalLM, AutoTokenizer++from auto_round import AutoRound+++class TestAutoRound(unittest.TestCase):+ @classmethod+ def setUpClass(self):+ self.save_dir = "./saved"++ @classmethod+ def tearDownClass(self):+ shutil.rmtree(self.save_dir, ignore_errors=True)+ shutil.rmtree("runs", ignore_errors=True)++ def test_small_model_rtn_generation(self):+ model_name = "/models/Qwen3-0.6B-FP8"+ ar = AutoRound(model=model_name, iters=0)+ ar.quantize_and_save(output_dir=self.save_dir)+ model = AutoModelForCausalLM.from_pretrained(self.save_dir, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree(self.save_dir, ignore_errors=True)++ def test_gguf_imatrix(self):+ model_name = "/models/Qwen3-0.6B-FP8"+ ar = AutoRound(model=model_name, iters=0)+ ar.quantize_and_save(format="gguf:q2_k_s", output_dir=self.save_dir)+ # from llama_cpp import Llama+ #+ # gguf_file = os.listdir("saved/Qwen3-0.6B-FP8/-gguf")[0]+ # llm = Llama(f"saved/Qwen2.5-0.5B-Instruct-gguf/{gguf_file}", n_gpu_layers=-1)+ # output = llm("There is a girl who likes adventure,", max_tokens=32)+ # print(output)+ # shutil.rmtree("./saved", ignore_errors=True)+ # model = AutoModelForCausalLM.from_pretrained(self.save_dir, torch_dtype="auto", trust_remote_code=True)+ # tokenizer = AutoTokenizer.from_pretrained(self.save_dir)+ # text = "There is a girl who likes adventure,"+ # inputs = tokenizer(text, return_tensors="pt").to(model.device)+ # print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))++ def test_small_model_rtn(self):+ model_name = "/models/Qwen3-0.6B-FP8"+ ar = AutoRound(model=model_name, iters=0)+ _, folder = ar.quantize_and_save(output_dir=self.save_dir)+ model_args = f"pretrained={self.save_dir}"+ result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.25)++ shutil.rmtree(self.save_dir, ignore_errors=True)++ def test_small_model_iters1(self):+ model_name = "/models/Qwen3-0.6B-FP8"+ ar = AutoRound(model=model_name, iters=1)+ _, folder = ar.quantize_and_save(output_dir=self.save_dir)+ model_args = f"pretrained={self.save_dir}"+ result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.25)++ shutil.rmtree(self.save_dir, ignore_errors=True)++ def test_medium_model_rtn(self):+ model_name = "/models/Qwen3-8B-FP8"+ ar = AutoRound(model=model_name, iters=0)+ _, folder = ar.quantize_and_save(output_dir=self.save_dir)+ model_args = f"pretrained={self.save_dir}"+ result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.55)++ shutil.rmtree(self.save_dir, ignore_errors=True)++ def test_medium_model_rtn_with_lm_head(self):+ model_name = "/models/Qwen3-8B-FP8"+ layer_config = {"lm_head": {"bits": 4}}+ ar = AutoRound(model=model_name, iters=0, layer_config=layer_config)+ _, folder = ar.quantize_and_save(output_dir=self.save_dir)+ model_args = f"pretrained={self.save_dir}"+ result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")+ print(result["results"]["lambada_openai"]["acc,none"])+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.55)++ shutil.rmtree(self.save_dir, ignore_errors=True)+++if __name__ == "__main__":+ unittest.main()