Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support fp8 model and str as input in llm quantization (#699)

Wenhua Cheng32777f7327f0
diff --git a/README.md b/README.md
index a2c5397d9..0e5c3539f 100644
--- a/README.md
+++ b/README.md
@@ -29,10 +29,10 @@ and [fbaldassarri](https://huggingface.co/fbaldassarri).
## 🆕 What's New
[2025/07] AutoRound now offers experimental support for **GGUF** format, and recommends using optimized RTN mode (--iters 0) for
- all bits other than 3 bits. **A more advanced algorithm** tailored for specific configurations may be available in
- v0.6.1. Example
+ all bits other than 3 bits. Example
models: [Intel/Qwen3-235B-A22B-q2ks-mixed-ar](https://huggingface.co/Intel/Qwen3-235B-A22B-q2ks-ar)
- and [Intel/DeepSeek-R1-0528-q2ks-mixed-ar](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-ar).
+ and [Intel/DeepSeek-R1-0528-q2ks-mixed-ar](https://huggingface.co/Intel/DeepSeek-R1-0528-q2ks-mixed-ar). **A more advanced algorithm** tailored for specific configurations may be available in
+ v0.6.1.
[2025/05] AutoRound provides some recipes for **DeepSeek-R1-0528**, please refer
to [Intel/DeepSeek-R1-0528-int2-mixed-ar](https://huggingface.co/Intel/DeepSeek-R1-0528-int2-mixed-ar), [Intel/DeepSeek-R1-0528-int4-ar](https://huggingface.co/Intel/DeepSeek-R1-0528-int4-ar)
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index bd3237d7a..f6d6b6ea2 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -31,6 +31,7 @@ from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType
from auto_round.low_cpu_mem.utils import get_layers_before_block
from auto_round.utils import (
+ INNER_SUPPORTED_LAYER_TYPES,
SUPPORTED_DTYPES,
SUPPORTED_LAYER_TYPES,
TORCH_VERSION_AT_LEAST_2_6,
@@ -46,6 +47,8 @@ from auto_round.utils import (
collect_best_params,
compile_func,
convert_dtype_str2torch,
+ convert_fp8_layer_to_linear,
+ convert_fp8_model_to_16b_model,
detect_device,
find_matching_blocks,
flatten_list,
@@ -55,6 +58,7 @@ from auto_round.utils import (
get_layer_names_in_block,
get_lm_head_name,
get_module,
+ get_quant_keys,
get_shared_keys,
htcore,
infer_bits_by_data_type,
@@ -141,15 +145,15 @@ class AutoRound(object):
def __init__(
self,
- model: torch.nn.Module,
- tokenizer,
+ model: Union[torch.nn.Module, str],
+ tokenizer=None,
bits: int = 4,
group_size: int = 128,
sym: bool = True,
layer_config: dict = None,
batch_size: int = 8,
amp: bool = True,
- device: str = None,
+ device: Union[str, torch.device, int] = 0,
lr_scheduler=None,
dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
enable_quanted_input: bool = True,
@@ -157,7 +161,7 @@ class AutoRound(object):
lr: float = None,
minmax_lr: float = None,
low_gpu_mem_usage: bool = False,
- low_cpu_mem_usage: bool = False,
+ low_cpu_mem_usage: int = 0,
iters: int = 200,
seqlen: int = 2048,
nsamples: int = 128,
@@ -188,14 +192,28 @@ class AutoRound(object):
if kwargs:
logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")
self.quantized = False
- self.model_orig_dtype = model.dtype
self.seed = seed
set_seed(self.seed)
+ if device is not None and "," in str(device):
+ raise ValueError(
+ "API does not support explicit set multiple devices,"
+ " please set CUDA_VISIBLE_DEVICES yourself and use `device=auto` instead"
+ )
+
+ if isinstance(model, str):
+ model, tokenizer, low_cpu_mem_usage = llm_load_model(
+ model, device=device, low_cpu_mem_mode=low_cpu_mem_usage
+ )
+ elif tokenizer is None and iters > 0:
+ raise ValueError("A tokenizer must be set for non-str model input")
+ self.low_cpu_mem_usage = bool(low_cpu_mem_usage)
if unsupport_meta_device(model):
raise RuntimeError(
"AutoRound does not support parameters on meta device. "
"Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."
)
+ # self.model_orig_dtype = model.dtype
+ self.device = detect_device(device) ##must place after llm_load_model, because this one will convert auto
## important tuning hype-parameters
self.amp = amp
@@ -208,7 +226,7 @@ class AutoRound(object):
self.sym = sym
self.low_gpu_mem_usage = low_gpu_mem_usage
- self.low_cpu_mem_usage = low_cpu_mem_usage
+
self.layer_config = {} if layer_config is None else layer_config
self.seqlen = seqlen
self.batch_size, self.gradient_accumulate_steps = batch_size, gradient_accumulate_steps
@@ -232,9 +250,10 @@ class AutoRound(object):
)
self.bits = tmp_bits
self.supported_types = SUPPORTED_LAYER_TYPES
+ self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES
self.model = model.eval()
self.tokenizer = tokenizer
- self.device = detect_device(device)
+
self.scale_dtype = convert_dtype_str2torch(scale_dtype)
self.set_amp_dtype()
self.to_quant_block_names = to_quant_block_names
@@ -837,7 +856,12 @@ class AutoRound(object):
# Load dataset
from .calib_dataset import get_dataloader
+ if hasattr(self.model, "is_fp8"):
+ convert_fp8_model_to_16b_model(self.model, self.amp_dtype)
+
if isinstance(self.dataset, str):
+ if self.tokenizer is None:
+ raise ValueError("A tokenizer must be set for the model when using a dataset string.")
dataset_name = self.dataset.replace(" ", "")
self.dataloader = get_dataloader(
self.tokenizer, self.seqlen, dataset_name, self.seed, self.batch_size, self.nsamples
@@ -1063,6 +1087,10 @@ class AutoRound(object):
"""
m = get_module(self.model, name)
+ if m.__class__.__name__ == "FP8Linear":
+ m = convert_fp8_layer_to_linear(m, self.amp_dtype)
+ set_module(self.model, name, m)
+
# Step 1: Use optimized RTN data type if available
if not self.disable_opt_rtn and not m.data_type.startswith("rtn_"):
from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
@@ -1192,7 +1220,9 @@ class AutoRound(object):
clear_memory()
cnt = 1
cnt += 1
-
+ ##convert remainning fp8
+ if hasattr(self.model, "is_fp8"):
+ convert_fp8_model_to_16b_model(self.model, self.amp_dtype)
self.quantized = True
return self.model, self.layer_config
@@ -1250,6 +1280,8 @@ class AutoRound(object):
pbar.set_description(f"Quantizing {block_name}")
block = get_module(self.model, block_name)
block = block.to(self.device)
+ if hasattr(self.model, "is_fp8"):
+ convert_fp8_model_to_16b_model(block, dtype=self.amp_dtype)
# Dispatch model if needed
if self.device_map is not None:
from accelerate import dispatch_model
@@ -1401,7 +1433,7 @@ class AutoRound(object):
self.batch_size = total_samples
logger.warning(f"force the train batch size to {total_samples}")
- self.quant_blocks(
+ self.quantize_blocks(
self.model,
inputs,
block_names,
@@ -1419,6 +1451,12 @@ class AutoRound(object):
self.quant_layers(layer_names, all_inputs) ##TODO pack layer immediately
+ if hasattr(self.model, "is_fp8"):
+ for n, m in self.model.named_modules():
+ if m.__class__.__name__ == "FP8Linear":
+ new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype).to("cpu")
+ set_module(self.model, n, new_layer)
+
end_time = time.time()
cost_time = end_time - self.start_time
logger.info(f"quantization tuning time {cost_time}")
@@ -1456,17 +1494,22 @@ class AutoRound(object):
"""
##TODO currently we take all the layers outside blocks as post block layers which is not optimal
## if there is no input for layer, we use rtn
+
for layer_name in copy.deepcopy(layer_names):
if layer_name not in layer_inputs:
logger.info(f"using rtn to quantize {layer_name}")
from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
layer = get_module(self.model, layer_name)
+ if layer.__class__.__name__ == "FP8Linear":
+ new_layer = convert_fp8_layer_to_linear(layer, self.amp_dtype).to(self.device)
+ set_module(self.model, layer_name, new_layer)
+ layer = new_layer
+
if not self.disable_opt_rtn and "rtn_" + layer.data_type in QUANT_FUNC_WITH_DTYPE:
layer.data_type = "rtn_" + layer.data_type
logger.info("using optimized rtn method for quantizing %s", layer_name)
self.layer_config[layer_name]["data_type"] = layer.data_type
- layer.to(self.device)
wrapper_layer = WrapperLinear(
layer,
enable_round_tuning=False,
@@ -1529,24 +1572,12 @@ class AutoRound(object):
otherwise returns False.
"""
# Get the names of layers in quantization blocks
- layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)
+ supported_types = self.supported_types + self.inner_supported_types
+ layers_in_blocks = get_layer_names_in_block(self.model, supported_types, self.quant_block_list)
##process regex in layer_config
all_supported_layer_names = []
# List of configuration keys
- keys = [
- "bits",
- "group_size",
- "sym",
- "data_type",
- "scale_dtype",
- "act_bits",
- "act_group_size",
- "act_sym",
- "act_dynamic",
- "act_data_type",
- "super_bits",
- "super_group_size",
- ]
+ keys = get_quant_keys()
for n, m in self.model.named_modules():
# Delete previous configuration to avoid conflicts with prior tuning
@@ -1554,9 +1585,6 @@ class AutoRound(object):
if hasattr(m, key):
delattr(m, key)
- # Skip unsupported types
- supported_types = self.supported_types
-
if not isinstance(m, supported_types):
continue
all_supported_layer_names.append(n)
@@ -1794,6 +1822,8 @@ class AutoRound(object):
Raises:
Exception: If caching on GPU fails, switches to CPU and caches there.
"""
+ if hasattr(self.model, "is_fp8"):
+ layer_names = []
if layer_names is None:
layer_names = []
@@ -2211,7 +2241,7 @@ class AutoRound(object):
continue
return hook_handles
- def quant_block(self, block, input_ids, input_others, q_input=None, device=torch.device("cpu")):
+ def quantize_block(self, block, input_ids, input_others, q_input=None, device=torch.device("cpu")):
"""Quantize the weights of a given block of the model.
Args:
@@ -2224,6 +2254,12 @@ class AutoRound(object):
Returns:
Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)
"""
+ if hasattr(self.model, "is_fp8"):
+ for n, m in block.named_modules():
+ if m.__class__.__name__ == "FP8Linear":
+ new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype).to(device)
+ set_module(block, n, new_layer)
+
if self.device_map is not None:
from accelerate import dispatch_model
@@ -2421,7 +2457,7 @@ class AutoRound(object):
clear_memory(input_ids)
return None, output
- def quant_blocks(
+ def quantize_blocks(
self, model: torch.nn.Module, inputs, block_names, q_input=None, nblocks=1, device="cpu", pbar=None
):
"""Quantize and dequantize the weights of the specified blocks in the model.
@@ -2459,9 +2495,9 @@ class AutoRound(object):
for i in range(len(input_others[key])):
to_dtype(input_others[key][i], tmp_dtype)
if self.enable_torch_compile:
- quant_block = compile_func(self.quant_block, device)
+ quant_block = compile_func(self.quantize_block, device)
else:
- quant_block = self.quant_block
+ quant_block = self.quantize_block
if pbar is None:
pbar = tqdm(range(0, len(block_names), nblocks))
@@ -2806,7 +2842,7 @@ class AutoRoundOPT(AutoRound):
def __init__(
self,
- model,
+ model: Union[torch.nn.Module, str],
tokenizer=None,
bits: int = 4,
group_size: int = 128,
@@ -2814,7 +2850,7 @@ class AutoRoundOPT(AutoRound):
layer_config=None,
batch_size: int = 8,
amp: bool = True,
- device=None,
+ device: Union[str, torch.device, int] = 0,
lr_scheduler=None,
dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
enable_quanted_input: bool = True,
@@ -2822,7 +2858,7 @@ class AutoRoundOPT(AutoRound):
lr: float = None,
minmax_lr: float = None,
low_gpu_mem_usage: bool = False,
- low_cpu_mem_usage: bool = False,
+ low_cpu_mem_usage: int = 0,
iters: int = 200,
seqlen: int = 2048,
nsamples: int = 128,
@@ -2985,7 +3021,7 @@ class AutoRoundAdam(AutoRoundOPT):
def __init__(
self,
- model,
+ model: Union[torch.nn.Module, str],
tokenizer=None,
bits: int = 4,
group_size: int = 128,
@@ -2993,7 +3029,7 @@ class AutoRoundAdam(AutoRoundOPT):
layer_config=None,
batch_size: int = 8,
amp: bool = True,
- device=None,
+ device: Union[str, torch.device, int] = 0,
lr_scheduler=None,
dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
enable_quanted_input: bool = True,
@@ -3001,7 +3037,7 @@ class AutoRoundAdam(AutoRoundOPT):
lr: float = None,
minmax_lr: float = None,
low_gpu_mem_usage: bool = False,
- low_cpu_mem_usage: bool = False,
+ low_cpu_mem_usage: int = 0,
iters: int = 200,
seqlen: int = 2048,
nsamples: int = 128,
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index 79fbafb23..7ba4fb8d1 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -15,7 +15,7 @@
import os
from typing import Optional, Union
-from lm_eval import simple_evaluate as lm_simple_evaluate
+from lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0611
os.environ["TOKENIZERS_PARALLELISM"] = "false"
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 52e97f557..1b042d31e 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -418,18 +418,13 @@ def tune(args):
if model_name[-1] == "/":
model_name = model_name[:-1]
logger.info(f"start to quantize {model_name}")
- torch_dtype = "auto"
- if device_str is not None and "hpu" in device_str:
- torch_dtype = torch.bfloat16
from auto_round.utils import llm_load_model
model, tokenizer, low_cpu_mem_usage = llm_load_model(
model_name,
- torch_dtype=torch_dtype,
- use_auto_mapping=use_auto_mapping,
trust_remote_code=not args.disable_trust_remote_code,
- device=device_str,
+ device=args.device,
low_cpu_mem_mode=args.low_cpu_mem_mode,
low_cpu_mem_tmp_dir=args.low_cpu_mem_tmp_dir,
model_dtype=args.model_dtype,
@@ -440,9 +435,7 @@ def tune(args):
if "bloom" in model_name:
args.low_gpu_mem_usage = False
- round = AutoRound
- if args.adam:
- round = AutoRoundAdam
+ round = AutoRound if not args.adam else AutoRoundAdam
layer_config = {}
not_quantize_layer_names = get_fp_layer_names(model, args.fp_layers)
@@ -516,7 +509,7 @@ def tune(args):
enable_minmax_tuning=not args.disable_minmax_tuning,
act_bits=args.act_bits,
act_group_size=args.act_group_size,
- low_cpu_mem_usage=low_cpu_mem_usage,
+ low_cpu_mem_usage=args.low_cpu_mem_mode,
data_type=args.data_type,
enable_norm_bias_tuning=args.enable_norm_bias_tuning,
not_use_best_mse=args.not_use_best_mse,
@@ -773,10 +766,10 @@ def eval_task_by_task(
set_cuda_visible_devices(device)
device_str, parallelism = get_device_and_parallelism(device)
- # load after _eval_int in order to make sure import torch after set CUDA_VISBILE_DEVICES
+ # load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICES
import traceback
- from lm_eval import simple_evaluate as lm_simple_evaluate
+ from lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0611
from lm_eval.models.huggingface import HFLM
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -827,7 +820,7 @@ def eval_task_by_task(
if isinstance(tasks, str):
tasks = tasks.replace(" ", "").split(",")
- from lm_eval.utils import make_table # pylint: disable=E0401
+ from lm_eval.utils import make_table # pylint: disable=E0611
res_all = {}
res_keys = ["results", "versions", "n-shot", "higher_is_better"]
diff --git a/auto_round/utils.py b/auto_round/utils.py
index f7fe856c8..f1d160757 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -67,6 +67,8 @@ SUPPORTED_FORMATS = SupportedFormats()
SUPPORTED_LAYER_TYPES = (torch.nn.Linear, transformers.pytorch_utils.Conv1D)
+INNER_SUPPORTED_LAYER_TYPES = (transformers.integrations.finegrained_fp8.FP8Linear,)
+
SUPPORTED_DTYPES = ("int", "mx_fp", "fp", "nv_fp")
@@ -571,6 +573,8 @@ def detect_device(device=None):
return str(device)
elif isinstance(device, torch.device):
device = str(device)
+ elif isinstance(device, str): ## for cuda:0
+ device = device
return device
@@ -1146,7 +1150,12 @@ def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=Non
def get_device_and_parallelism(device):
from auto_round.utils import detect_device
- devices = device.replace(" ", "").split(",")
+ if isinstance(device, str):
+ devices = device.replace(" ", "").split(",")
+ elif isinstance(device, int):
+ devices = str(device)
+ else:
+ devices = device
if all(s.isdigit() for s in devices) and len(devices) > 1 and torch.cuda.is_available():
device = "cuda"
parallelism = True
@@ -1302,10 +1311,21 @@ def _to_model_dtype(model, model_dtype):
return model
+def set_fake_cuda_device_capability(func=None):
+ if func is not None:
+ torch.cuda.get_device_capability = func
+ return func
+
+ def fake_cuda():
+ return 100, 1
+
+ orig_func = torch.cuda.get_device_capability
+ torch.cuda.get_device_capability = fake_cuda
+ return orig_func
+
+
def llm_load_model(
pretrained_model_name_or_path,
- torch_dtype="auto",
- use_auto_mapping=True,
trust_remote_code=True,
model_dtype=None,
device="cpu",
@@ -1315,6 +1335,11 @@ def llm_load_model(
):
from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer
+ device_str, use_auto_mapping = get_device_and_parallelism(device)
+ torch_dtype = "auto"
+ if device_str is not None and "hpu" in device_str:
+ torch_dtype = torch.bfloat16
+
is_glm = bool(re.search("chatglm", pretrained_model_name_or_path.lower()))
low_cpu_mem_usage = False
@@ -1365,6 +1390,19 @@ def llm_load_model(
trust_remote_code=trust_remote_code,
device_map="auto" if use_auto_mapping else None,
)
+ except ValueError as e:
+ if "FP8 quantized" in str(e):
+ orig_func = set_fake_cuda_device_capability()
+ model = model_cls.from_pretrained(
+ pretrained_model_name_or_path,
+ torch_dtype=torch_dtype,
+ trust_remote_code=trust_remote_code,
+ device_map="auto" if use_auto_mapping else None,
+ )
+ torch.cuda.get_device_capability = orig_func
+ model.is_fp8 = True ##tricky setting
+ logger.warning("the support for fp8 model as input is experimental, please use with caution.")
+
except OSError as e:
logger.warning(
f"fail to load {pretrained_model_name_or_path}, set trust_remote_code to False and retry."
@@ -2055,6 +2093,78 @@ def check_need_act_calibration(is_act_dynamic, act_data_type=None):
return False
+def dequant_block_fp8_weight(weight, weight_scale, block_size):
+ dtype = torch.bfloat16
+ if weight_scale is None:
+ return weight
+ assert len(block_size) == 2
+
+ weight_shape_len = len(weight.shape)
+
+ block_size_m, block_size_n = block_size
+
+ # mul scale
+ if weight_shape_len == 2:
+ weight_scale_m, weight_scale_n = weight_scale.shape
+ weight_scale = weight_scale.view(weight_scale_m, 1, weight_scale_n, 1)
+ weight = weight.view(weight_scale_m, block_size_m, weight_scale_n, block_size_n)
+ dequant_weight = weight.to(dtype) * weight_scale.to(dtype)
+ dequant_weight = dequant_weight.view(weight_scale_m * block_size_m, weight_scale_n * block_size_n)
+ elif weight_shape_len == 3:
+ fd, weight_scale_m, weight_scale_n = weight_scale.shape
+ weight_scale = weight_scale.view(fd, weight_scale_m, 1, weight_scale_n, 1)
+ weight = weight.view(fd, weight_scale_m, block_size_m, weight_scale_n, block_size_n)
+ dequant_weight = weight.to(dtype) * weight_scale.to(dtype)
+ dequant_weight = dequant_weight.view(fd, weight_scale_m * block_size_m, weight_scale_n * block_size_n)
+ else:
+ raise ValueError("Only support original weight shape is either 2 or 3")
+
+ return dequant_weight
+
+
+def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16):
+ """ """
+ new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)
+ if layer.bias:
+ new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))
+ keys = get_quant_keys() + ["tmp_name"]
+ for key in keys:
+ setattr(new_layer, key, getattr(layer, key, None))
+ dq_weight = dequant_block_fp8_weight(layer.weight, layer.weight_scale_inv, layer.block_size)
+ new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
+ return new_layer
+
+
+def convert_fp8_model_to_16b_model(model, dtype=torch.bfloat16):
+ """
+ Convert a model with FP8 quantized layers to a model with 16-bit linear layers.
+ This is useful for compatibility with other frameworks or for further processing.
+ """
+ for n, m in model.named_modules():
+ if m.__class__.__name__ == "FP8Linear":
+ new_module = convert_fp8_layer_to_linear(m, dtype=dtype)
+ set_module(model, n, new_module)
+ return model
+
+
+def get_quant_keys():
+ keys = [
+ "bits",
+ "group_size",
+ "sym",
+ "data_type",
+ "scale_dtype",
+ "act_bits",
+ "act_group_size",
+ "act_sym",
+ "act_dynamic",
+ "act_data_type",
+ "super_bits",
+ "super_group_size",
+ ]
+ return keys
+
+
def out_of_vram(error_msg):
error_msg = str(error_msg)
# CUDA
diff --git a/requirements.txt b/requirements.txt
index 13809cc16..282f3a5b6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,6 +8,7 @@ packaging
pillow
numba
tbb
+triton ## for fp8 model
torch
transformers>=4.38
threadpoolctl
diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.py
index 94097d01b..84cf50f7e 100644
--- a/test/test_cpu/test_autoround.py
+++ b/test/test_cpu/test_autoround.py
@@ -596,6 +596,16 @@ class TestAutoRound(unittest.TestCase):
)
autoround.quantize()
+ def test_fp8_model_input_rtn_generation(self):
+ model_name = "Qwen/Qwen3-0.6B-FP8"
+ ar = AutoRound(model=model_name, iters=0)
+ ar.quantize_and_save(output_dir=self.save_folder)
+ model = AutoModelForCausalLM.from_pretrained(self.save_folder, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/test_cuda/test_fp8_input.py b/test/test_cuda/test_fp8_input.py
new file mode 100644
index 000000000..00af4ee14
--- /dev/null
+++ b/test/test_cuda/test_fp8_input.py
@@ -0,0 +1,101 @@
+import os
+import shutil
+import sys
+import unittest
+
+from auto_round.eval.evaluation import simple_evaluate
+
+sys.path.insert(0, "../..")
+import torch
+import transformers
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from auto_round import AutoRound
+
+
+class TestAutoRound(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.save_dir = "./saved"
+
+ @classmethod
+ def tearDownClass(self):
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_small_model_rtn_generation(self):
+ model_name = "/models/Qwen3-0.6B-FP8"
+ ar = AutoRound(model=model_name, iters=0)
+ ar.quantize_and_save(output_dir=self.save_dir)
+ model = AutoModelForCausalLM.from_pretrained(self.save_dir, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.save_dir)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+ def test_gguf_imatrix(self):
+ model_name = "/models/Qwen3-0.6B-FP8"
+ ar = AutoRound(model=model_name, iters=0)
+ ar.quantize_and_save(format="gguf:q2_k_s", output_dir=self.save_dir)
+ # from llama_cpp import Llama
+ #
+ # gguf_file = os.listdir("saved/Qwen3-0.6B-FP8/-gguf")[0]
+ # llm = Llama(f"saved/Qwen2.5-0.5B-Instruct-gguf/{gguf_file}", n_gpu_layers=-1)
+ # output = llm("There is a girl who likes adventure,", max_tokens=32)
+ # print(output)
+ # shutil.rmtree("./saved", ignore_errors=True)
+ # model = AutoModelForCausalLM.from_pretrained(self.save_dir, torch_dtype="auto", trust_remote_code=True)
+ # tokenizer = AutoTokenizer.from_pretrained(self.save_dir)
+ # text = "There is a girl who likes adventure,"
+ # inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ # print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+
+ def test_small_model_rtn(self):
+ model_name = "/models/Qwen3-0.6B-FP8"
+ ar = AutoRound(model=model_name, iters=0)
+ _, folder = ar.quantize_and_save(output_dir=self.save_dir)
+ model_args = f"pretrained={self.save_dir}"
+ result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")
+ print(result["results"]["lambada_openai"]["acc,none"])
+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.25)
+
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+ def test_small_model_iters1(self):
+ model_name = "/models/Qwen3-0.6B-FP8"
+ ar = AutoRound(model=model_name, iters=1)
+ _, folder = ar.quantize_and_save(output_dir=self.save_dir)
+ model_args = f"pretrained={self.save_dir}"
+ result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")
+ print(result["results"]["lambada_openai"]["acc,none"])
+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.25)
+
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+ def test_medium_model_rtn(self):
+ model_name = "/models/Qwen3-8B-FP8"
+ ar = AutoRound(model=model_name, iters=0)
+ _, folder = ar.quantize_and_save(output_dir=self.save_dir)
+ model_args = f"pretrained={self.save_dir}"
+ result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")
+ print(result["results"]["lambada_openai"]["acc,none"])
+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.55)
+
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+ def test_medium_model_rtn_with_lm_head(self):
+ model_name = "/models/Qwen3-8B-FP8"
+ layer_config = {"lm_head": {"bits": 4}}
+ ar = AutoRound(model=model_name, iters=0, layer_config=layer_config)
+ _, folder = ar.quantize_and_save(output_dir=self.save_dir)
+ model_args = f"pretrained={self.save_dir}"
+ result = simple_evaluate(model="hf", model_args=model_args, tasks="lambada_openai", batch_size="auto")
+ print(result["results"]["lambada_openai"]["acc,none"])
+ self.assertGreater(result["results"]["lambada_openai"]["acc,none"], 0.55)
+
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+
+if __name__ == "__main__":
+ unittest.main()