Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Enable mxfp exporting (#649)

Weiwei64b8afee0922
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 3c0b07247..854f3f460 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -64,7 +64,10 @@ from auto_round.utils import (
infer_bits_by_data_type,
init_cache,
is_debug_mode,
+ is_mx_fp,
+ is_nv_fp,
is_optimum_habana_available,
+ is_standard_fp,
llm_load_model,
logger,
mv_module_from_gpu,
@@ -480,16 +483,10 @@ class AutoRound(object):
"And please save the quantized model to fake format as real deployment is not supported currently"
)
- if "mx_fp" in self.data_type or "nv_fp" in self.data_type:
- logger.warning(
- "please save the quantized model to fake format "
- "as real deployment is not supported for mx_fp/nv_fp datatype currently"
- )
-
- if "mx_fp" in self.data_type and self.group_size != 32:
+ if is_mx_fp(self.data_type) and self.group_size != 32:
logger.warning("mx_fp should only support group_size of 32 in real deployment")
- if "nv_fp" in self.data_type and (self.group_size != 16):
+ if is_nv_fp(self.data_type) and (self.group_size != 16):
logger.warning("nv_fp should only support group_size of 16 in real deployment")
if self.nsamples < self.gradient_accumulate_steps * self.batch_size:
@@ -620,9 +617,20 @@ class AutoRound(object):
)
if enable_awq:
formats[index] = format.replace("auto_round", "auto_round:auto_awq")
- if "fp8" in self.data_type:
+ if is_nv_fp(self.data_type) or is_mx_fp(self.data_type) or is_standard_fp(self.data_type):
format = format.replace("auto_round", f"auto_round:{self.data_type}")
formats[index] = format
+ elif format == "llmcompressor":
+ from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported
+
+ if check_compressed_tensors_supported() and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):
+ format = format.replace("llmcompressor", f"llmcompressor:{self.data_type}")
+ formats[index] = format
+ elif not is_standard_fp(self.data_type):
+ logger.error(
+ "Currently, the llmcompressor format only supports MXFP/NVFP/StandardFP8. "
+ "Please change format to fake or auto_round etc."
+ )
# Remove duplicates from formats list
def remove_duplicates(lst):
@@ -651,10 +659,12 @@ class AutoRound(object):
if format == "fake":
return format
format = format.replace("q*_", f"q{self.bits}_")
- # only support to export afp8
+ # only support to export afp8/nv_fp
if self.act_bits <= 8:
- if "fp8" not in self.act_data_type or self.act_dynamic:
+ if not is_standard_fp(self.act_data_type) or self.act_dynamic:
if format == "llmcompressor":
+ if is_nv_fp(self.act_data_type):
+ return format
bits, group_size, sym, act_bits = 8, -1, True, 8
assert (
self.bits == bits
@@ -667,10 +677,10 @@ class AutoRound(object):
f" W{self.bits}A{self.act_bits} model, but got bits={self.bits},"
f" group_size={self.group_size}, sym={self.sym}, act_bits={self.act_bits}"
)
- elif format != "fake":
+ elif format != "fake" and not is_nv_fp(format):
logger.warning(
"Currently only support to export auto_round format quantized model"
- " with fp8 dtype activation for activation quantization."
+ " with fp8 or nv_fp4 dtype activation for activation quantization."
" Change format to fake and save."
)
format = "fake"
@@ -1130,6 +1140,7 @@ class AutoRound(object):
m.to("cpu")
except RuntimeError as e:
cuda_error_msg = traceback.format_exc()
+ m = m.orig_layer if hasattr(m, "orig_layer") else m
try:
logger.warning("Out of VRAM, falling back to CPU.")
m.to("cpu")
@@ -1191,6 +1202,21 @@ class AutoRound(object):
all_to_quantized_module_names: list[str] = [n for n, m in self.model.named_modules() if check_to_quantized(m)]
+ if is_nv_fp(self.data_type):
+ from auto_round.data_type.nvfp import calculate_gparam
+ from auto_round.data_type.utils import update_fused_layer_global_scales
+
+ pbar = tqdm(all_to_quantized_module_names)
+ for name in pbar:
+ pbar.set_description(f"Calculate weight global scale: {name}")
+ m = get_module(self.model, name)
+ weight_global_scale = calculate_gparam(m.weight, self.group_size)
+ setattr(m, "weight_global_scale", weight_global_scale)
+
+ modules = list(self.model.modules())
+ for module in tqdm(modules, desc="Update weight global scale for fuse module"):
+ update_fused_layer_global_scales(module)
+
has_gguf_k = any("gguf" in fmt and "k" in fmt for fmt in getattr(self, "formats", []))
self.quantize_embedding_layer()
@@ -1318,6 +1344,11 @@ class AutoRound(object):
if self.device_map is not None:
accelerate.hooks.remove_hook_from_submodules(block)
+ if is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats):
+ from auto_round.utils import set_amax_for_all_moe_layers
+
+ # enable moe experts act_max automatic generation for linears
+ set_amax_for_all_moe_layers(block, attr_name="act_max")
# Normalize imatrix and quantize layers
for _, m in block.named_modules():
if hasattr(m, "imatrix"):
@@ -1374,7 +1405,13 @@ class AutoRound(object):
formats = self.formats
if (
len(formats) == 1
- and ("awq" in formats[0] or "gptq" in formats[0] or "auto_round" in formats[0] or "gguf" in formats[0])
+ and (
+ "awq" in formats[0]
+ or "gptq" in formats[0]
+ or "auto_round" in formats[0]
+ or "gguf" in formats[0]
+ or "llmcompressor" in formats[0]
+ )
and self.inplace
):
self.is_packing_immediate = True
@@ -2217,16 +2254,23 @@ class AutoRound(object):
logger.info(dump_info)
def register_act_max_hook(self, model):
-
def get_act_max_hook(module, input, output):
if isinstance(input, (tuple, list)):
input = input[0]
+ if input.numel() == 0:
+ return # as no needs for act_max update
input, _, _ = reshape_pad_tensor_by_group_size(input, self.act_group_size)
act_max = torch.max(torch.abs(input), dim=-1).values
- if not hasattr(module, "act_max"):
+ if not hasattr(module, "act_max") or module.act_max.numel() == 0:
module.act_max = act_max
else:
- module.act_max = torch.max(act_max.to(module.act_max.device), module.act_max)
+ act_max = act_max.to(module.act_max.device)
+ if is_nv_fp(self.data_type): ## for nvfp per-tensor input_global_scale calculation usage
+ module.act_max = torch.max(
+ torch.tensor([act_max.max(), module.act_max.max()], device=act_max.device)
+ )
+ else:
+ module.act_max = torch.max(act_max, module.act_max)
hook_handles = []
@@ -2323,7 +2367,12 @@ class AutoRound(object):
quantized_layer_names, unquantized_layer_names = wrapper_block(
block, self.enable_minmax_tuning, self.enable_norm_bias_tuning, device=self.device
)
+ if is_nv_fp(self.data_type): # enable qkv and moe structure global_scale fuse
+ from auto_round.data_type.utils import update_fused_layer_global_scales
+ modules = block.modules()
+ for module in modules:
+ update_fused_layer_global_scales(module)
round_params = []
minmax_params = []
for n, m in block.named_modules():
@@ -2446,6 +2495,11 @@ class AutoRound(object):
with torch.no_grad():
unwrapper_block(block, best_params)
if self.enable_quanted_input:
+ if is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats):
+ from auto_round.utils import set_amax_for_all_moe_layers
+
+ # enable moe experts act_max automatic generation for WrapperWALayer
+ set_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")
if self.low_cpu_mem_usage:
block = block.to(device)
clear_memory()
@@ -2613,6 +2667,8 @@ class AutoRound(object):
"Support for exporting activation quantization is limited. "
"Please ensure that your configuration is supported."
)
+ if format == "llmcompressor" and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):
+ format = format.replace("llmcompressor", f"llmcompressor:{self.data_type}")
from auto_round.export import EXPORT_FORMAT
@@ -2649,6 +2705,7 @@ class AutoRound(object):
layer_config=self.layer_config,
inplace=inplace,
bits=self.bits,
+ act_bits=self.act_bits,
group_size=self.group_size,
sym=self.sym,
iters=self.iters,
@@ -2660,6 +2717,7 @@ class AutoRound(object):
tokenizer=self.tokenizer,
supported_types=self.supported_types,
data_type=self.data_type,
+ act_data_type=self.act_data_type,
serialization_dict=serialization_dict,
backend=backend,
to_quant_block_names=self.to_quant_block_names,
diff --git a/auto_round/data_type/__init__.py b/auto_round/data_type/__init__.py
index 03e9160e1..ed446f314 100644
--- a/auto_round/data_type/__init__.py
+++ b/auto_round/data_type/__init__.py
@@ -17,6 +17,6 @@ import auto_round.data_type.mxfp
import auto_round.data_type.fp8
from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE
import auto_round.data_type.w4fp8
-from auto_round.data_type.utils import get_quant_func
+from auto_round.data_type.utils import get_quant_func, update_fused_layer_global_scales
import auto_round.data_type.nvfp
import auto_round.data_type.gguf
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.py
index a5b7bfebc..058c5345c 100644
--- a/auto_round/data_type/nvfp.py
+++ b/auto_round/data_type/nvfp.py
@@ -57,6 +57,16 @@ FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max if hasattr(torch, "float8
FLOAT8_E4M3_MIN = torch.finfo(torch.float8_e4m3fn).min if hasattr(torch, "float8_e4m3fn") else -448
+def calculate_gparam(tensor, group_size=16, device="cpu"):
+ assert group_size == 16
+ if isinstance(tensor, (float, int)):
+ tensor_amax = torch.ones((1), device=device) * tensor
+ elif isinstance(tensor, torch.Tensor):
+ tensor_amax = tensor.abs().max().to(torch.float32)
+ global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_amax)
+ return global_scale
+
+
def ref_nvfp4_quant(x, global_scale, block_size=16, v=0):
assert global_scale.dtype == torch.float32
assert x.ndim == 2
@@ -68,22 +78,26 @@ def ref_nvfp4_quant(x, global_scale, block_size=16, v=0):
output_scale = get_reciprocal(scale * get_reciprocal(global_scale))
scaled_x = x.to(torch.float32) * output_scale + v
clipped_x = torch.clamp(scaled_x, -6.0, 6.0)
- return (cast_to_fp4_ste(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), output_scale
+ return (cast_to_fp4_ste(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), scale
@register_dtype("nv_fp4")
-def nv_fp4(tensor, bits=4, group_size=16, v=0, **kwargs):
+def nv_fp4(tensor, bits=4, group_size=16, v=0, global_scale=None, **kwargs):
orig_dtype = tensor.dtype
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
- tensor_max = tensor.abs().max().to(torch.float32)
- global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)
- qdq_res, output_scale = ref_nvfp4_quant(tensor, global_scale, group_size, v)
+ if global_scale is None:
+ tensor_max = tensor.abs().max().to(torch.float32)
+ global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)
+ global_scale = global_scale.to(tensor.device)
+ qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v)
qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
- return qdq_res.to(orig_dtype), output_scale, None
+ return qdq_res.to(orig_dtype), scale, None
@register_dtype("nv_fp4_with_static_gs")
def nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, **kwargs):
+ if tensor is None or tensor.numel() == 0:
+ return tensor, None, None
orig_dtype = tensor.dtype
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
if tensor_max is None:
@@ -94,9 +108,10 @@ def nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, *
if tensor_max.numel() != 1:
tensor_max = tensor.abs().max().to(torch.float32)
global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)
- qdq_res, output_scale = ref_nvfp4_quant(tensor, global_scale, group_size, v)
+ global_scale = global_scale.to(tensor.device)
+ qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v)
qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
- return qdq_res.to(orig_dtype), output_scale, None
+ return qdq_res.to(orig_dtype), scale, None
FLOAT8_UE5M3_MAX = 114688
@@ -109,7 +124,7 @@ def float_to_e5m3_frexp(x: torch.Tensor) -> torch.Tensor:
mask = x > 0
x_masked = x[mask]
- # 正常数:x >= 2^-14
+ # normal number: x >= 2^-14
normal_mask = x_masked >= 2**-14
x_normal = x_masked[normal_mask]
mantissa, exponent = torch.frexp(x_normal)
@@ -178,6 +193,7 @@ def ref_fp4_quant(x, global_scale, block_size=16, v=0, max_scale=1.0):
assert (not isinstance(global_scale, torch.Tensor)) or global_scale.dtype == torch.float32
assert x.ndim == 2
m, n = x.shape
+ device = x.device
if isinstance(max_scale, torch.Tensor):
max_scale = max_scale.unsqueeze(dim=-1).to(x.device)
vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32) * max_scale
@@ -185,19 +201,16 @@ def ref_fp4_quant(x, global_scale, block_size=16, v=0, max_scale=1.0):
scale = torch.clip(scale, 0, FLOAT8_UE5M3_MAX)
scale = cast_to_ue5m3_ste(scale).to(torch.float32)
output_scale = get_reciprocal(scale * get_reciprocal(global_scale))
-
scaled_x = x.to(torch.float32) * output_scale + v
clipped_x = torch.clamp(scaled_x, -6.0, 6.0)
- return (cast_to_fp4_ste(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), output_scale
+ return (cast_to_fp4_ste(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), scale
@register_dtype("fp4_v2_with_global_scale")
def fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, tensor_max=None, max_scale=1.0, **kwargs):
assert group_size == 32 or group_size == 16
orig_dtype = tensor.dtype
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
-
if tensor_max is None:
tensor_max = tensor.abs().max().to(torch.float32)
elif tensor_max is not None:
@@ -206,21 +219,20 @@ def fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, tensor_max=None
if tensor_max.numel() != 1:
tensor_max = tensor.abs().max().to(torch.float32)
global_scale = FLOAT8_UE5M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)
- qdq_res, output_scale = ref_fp4_quant(tensor, global_scale, group_size, v)
+ qdq_res, scale = ref_fp4_quant(tensor, global_scale, group_size, v)
qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
- return qdq_res.to(orig_dtype), output_scale, None
+ return qdq_res.to(orig_dtype), scale, None
@register_dtype("fp4_v2")
def fp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):
assert group_size == 32 or group_size == 16
orig_dtype = tensor.dtype
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
global_scale = 1.0
- qdq_res, output_scale = ref_fp4_quant(tensor, global_scale, group_size, v, max_scale)
+ qdq_res, scale = ref_fp4_quant(tensor, global_scale, group_size, v, max_scale)
qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
- return qdq_res.to(orig_dtype), output_scale, None
+ return qdq_res.to(orig_dtype), scale, None
if __name__ == "__main__":
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index 09178ed94..371104b9c 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -12,9 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import math
from functools import lru_cache
+from typing import List
import torch
+from torch.nn import Linear, Module
from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE
from auto_round.utils import logger
@@ -23,8 +26,8 @@ from auto_round.utils import logger
def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int):
"""Reshapes and pads the tensor to ensure that it can be quantized in groups of `group_size`.
- This function adjusts t
- he input tensor's shape so that its last dimension is a multiple
+ This function adjusts the
+ input tensor's shape so that its last dimension is a multiple
of the specified `group_size`. If padding is required, it adds padding to the tensor
to achieve this. If the tensor's last dimension is already divisible by `group_size`,
no padding is applied.
@@ -111,7 +114,7 @@ def get_quant_func(dtype, bits, sym):
if key in QUANT_FUNC_WITH_DTYPE.keys():
return QUANT_FUNC_WITH_DTYPE[key], key
- ##need to add bits
+ ##need to add bits and sym infos
if sym:
key = dtype + str(bits) + "_sym"
else:
@@ -120,14 +123,6 @@ def get_quant_func(dtype, bits, sym):
if key in QUANT_FUNC_WITH_DTYPE.keys():
return QUANT_FUNC_WITH_DTYPE[key], key
- if sym:
- key = dtype + "_sym"
- else:
- key = dtype + "_asym"
-
- if key in QUANT_FUNC_WITH_DTYPE.keys():
- return QUANT_FUNC_WITH_DTYPE[key], key
-
if sym:
key = dtype + str(bits)
else:
@@ -237,3 +232,74 @@ def get_gaudi_fp8_ste_func():
fn = float8_e4m3fn_ste
logger.warning_once("Using CUDA/CPU STE for FP8")
return fn
+
+
+# please refer from https://github.com/vllm-project/llm-compressor/blob/
+# 29f4d5644b48e9c8ebb7e36d5be9f7c92747ceb7/src/llmcompressor/modifiers/utils/helpers.py#L11
+def update_fused_layer_global_scales(submodule: torch.nn.Module, base_name="weight"):
+ """
+ When running NVFP4 quantization, update the global scale
+ such that q,k,v layers are treated as one tensor with the same
+ global_scale and gate_proj/up_proj layers are treated as one tensor
+ with the same global scale. This is requirement currently being set
+ by vLLM and may be removed in the future OR potentially make it
+ an optional step.
+
+ :param model: model to quantize
+ base_name: op name for fuse usage, option: weight, input
+ """
+ global_scale_name = f"{base_name}_global_scale"
+ max_value_tensor = torch.ones(1, device="cpu", dtype=torch.float32) * torch.finfo(torch.float32).max
+
+ def _is_attention_module(module: Module):
+ return "attention" in module.__class__.__name__.lower() and (
+ hasattr(module, "k_proj") or hasattr(module, "v_proj") or hasattr(module, "qkv_proj")
+ )
+
+ def _is_mlp_module(module: Module):
+ return "mlp" in module.__class__.__name__.lower() and (
+ hasattr(module, "gate_proj") or hasattr(module, "up_proj")
+ )
+
+ if _is_attention_module(submodule):
+ # already fused/treated as one layer
+ if hasattr(submodule, "qkv_proj"):
+ return
+ global_scale = torch.min(
+ torch.cat(
+ (
+ getattr(submodule.q_proj, global_scale_name, max_value_tensor).reshape(1),
+ getattr(submodule.k_proj, global_scale_name, max_value_tensor).reshape(1),
+ getattr(submodule.v_proj, global_scale_name, max_value_tensor).reshape(1),
+ )
+ )
+ ).reshape([1])
+
+ if math.isclose(global_scale.data, max_value_tensor.data, rel_tol=1e-9):
+ return
+ if hasattr(submodule.q_proj, global_scale_name):
+ setattr(submodule.q_proj, global_scale_name, global_scale.clone())
+ if hasattr(submodule.k_proj, global_scale_name):
+ setattr(submodule.k_proj, global_scale_name, global_scale.clone())
+ if hasattr(submodule.v_proj, global_scale_name):
+ setattr(submodule.v_proj, global_scale_name, global_scale.clone())
+
+ del global_scale
+
+ if _is_mlp_module(submodule):
+ global_scale = torch.min(
+ torch.cat(
+ (
+ getattr(submodule.gate_proj, global_scale_name, max_value_tensor).reshape(1),
+ getattr(submodule.up_proj, global_scale_name, max_value_tensor).reshape(1),
+ )
+ )
+ ).reshape([1])
+ if math.isclose(global_scale.data, max_value_tensor.data, rel_tol=1e-9):
+ return
+ if hasattr(submodule.gate_proj, global_scale_name):
+ setattr(submodule.gate_proj, global_scale_name, global_scale.clone())
+ if hasattr(submodule.up_proj, global_scale_name):
+ setattr(submodule.up_proj, global_scale_name, global_scale.clone())
+
+ del global_scale
diff --git a/auto_round/export/__init__.py b/auto_round/export/__init__.py
index 99c19822a..5956238d6 100644
--- a/auto_round/export/__init__.py
+++ b/auto_round/export/__init__.py
@@ -83,3 +83,10 @@ def _save_quantized_as_llmcompressor(*args, **kwargs):
from auto_round.export.export_to_llmcompressor.export import save_quantized_as_llmcompressor
return save_quantized_as_llmcompressor(*args, **kwargs)
+
+
+@register_layer_packing("llmcompressor")
+def _packing_layer_with_llmcompressor(*args, **kwargs):
+ from auto_round.export.export_to_llmcompressor.export import pack_layer
+
+ return pack_layer(*args, **kwargs)
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 3e72c9590..22ffe967d 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -33,27 +33,14 @@ from auto_round.utils import (
filter_quantization_config,
get_autogptq_packing_qlinear,
get_module,
+ is_mx_fp,
+ is_nv_fp,
+ is_standard_fp,
logger,
set_module,
)
-
-def check_neq_config(config, data_type, bits, group_size, sym):
- """
- Checks if the provided configuration parameters are not equal to the values in the config dictionary.
-
- Args:
- config (dict): A dictionary containing the configuration parameters.
- data_type (str): The expected data type.
- bits (int): The expected number of bits.
- group_size (int): The expected group size.
- sym (bool): The expected symmetry flag.
-
- Returns:
- list: A list of strings indicating which configuration parameters do not match.
- """
- expected_config = {"data_type": data_type, "bits": bits, "group_size": group_size, "sym": sym}
- return [key for key, expected_value in expected_config.items() if config.get(key) != expected_value]
+from .utils import check_neq_config
def dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_bits=16):
@@ -160,6 +147,16 @@ def pack_layer(layer_name, model, backend):
Returns:
None: The function modifies the model in place.
"""
+ if is_nv_fp(backend) or is_mx_fp(backend):
+ from auto_round.export.export_to_autoround.export_to_fp import pack_layer
+
+ return pack_layer(layer_name, model, backend)
+
+ if is_standard_fp(backend):
+ from auto_round.export.export_to_autoround.export_to_fp8_woq import pack_layer
+
+ return pack_layer(layer_name, model, backend)
+
layer = get_module(model, layer_name)
if hasattr(layer, "orig_layer"):
layer = layer.orig_layer
@@ -167,11 +164,6 @@ def pack_layer(layer_name, model, backend):
if not isinstance(layer, SUPPORTED_LAYER_TYPES): ##already packed
return
- if "fp8" in backend:
- from auto_round.export.export_to_autoround.export_to_fp8_woq import pack_layer
-
- return pack_layer(layer_name, model, backend)
-
if int(layer.act_bits) <= 8:
return pack_qact_layer(layer_name, model)
@@ -267,7 +259,13 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
Raises:
ValueError: If the backend is not supported.
"""
- if "fp8" in kwargs.get("data_type", None) and kwargs.get("act_bits", 16) >= 16:
+ data_type = kwargs.get("data_type", None)
+ if is_nv_fp(data_type) or is_mx_fp(data_type): ## detect nvfp & mxfp first
+ from auto_round.export.export_to_autoround.export_to_fp import save_quantized_as_fp
+
+ return save_quantized_as_fp(output_dir, inplace=inplace, backend="auto_round", **kwargs)
+
+ if is_standard_fp(data_type) and kwargs.get("act_bits", 16) >= 16:
from auto_round.export.export_to_autoround.export_to_fp8_woq import save_quantized_as_autoround
return save_quantized_as_autoround(output_dir, inplace=inplace, backend="auto_round", **kwargs)
@@ -314,6 +312,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
layer_config[layer_name],
data_type=quantization_config["data_type"],
bits=quantization_config["bits"],
+ act_bits=quantization_config["act_bits"],
group_size=quantization_config["group_size"],
sym=quantization_config["sym"],
)
diff --git a/auto_round/export/export_to_autoround/export_to_fp.py b/auto_round/export/export_to_autoround/export_to_fp.py
new file mode 100644
index 000000000..a3c724ad9
--- /dev/null
+++ b/auto_round/export/export_to_autoround/export_to_fp.py
@@ -0,0 +1,294 @@
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import inspect
+import json
+import os
+from concurrent.futures import ThreadPoolExecutor
+
+import threadpoolctl as tctl
+import torch
+import torch.nn as nn
+import transformers
+from tqdm import tqdm
+
+from auto_round.utils import (
+ SUPPORTED_LAYER_TYPES,
+ check_start_with_block_name,
+ check_to_quantized,
+ filter_quantization_config,
+ get_block_names,
+ get_module,
+ is_mx_fp,
+ is_nv_fp,
+ logger,
+ set_amax_for_all_moe_layers,
+ set_module,
+)
+from auto_round.wrapper import WrapperWALayer
+
+from .qlinear_fp import QuantLinear
+from .utils import check_neq_config
+
+__all__ = [
+ "pack_layer",
+ "save_quantized_as_fp",
+]
+
+
+def pack_layer(name, model, backend):
+ if name == "lm_head": # TODO: Check vLLM inference status to determine whether to enable this feature
+ return
+ layer = get_module(model, name)
+
+ if not isinstance(layer, SUPPORTED_LAYER_TYPES) and not isinstance(layer, WrapperWALayer): ##already packed
+ return
+
+ if isinstance(layer, WrapperWALayer): # revert WrapperWALayer for offline usage
+ wp_layer = layer
+ layer = wp_layer.orig_layer
+ set_module(model, name, layer)
+
+ data_type = layer.data_type
+ act_bits = layer.act_bits
+ act_data_type = layer.act_data_type
+ bits = layer.bits
+ if bits > 8:
+ return
+ group_size = layer.group_size
+ sym = layer.sym
+ device = layer.weight.device
+
+ if is_nv_fp(act_data_type) and act_bits <= 8:
+ if not getattr(layer, "input_global_scale", None):
+ assert hasattr(layer, "act_max")
+ from auto_round.data_type.nvfp import calculate_gparam
+
+ input_global_scale = calculate_gparam(layer.act_max, layer.group_size, "cpu") # , model.device
+ setattr(layer, "input_global_scale", input_global_scale)
+ delattr(layer, "act_max")
+
+ # QuantLinear = get_fp_qlinear(backend, bits, group_size, sym)
+
+ if isinstance(layer, nn.Linear):
+ in_features = layer.in_features
+ out_features = layer.out_features
+ elif isinstance(layer, nn.Conv2d):
+ in_features = layer.in_channels
+ out_features = layer.out_channels
+ elif isinstance(layer, transformers.pytorch_utils.Conv1D):
+ in_features = layer.weight.shape[0]
+ out_features = layer.weight.shape[1]
+
+ bias = layer.bias is not None
+ ##bias = True ## if using the above, llama3 lambada RTN will be NAN , TODO why?
+ new_layer = QuantLinear( ##pylint: disable=E1123
+ bits,
+ group_size,
+ in_features,
+ out_features,
+ bias,
+ weight_dtype=layer.weight.dtype,
+ sym=sym,
+ data_type=data_type,
+ act_bits=act_bits,
+ act_data_type=act_data_type,
+ )
+
+ new_layer.device = device
+ set_module(model, name, new_layer)
+ qlayer = new_layer
+ scale = layer.scale
+ global_scale = getattr(layer, "weight_global_scale", None)
+ input_global_scale = getattr(layer, "input_global_scale", None)
+ # zero = layer.zp
+ # so far can only pack layer on CPU
+ qlayer.to("cpu")
+ layer, scale = layer.to("cpu"), scale.to("cpu")
+ qlayer.pack(layer, scale, global_scale=global_scale, input_global_scale=input_global_scale)
+ ## no zeros to handle, as mxfp not support asym quantization
+ qlayer.to(device)
+
+
+def save_quantized_as_fp(output_dir, inplace=True, **kwargs):
+ """
+ Saves a quantized model of mxfp/nvfp data_type in the auto-round format.
+
+ Args:
+ output_dir (str): The directory where the quantized model will be saved.
+ inplace (bool, optional): If True, modifies the model in place. Otherwise, creates a deepcopy of the model.
+ Default is True.
+ backend (str, optional): The backend to be used for quantization.
+ Default is "autoround:exllamav2".
+ **kwargs: Additional keyword arguments including:
+ - model (nn.Module): The model to be quantized.
+ - layer_config (dict): The layer configuration for each layer.
+ - serialization_dict (dict): The serialization configuration.
+ - tokenizer (Tokenizer, optional): The tokenizer to be saved.
+
+ Returns:
+ None
+
+ Raises:
+ ValueError: If the backend is not supported.
+ """
+ model = kwargs["model"]
+ backend = kwargs.get("backend", None)
+ bits = kwargs.get("bits", None)
+ data_type = kwargs.get("data_type", None)
+ act_bits = kwargs.get("act_bits", None)
+ act_data_type = kwargs.get("act_data_type", None)
+ safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]
+ if not inplace:
+ model = copy.deepcopy(model.to("cpu"))
+ layer_config = kwargs["layer_config"]
+ quantization_config = kwargs["serialization_dict"]
+ quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)
+ quantization_config["quant_method"] = "auto-round"
+ quantization_config["packing_format"] = backend
+ quantization_config["scale_format"] = ("e8m0",)
+ quantization_config["scale_calculation_mode"] = ("even",)
+
+ tokenizer = kwargs.get("tokenizer", None)
+ processor = kwargs.get("processor", None)
+ extra_config = {}
+
+ if act_bits <= 8:
+ # revert WrapperWALayer for offline usage
+ for n, m in model.named_modules():
+ if isinstance(m, WrapperWALayer):
+ orig_layer = m.orig_layer
+ if not getattr(orig_layer, "input_global_scale", None):
+ assert hasattr(orig_layer, "act_max")
+ from auto_round.data_type.nvfp import calculate_gparam
+
+ input_global_scale = calculate_gparam(orig_layer.act_max, orig_layer.group_size, model.device)
+ setattr(orig_layer, "input_global_scale", input_global_scale)
+ delattr(orig_layer, "act_max")
+ set_module(model, n, orig_layer)
+
+ # update input_global_scale
+ from auto_round.data_type.utils import update_fused_layer_global_scales
+
+ modules = list(model.modules())
+ for module in tqdm(modules, desc="Update input global scale for fuse modules"):
+ update_fused_layer_global_scales(module, base_name="input")
+
+ block_name_to_quantize = quantization_config["block_name_to_quantize"]
+ if isinstance(block_name_to_quantize, str):
+ block_name_to_quantize = block_name_to_quantize.split(",")
+ elif isinstance(block_name_to_quantize, list):
+ for i in range(len(block_name_to_quantize)):
+ block_name_to_quantize[i] = os.path.commonprefix(block_name_to_quantize[i]).rstrip(".")
+
+ for layer_name in layer_config:
+ if (
+ not layer_config[layer_name]["in_blocks"] and layer_config[layer_name]["bits"] <= 8
+ ): ##lm head ##TODO fix act and so on
+ extra_config[layer_name] = {}
+ extra_config[layer_name]["bits"] = layer_config[layer_name]["bits"]
+ extra_config[layer_name]["data_type"] = layer_config[layer_name]["data_type"]
+ extra_config[layer_name]["group_size"] = layer_config[layer_name]["group_size"]
+ extra_config[layer_name]["sym"] = layer_config[layer_name]["sym"]
+ elif layer_config[layer_name]["in_blocks"] or (
+ block_name_to_quantize is not None and check_start_with_block_name(layer_name, block_name_to_quantize)
+ ):
+ neq_keys = check_neq_config(
+ layer_config[layer_name],
+ data_type=quantization_config["data_type"],
+ bits=quantization_config["bits"],
+ act_bits=quantization_config["act_bits"],
+ group_size=quantization_config["group_size"],
+ sym=quantization_config["sym"],
+ )
+ if len(neq_keys) > 0:
+ extra_config[layer_name] = {}
+ for key in neq_keys:
+ if layer_config[layer_name][key] is not None:
+ extra_config[layer_name][key] = layer_config[layer_name][key]
+ if len(extra_config) > 0:
+ quantization_config["extra_config"] = extra_config
+ names = list(layer_config.keys())
+ max_workers = 1
+ if not torch.cuda.is_available() or not torch.xpu.is_available():
+ max_workers = 2 ## 2 with cuda packing will cause hang occasionally
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ with tqdm(total=len(names), leave=True) as pbar:
+
+ def wrapper(name):
+ pbar.set_description(f"packing {name}")
+ with tctl.threadpool_limits(limits=1):
+ pack_layer(name, model, backend)
+ pbar.update(1)
+
+ for _ in executor.map(wrapper, names):
+ pass
+ filter_quantization_config(quantization_config)
+
+ if hasattr(model, "config"):
+ model.config.quantization_config = quantization_config
+ if output_dir is None:
+ return model
+
+ if output_dir is None:
+ model.tokenizer = tokenizer
+ return model
+ if os.path.exists(output_dir):
+ logger.warning(f"{output_dir} already exists, this may cause model conflict")
+ if tokenizer is not None:
+ tokenizer.save_pretrained(output_dir)
+
+ if processor is not None:
+ processor.save_pretrained(output_dir)
+
+ dtype = None
+ save(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)
+
+ return model
+
+
+def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True, dtype=None):
+ """Save model state dict and configs.
+
+ Args:
+ model (`nn.Module`):
+ Model to be saved. The model can be wrapped or unwrapped.
+ save_dir (`str`):
+ Directory to which to save. Will be created if it doesn't exist.
+ max_shard_size (`str`, defaults to `"10GB"`):
+ The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size
+ lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`).
+ <Tip warning={true}>
+
+ If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard
+ which will be bigger than `max_shard_size`.
+
+ </Tip>
+ safe_serialization (`bool`, defaults to `True`):
+ Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ """
+ os.makedirs(save_dir, exist_ok=True)
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ config_path = os.path.join(save_dir, "config.json")
+ if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):
+ with open(config_path, "r") as file:
+ data = json.load(file)
+ data["torch_dtype"] = str(dtype).split(".")[-1]
+ with open(config_path, "w") as file:
+ json.dump(data, file, indent=2)
+ config_file = "quantization_config.json"
+ if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
+ with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
+ json.dump(model.config.quantization_config, f, indent=2)
diff --git a/auto_round/export/export_to_autoround/export_to_fp8_woq.py b/auto_round/export/export_to_autoround/export_to_fp8_woq.py
index 5b6a4c400..b543c4390 100644
--- a/auto_round/export/export_to_autoround/export_to_fp8_woq.py
+++ b/auto_round/export/export_to_autoround/export_to_fp8_woq.py
@@ -32,23 +32,7 @@ from auto_round.utils import (
set_module,
)
-
-def check_neq_config(config, data_type, bits, group_size, sym):
- """
- Checks if the provided configuration parameters are not equal to the values in the config dictionary.
-
- Args:
- config (dict): A dictionary containing the configuration parameters.
- data_type (str): The expected data type.
- bits (int): The expected number of bits.
- group_size (int): The expected group size.
- sym (bool): The expected symmetry flag.
-
- Returns:
- list: A list of strings indicating which configuration parameters do not match.
- """
- expected_config = {"data_type": data_type, "bits": bits, "group_size": group_size, "sym": sym}
- return [key for key, expected_value in expected_config.items() if config.get(key) != expected_value]
+from .utils import check_neq_config
class FP8WOQLinear(torch.nn.Module):
@@ -199,6 +183,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",
layer_config[layer_name],
data_type=quantization_config["data_type"],
bits=quantization_config["bits"],
+ act_bits=quantization_config["act_bits"],
group_size=quantization_config["group_size"],
sym=quantization_config["sym"],
)
diff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.py
new file mode 100644
index 000000000..bd70a43ba
--- /dev/null
+++ b/auto_round/export/export_to_autoround/qlinear_fp.py
@@ -0,0 +1,258 @@
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from logging import getLogger
+
+import numpy as np
+import torch
+import torch.nn as nn
+import transformers
+
+from auto_round.data_type.mxfp import FP32_EXPONENT_BIAS, FP32_MIN_NORMAL
+from auto_round.data_type.nvfp import cast_to_fp4, get_reciprocal
+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
+from auto_round.utils import is_mx_fp, is_nv_fp, is_standard_fp
+
+# from auto_round.utils import get_weight_compress_dtype
+logger = getLogger(__name__)
+E8M0_EXPONENT_BIAS = 127
+E8M0_EXPONENT_NAN_VAL = 255
+
+__all__ = ["QuantLinear"]
+
+FLOAT_TO_E2M1 = [
+ 0.0,
+ 0.5,
+ 1.0,
+ 1.5,
+ 2.0,
+ 3.0,
+ 4.0,
+ 6.0,
+]
+
+
+class QuantLinear(nn.Module):
+ """
+ MXFP quantized linear layer.
+ """
+
+ QUANT_TYPE = "MXFP"
+
+ def __init__(
+ self, bits, group_size, infeatures, outfeatures, bias, trainable=False, data_type="mx_fp8e4m3", **kwargs
+ ):
+ super().__init__()
+ if bits not in [4, 8]:
+ raise NotImplementedError("Only 4,8 bits are supported.")
+ if infeatures % 32 != 0 or outfeatures % 32 != 0:
+ raise NotImplementedError("in_feature and out_feature must be divisible by 32.")
+ self.is_mx = is_mx_fp(data_type)
+ self.is_nv = is_nv_fp(data_type)
+ if self.is_mx and group_size != 32:
+ raise NotImplementedError("Only group_size 32 are supported for mxfp.")
+ if self.is_nv and group_size not in [16, 32]:
+ raise NotImplementedError("Only group_size 16 are supported for nvfp.")
+ self.infeatures = infeatures
+ self.outfeatures = outfeatures
+ self.bits = bits
+ self.data_type = data_type
+ self.sym = kwargs.get("sym", True)
+ self.group_size = group_size if group_size != -1 else infeatures
+ self.maxq = 2**self.bits - 1
+ self.act_bits = kwargs.get("act_bits", None)
+
+ weight_name = "weight" if self.bits == 8 and self.is_mx else "weight_packed"
+ ## TODO check the dtype of weight_packed and weight_scale
+ self.register_buffer(
+ weight_name,
+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),
+ )
+ if not self.sym:
+ ## TODO Currently only sym quant is supported for mxfp dtype. Is weight_zero_point param necessary?
+ self.register_buffer(
+ "weight_zero_point",
+ torch.zeros(
+ (
+ math.ceil(infeatures / self.group_size),
+ outfeatures // 32 * self.bits,
+ ),
+ dtype=torch.int32,
+ ),
+ )
+ self.register_buffer(
+ "weight_scale",
+ torch.zeros(
+ (math.ceil(infeatures / self.group_size), outfeatures),
+ dtype=torch.float16, ## TODO update to correct scale dtype for different bits
+ ),
+ )
+ if self.is_nv and self.bits == 4:
+ self.register_buffer(
+ "weight_global_scale",
+ torch.zeros(
+ (1),
+ dtype=torch.float32,
+ ),
+ )
+ if self.is_nv and self.act_bits <= 8:
+ self.register_buffer(
+ "input_global_scale",
+ torch.zeros(
+ (1),
+ dtype=torch.float32,
+ ),
+ )
+ if bias:
+ self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))
+ else:
+ self.bias = None
+
+ self.trainable = trainable
+
+ def post_init(self):
+ pass
+
+ def pack(self, linear, scales, zeros=None, g_idx=None, global_scale=None, input_global_scale=None):
+ if linear.bias is not None:
+ self.bias = linear.bias.clone().half()
+ device = "cpu"
+ if torch.cuda.is_available():
+ device = "cuda:0"
+ elif torch.xpu.is_available():
+ device = "xpu:0"
+
+ W = linear.weight.data.to(device).clone() # TODO check is nesscessory
+ if isinstance(linear, nn.Conv2d):
+ W = W.flatten(1)
+ if isinstance(linear, transformers.pytorch_utils.Conv1D):
+ W = W.t()
+
+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(linear.weight, self.group_size)
+ if self.is_nv:
+ assert global_scale is not None and global_scale.numel() == 1
+ scaled_tensor = tensor.to(global_scale.dtype) * get_reciprocal(
+ scales.reshape(tensor.shape[0], -1) * get_reciprocal(global_scale)
+ )
+ scaled_tensor = cast_to_fp4(torch.clamp(scaled_tensor, -6.0, 6.0))
+ else:
+ scaled_tensor = tensor / (2 ** scales.reshape(tensor.shape[0], -1))
+ scaled_tensor = revert_tensor_by_pad(scaled_tensor, orig_shape=orig_shape, pad_len=pad_len)
+ if self.is_mx:
+ final_scale = (scales + E8M0_EXPONENT_BIAS).clamp(0, E8M0_EXPONENT_NAN_VAL).to(torch.uint8)
+ else:
+ final_scale = scales.to(torch.float8_e4m3fn)
+ self.weight_scale = final_scale
+ # self.weight = get_compressed_weight(scaled_tensor, self.bits, self.data_type) ## TODO
+ if self.bits == 8:
+ compress_dtype = torch.float8_e4m3fn
+ self.weight = scaled_tensor.to(compress_dtype)
+ else:
+ compress_dtype = torch.uint8
+ self.weight_packed = self.pack_fp4_to_uint8(scaled_tensor)
+
+ if global_scale is not None:
+ self.weight_global_scale = global_scale.to(torch.float32).to(device)
+
+ if input_global_scale is not None:
+ self.input_global_scale = input_global_scale.to(torch.float32).to(device)
+ return
+
+ def pack_fp4_to_uint8(self, x: torch.Tensor) -> torch.Tensor:
+ """
+ Packs a tensor with values in the fp4 range into uint8.
+ As there are 16 valid fp4 values, two fp4 values can be
+ packed into one uint8. Each fp4 value is mapped to its
+ particular index (e.g. 0.5 is mapped to index 1, 6.0 is mapped
+ to index 7) which is then represented using 4 bits. Consecutive
+ pairs of 4 bits are then packed into an uint8.
+
+ :param x: tensor to pack
+ returns: a packed tensor in uint8
+ """
+
+ m, n = x.shape
+ device = x.device
+
+ # Create lookup table for FP4 values to indices
+ # Map the absolute values to 0-7 indices
+ kE2M1 = torch.tensor(FLOAT_TO_E2M1, device=device, dtype=x.dtype)
+
+ # Find closest valid FP4 value index for each element
+ abs_x = torch.abs(x)
+ abs_indices = torch.zeros_like(abs_x, dtype=torch.long)
+ for i, val in enumerate(kE2M1): # TODO any optimize?
+ abs_indices = torch.where(torch.isclose(abs_x, val), i, abs_indices)
+
+ # Apply sign bit (bit 3) to get final 4-bit representation
+ indices = abs_indices + (torch.signbit(x) << 3).to(torch.long)
+
+ # Reshape to prepare for packing pairs of values
+ indices = indices.reshape(-1)
+
+ # Handle odd length by padding if necessary
+ if indices.numel() % 2 != 0:
+ indices = torch.cat([indices, torch.zeros(1, dtype=torch.long, device=device)])
+
+ # Reshape to pair consecutive elements
+ indices = indices.reshape(-1, 2)
+
+ # Pack pairs of 4-bit values into 8-bit values
+ packed = (indices[:, 0] | (indices[:, 1] << 4)).to(torch.uint8)
+
+ return packed.reshape(m, n // 2)
+
+ # def get_fp_scale(self, scale_e8m0):
+ # E8M0_EXPONENT_BIAS = 127
+ # E8M0_EXPONENT_NAN_VAL = 255
+
+ # scale_e8m0 = scale_e8m0.view(torch.uint8)
+ # s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS
+ # # TODO(later): it would be nice if there was a way to do the 2^x operation
+ # # in PyTorch without creating a tensor of twos
+ # two = torch.full(s_offset.size(), 2.0, device=scale_e8m0.device)
+ # # pow(two, s_offset) can be out of range of floating point formats.
+ # # TODO(later): handle this for float16 if we decide to support float16
+ # # scales.
+ # s_fp = torch.pow(two, s_offset)
+
+ # # If a block exponent was 255, set values of that block to NaN
+ # s_fp = torch.where(scale_e8m0 != E8M0_EXPONENT_NAN_VAL, s_fp, float("nan"))
+
+ # return s_fp
+
+ # def get_compressed_weight(self, tensor, bits, data_type):
+ # data_type = str(data_type)
+ # compress_dtype = None
+ # if self.is_mx:
+ # if bits == 4:
+
+ # if self.is_nv:
+ # pass ## TODO
diff --git a/auto_round/export/export_to_autoround/utils.py b/auto_round/export/export_to_autoround/utils.py
new file mode 100644
index 000000000..11b217424
--- /dev/null
+++ b/auto_round/export/export_to_autoround/utils.py
@@ -0,0 +1,37 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def check_neq_config(config, data_type, bits, act_bits, group_size, sym):
+ """
+ Checks if the provided configuration parameters are not equal to the values in the config dictionary.
+
+ Args:
+ config (dict): A dictionary containing the configuration parameters.
+ data_type (str): The expected data type.
+ bits (int): The expected number of bits.
+ group_size (int): The expected group size.
+ sym (bool): The expected symmetry flag.
+
+ Returns:
+ list: A list of strings indicating which configuration parameters do not match.
+ """
+ expected_config = {
+ "data_type": data_type,
+ "bits": bits,
+ "group_size": group_size,
+ "sym": sym,
+ "act_bits": act_bits,
+ }
+ return [key for key, expected_value in expected_config.items() if config.get(key) != expected_value]
diff --git a/auto_round/export/export_to_llmcompressor/__init__.py b/auto_round/export/export_to_llmcompressor/__init__.py
index 0d2740cdb..1ab77d2e5 100644
--- a/auto_round/export/export_to_llmcompressor/__init__.py
+++ b/auto_round/export/export_to_llmcompressor/__init__.py
@@ -11,3 +11,4 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+from .config import check_compressed_tensors_supported
diff --git a/auto_round/export/export_to_llmcompressor/config.py b/auto_round/export/export_to_llmcompressor/config.py
index 6b3145c98..9b205123e 100644
--- a/auto_round/export/export_to_llmcompressor/config.py
+++ b/auto_round/export/export_to_llmcompressor/config.py
@@ -52,3 +52,87 @@ quantization_config = {
"quantization_status": "compressed",
"provider": "auto-round", # This is the key to indicate the model provider
}
+
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from auto_round.utils import logger
+
+
+def check_compressed_tensors_supported(): # pragma: no cover
+ try:
+ import compressed_tensors # noqa: F401
+
+ return True
+ except ImportError:
+ logger.warning(
+ "Please install compressed-tensors via 'pip install compressed-tensors'" " to save as llm-compressor format"
+ )
+ return False
+
+
+if check_compressed_tensors_supported():
+ from compressed_tensors.quantization import ( # pylint: disable=E0401
+ QuantizationConfig,
+ QuantizationScheme,
+ QuantizationStatus,
+ is_preset_scheme,
+ preset_name_to_scheme,
+ )
+
+
+# please refer to https://github.com/vllm-project/llm-compressor/blob/
+# 29f4d5644b48e9c8ebb7e36d5be9f7c92747ceb7/src/llmcompressor/modifiers/quantization/quantization/mixin.py#L168
+def initialize_quantization(scheme, targets=["Linear"], config_groups=None, kv_cache_scheme=None, ignore=["lm_head"]):
+ """
+ Attach quantization schemes to modules in the model and initialize the quantization config
+ """
+
+ # apply scheme and status to model
+ scheme = scheme
+ targets = targets
+ config_groups = config_groups
+ kv_cache_scheme = kv_cache_scheme
+ ignore = ignore
+ check_compressed_tensors_supported()
+ if scheme is not None and config_groups is not None:
+ raise ValueError("Please specify either `scheme` or `config_groups`")
+
+ if scheme is not None:
+ # takes precedence over config_groups
+
+ if isinstance(scheme, str) and is_preset_scheme(scheme):
+ # attach targets to scheme
+ scheme = {scheme: targets}
+
+ config_groups = {}
+ for idx, key in enumerate(scheme.keys()):
+ if is_preset_scheme(key):
+ scheme = preset_name_to_scheme(key, scheme[key])
+ else:
+ scheme = QuantizationScheme.model_validate({"targets": scheme[key], **scheme})
+
+ group_name = f"group_{idx}"
+ config_groups[group_name] = scheme
+
+ if config_groups is None or len(config_groups) == 0:
+ default_quant_scheme = QuantizationScheme(targets=targets)
+ config_groups = {"group_0": default_quant_scheme}
+
+ return QuantizationConfig(
+ config_groups=config_groups,
+ kv_cache_scheme=kv_cache_scheme,
+ quantization_status=QuantizationStatus.COMPRESSED,
+ ignore=ignore,
+ )
diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.py
index e5863f374..0f759384b 100644
--- a/auto_round/export/export_to_llmcompressor/export.py
+++ b/auto_round/export/export_to_llmcompressor/export.py
@@ -17,9 +17,11 @@ import os
import torch
from auto_round.export.export_to_llmcompressor.config import quantization_config
-from auto_round.utils import detect_device, get_module, logger, set_module
+from auto_round.utils import detect_device, get_module, is_mx_fp, is_nv_fp, is_standard_fp, logger, set_module
from auto_round.wrapper import WrapperWALayer
+from .export_to_fp import save_quantized_as_fp
+
@torch.no_grad()
def recover_qweight(qdq_weight, scale):
@@ -36,8 +38,40 @@ def recover_qweight(qdq_weight, scale):
return (qdq_weight / scale).to(torch.int8)
+def pack_layer(layer_name, model, backend):
+ """
+ Packs a model layer for quantization based on its type and configuration.
+
+ This function retrieves the specified layer from the model, checks its
+ compatibility for quantization, and replaces it with a quantized version
+ if applicable. The quantization process depends on the layer's bit-width,
+ group size, symmetry, and activation bits.
+
+ Args:
+ layer_name (str): The name of the layer to be packed.
+ model (torch.nn.Module): The model containing the layer.
+ backend (str): The backend framework to be used for quantization.
+
+ Returns:
+ None: The function modifies the model in place.
+ """
+ if is_nv_fp(backend) or is_mx_fp(backend):
+ from auto_round.export.export_to_llmcompressor.export_to_fp import pack_layer
+
+ return pack_layer(layer_name, model, backend)
+
+ ## passed as no other llmcompressor format is supported yet
+ logger.warning("No other llmcompressor packing format(except NVFP&MXFP) is supported yet, skip packing")
+ return
+
+
@torch.no_grad()
-def save_quantized_as_llmcompressor(output_dir, model=None, **kwargs):
+def save_quantized_as_llmcompressor(output_dir, **kwargs):
+ backend = kwargs.get("backend", None)
+ if is_nv_fp(backend) or is_mx_fp(backend):
+ return save_quantized_as_fp(output_dir, **kwargs)
+
+ model = kwargs.get("model", None)
safe_serialization = kwargs.get("safe_serialization", True)
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.py
new file mode 100644
index 000000000..0b989f65d
--- /dev/null
+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py
@@ -0,0 +1,278 @@
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import inspect
+import json
+import os
+from concurrent.futures import ThreadPoolExecutor
+
+import threadpoolctl as tctl
+import torch
+import torch.nn as nn
+import transformers
+from tqdm import tqdm
+
+from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear
+from auto_round.utils import (
+ SUPPORTED_LAYER_TYPES,
+ check_start_with_block_name,
+ check_to_quantized,
+ filter_quantization_config,
+ get_block_names,
+ get_module,
+ is_mx_fp,
+ is_nv_fp,
+ logger,
+ set_amax_for_all_moe_layers,
+ set_module,
+)
+from auto_round.wrapper import WrapperWALayer
+
+from .config import check_compressed_tensors_supported
+
+__all__ = [
+ "pack_layer",
+ "save_quantized_as_fp",
+]
+
+
+def pack_layer(name, model, backend):
+ if name == "lm_head": # TODO: Check vLLM inference status to determine whether to enable this feature
+ return
+ layer = get_module(model, name)
+
+ if not isinstance(layer, SUPPORTED_LAYER_TYPES) and not isinstance(layer, WrapperWALayer): ##already packed
+ return
+
+ if isinstance(layer, WrapperWALayer): # revert WrapperWALayer for offline usage
+ wp_layer = layer
+ layer = wp_layer.orig_layer
+ set_module(model, name, layer)
+
+ data_type = layer.data_type
+ act_bits = layer.act_bits
+ act_data_type = layer.act_data_type
+ bits = layer.bits
+ if bits > 8:
+ return
+ group_size = layer.group_size
+ sym = layer.sym
+ device = layer.weight.device
+
+ if is_nv_fp(act_data_type) and act_bits <= 8:
+ if not getattr(layer, "input_global_scale", None):
+ assert hasattr(layer, "act_max")
+ from auto_round.data_type.nvfp import calculate_gparam
+
+ input_global_scale = calculate_gparam(layer.act_max, layer.group_size) # , model.device
+ setattr(layer, "input_global_scale", input_global_scale)
+ delattr(layer, "act_max")
+
+ # QuantLinear = get_fp_qlinear(backend, bits, group_size, sym)
+
+ if isinstance(layer, nn.Linear):
+ in_features = layer.in_features
+ out_features = layer.out_features
+ elif isinstance(layer, nn.Conv2d):
+ in_features = layer.in_channels
+ out_features = layer.out_channels
+ elif isinstance(layer, transformers.pytorch_utils.Conv1D):
+ in_features = layer.weight.shape[0]
+ out_features = layer.weight.shape[1]
+
+ bias = layer.bias is not None
+ ##bias = True ## if using the above, llama3 lambada RTN will be NAN , TODO why?
+ new_layer = QuantLinear( ##pylint: disable=E1123
+ bits,
+ group_size,
+ in_features,
+ out_features,
+ bias,
+ weight_dtype=layer.weight.dtype,
+ sym=sym,
+ data_type=data_type,
+ act_bits=act_bits,
+ act_data_type=act_data_type,
+ )
+
+ new_layer.device = device
+ set_module(model, name, new_layer)
+ qlayer = new_layer
+ scale = layer.scale
+ global_scale = getattr(layer, "weight_global_scale", None)
+ input_global_scale = getattr(layer, "input_global_scale", None)
+ # zero = layer.zp
+ # so far can only pack layer on CPU
+ qlayer.to("cpu")
+ layer, scale = layer.to("cpu"), scale.to("cpu")
+ qlayer.pack(layer, scale, global_scale=global_scale, input_global_scale=input_global_scale)
+ ## no zeros to handle, as mxfp not support asym quantization
+ qlayer.to(device)
+
+
+def save_quantized_as_fp(output_dir, inplace=True, **kwargs):
+ """
+ Saves a quantized model of mxfp/nvfp data_type in the llm-compressor format.
+
+ Args:
+ output_dir (str): The directory where the quantized model will be saved.
+ inplace (bool, optional): If True, modifies the model in place. Otherwise, creates a deepcopy of the model.
+ Default is True.
+ backend (str, optional): The backend to be used for quantization.
+ Default is "autoround:exllamav2".
+ **kwargs: Additional keyword arguments including:
+ - model (nn.Module): The model to be quantized.
+ - layer_config (dict): The layer configuration for each layer.
+ - serialization_dict (dict): The serialization configuration.
+ - tokenizer (Tokenizer, optional): The tokenizer to be saved.
+
+ Returns:
+ None
+
+ Raises:
+ ValueError: If the backend is not supported.
+ """
+ model = kwargs["model"]
+ backend = kwargs.get("backend", None)
+ bits = kwargs.get("bits", None)
+ data_type = kwargs.get("data_type", None)
+ act_bits = kwargs.get("act_bits", None)
+ act_data_type = kwargs.get("act_data_type", None)
+ safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]
+ if not inplace:
+ model = copy.deepcopy(model.to("cpu"))
+ layer_config = kwargs["layer_config"]
+ tokenizer = kwargs.get("tokenizer", None)
+ processor = kwargs.get("processor", None)
+ extra_config = {}
+
+ if act_bits <= 8:
+ # revert WrapperWALayer for offline usage
+ for n, m in model.named_modules():
+ if isinstance(m, WrapperWALayer):
+ orig_layer = m.orig_layer
+ if not getattr(orig_layer, "input_global_scale", None):
+ assert hasattr(orig_layer, "act_max")
+ from auto_round.data_type.nvfp import calculate_gparam
+
+ input_global_scale = calculate_gparam(orig_layer.act_max, orig_layer.group_size, model.device)
+ setattr(orig_layer, "input_global_scale", input_global_scale)
+ delattr(orig_layer, "act_max")
+ set_module(model, n, orig_layer)
+
+ # update input_global_scale
+ from auto_round.data_type.utils import update_fused_layer_global_scales
+
+ modules = list(model.modules())
+ for module in tqdm(modules, desc="Update input global scale for fuse modules"):
+ update_fused_layer_global_scales(module, base_name="input")
+
+ names = list(layer_config.keys())
+ max_workers = 1
+ if not torch.cuda.is_available() or not torch.xpu.is_available():
+ max_workers = 2 ## 2 with cuda packing will cause hang occasionally
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
+ with tqdm(total=len(names), leave=True) as pbar:
+
+ def wrapper(name):
+ pbar.set_description(f"packing {name}")
+ with tctl.threadpool_limits(limits=1):
+ pack_layer(name, model, backend)
+ pbar.update(1)
+
+ for _ in executor.map(wrapper, names):
+ pass
+
+ # TODO fix the ignore re match issue, compile with fp8 & int8 config
+ ignore = ["lm_head"]
+ for layer_name in layer_config:
+ if layer_config[layer_name]["bits"] > 8: ## find ignore layers
+ ignore.append(layer_name)
+ ignore = list(set(ignore))
+
+ # get llm-compressor format config
+ check_compressed_tensors_supported()
+ from .config import initialize_quantization
+
+ scheme = "NVFP4"
+ quantization_config = initialize_quantization(scheme=scheme)
+ setattr(quantization_config, "format", "nvfp4-pack-quantized")
+ setattr(quantization_config, "ignore", ignore)
+ quantization_config = quantization_config.to_dict()
+ if is_mx_fp(data_type): # Manually replace some parameters, as compressed-tensor is not support MXFP scheme yet
+ quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] = bits
+ quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] = (
+ act_bits if act_bits <= 8 else bits
+ )
+ quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] = True
+ quantization_config["config_groups"]["group_0"]["input_activations"]["is_mx"] = True
+ quantization_config["config_groups"]["group_0"]["weights"]["group_size"] = 32
+ quantization_config["config_groups"]["group_0"]["input_activations"]["group_size"] = 32
+ quantization_config["format"] = "float-quantized"
+ if hasattr(model, "config"):
+ model.config.quantization_config = quantization_config
+ if output_dir is None:
+ return model
+
+ if output_dir is None:
+ model.tokenizer = tokenizer
+ return model
+ if os.path.exists(output_dir):
+ logger.warning(f"{output_dir} already exists, this may cause model conflict")
+ if tokenizer is not None:
+ tokenizer.save_pretrained(output_dir)
+
+ if processor is not None:
+ processor.save_pretrained(output_dir)
+
+ dtype = None
+ save(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)
+
+ return model
+
+
+def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True, dtype=None):
+ """Save model state dict and configs.
+
+ Args:
+ model (`nn.Module`):
+ Model to be saved. The model can be wrapped or unwrapped.
+ save_dir (`str`):
+ Directory to which to save. Will be created if it doesn't exist.
+ max_shard_size (`str`, defaults to `"10GB"`):
+ The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size
+ lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`).
+ <Tip warning={true}>
+
+ If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard
+ which will be bigger than `max_shard_size`.
+
+ </Tip>
+ safe_serialization (`bool`, defaults to `True`):
+ Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
+ """
+ os.makedirs(save_dir, exist_ok=True)
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ config_path = os.path.join(save_dir, "config.json")
+ if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):
+ with open(config_path, "r") as file:
+ data = json.load(file)
+ data["torch_dtype"] = str(dtype).split(".")[-1]
+ with open(config_path, "w") as file:
+ json.dump(data, file, indent=2)
+ config_file = "quantization_config.json"
+ if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
+ with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
+ json.dump(model.config.quantization_config, f, indent=2)
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 9b83aa371..08b693c26 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -483,7 +483,7 @@ def tune(args):
layer_config = {}
not_quantize_layer_names = get_fp_layer_names(model, args.fp_layers)
for name in not_quantize_layer_names:
- layer_config[name] = {"bits": 16}
+ layer_config[name] = {"bits": 16, "act_bits": 16}
if len(not_quantize_layer_names) > 0:
logger.info(f"{not_quantize_layer_names} will not be quantized.")
for format in formats:
@@ -508,7 +508,7 @@ def tune(args):
break
if args.quant_lm_head:
- layer_config[lm_head_layer_name] = {"bits": args.bits}
+ layer_config[lm_head_layer_name] = {"bits": args.bits, "act_bits": args.act_bits}
for format in formats:
if "auto_round" not in format and "fake" not in format:
auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 86c850628..1f55402df 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -369,7 +369,7 @@ def tune(args):
layer_config = {}
not_quantize_layer_names = get_fp_layer_names(model, args.fp_layers)
for name in not_quantize_layer_names:
- layer_config[name] = {"bits": 16}
+ layer_config[name] = {"bits": 16, "act_bits": 16}
if len(not_quantize_layer_names) > 0:
logger.info(f"{not_quantize_layer_names} will not be quantized.")
for format in formats:
@@ -408,7 +408,7 @@ def tune(args):
)
break
if args.quant_lm_head:
- layer_config[lm_head_layer_name] = {"bits": args.bits}
+ layer_config[lm_head_layer_name] = {"bits": args.bits, "act_bits": args.act_bits}
for format in formats:
if "auto_round" not in format and "fake" not in format:
auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 8b8530cb4..2b8ef540e 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -12,14 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import collections.abc
import copy
import gc
+import json
import logging
import os
import re
import subprocess
import sys
from collections import UserDict
+from enum import Enum
from functools import lru_cache
import cpuinfo
@@ -48,6 +51,7 @@ class SupportedFormats:
"itrex",
"itrex_xpu",
"fake",
+ "llmcompressor",
)
self._gguf_format = tuple(sorted(GGUF_CONFIG.keys()))
self._support_list = self._support_format + self._gguf_format
@@ -1449,7 +1453,6 @@ def mllm_load_model(
model_dtype=None,
**kwargs,
):
- import json
import transformers
from huggingface_hub import HfApi, HfFileSystem, hf_hub_download
@@ -2249,3 +2252,167 @@ def download_hf_model(repo_id, cache_dir=None, repo_type=None, revision=None):
model_path = snapshot_download(repo_id)
return model_path
+
+
+def is_moe(module: torch.nn.Module) -> bool:
+ """Returns whether the module is an MOE layer."""
+ return any(
+ key in type(module).__name__.lower()
+ for key in [
+ "MixtralSparseMoeBlock".lower(),
+ "ArcticMoE".lower(),
+ "DbrxFFN".lower(),
+ "MoELayer".lower(),
+ "PhimoeSparseMoeBlock".lower(),
+ "DeepseekMoE".lower(),
+ "DeepseekV2MoE".lower(),
+ "DeepseekV3MoE".lower(),
+ "Qwen2MoeSparseMoeBlock".lower(),
+ "Qwen3MoeSparseMoeBlock".lower(),
+ ]
+ )
+
+
+# please refer to https://github.com/NVIDIA/TensorRT-Model-Optimizer
+# /blob/4c611e47a60084a86e1de7e48690a692a1b8170c/modelopt/torch/export/layer_utils.py#L976
+def get_expert_linear_names(module: torch.nn.Module) -> list[str]:
+ """Get the list of linear names for the experts."""
+
+ def module_match_name_list(module, name_list):
+ """Check if the module name matches any of the names in the list.
+
+ e.g. module_match_name_list(QuantQwen3MoeSparseMoeBlock, ['Qwen3MoeSparseMoeBlock']) -> True
+
+ """
+ return any(name.lower() in type(module).__name__.lower() for name in name_list)
+
+ if module_match_name_list(
+ module, ["Qwen2MoeSparseMoeBlock", "Qwen3MoeSparseMoeBlock", "DeepseekMoE", "DeepseekV2MoE", "DeepseekV3MoE"]
+ ):
+ return ["gate_proj", "down_proj", "up_proj"]
+ elif module_match_name_list(module, ["MixtralMoeSparseMoeBlock"]):
+ return ["linear_fc1", "linear_fc2"]
+ elif module_match_name_list(module, ["DBRXMoeSparseMoeBlock"]):
+ return ["w1_linear", "w2_linear", "v1_linear"]
+ else:
+ # assuing w1, w2, w3 by default
+ return ["w1", "w2", "w3"]
+
+
+def get_nested_attr(module, attr_name: str):
+ """Recursively get nested attribute (e.g., 'orig_layer.act_max')."""
+ attrs = attr_name.split(".")
+ for attr in attrs:
+ if not hasattr(module, attr):
+ return None
+ module = getattr(module, attr)
+ return module
+
+
+def set_nested_attr(module, attr_name: str, value):
+ """Recursively set nested attribute (e.g., 'orig_layer.act_max' = value)."""
+ attrs = attr_name.split(".")
+ for attr in attrs[:-1]:
+ if not hasattr(module, attr):
+ raise AttributeError(f"{module} has no attribute '{attr}'")
+ module = getattr(module, attr)
+ setattr(module, attrs[-1], value)
+
+
+def set_amax_for_uncalibrated_experts(
+ experts: torch.nn.Module, set_amax_value: float | None = None, attr_name="act_max"
+):
+ """Set amax of uncalibrated experts to a given value or the max of existing amax value from other experts.
+
+ Args:
+ experts: a list of experts
+ set_amax_value: set amax value to the given value.
+ If None, set amax value to the max of existing amax value from other experts.
+
+ Returns:
+ uncalibrated_experts: a list of uncalibrated experts
+ """
+ uncalibrated_experts = []
+ # get the max amax value from all experts
+ if set_amax_value is None:
+ amax_values = [
+ get_nested_attr(module, attr_name) for module in experts if get_nested_attr(module, attr_name) is not None
+ ]
+ if len(amax_values) == 0:
+ return uncalibrated_experts
+ # Flatten all tensors to 1D before concatenation
+ flat_values = [t.reshape(-1) for t in amax_values]
+ all_values = torch.cat(flat_values)
+ set_amax_value = torch.max(all_values)
+
+ for module in experts:
+ if get_nested_attr(module, attr_name) is None:
+ logger.warning_once(
+ "Missing amax value of expert layers."
+ "This typically occurs in MoE models when certain experts are not activated during calibration. "
+ "Consider increasing your calibration dataset size to ensure all experts are exercised."
+ )
+ # Use float32 dtype explicitly to ensure we create a floating point tensor
+ if not isinstance(set_amax_value, torch.Tensor):
+ set_amax_value = torch.tensor(set_amax_value, dtype=torch.float32)
+ set_nested_attr(module, attr_name, set_amax_value)
+ # uncalibrated_experts.append(module)
+
+
+# Please refer to: https://github.com/NVIDIA/TensorRT-Model-Optimizer/blob/
+# 4c611e47a60084a86e1de7e48690a692a1b8170c/modelopt/torch/export/unified_export_hf.py#L195-L207
+def set_amax_for_all_moe_layers(model: torch.nn.Module, layer_name=None, attr_name="act_max"):
+ if layer_name is not None:
+ parts = layer_name.split(".")
+ if "experts" not in parts:
+ raise ValueError
+ idx = parts.index("experts")
+ moe_name = ".".join(parts[:idx])
+ model = get_module(model, moe_name)
+ # Handle input quantizers of experts that are not calibrated
+ for name, sub_module in model.named_modules():
+ if is_moe(sub_module) and hasattr(sub_module, "experts"):
+ expert_linear_names = get_expert_linear_names(sub_module)
+ for linear_name in expert_linear_names:
+ if isinstance(sub_module.experts, collections.abc.Iterable):
+ # For other MoE models (like Mixtral) with iterable experts
+ try:
+ set_amax_for_uncalibrated_experts(
+ [getattr(expert, linear_name) for expert in sub_module.experts], attr_name=attr_name
+ )
+ except AttributeError as e:
+ # Provide more helpful debugging information
+ expert_types = list(set(type(expert).__name__ for expert in sub_module.experts))
+ raise AttributeError(
+ f"Failed to access attribute '{linear_name}' on experts. "
+ f"MoE module type: {type(sub_module).__name__}, "
+ f"Expert types: {expert_types}, "
+ f"Expected linear names: {expert_linear_names}. "
+ f"This suggests the get_expert_linear_names function may need "
+ f"to be updated for this model architecture. "
+ f"Original error: {e}"
+ ) from e
+ else:
+ # Unsupported MoE model structure
+ raise NotImplementedError(
+ f"MoE model with experts type '{type(sub_module.experts).__name__}' is not supported in export."
+ f"Please file an issue or add support for this model architecture."
+ )
+
+
+class BackendDataType(str, Enum):
+ STANDARD_FP8 = "fp8"
+ MX_FP = "mx_fp"
+ NV_FP = "nv_fp"
+
+
+def is_standard_fp(backend):
+ return BackendDataType.STANDARD_FP8 in backend and not is_mx_fp(backend) and not is_nv_fp(backend)
+
+
+def is_mx_fp(backend):
+ return BackendDataType.MX_FP in backend
+
+
+def is_nv_fp(backend):
+ return BackendDataType.NV_FP in backend
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index be3427424..591e5c567 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -18,7 +18,7 @@ from torch.functional import F
from auto_round.data_type import get_quant_func
-from .utils import SUPPORTED_LAYER_TYPES, check_to_quantized, get_scale_shape, logger, set_module
+from .utils import SUPPORTED_LAYER_TYPES, check_to_quantized, get_scale_shape, is_mx_fp, is_nv_fp, logger, set_module
def reshape_and_pad_tensor(v, group_size=-1):
@@ -82,6 +82,13 @@ class WrapperLinear(torch.nn.Module):
self.enable_round_tuning = enable_round_tuning
self.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)
self.enable_act_quant = self.orig_layer.act_bits <= 8
+ self.weight_global_scale = getattr(self.orig_layer, "weight_global_scale", None)
+ if is_nv_fp(self.orig_layer.data_type) and self.weight_global_scale is None:
+ from auto_round.data_type.nvfp import calculate_gparam
+
+ weight_global_scale = calculate_gparam(self.orig_layer.weight, self.orig_layer.group_size)
+ setattr(self, "weight_global_scale", weight_global_scale)
+ self.weight_global_scale = self.weight_global_scale.to(self.orig_layer.weight.device)
if hasattr(self.orig_layer, "scale_dtype") and self.orig_layer.scale_dtype == torch.float32:
self.q_scale_thresh = 1e-8
else:
@@ -185,6 +192,7 @@ class WrapperLinear(torch.nn.Module):
data_type=self.data_type,
q_scale_thresh=self.q_scale_thresh,
imatrix=self.orig_layer.imatrix if hasattr(self.orig_layer, "imatrix") else None,
+ global_scale=getattr(self, "weight_global_scale", None),
**quant_kwargs,
)
weight_q = weight_q.to(weight.dtype)
@@ -213,6 +221,7 @@ class WrapperLinear(torch.nn.Module):
data_type=self.act_data_type,
max_scale=act_max_scale,
tensor_max=act_max,
+ global_scale=getattr(self, "weight_global_scale", None),
)
return x, scale, zp
@@ -229,7 +238,12 @@ class WrapperLinear(torch.nn.Module):
bias_bits = 4 ## hard code
bias_group_size = -1
bias, scale, zp = self.bias_quant_func(
- bias, bits=bias_bits, group_size=bias_group_size, v=bias_v, q_scale_thresh=self.q_scale_thresh
+ bias,
+ bits=bias_bits,
+ group_size=bias_group_size,
+ v=bias_v,
+ q_scale_thresh=self.q_scale_thresh,
+ global_scale=getattr(self, "weight_global_scale", None),
)
return bias, scale, zp
@@ -280,8 +294,6 @@ class WrapperLinear(torch.nn.Module):
if zp is not None:
if isinstance(zp, dict):
_set_dict_attr(zp, "zp")
- elif zp is None:
- self.orig_layer.zp = None
elif zp.numel() > 1:
zp = zp.reshape(shape[0], -1)
self.orig_layer.zp = zp.to("cpu")
@@ -290,6 +302,11 @@ class WrapperLinear(torch.nn.Module):
else:
self.orig_layer.zp = None
+ if self.weight_global_scale is not None:
+ global_scale = self.weight_global_scale
+ assert global_scale.numel() == 1
+ self.orig_layer.weight_global_scale = global_scale.to("cpu")
+
##unwrapper bias
if self.enable_norm_bias_tuning and "bias_v" in best_params.keys(): ##fake quant
bias_v = best_params["bias_v"].to(self.device)
@@ -373,7 +390,7 @@ class WrapperLinear(torch.nn.Module):
torch.Tensor: Output tensor after applying the wrapped layer.
"""
x = x.to(self.device)
- weight_q, _, _ = self._qdq_weight(self.value, self.min_scale, self.max_scale)
+ weight_q, *_ = self._qdq_weight(self.value, self.min_scale, self.max_scale)
if self.enable_act_quant:
act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None
@@ -584,7 +601,6 @@ def wrapper_block(block, enable_minmax_tuning, enable_norm_bias_tuning, device="
set_module(block, n, new_m)
else:
logger.warning_once(f"{m.__class__.__name__} is not supported")
-
return quantized_layers, unquantized_layers
diff --git a/test/test_cpu/requirements.txt b/test/test_cpu/requirements.txt
index bc6615906..265ba7e94 100644
--- a/test/test_cpu/requirements.txt
+++ b/test/test_cpu/requirements.txt
@@ -2,3 +2,4 @@ addict
modelscope
gguf
torchvision
+compressed-tensors
diff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.py
index bbce4036b..b60c87f26 100644
--- a/test/test_cpu/test_export.py
+++ b/test/test_cpu/test_export.py
@@ -252,6 +252,169 @@ class TestAutoRound(unittest.TestCase):
self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype, torch.float8_e4m3fn)
shutil.rmtree(quantized_model_path, ignore_errors=True)
+ def test_mxfp4_llmcompressor_format(self):
+ model_name = "facebook/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ from transformers import AutoConfig
+
+ bits = 4
+ data_type = "mx_fp4e2m1"
+ group_size = 32
+ sym = True
+ autoround = AutoRound(
+ model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ data_type=data_type,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_dir
+ autoround.quantize()
+ compressed_model = autoround.save_quantized(
+ output_dir=quantized_model_path, inplace=True, format="llmcompressor"
+ )
+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj
+ assert (
+ hasattr(tmp_layer, "weight_scale")
+ and hasattr(tmp_layer, "weight_packed")
+ and tmp_layer.weight_scale.dtype is torch.uint8
+ and tmp_layer.weight_scale.shape[0] == 768
+ ), "Illegal MXFP4 packing name or data_type or shape"
+ quantization_config = AutoConfig.from_pretrained(
+ quantized_model_path, trust_remote_code=True
+ ).quantization_config
+ assert (
+ quantization_config["format"] == "float-quantized"
+ and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True
+ and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4
+ ), f"Invalid MXFP4 quantization configuration: {quantization_config}"
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_mxfp8_llmcompressor_format(self):
+ model_name = "facebook/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ from transformers import AutoConfig
+
+ bits = 8
+ data_type = "mx_fp8e4m3_rceil"
+ group_size = 32
+ sym = True
+ autoround = AutoRound(
+ model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ data_type=data_type,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_dir
+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llmcompressor")
+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj
+ assert (
+ hasattr(tmp_layer, "weight_scale")
+ and hasattr(tmp_layer, "weight")
+ and tmp_layer.weight.dtype is torch.float8_e4m3fn
+ and tmp_layer.weight_scale.dtype is torch.uint8
+ and tmp_layer.weight_scale.shape[0] == 768
+ ), "Illegal MXFP8 packing name or data_type or shape"
+ quantization_config = AutoConfig.from_pretrained(
+ quantized_model_path, trust_remote_code=True
+ ).quantization_config
+ assert (
+ quantization_config["format"] == "float-quantized"
+ and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True
+ and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 8
+ ), f"Invalid MXFP8 quantization configuration: {quantization_config}"
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_nvfp4_llmcompressor_format(self):
+ model_name = "facebook/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ from transformers import AutoConfig
+
+ bits = 4
+ act_bits = 4
+ data_type = "nv_fp"
+ act_data_type = "nv_fp4_with_static_gs"
+ group_size = 16
+ sym = True
+ autoround = AutoRound(
+ model,
+ self.tokenizer,
+ bits=bits,
+ act_bits=act_bits,
+ data_type=data_type,
+ act_data_type=act_data_type,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_dir
+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llmcompressor")
+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj
+ assert (
+ hasattr(tmp_layer, "weight_scale")
+ and hasattr(tmp_layer, "weight_global_scale")
+ and hasattr(tmp_layer, "input_global_scale")
+ and tmp_layer.weight_packed.dtype is torch.uint8
+ and tmp_layer.weight_scale.dtype is torch.float8_e4m3fn
+ and tmp_layer.weight_scale.shape[0] == 768
+ ), "Illegal NVFP4 packing name or data_type or shape"
+ quantization_config = AutoConfig.from_pretrained(
+ quantized_model_path, trust_remote_code=True
+ ).quantization_config
+ assert (
+ quantization_config["format"] == "nvfp4-pack-quantized"
+ and quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] == 4
+ ), f"Invalid NVFP4 quantization configuration: {quantization_config}"
+ shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_nvfp4_autoround_format(self):
+ model_name = "facebook/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ from transformers import AutoConfig
+
+ bits = 4
+ act_bits = 4
+ data_type = "nv_fp"
+ act_data_type = "nv_fp4_with_static_gs"
+ group_size = 16
+ sym = True
+ autoround = AutoRound(
+ model,
+ self.tokenizer,
+ bits=bits,
+ act_bits=act_bits,
+ data_type=data_type,
+ act_data_type=act_data_type,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_dir
+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj
+ assert (
+ hasattr(tmp_layer, "weight_scale")
+ and hasattr(tmp_layer, "weight_global_scale")
+ and hasattr(tmp_layer, "input_global_scale")
+ and tmp_layer.weight_packed.dtype is torch.uint8
+ and tmp_layer.weight_scale.dtype is torch.float8_e4m3fn
+ and tmp_layer.weight_scale.shape[0] == 768
+ ), "Illegal NVFP4 packing name or data_type or shape"
+ shutil.rmtree("./saved", ignore_errors=True)
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.py
index 028a6fedd..9e3fc4b26 100644
--- a/test/test_cpu/test_gguf_format.py
+++ b/test/test_cpu/test_gguf_format.py
@@ -59,7 +59,7 @@ class TestGGUF(unittest.TestCase):
quantized_model_path = "./saved"
autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0")
- gguf_file = os.listdir(quantized_model_path)[0]
+ gguf_file = "Qwen2.5-0.5B-Instruct-494M-Q4_0.gguf"
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
text = "There is a girl who likes adventure,"
inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
@@ -79,7 +79,7 @@ class TestGGUF(unittest.TestCase):
quantized_model_path = "./saved"
autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_1")
- gguf_file = os.listdir(quantized_model_path)[0]
+ gguf_file = "Qwen2.5-0.5B-Instruct-494M-Q4_1.gguf"
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
text = "There is a girl who likes adventure,"
inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
@@ -317,9 +317,13 @@ class TestGGUF(unittest.TestCase):
quantized_model_path = "./saved"
autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")
self.assertTrue("mmproj-model.gguf" in os.listdir("./saved"))
- file_name = os.listdir(quantized_model_path)[0]
- file_size = os.path.getsize(os.path.join(quantized_model_path, file_name)) / 1024**2
- self.assertAlmostEqual(file_size, 892, delta=1.0)
+ for file_name in os.listdir(quantized_model_path):
+ file_name = os.listdir(quantized_model_path)[0]
+ file_size = os.path.getsize(os.path.join(quantized_model_path, file_name)) / 1024**2
+ if file_name == "mmproj-model.gguf":
+ self.assertAlmostEqual(file_size, 2535, delta=1.0)
+ else:
+ self.assertAlmostEqual(file_size, 892, delta=1.0)
shutil.rmtree("./saved", ignore_errors=True)
diff --git a/test/test_cuda/requirements.txt b/test/test_cuda/requirements.txt
index 1d9a5cb65..88b031217 100644
--- a/test/test_cuda/requirements.txt
+++ b/test/test_cuda/requirements.txt
@@ -18,4 +18,6 @@ torchvision
tqdm
transformers
vllm>=0.8.5.post1
+compressed-tensors
+
diff --git a/test/test_cuda/test_export.py b/test/test_cuda/test_export.py
index 36021d3d5..9e4711915 100644
--- a/test/test_cuda/test_export.py
+++ b/test/test_cuda/test_export.py
@@ -6,7 +6,7 @@ import unittest
sys.path.insert(0, "../..")
import torch
import transformers
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
from auto_round.testing_utils import require_awq, require_optimum
@@ -25,7 +25,7 @@ class TestAutoRound(unittest.TestCase):
@classmethod
def setUpClass(self):
self.model_name = "facebook/opt-125m"
-
+ self.save_dir = "./saved"
self.llm_dataloader = LLMDataLoader()
@classmethod
@@ -320,6 +320,71 @@ class TestAutoRound(unittest.TestCase):
print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
shutil.rmtree("./saved", ignore_errors=True)
+ def test_nvfp4_llmcompressor_format(self):
+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
+
+ bits = 4
+ act_bits = 4
+ data_type = "nv_fp"
+ act_data_type = "nv_fp4_with_static_gs"
+ group_size = 16
+ sym = True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ act_bits=act_bits,
+ data_type=data_type,
+ act_data_type=act_data_type,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ quantized_model_path = self.save_dir
+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llmcompressor")
+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj
+ assert (
+ hasattr(tmp_layer, "weight_scale")
+ and hasattr(tmp_layer, "weight_global_scale")
+ and hasattr(tmp_layer, "input_global_scale")
+ and tmp_layer.weight_packed.dtype is torch.uint8
+ and tmp_layer.weight_scale.dtype is torch.float8_e4m3fn
+ and tmp_layer.weight_scale.shape[0] == 768
+ ), "Illegal NVFP4 packing name or data_type or shape"
+ quantization_config = AutoConfig.from_pretrained(
+ quantized_model_path, trust_remote_code=True
+ ).quantization_config
+ assert (
+ quantization_config["format"] == "nvfp4-pack-quantized"
+ and quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] == 4
+ ), f"Invalid NVFP4 quantization configuration: {quantization_config}"
+ shutil.rmtree("./saved", ignore_errors=True)
+ # from vllm import LLM, SamplingParams
+ # prompts = [
+ # "The capital of France is",
+ # "The future of AI is",
+ # ]
+ ## Create a sampling params object.
+ # sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=20)
+ # QUANTIZATION = "compressed-tensors"
+ # llm = LLM(model=quantized_model_path,
+ # # quantization=QUANTIZATION,
+ # trust_remote_code=True,
+ # tensor_parallel_size=1,
+ # enforce_eager=True,
+ # gpu_memory_utilization=0.7,
+ # )
+ # outputs = llm.generate(prompts, sampling_params)
+ # # Print the outputs.
+ # for output in outputs:
+ # prompt = output.prompt
+ # generated_text = output.outputs[0].text
+ # if "France" in prompt:
+ # assert "Paris" in generated_text
+
if __name__ == "__main__":
unittest.main()