Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

disable compile packing by default (#975)

Yi Liu8c767db6f2aa
diff --git a/auto_round/envs.py b/auto_round/envs.py
index 3d9b0dfa3..1a1f51de0 100644
--- a/auto_round/envs.py
+++ b/auto_round/envs.py
@@ -23,6 +23,7 @@ if TYPE_CHECKING:
environment_variables: dict[str, Callable[[], Any]] = {
# this is used for configuring the default logging level
"AR_LOG_LEVEL": lambda: os.getenv("AR_LOG_LEVEL", "INFO").upper(),
+ "AR_ENABLE_COMPILE_PACKING": lambda: os.getenv("AR_ENABLE_COMPILE_PACKING", "0").lower() in ("1", "true", "yes"),
"AR_USE_MODELSCOPE": lambda: os.getenv("AR_USE_MODELSCOPE", "False").lower() in ["1", "true"],
}
diff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.py
index 5f8f2f6f4..338d624a7 100644
--- a/auto_round/export/export_to_autoround/qlinear_fp.py
+++ b/auto_round/export/export_to_autoround/qlinear_fp.py
@@ -27,21 +27,20 @@
# limitations under the License.
import math
-from logging import getLogger
import numpy as np
import torch
import torch.nn as nn
import transformers
+import auto_round.envs as envs
from auto_round.compressors.utils import BackendDataType, is_mx_fp, is_nv_fp
from auto_round.data_type.mxfp import FP32_EXPONENT_BIAS, FP32_MIN_NORMAL
from auto_round.data_type.nvfp import cast_to_fp4, get_reciprocal
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
-from auto_round.utils import get_packing_device
+from auto_round.utils import get_packing_device, logger
# from auto_round.utils import get_weight_compress_dtype
-logger = getLogger(__name__)
E8M0_EXPONENT_BIAS = 127
E8M0_EXPONENT_NAN_VAL = 255
@@ -202,7 +201,19 @@ def pack_fp4_to_uint8_cpu(x: torch.Tensor) -> torch.Tensor:
# Adapted from https://github.com/neuralmagic/compressed-tensors/pull/400
[email protected](fullgraph=True, dynamic=True)
+
+
+def _get_packing_fn():
+ if envs.AR_ENABLE_COMPILE_PACKING:
+ logger.warning_once(
+ "Compiled FP4 to UINT8 packing may be incompatible with multi-threading."
+ " Disable it by setting AR_ENABLE_COMPILE_PACKING=0"
+ )
+ return torch.compile(fullgraph=True, dynamic=True)(_pack_fp4_to_uint8)
+ else:
+ return torch.compiler.disable()(_pack_fp4_to_uint8)
+
+
def pack_fp4_to_uint8_cuda(x: torch.Tensor) -> torch.Tensor:
"""
Packs a tensor with values in the fp4 range into uint8.
@@ -210,7 +221,8 @@ def pack_fp4_to_uint8_cuda(x: torch.Tensor) -> torch.Tensor:
:param x: tensor to pack
returns: a packed tensor in uint8
"""
- return _pack_fp4_to_uint8(x)
+ pack_fn = _get_packing_fn()
+ return pack_fn(x)
def _pack_fp4_to_uint8(x: torch.Tensor) -> torch.Tensor: