Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix cpu ut for transformers v5 (#1333)

Tang Kaihuif6e7e3030d55
diff --git a/auto_round/eval/eval_cli.py b/auto_round/eval/eval_cli.py
index 863a9110c..5d6d965b0 100644
--- a/auto_round/eval/eval_cli.py
+++ b/auto_round/eval/eval_cli.py
@@ -279,11 +279,7 @@ def eval_task_by_task(
device_str, parallelism = get_device_and_parallelism(device)
# load after _eval_int in order to make sure import torch after set CUDA_VISIBLE_DEVICES
- import traceback
-
- from lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0401
from lm_eval.models.huggingface import HFLM # pylint: disable=E0401
- from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round.utils import logger
@@ -402,7 +398,7 @@ def _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retry
import time
import traceback
- from lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0401
+ import lm_eval # pylint: disable=E0401
from lm_eval.utils import make_table # pylint: disable=E0401
from auto_round.utils import logger
@@ -418,7 +414,7 @@ def _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retry
current_retry_times = retry_times
while current_retry_times:
try:
- res = lm_simple_evaluate(
+ res = lm_eval.simple_evaluate(
model=hflm, model_args=None, device=device_str, tasks=task, batch_size=batch_size, limit=limit
)
break
@@ -430,7 +426,7 @@ def _evaluate_tasks_with_retry(tasks, hflm, device_str, batch_size, limit, retry
for k, v in hflm.batch_sizes.items():
hflm.batch_sizes[k] = max(v // 2, 1)
logger.warning(f"Out of memory, reset batch_size to {hflm.batch_sizes} and re-try.")
- res = lm_simple_evaluate(
+ res = lm_eval.simple_evaluate(
model=hflm, model_args=None, device=device_str, tasks=task, batch_size=1, limit=limit
)
hflm.batch_sizes = ori_batch_sizes
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index 1f4df7b69..cff49b371 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -31,7 +31,7 @@ def simple_evaluate_user_model(
mllm: bool = False,
**kwargs,
):
- from lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0401
+ import lm_eval # pylint: disable=E0401
from lm_eval.models.huggingface import HFLM # pylint: disable=E0401
if mllm:
@@ -57,7 +57,7 @@ def simple_evaluate_user_model(
dtype=eval_model_dtype,
add_bos_token=add_bos_token,
)
- return lm_simple_evaluate(
+ return lm_eval.simple_evaluate(
model=hflm, model_args=None, batch_size=batch_size, max_batch_size=max_batch_size, limit=limit, **kwargs
)
@@ -71,9 +71,9 @@ def simple_evaluate(
device: Optional[str] = None,
**kwargs,
):
- from lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0401
+ import lm_eval # pylint: disable=E0401
- return lm_simple_evaluate(
+ return lm_eval.simple_evaluate(
model=model,
model_args=model_args,
batch_size=batch_size,
diff --git a/auto_round/modelling/gpt_oss.py b/auto_round/modelling/gpt_oss.py
index 83d980d42..a71b5b610 100644
--- a/auto_round/modelling/gpt_oss.py
+++ b/auto_round/modelling/gpt_oss.py
@@ -14,8 +14,16 @@
import torch
+import transformers
+from packaging import version
from torch import nn
-from transformers.modeling_utils import no_init_weights as skip_weights_initialize
+
+transformers_version = version.parse(transformers.__version__)
+if transformers_version < version.parse("5.0.0"):
+ from transformers.modeling_utils import no_init_weights
+else:
+ from transformers.initialization import no_init_weights
+
from transformers.models.gpt_oss.configuration_gpt_oss import GptOssConfig
from transformers.models.gpt_oss.modeling_gpt_oss import GptOssMLP
@@ -71,7 +79,7 @@ class SequentialGPTOSSMoE(ReplacementModuleBase):
# Build per-expert MLPs
self.experts = nn.ModuleList()
target_device = next(original.experts.parameters()).device
- with skip_weights_initialize(), torch.device("meta"):
+ with no_init_weights(), torch.device("meta"):
for _ in range(E):
self.experts.append(GPTOssSingleExpert(hidden_size, intermediate_size, dtype=dtype))
diff --git a/auto_round/modelling/llama4.py b/auto_round/modelling/llama4.py
index 795431554..7057f777d 100644
--- a/auto_round/modelling/llama4.py
+++ b/auto_round/modelling/llama4.py
@@ -13,7 +13,14 @@
# limitations under the License.
# Note: adapted from # https://github.com/vllm-project/llm-compressor/blob/main/src/llmcompressor/modeling/llama4.py
import torch
-from transformers.modeling_utils import no_init_weights
+import transformers
+from packaging import version
+
+transformers_version = version.parse(transformers.__version__)
+if transformers_version < version.parse("5.0.0"):
+ from transformers.modeling_utils import no_init_weights
+else:
+ from transformers.initialization import no_init_weights
from transformers.models.llama4.modeling_llama4 import Llama4Config, Llama4TextMLP, Llama4TextMoe
from auto_round.modelling.replace_modules import ReplacementModuleBase
diff --git a/auto_round_extension/torch/qlinear_torch.py b/auto_round_extension/torch/qlinear_torch.py
index e223f2aaa..d5389cfa6 100644
--- a/auto_round_extension/torch/qlinear_torch.py
+++ b/auto_round_extension/torch/qlinear_torch.py
@@ -25,6 +25,25 @@ from auto_round.utils import get_packing_device
logger = getLogger(__name__)
+_DEVICE_WF_3BITS_TENSORS = {}
+# Constants for FP4 values (E2M1 format)
+_wf_3bits = [
+ [0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 0],
+ [0, 1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31],
+ [0, 2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 0],
+]
+
+
+def get_wf_3bits_tensor(device):
+ """Get device-specific wf_3bits tensor, creating it if needed."""
+ device_str = str(device)
+ if device_str not in _DEVICE_WF_3BITS_TENSORS:
+ _DEVICE_WF_3BITS_TENSORS[device_str] = torch.tensor(_wf_3bits, dtype=torch.int32, device=device).reshape(
+ 1, 3, 12
+ )
+ return _DEVICE_WF_3BITS_TENSORS[device_str]
+
+
class QuantLinear(nn.Module):
"""
Torch quantized linear layer.
@@ -72,17 +91,11 @@ class QuantLinear(nn.Module):
# is performed by unpacking the weights and using torch.matmul
if self.bits in [2, 4, 8]:
- self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32).unsqueeze(0)
- elif self.bits == 3:
self.wf = torch.tensor(
- [
- [0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 0],
- [0, 1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31],
- [0, 2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 0],
- ],
- dtype=torch.int32,
- ).reshape(1, 3, 12)
-
+ list(range(0, 32, self.bits)), dtype=torch.int32, device=self.qweight.device
+ ).unsqueeze(0)
+ elif self.bits == 3:
+ self.wf = get_wf_3bits_tensor(device=self.qweight.device)
self.dequant_dtype = torch.int16 if self.bits == 8 else torch.int8
def post_init(self):
@@ -277,7 +290,9 @@ class QuantLinear(nn.Module):
if self.bits in [2, 4, 8]:
if self.wf.device != self.qzeros.device:
- self.wf = self.wf.to(self.qzeros.device)
+ self.wf = torch.tensor(
+ list(range(0, 32, self.bits)), dtype=torch.int32, device=self.qzeros.device
+ ).unsqueeze(0)
zeros = torch.bitwise_right_shift(
torch.unsqueeze(self.qzeros, 2).expand(-1, -1, 32 // self.bits),
self.wf.unsqueeze(0),
@@ -293,7 +308,7 @@ class QuantLinear(nn.Module):
)
elif self.bits == 3:
if self.wf.device != self.qzeros.device:
- self.wf = self.wf.to(self.qzeros.device)
+ self.wf = get_wf_3bits_tensor(self.qzeros.device)
zeros = self.qzeros.reshape(self.qzeros.shape[0], self.qzeros.shape[1] // 3, 3, 1).expand(-1, -1, -1, 12)
zeros = zeros >> self.wf.unsqueeze(0)
zeros[:, :, 0, 10] = (zeros[:, :, 0, 10] & 0x3) | ((zeros[:, :, 1, 0] << 2) & 0x4)
@@ -335,7 +350,7 @@ class QuantLinear(nn.Module):
out = torch.matmul(x, weights)
out = out.to(x_dtype)
out = out.reshape(out_shape)
- out = out + self.bias if self.bias is not None else out
+ out = (out + self.bias).to(x_dtype) if self.bias is not None else out
return out
diff --git a/auto_round_extension/torch/qlinear_torch_zp.py b/auto_round_extension/torch/qlinear_torch_zp.py
index f0701fc1f..e501066b3 100644
--- a/auto_round_extension/torch/qlinear_torch_zp.py
+++ b/auto_round_extension/torch/qlinear_torch_zp.py
@@ -21,6 +21,7 @@ import torch.nn as nn
import transformers
from auto_round.utils import get_packing_device
+from auto_round_extension.torch.qlinear_torch import get_wf_3bits_tensor
logger = getLogger(__name__)
@@ -72,16 +73,11 @@ class QuantLinear(nn.Module):
# is performed by unpacking the weights and using torch.matmul
if self.bits in [2, 4, 8]:
- self.wf = torch.tensor(list(range(0, 32, self.bits)), dtype=torch.int32).unsqueeze(0)
- else: ## bits == 3
self.wf = torch.tensor(
- [
- [0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 0],
- [0, 1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31],
- [0, 2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 0],
- ],
- dtype=torch.int32,
- ).reshape(1, 3, 12)
+ list(range(0, 32, self.bits)), dtype=torch.int32, device=self.qweight.device
+ ).unsqueeze(0)
+ else: ## bits == 3
+ self.wf = get_wf_3bits_tensor(device=self.qweight.device)
self.dequant_dtype = torch.int16 if self.bits == 8 else torch.int8
@@ -276,7 +272,9 @@ class QuantLinear(nn.Module):
if self.bits in [2, 4, 8]:
if self.wf.device != self.qzeros.device:
- self.wf = self.wf.to(self.qzeros.device)
+ self.wf = torch.tensor(
+ list(range(0, 32, self.bits)), dtype=torch.int32, device=self.qzeros.device
+ ).unsqueeze(0)
zeros = torch.bitwise_right_shift(
torch.unsqueeze(self.qzeros, 2).expand(-1, -1, 32 // self.bits),
self.wf.unsqueeze(0),
@@ -292,7 +290,7 @@ class QuantLinear(nn.Module):
)
elif self.bits == 3:
if self.wf.device != self.qzeros.device:
- self.wf = self.wf.to(self.qzeros.device)
+ self.wf = get_wf_3bits_tensor(device=self.qzeros.device)
zeros = self.qzeros.reshape(self.qzeros.shape[0], self.qzeros.shape[1] // 3, 3, 1).expand(-1, -1, -1, 12)
zeros = zeros >> self.wf.unsqueeze(0)
zeros[:, :, 0, 10] = (zeros[:, :, 0, 10] & 0x3) | ((zeros[:, :, 1, 0] << 2) & 0x4)
@@ -335,7 +333,7 @@ class QuantLinear(nn.Module):
out = torch.matmul(x, weights)
out = out.to(x_dtype)
out = out.reshape(out_shape)
- out = out + self.bias if self.bias is not None else out
+ out = (out + self.bias).to(x_dtype) if self.bias is not None else out
return out
diff --git a/test/fixtures.py b/test/fixtures.py
index 624170710..cf9ad806e 100644
--- a/test/fixtures.py
+++ b/test/fixtures.py
@@ -64,7 +64,7 @@ def tiny_phi2_model_path():
def tiny_deepseek_v2_model_path():
model_name_or_path = deepseek_v2_name_or_path
tiny_model_path = "./tmp/tiny_deepseek_v2_model_path"
- tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2)
+ tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2, trust_remote_code=False)
yield tiny_model_path
shutil.rmtree(tiny_model_path, ignore_errors=True)
diff --git a/test/helpers.py b/test/helpers.py
index e8630e1fa..efd04dff5 100644
--- a/test/helpers.py
+++ b/test/helpers.py
@@ -4,9 +4,12 @@ import os
import pytest
import torch
import transformers
+from packaging import version
from auto_round.utils import get_attr, llm_load_model, mllm_load_model, set_attr
+transformers_version = version.parse(transformers.__version__)
+
# Automatic choose local path or model name.
def get_model_path(model_name: str) -> str:
@@ -98,14 +101,14 @@ def save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2, is_mllm=F
for key in model._tied_weights_keys:
weight = get_attr(model, key)
set_attr(model, key, copy.deepcopy(weight))
- tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True)
+ tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path, **kwargs)
test_path = os.path.dirname(__file__)
tiny_model_path = os.path.join(test_path, tiny_model_path.removeprefix("./"))
model.save_pretrained(tiny_model_path)
tokenizer.save_pretrained(tiny_model_path)
if is_mllm:
- processor = transformers.AutoProcessor.from_pretrained(model_name_or_path, trust_remote_code=True)
- image_processor = transformers.AutoImageProcessor.from_pretrained(model_name_or_path, trust_remote_code=True)
+ processor = transformers.AutoProcessor.from_pretrained(model_name_or_path, **kwargs)
+ image_processor = transformers.AutoImageProcessor.from_pretrained(model_name_or_path, **kwargs)
processor.save_pretrained(tiny_model_path)
image_processor.save_pretrained(tiny_model_path)
print(f"[Fixture]: built tiny model path:{tiny_model_path} for testing in session")
diff --git a/test/test_ark/requirements.txt b/test/test_ark/requirements.txt
index 07605745d..286955dfd 100644
--- a/test/test_ark/requirements.txt
+++ b/test/test_ark/requirements.txt
@@ -1,2 +1,2 @@
auto-round-kernel
-lm-eval
\ No newline at end of file
+lm_eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@main
\ No newline at end of file
diff --git a/test/test_cpu/core/test_autoround.py b/test/test_cpu/core/test_autoround.py
index 79458e237..6e77a7b2f 100644
--- a/test/test_cpu/core/test_autoround.py
+++ b/test/test_cpu/core/test_autoround.py
@@ -3,13 +3,14 @@ import shutil
import pytest
import torch
+from packaging import version
from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizer
from auto_round import AutoRound
from auto_round.eval.evaluation import simple_evaluate_user_model
from auto_round.utils import get_module
-from ...helpers import get_model_path, model_infer, opt_name_or_path, qwen_name_or_path
+from ...helpers import get_model_path, model_infer, opt_name_or_path, qwen_name_or_path, transformers_version
class TestAutoRound:
@@ -72,6 +73,10 @@ class TestAutoRound:
)
autoround.quantize()
+ @pytest.mark.skipif(
+ transformers_version >= version.parse("5.0"),
+ reason="PhiConfig missing pad_token_id, https://github.com/huggingface/transformers/pull/43453",
+ )
def test_consecutive_quant(self, tiny_opt_model_path, tiny_phi2_model_path, dataloader):
bits, group_size, sym = 4, -1, False
autoround = AutoRound(
@@ -456,8 +461,12 @@ class TestAutoRound:
model = Qwen2VLForConditionalGeneration.from_pretrained(
model_name, quantization_config=quantization_config, device_map="cpu", torch_dtype=torch.float16
)
- assert isinstance(model.visual.blocks[0].attn.qkv, torch.nn.Linear)
- assert not isinstance(model.visual.merger.mlp[0], QuantLinear)
+ if transformers_version < version.parse("5.0.0"):
+ assert isinstance(model.visual.blocks[0].attn.qkv, torch.nn.Linear)
+ assert not isinstance(model.visual.merger.mlp[0], QuantLinear)
+ else:
+ assert isinstance(model.model.visual.blocks[0].attn.qkv, torch.nn.Linear)
+ assert not isinstance(model.model.visual.merger.mlp[0], QuantLinear)
if hasattr(model.model, "language_model"):
assert isinstance(model.model.language_model.layers[0].self_attn.v_proj, QuantLinear)
else:
diff --git a/test/test_cpu/export/test_export.py b/test/test_cpu/export/test_export.py
index cb6033de9..f7702108f 100644
--- a/test/test_cpu/export/test_export.py
+++ b/test/test_cpu/export/test_export.py
@@ -3,11 +3,12 @@ import shutil
import pytest
import torch
+from packaging import version
from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizer
from auto_round import AutoRound
-from ...helpers import get_model_path, opt_name_or_path
+from ...helpers import get_model_path, opt_name_or_path, transformers_version
def _get_folder_size(path: str) -> float:
@@ -262,7 +263,10 @@ class TestAutoRound:
assert "model.decoder.layers.8.self_attn.v_scale" in f.keys()
assert f.get_tensor("model.decoder.layers.5.self_attn.v_scale").shape == torch.Size([1])
assert f.get_tensor("model.decoder.layers.5.self_attn.k_scale").shape == torch.Size([1])
- assert f.get_tensor("model.decoder.layers.5.self_attn.k_scale").dtype == torch.float32
+ assert (
+ f.get_tensor("model.decoder.layers.5.self_attn.k_scale").dtype == torch.float32
+ or f.get_tensor("model.decoder.layers.5.self_attn.k_scale").dtype == torch.bfloat16
+ )
shutil.rmtree(quantized_model_path, ignore_errors=True)
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
@@ -318,10 +322,14 @@ class TestAutoRound:
weight_name = f"model.decoder.layers.8.self_attn.{attr}"
assert weight_name in f.keys()
assert f.get_tensor(weight_name).shape == torch.Size([1])
- assert f.get_tensor(weight_name).dtype == torch.float32
+ assert f.get_tensor(weight_name).dtype == torch.float32 or f.get_tensor(weight_name).dtype == torch.bfloat16
shutil.rmtree(quantized_model_path, ignore_errors=True)
+ @pytest.mark.skipif(
+ transformers_version >= version.parse("5.0"),
+ reason="PhiConfig missing pad_token_id, https://github.com/huggingface/transformers/pull/43453",
+ )
def test_awq_lmhead_export(self, dataloader):
bits, sym, group_size = 4, False, 128
model_name = get_model_path("microsoft/phi-2")
@@ -348,6 +356,10 @@ class TestAutoRound:
assert isinstance(lm_head, WQLinear_GEMM), "Illegal AWQ quantization for lm_head layer"
shutil.rmtree(quantized_model_path, ignore_errors=True)
+ @pytest.mark.skipif(
+ transformers_version >= version.parse("5.0"),
+ reason="PhiConfig missing pad_token_id, https://github.com/huggingface/transformers/pull/43453",
+ )
def test_gptq_lmhead_export(self, dataloader):
bits, sym, group_size = 4, True, 128
# Note that, to save UT tuning time, the local model is intentionally kept lightweight, using only 2 hidden layers.
diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.py
index 2e3cde6f0..f1980c455 100644
--- a/test/test_cpu/export/test_gguf_format.py
+++ b/test/test_cpu/export/test_gguf_format.py
@@ -4,16 +4,22 @@ import sys
import pytest
import torch
+from packaging import version
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
-from ...helpers import get_model_path, get_tiny_model, save_tiny_model
+from ...helpers import get_model_path, get_tiny_model, save_tiny_model, transformers_version
AUTO_ROUND_PATH = __file__.split("/")
AUTO_ROUND_PATH = "/".join(AUTO_ROUND_PATH[: AUTO_ROUND_PATH.index("test")])
+ transformers_version >= version.parse("5.0.0"),
+ reason="GGUF format saving and loading failed in transformers v5, \
+ https://github.com/huggingface/transformers/issues/43482",
+)
class TestGGUF:
@classmethod
@@ -60,6 +66,12 @@ class TestGGUF:
autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0")
gguf_file = os.listdir(quantized_model_path)[0]
+
+ # TODO: fix the issue of gguf loading error in transformers v5
+ # cls = transformers.generation.configuration_utils.GenerationConfig'>, json_file = None
+ # def _dict_from_json_file(cls, json_file: str | os.PathLike):
+ # > with open(json_file, "r", encoding="utf-8") as reader:
+ # E TypeError: expected str, bytes or os.PathLike object, not NoneType
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")
text = "There is a girl who likes adventure,"
inputs = self.tokenizer(text, return_tensors="pt").to(model.device)
diff --git a/test/test_cpu/integrations/test_llmc_integration.py b/test/test_cpu/integrations/test_llmc_integration.py
index cea412327..069d4fe06 100644
--- a/test/test_cpu/integrations/test_llmc_integration.py
+++ b/test/test_cpu/integrations/test_llmc_integration.py
@@ -3,10 +3,13 @@ import torch
from compressed_tensors.quantization import QuantizationArgs, QuantizationScheme
from llmcompressor import oneshot
from llmcompressor.modifiers.autoround import AutoRoundModifier
+from packaging import version
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round.calib_dataset import get_dataset
+from ...helpers import transformers_version
+
recipe_str = """
quant_stage:
quant_modifiers:
@@ -39,6 +42,11 @@ recipe_modifier_full = AutoRoundModifier(
)
+ transformers_version >= version.parse("5.0.0"),
+ reason="transformers 5.0 use_auth_token is deprecated and llmcompressor oneshot has not been updated yet, \
+ https://github.com/vllm-project/llm-compressor/issues/2289",
+)
@pytest.mark.parametrize(
"recipe",
[
diff --git a/test/test_cpu/models/test_diffusion.py b/test/test_cpu/models/test_diffusion.py
index 32848ce85..ca495500a 100644
--- a/test/test_cpu/models/test_diffusion.py
+++ b/test/test_cpu/models/test_diffusion.py
@@ -1,11 +1,11 @@
import shutil
import pytest
-from diffusers import AutoPipelineForText2Image
+from packaging import version
from auto_round import AutoRound
-from ...helpers import get_model_path
+from ...helpers import get_model_path, transformers_version
flux_name_or_path = get_model_path("black-forest-labs/FLUX.1-dev")
@@ -13,12 +13,18 @@ flux_name_or_path = get_model_path("black-forest-labs/FLUX.1-dev")
@pytest.fixture
def setup_flux():
"""Fixture to set up the Flux model and tokenizer."""
+ from diffusers import AutoPipelineForText2Image
+
model_name = flux_name_or_path
pipe = AutoPipelineForText2Image.from_pretrained(model_name)
output_dir = "./tmp/test_quantized_flux"
return pipe, output_dir
+ transformers_version >= version.parse("5.0.0"),
+ reason="cannot import name 'MT5Tokenizer' from 'transformers', https://github.com/huggingface/diffusers/issues/13035",
+)
def test_flux(setup_flux):
pipe, output_dir = setup_flux
autoround = AutoRound(
diff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.py
index 4934d619e..8872edccb 100644
--- a/test/test_cpu/models/test_moe_model.py
+++ b/test/test_cpu/models/test_moe_model.py
@@ -1,13 +1,14 @@
import shutil
import pytest
+from packaging import version
from transformers import AutoConfig, AutoProcessor, AutoTokenizer, Llama4ForConditionalGeneration
from transformers.models.gpt_oss.modeling_gpt_oss import GptOssForCausalLM
from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import Qwen3VLMoeForConditionalGeneration
from auto_round import AutoRound
-from ...helpers import get_model_path
+from ...helpers import get_model_path, transformers_version
gpt_oss_name_or_path = get_model_path("unsloth/gpt-oss-20b-BF16")
llama4_name_or_path = get_model_path("meta-llama/Llama-4-Scout-17B-16E-Instruct")
@@ -36,6 +37,7 @@ def setup_llama4():
config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
config.vision_config.num_hidden_layers = 1 # Reduce layers for testing
config.text_config.num_hidden_layers = 1
+ # config.vision_config.rope_theta = config.vision_config.rope_parameters["rope_theta"] # for transformers >= 5.0
model = Llama4ForConditionalGeneration(config)
output_dir = "./tmp/test_quantized_llama4"
return model, tokenizer, output_dir, config
@@ -121,6 +123,10 @@ def test_gptoss(setup_gpt_oss, scheme):
shutil.rmtree(output_dir, ignore_errors=True)
+ transformers_version >= version.parse("5.0.0"),
+ reason="transformers v5 'Llama4VisionConfig' object has no attribute 'rope_theta'",
+)
def test_llama4(setup_llama4):
model, tokenizer, output_dir, config = setup_llama4
diff --git a/test/test_cpu/quantization/test_mxfp_nvfp.py b/test/test_cpu/quantization/test_mxfp_nvfp.py
index f5044bc73..3c2e9bcce 100644
--- a/test/test_cpu/quantization/test_mxfp_nvfp.py
+++ b/test/test_cpu/quantization/test_mxfp_nvfp.py
@@ -3,11 +3,12 @@ import shutil
import pytest
import torch
+from packaging import version
from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizer
from auto_round import AutoRound
-from ...helpers import is_model_outputs_similar
+from ...helpers import is_model_outputs_similar, transformers_version
def _get_folder_size(path: str) -> float:
@@ -31,6 +32,9 @@ class TestAutoRoundFP:
shutil.rmtree("./saved", ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
+ @pytest.mark.skipif(
+ transformers_version >= version.parse("5.0.0"), reason="transformers v5 MOE model has breaking changes"
+ )
def test_nvfp4_moe_actmax_rtn(self, tiny_deepseek_v2_model_path, dataloader):
model_name = tiny_deepseek_v2_model_path
layer_config = {
@@ -58,6 +62,9 @@ class TestAutoRoundFP:
), "Illegal NVFP4 quantization for lm_head layer"
shutil.rmtree(self.save_dir, ignore_errors=True)
+ @pytest.mark.skipif(
+ transformers_version >= version.parse("5.0.0"), reason="transformers v5 MOE model has breaking changes"
+ )
def test_nvfp4_moe_actmax_ar(self, tiny_deepseek_v2_model_path, dataloader):
model_name = tiny_deepseek_v2_model_path
layer_config = {
@@ -90,6 +97,9 @@ class TestAutoRoundFP:
assert is_model_outputs_similar(model_name, quantized_model_path)
shutil.rmtree(self.save_dir, ignore_errors=True)
+ @pytest.mark.skipif(
+ transformers_version >= version.parse("5.0.0"), reason="transformers v5 MOE model has breaking changes"
+ )
def test_mxfp4_moe_ar(self, tiny_deepseek_v2_model_path, dataloader):
model_name = tiny_deepseek_v2_model_path
layer_config = {
@@ -323,6 +333,9 @@ class TestAutoRoundFP:
), "Illegal NVFP4 packing name or data_type or shape"
shutil.rmtree(quantized_model_path, ignore_errors=True)
+ @pytest.mark.skipif(
+ transformers_version >= version.parse("5.0.0"), reason="transformers v5 MOE model has breaking changes"
+ )
def test_qwen_moe_quant_infer(self, tiny_qwen_moe_model_path, dataloader):
model_name = tiny_qwen_moe_model_path
layer_config = {
diff --git a/test/test_cpu/requirements.txt b/test/test_cpu/requirements.txt
index 7f8f469f8..dbe2d5014 100644
--- a/test/test_cpu/requirements.txt
+++ b/test/test_cpu/requirements.txt
@@ -6,6 +6,6 @@ torchvision
pillow
numba
llmcompressor @ git+https://github.com/vllm-project/llm-compressor.git@main
-lm_eval
+lm_eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@main
diffusers
protobuf
diff --git a/test/test_cuda/models/test_moe_model.py b/test/test_cuda/models/test_moe_model.py
index 66b0dc827..0cd2bd5e9 100644
--- a/test/test_cuda/models/test_moe_model.py
+++ b/test/test_cuda/models/test_moe_model.py
@@ -31,6 +31,7 @@ def setup_llama4():
model_name = "/dataset/Llama-4-Scout-17B-16E-Instruct"
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
+ config.pad_token_id = None # TODO: https://github.com/huggingface/transformers/issues/43525
config.vision_config.num_hidden_layers = 1 # Reduce layers for testing
config.text_config.num_hidden_layers = 1
model = Llama4ForConditionalGeneration(config)
diff --git a/test/test_cuda/quantization/test_mix_bits.py b/test/test_cuda/quantization/test_mix_bits.py
index 614a5d2b1..9daa9b727 100644
--- a/test/test_cuda/quantization/test_mix_bits.py
+++ b/test/test_cuda/quantization/test_mix_bits.py
@@ -263,7 +263,13 @@ class TestAutoRound:
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
# Create an LLM.
QUANTIZATION = "auto-round" # quantized_model_path
- llm = LLM(model=quantized_model_path, quantization=QUANTIZATION, trust_remote_code=True, tensor_parallel_size=1)
+ llm = LLM(
+ model=quantized_model_path,
+ quantization=QUANTIZATION,
+ trust_remote_code=True,
+ tensor_parallel_size=1,
+ allow_deprecated_quantization=True,
+ )
outputs = llm.generate(prompts, sampling_params)
# Print the outputs.
for output in outputs:
@@ -302,8 +308,12 @@ class TestAutoRound:
# Create a sampling params object.
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
# Create an LLM.
- QUANTIZATION = "auto-round" # quantized_model_path
- llm = LLM(model=quantized_model_path, trust_remote_code=True, tensor_parallel_size=1)
+ llm = LLM(
+ model=quantized_model_path,
+ trust_remote_code=True,
+ tensor_parallel_size=1,
+ allow_deprecated_quantization=True,
+ )
outputs = llm.generate(prompts, sampling_params)
# Print the outputs.
for output in outputs:
diff --git a/test/test_cuda/quantization/test_mxfp_nvfp.py b/test/test_cuda/quantization/test_mxfp_nvfp.py
index 922cb9f9b..7aa3746a0 100644
--- a/test/test_cuda/quantization/test_mxfp_nvfp.py
+++ b/test/test_cuda/quantization/test_mxfp_nvfp.py
@@ -150,12 +150,12 @@ class TestAutoRound:
layer_config=layer_config,
)
quantized_model_path = self.save_dir
- autoround.quantize_and_save(output_dir=quantized_model_path, inplace=True, format="auto_round")
+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="auto_round")
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
from auto_round.eval.evaluation import simple_evaluate_user_model
- result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="piqa")
+ result = simple_evaluate_user_model(model, tokenizer, batch_size=16, tasks="piqa", limit=10)
print(result["results"]["piqa"]["acc,none"])
assert result["results"]["piqa"]["acc,none"] > 0.49
shutil.rmtree(quantized_model_path, ignore_errors=True)
diff --git a/test/test_cuda/requirements.txt b/test/test_cuda/requirements.txt
index fa36ffcc4..3d4e3e101 100644
--- a/test/test_cuda/requirements.txt
+++ b/test/test_cuda/requirements.txt
@@ -1,11 +1,12 @@
# pip install -v git+https://github.com/casper-hansen/AutoAWQ.git --no-build-isolation
+--extra-index-url https://wheels.vllm.ai/nightly
auto-gptq
einops
# pip install -v git+https://github.com/ModelCloud/GPTQModel.git --no-build-isolation
-lm-eval>=0.4.9.1
+lm_eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@main
optimum
pandas
pillow
torchvision
numba
-vllm>=0.8.5.post1
+vllm
\ No newline at end of file
diff --git a/test/test_cuda/requirements_sglang.txt b/test/test_cuda/requirements_sglang.txt
index db0e61c5d..384c162b5 100644
--- a/test/test_cuda/requirements_sglang.txt
+++ b/test/test_cuda/requirements_sglang.txt
@@ -1 +1 @@
-sglang>=0.5.4.post2
+sglang @ git+https://github.com/sgl-project/sglang.git@main#subdirectory=python
\ No newline at end of file