Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Update torch to 2.11.0 in CI (#1606)

Sun, Xuehaoae7661ba26f0
diff --git a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
index 398a7da4c..0fe6f4440 100644
--- a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
+++ b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
@@ -62,27 +62,15 @@ function run_unit_test() {
# install unit test dependencies
echo "##[group]set up UT env..."
cd "${BUILD_SOURCESDIRECTORY}" || exit 1
- uv pip install torch==2.10.0 torchvision
- uv pip install git+https://github.com/casper-hansen/AutoAWQ.git --no-build-isolation
-
- # install gptqmodel
- CUDA_VER=$(python -c 'import torch; print(f"cu{torch.version.cuda.replace(".", "")}")')
- PY_VER=$(python -c 'import sys; print(f"cp{sys.version_info.major}{sys.version_info.minor}")')
- TORCH_VER="torch2.10"
- WHEEL="gptqmodel-5.7.0-${CUDA_VER}${TORCH_VER}-${PY_VER}-${PY_VER}-linux_x86_64.whl"
- URL="https://pkgs.dev.azure.com/lpot-inc/b7121868-d73a-4794-90c1-23135f974d09/_packaging/4728fbab-e069-4cbd-bcca-d35f4d42256b/pypi/download/gptqmodel/5.7/${WHEEL}"
- wget -q "$URL" -O "$WHEEL" || { echo "Download failed. Check CUDA/PyTorch/Python versions match (cu126/cu128/cu130, torch2.10, cp310-cp313)"; exit 1; }
- mv "$WHEEL" "${WHEEL/-${CUDA_VER}${TORCH_VER}-/+${CUDA_VER}.${TORCH_VER}-}"
- uv pip install "./${WHEEL/-${CUDA_VER}${TORCH_VER}-/+${CUDA_VER}.${TORCH_VER}-}" --no-build-isolation
- rm -f "./${WHEEL/-${CUDA_VER}${TORCH_VER}-/+${CUDA_VER}.${TORCH_VER}-}"
-
- uv pip install gptqmodel --extra-index-url https://pkgs.dev.azure.com/lpot-inc/neural-compressor/_packaging/gptqmodel-wheels/pypi/simple/
- uv pip install -r https://raw.githubusercontent.com/ModelCloud/GPTQModel/refs/tags/v5.7.0/requirements.txt
+ uv pip install https://github.com/XuehaoSun/GPTQModel/releases/download/v5.8.0/gptqmodel-5.8.0+cu128torch2.11-cp312-cp312-linux_x86_64.whl
+ uv pip install -r https://raw.githubusercontent.com/ModelCloud/GPTQModel/refs/tags/v5.8.0/requirements.txt
uv pip install https://github.com/XuehaoSun/llama-cpp-python/releases/download/v0.3.16/llama_cpp_python-0.3.16-cp312-cp312-linux_x86_64.whl
uv pip install 'git+https://github.com/ggml-org/llama.cpp.git#subdirectory=gguf-py'
uv pip install -r test/test_cuda/requirements.txt
uv pip install -r test/test_cuda/requirements_diffusion.txt
uv pip install -U transformers
+ uv pip uninstall torch torchvision
+ uv pip install torch==2.11.0 torchvision --index-url https://download.pytorch.org/whl/cu128
uv pip install .
echo "##[endgroup]"
@@ -126,8 +114,9 @@ function run_unit_test_llmc() {
uv pip install -U pytest-cov pytest-html
BUILD_TYPE="nightly" uv pip install -r test/test_cuda/requirements_llmc.txt --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match
uv pip install .
- echo "##[endgroup]"
uv pip list
+ echo "##[endgroup]"
+
cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1
export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coverage"
@@ -151,9 +140,9 @@ function run_unit_test_sglang() {
uv pip install -U pytest-cov pytest-html
uv pip install -r test/test_cuda/requirements_sglang.txt
uv pip install .
+ uv pip list
echo "##[endgroup]"
- uv pip list
cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1
export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coverage"
@@ -174,11 +163,11 @@ function run_unit_test_vllm() {
rm -rf /root/.venv
uv venv --python=3.12 /root/.venv
uv pip install -U pytest-cov pytest-html
- uv pip install -r test/test_cuda/requirements_vllm.txt
+ uv pip install -r test/test_cuda/requirements_vllm.txt --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match
uv pip install .
+ uv pip list
echo "##[endgroup]"
- uv pip list
cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1
export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coverage"
@@ -197,16 +186,14 @@ function main() {
setup_environment
if [ "${test_case}" == "vlm" ]; then
run_unit_test_vlm
- elif [ "${test_case}" == "llmc" ]; then
- run_unit_test_llmc
- elif [ "${test_case}" == "sglang" ]; then
+ elif [ "${test_case}" == "specific" ]; then
run_unit_test_sglang
- elif [ "${test_case}" == "vllm" ]; then
+ run_unit_test_llmc
run_unit_test_vllm
elif [ "${test_case}" == "all" ]; then
run_unit_test
else
- echo "##[error]Invalid test case specified: ${test_case}. Please use 'vlm', 'llmc', 'sglang', 'vllm', or 'all'."
+ echo "##[error]Invalid test case specified: ${test_case}. Please use 'vlm', 'specific', or 'all'."
exit 1
fi
check_storage_usage
diff --git a/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py b/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py
index 1959433cd..fcdfc8ab0 100644
--- a/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py
+++ b/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py
@@ -13,6 +13,36 @@ TARGET_GPUS = [
"NVIDIA L40S",
"NVIDIA RTX PRO 6000 Blackwell Server Edition",
]
+DATA_CENTER_IDS = [
+ "AP-JP-1",
+ "CA-MTL-1",
+ "CA-MTL-2",
+ "CA-MTL-3",
+ "EU-CZ-1",
+ "EU-FR-1",
+ "EU-NL-1",
+ "EU-RO-1",
+ "EU-SE-1",
+ "EUR-IS-1",
+ "EUR-IS-2",
+ "EUR-IS-3",
+ "EUR-NO-1",
+ "OC-AU-1",
+ "US-CA-2",
+ "US-DE-1",
+ "US-GA-1",
+ "US-GA-2",
+ "US-IL-1",
+ "US-KS-2",
+ "US-KS-3",
+ "US-NC-1",
+ "US-TX-1",
+ "US-TX-3",
+ "US-TX-4",
+ "US-WA-1",
+]
+DATA_CENTER_BAN_LIST = ["EUR-IS-2", "US-IL-1"]
+DATA_CENTER_SELECT_LIST = [dc for dc in DATA_CENTER_IDS if dc not in DATA_CENTER_BAN_LIST]
REQUIRED_COUNT = 1
IMAGES_NAME = "xuehaosu/azure-agent:v0.1"
@@ -46,7 +76,17 @@ def check_gpu_count(token):
}
}
""" % ids_string
- variables = {"input": {"gpuCount": 1, "secureCloud": True, "minMemoryInGb": 0, "minVcpuCount": 0}}
+
+ datacenter_id_string = ",".join(DATA_CENTER_SELECT_LIST)
+ variables = {
+ "input": {
+ "gpuCount": 1,
+ "secureCloud": True,
+ "minMemoryInGb": 0,
+ "minVcpuCount": 0,
+ "dataCenterId": datacenter_id_string,
+ }
+ }
try:
response = requests.post(
@@ -94,15 +134,27 @@ def check_gpu_count(token):
def run_create_pod(api_key, payload):
url = "https://rest.runpod.io/v1/pods"
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
- response = requests.post(url, json=payload, headers=headers)
+ max_retries = 3
+
+ for attempt in range(max_retries + 1):
+ response = requests.post(url, json=payload, headers=headers)
+
+ if response.status_code >= 500 or response.status_code == 400:
+ if attempt < max_retries:
+ print(f"⚠️ {response.status_code} Error, Retrying in 60 seconds ({attempt + 1}/{max_retries})...")
+ time.sleep(60)
+ continue
+ else:
+ print(f"❌ {response.status_code} Error, Reached maximum retry attempts ({max_retries}), giving up.")
+ response.raise_for_status()
- response.raise_for_status()
- result = response.json()
- if "errors" in result:
- print("❌ Errors:")
- print(json.dumps(result["errors"], indent=2))
- sys.exit(1)
- return result
+ result = response.json()
+ if "errors" in result:
+ print("❌ Errors:")
+ print(json.dumps(result["errors"], indent=2))
+ sys.exit(1)
+
+ return result
def create_pod(args):
@@ -125,6 +177,8 @@ def create_pod(args):
payload = {
"cloudType": "SECURE",
"containerDiskInGb": args.container_disk_size,
+ "dataCenterIds": DATA_CENTER_SELECT_LIST,
+ "dataCenterPriority": "availability",
"env": env_dict,
"gpuCount": args.gpu_count,
"gpuTypeIds": [gpu_type],
@@ -186,11 +240,23 @@ def terminate_pod(args):
url = f"https://rest.runpod.io/v1/pods/{pod_id}"
headers = {"Authorization": f"Bearer {args.api_key}"}
- response = requests.delete(url, headers=headers)
- response.raise_for_status()
max_tries = 30
+ for attempt in range(max_tries + 1):
+ response = requests.delete(url, headers=headers)
+ if response.status_code >= 500 or response.status_code == 400:
+ if attempt < max_tries:
+ print(f"⚠️ {response.status_code} Error, Retrying in 5 seconds ({attempt + 1}/{max_tries})...")
+ time.sleep(5)
+ continue
+ else:
+ print(f"❌ {response.status_code} Error, Reached maximum retry attempts ({max_tries}), giving up.")
+ else:
+ break
+
+ response.raise_for_status()
+
for i in range(max_tries): # Wait up to 5 minutes for termination
pod_id = get_pod_id(args)
if pod_id:
diff --git a/.azure-pipelines/scripts/ut/run_ut_cuda.sh b/.azure-pipelines/scripts/ut/run_ut_cuda.sh
index b11e17abd..1bdc63f5b 100644
--- a/.azure-pipelines/scripts/ut/run_ut_cuda.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_cuda.sh
@@ -2,7 +2,7 @@
set -xe
CONDA_ENV_NAME="unittest_cuda"
-PYTHON_VERSION="3.10"
+PYTHON_VERSION="3.12"
REPO_PATH=$(git rev-parse --show-toplevel)
LOG_DIR=${REPO_PATH}/ut_log_dir
SUMMARY_LOG=${LOG_DIR}/results_summary.log
@@ -91,15 +91,16 @@ function run_unit_test() {
cd ${REPO_PATH}/test
rm -rf .coverage* *.xml *.html
- uv pip install torch==2.10.0 torchvision
- uv pip install -v git+https://github.com/casper-hansen/AutoAWQ.git --no-build-isolation
+ uv pip install torch==2.11.0 torchvision --index-url https://download.pytorch.org/whl/cu128
uv pip install gptqmodel --no-build-isolation
uv pip install -r https://raw.githubusercontent.com/ModelCloud/GPTQModel/refs/heads/main/requirements.txt
- CMAKE_ARGS="-DGGML_CUDA=on -DLLAVA_BUILD=off" uv pip install llama-cpp-python
+ uv pip install https://github.com/XuehaoSun/llama-cpp-python/releases/download/v0.3.16/llama_cpp_python-0.3.16-cp312-cp312-linux_x86_64.whl
uv pip install 'git+https://github.com/ggml-org/llama.cpp.git#subdirectory=gguf-py'
uv pip install -r test_cuda/requirements.txt
uv pip install -r test_cuda/requirements_diffusion.txt
uv pip install -U transformers
+ uv pip uninstall torch torchvision
+ uv pip install torch==2.11.0 torchvision --index-url https://download.pytorch.org/whl/cu128
cd ${REPO_PATH} && uv pip install . && cd ${REPO_PATH}/test
pip list > ${LOG_DIR}/ut_pip_list.txt
@@ -132,15 +133,12 @@ function run_unit_test_vlm() {
cd ${REPO_PATH}/test
rm -rf .coverage* *.xml *.html
- uv pip install torch==2.10.0 torchvision
+ uv pip install torch==2.11.0 torchvision --index-url https://download.pytorch.org/whl/cu128
+ uv pip install https://github.com/XuehaoSun/GPTQModel/releases/download/v5.8.0/gptqmodel-5.8.0+cu128torch2.11-cp312-cp312-linux_x86_64.whl
uv pip install git+https://github.com/haotian-liu/[email protected] --no-deps
- local site_path=$(python -c "import site; print(site.getsitepackages()[0])")
- # reference https://github.com/haotian-liu/LLaVA/issues/1448#issuecomment-2119845242
- sed -i '/inputs\[.*image_sizes.*\] = image_sizes/a\ inputs.pop("cache_position")' ${site_path}/llava/model/language_model/llava_llama.py
- uv pip install git+https://github.com/deepseek-ai/DeepSeek-VL2.git timm attrdict --no-deps
uv pip install -v git+https://github.com/casper-hansen/[email protected] --no-build-isolation
- uv pip install flash-attn==2.7.4.post1 --no-build-isolation
- uv pip install -r test_cuda/requirements_vlm.txt
+ uv pip install flash-attn==2.8.3 --no-build-isolation
+ uv pip install -r test_cuda/requirements_vlm.txt --extra-index-url https://download.pytorch.org/whl/cu128
cd ${REPO_PATH} && uv pip install . && cd ${REPO_PATH}/test
pip list > ${LOG_DIR}/vlm_ut_pip_list.txt
@@ -173,7 +171,7 @@ function run_unit_test_llmc() {
cd ${REPO_PATH}/test
rm -rf .coverage* *.xml *.html
- BUILD_TYPE="nightly" uv pip install -r test_cuda/requirements_llmc.txt
+ BUILD_TYPE="nightly" uv pip install -r test_cuda/requirements_llmc.txt --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match
cd ${REPO_PATH} && uv pip install . && cd ${REPO_PATH}/test
pip list > ${LOG_DIR}/llmc_ut_pip_list.txt
@@ -234,10 +232,11 @@ function run_unit_test_sglang() {
function run_unit_test_vllm() {
# install unit test dependencies
create_conda_env
+ unset LD_LIBRARY_PATH
cd ${REPO_PATH}/test
rm -rf .coverage* *.xml *.html
- uv pip install -r test_cuda/requirements_vllm.txt
+ uv pip install -r test_cuda/requirements_vllm.txt --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match
cd ${REPO_PATH} && uv pip install . && cd ${REPO_PATH}/test
pip list > ${LOG_DIR}/vllm_ut_pip_list.txt
diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.yml
index f8fd5c826..e7508a7b7 100644
--- a/.azure-pipelines/template/ut-template.yml
+++ b/.azure-pipelines/template/ut-template.yml
@@ -46,8 +46,8 @@ steps:
- script: |
if [ "${{ parameters.utScriptFileName }}" == "run_ut_xpu" ];then
docker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round \
- && uv pip install torch==2.10.0 torchvision --index-url https://download.pytorch.org/whl/xpu \
- && uv pip install torch==2.10.0 auto-round-lib \
+ && uv pip install torch==2.11.0 torchvision --index-url https://download.pytorch.org/whl/xpu \
+ && uv pip install torch==2.11.0 auto-round-lib \
&& uv pip install -r requirements.txt \
&& uv pip install -r test/test_ark/requirements.txt \
&& uv pip install -r test/test_xpu/requirements.txt \
@@ -55,8 +55,8 @@ steps:
&& uv pip list"
else
docker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round \
- && uv pip install torch==2.10.0 torchvision --index-url https://download.pytorch.org/whl/cpu \
- && uv pip install torch==2.10.0 auto-round-lib \
+ && uv pip install torch==2.11.0 torchvision --index-url https://download.pytorch.org/whl/cpu \
+ && uv pip install torch==2.11.0 auto-round-lib \
&& uv pip install -r requirements.txt \
&& uv pip install -r requirements-cpu.txt \
&& uv pip install -r test/test_cpu/requirements.txt \
diff --git a/.azure-pipelines/unit-test-cuda.yml b/.azure-pipelines/unit-test-cuda.yml
index 8c0a77341..b3f7f354d 100644
--- a/.azure-pipelines/unit-test-cuda.yml
+++ b/.azure-pipelines/unit-test-cuda.yml
@@ -138,11 +138,7 @@ stages:
if [ ${{ pair.value.PART }} -eq 0 ]; then
export UV_NO_CACHE=0
bash .azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh \
- --test_case=sglang
- bash .azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh \
- --test_case=llmc
- bash .azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh \
- --test_case=vllm
+ --test_case=specific
else
bash .azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh \
--test_case=all \
diff --git a/test/fixtures.py b/test/fixtures.py
index 147c0663c..fe74030f6 100644
--- a/test/fixtures.py
+++ b/test/fixtures.py
@@ -12,6 +12,7 @@ from auto_round.utils import is_transformers_version_greater_or_equal_5_4_0
from .helpers import (
DataLoader,
deepseek_v2_name_or_path,
+ flux_name_or_path,
gemma_name_or_path,
get_model_path,
get_tiny_model,
@@ -70,7 +71,9 @@ def tiny_phi2_model_path():
def tiny_deepseek_v2_model_path():
model_name_or_path = deepseek_v2_name_or_path
tiny_model_path = "./tmp/tiny_deepseek_v2_model_path"
- tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2, trust_remote_code=False)
+ tiny_model_path = save_tiny_model(
+ model_name_or_path, tiny_model_path, num_layers=2, trust_remote_code=False, use_config=True
+ )
yield tiny_model_path
shutil.rmtree(tiny_model_path, ignore_errors=True)
@@ -105,6 +108,21 @@ def tiny_fp8_qwen_model_path():
shutil.rmtree(tiny_model_path, ignore_errors=True)
[email protected](scope="session")
+def tiny_flux_model_path():
+ model_name_or_path = flux_name_or_path
+ tiny_model_path = "./tmp/tiny_flux_model_path"
+ tiny_model_path = save_tiny_model(
+ model_name_or_path,
+ tiny_model_path,
+ num_layers=1,
+ is_diffusion=True,
+ from_config=True,
+ )
+ yield tiny_model_path
+ shutil.rmtree(tiny_model_path, ignore_errors=True)
+
+
@pytest.fixture(scope="session")
def tiny_untied_qwen_model_path():
model_name_or_path = qwen_name_or_path
diff --git a/test/helpers.py b/test/helpers.py
index f7beaf51f..d65bdb8c3 100644
--- a/test/helpers.py
+++ b/test/helpers.py
@@ -9,7 +9,7 @@ import transformers
from packaging import version
from auto_round.eval.evaluation import simple_evaluate, simple_evaluate_user_model
-from auto_round.utils import detect_device, get_attr, llm_load_model, mllm_load_model, set_attr
+from auto_round.utils import detect_device, diffusion_load_model, get_attr, llm_load_model, mllm_load_model, set_attr
transformers_version = version.parse(transformers.__version__)
@@ -157,6 +157,7 @@ qwen_2_5_vl_name_or_path = get_model_path("Qwen/Qwen2.5-VL-3B-Instruct")
gemma_name_or_path = get_model_path("benzart/gemma-2b-it-fine-tuning-for-code-test")
qwen2_5_omni_name_or_path = get_model_path("Qwen/Qwen2.5-Omni-3B")
qwen3_omni_name_or_path = get_model_path("Qwen/Qwen3-Omni-30B-A3B-Instruct")
+flux_name_or_path = get_model_path("black-forest-labs/FLUX.1-dev")
# Slice model into tiny model for speedup
@@ -167,18 +168,35 @@ def _reduce_config_layers(config, num_layers, num_experts=None):
thinker_config, talker_config, etc.) commonly found in multi-modal and
MoE architectures.
"""
- n_block_keys = ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth", "encoder_layers"]
+ n_block_keys = [
+ "n_layers",
+ "num_hidden_layers",
+ "n_layer",
+ "num_layers",
+ "depth",
+ "encoder_layers",
+ "num_single_layers",
+ "num_decoder_layers",
+ ]
n_expert_keys = ["num_experts", "num_local_experts"]
def _apply(cfg):
for key in n_block_keys:
- if hasattr(cfg, key):
+ if isinstance(cfg, dict) and key in cfg:
+ cfg[key] = num_layers
+ elif hasattr(cfg, key):
setattr(cfg, key, num_layers)
- if hasattr(cfg, "layer_types"):
+
+ if isinstance(cfg, dict) and "layer_types" in cfg:
+ cfg["layer_types"] = cfg["layer_types"][:num_layers]
+ elif hasattr(cfg, "layer_types"):
cfg.layer_types = cfg.layer_types[:num_layers]
+
if num_experts is not None:
for key in n_expert_keys:
- if hasattr(cfg, key):
+ if isinstance(cfg, dict) and key in cfg:
+ cfg[key] = num_experts
+ elif hasattr(cfg, key):
setattr(cfg, key, num_experts)
# Top-level config
@@ -202,7 +220,9 @@ def _reduce_config_layers(config, num_layers, num_experts=None):
_apply(inner_cfg)
-def get_tiny_model(model_name_or_path, num_layers=2, num_experts=None, is_mllm=False, from_config=False, **kwargs):
+def get_tiny_model(
+ model_name_or_path, num_layers=2, num_experts=None, is_mllm=False, from_config=False, is_diffusion=False, **kwargs
+):
"""Generate a tiny model by slicing layers from the original model.
Args:
@@ -220,29 +240,81 @@ def get_tiny_model(model_name_or_path, num_layers=2, num_experts=None, is_mllm=F
if from_config:
# ---- lightweight path: config-only, random weights ----
- trust_remote_code = kwargs.pop("trust_remote_code", True)
- config = transformers.AutoConfig.from_pretrained(model_name_or_path, trust_remote_code=trust_remote_code)
- # Special cases
- if config.model_type == "qwen3_omni_moe":
- config.initializer_range = 0.02 # Default initializer range for weight initialization
-
- _reduce_config_layers(config, num_layers, num_experts)
-
- # Pick the right model class
- base_lib = transformers
- architectures = getattr(config, "architectures", [None])[0]
- if (
- is_mllm
- and architectures.endswith("Model")
- and hasattr(base_lib, n := architectures.replace("Model", "ForConditionalGeneration"))
- ):
- model_cls = getattr(base_lib, n)
- elif hasattr(base_lib, architectures):
- model_cls = getattr(base_lib, architectures)
+ if is_diffusion:
+ import importlib
+ import json
+
+ from diffusers import AutoPipelineForText2Image
+ from huggingface_hub import snapshot_download
+
+ diffusers_module = importlib.import_module("diffusers")
+ transformers_module = importlib.import_module("transformers")
+
+ existing = False
+ if os.path.exists(model_name_or_path):
+ local_dir = model_name_or_path
+ existing = True
+ else:
+ local_dir = snapshot_download(
+ repo_id=model_name_or_path, ignore_patterns=["*.safetensors", "*.safetensors.index.json"]
+ )
+
+ def _get_module(cls_name, mod_name, folder_name):
+ if cls_name == "diffusers":
+ with open(os.path.join(local_dir, folder_name, "config.json"), "r", encoding="utf-8") as f:
+ config = json.load(f)
+ _reduce_config_layers(config, num_layers, num_experts)
+ return getattr(getattr(diffusers_module, mod_name), "from_config")(config)
+ else:
+ config = transformers.AutoConfig.from_pretrained(
+ os.path.join(local_dir, folder_name, "config.json")
+ )
+ _reduce_config_layers(config, num_layers, num_experts)
+ return getattr(getattr(transformers_module, mod_name), "_from_config")(config)
+
+ with open(os.path.join(local_dir, "model_index.json"), "r", encoding="utf-8") as f:
+ model_index = json.load(f)
+
+ if not existing:
+ for k, v in model_index.items():
+ if k in ["scheduler", "tokenizer", "tokenizer_2"]:
+ continue
+ if isinstance(v, list) and v[0] in ["diffusers", "transformers"]:
+ module = _get_module(v[0], v[1], k)
+ module.save_pretrained(os.path.join(local_dir, k))
+ model = AutoPipelineForText2Image.from_pretrained(local_dir)
+ else:
+ model = AutoPipelineForText2Image.from_pretrained(local_dir)
+ for k, v in model_index.items():
+ if (
+ k not in ["scheduler", "tokenizer", "tokenizer_2"]
+ and isinstance(v, list)
+ and v[0] in ["diffusers", "transformers"]
+ ):
+ _reduce_config_layers(getattr(model, k).config, num_layers, num_experts)
else:
- model_cls = transformers.AutoModelForCausalLM # default to causal LM if we can't find a better match
- model = model_cls._from_config(config)
- model = model.eval()
+ trust_remote_code = kwargs.pop("trust_remote_code", True)
+ config = transformers.AutoConfig.from_pretrained(model_name_or_path, trust_remote_code=trust_remote_code)
+ # Special cases, for transformers == 5.4.0
+ if config.model_type == "qwen3_omni_moe":
+ config.initializer_range = 0.02 # Default initializer range for weight initialization
+ _reduce_config_layers(config, num_layers, num_experts)
+
+ # Pick the right model class
+ base_lib = transformers
+ architectures = getattr(config, "architectures", [None])[0]
+ if (
+ is_mllm
+ and architectures.endswith("Model")
+ and hasattr(base_lib, n := architectures.replace("Model", "ForConditionalGeneration"))
+ ):
+ model_cls = getattr(base_lib, n)
+ elif hasattr(base_lib, architectures):
+ model_cls = getattr(base_lib, architectures)
+ else:
+ model_cls = transformers.AutoModelForCausalLM # default to causal LM if we can't find a better match
+ model = model_cls._from_config(config)
+ model = model.eval()
return model
# ---- original path: load pretrained weights then slice ----
@@ -267,6 +339,15 @@ def get_tiny_model(model_name_or_path, num_layers=2, num_experts=None, is_mllm=F
model.config.num_hidden_layers = num_layers
elif hasattr(model.config.vision_config, "depth"): # qwen vl
model.config.vision_config.depth = num_layers
+ elif is_diffusion:
+ pipe, model = diffusion_load_model(model_name_or_path, **kwargs)
+ if (
+ hasattr(model, "config")
+ and hasattr(model.config, "num_layers")
+ and hasattr(model.config, "num_single_layers")
+ ):
+ model.config.num_layers = num_layers
+ model.config.num_single_layers = num_layers
else:
model, tokenizer = llm_load_model(model_name_or_path, **kwargs)
@@ -286,6 +367,7 @@ def save_tiny_model(
is_mllm=False,
force_untie=False,
from_config=False,
+ is_diffusion=False,
**kwargs,
):
"""Generate a tiny model and save to the specified path.
@@ -307,6 +389,7 @@ def save_tiny_model(
num_experts=num_experts,
is_mllm=is_mllm,
from_config=from_config,
+ is_diffusion=is_diffusion,
**kwargs,
)
if force_untie:
@@ -315,11 +398,13 @@ def save_tiny_model(
for key in model._tied_weights_keys:
weight = get_attr(model, key)
set_attr(model, key, copy.deepcopy(weight))
- tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path, **kwargs)
test_path = os.path.dirname(__file__)
tiny_model_path = os.path.join(test_path, tiny_model_path.removeprefix("./"))
+
model.save_pretrained(tiny_model_path)
- tokenizer.save_pretrained(tiny_model_path)
+ if not is_diffusion:
+ tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path, **kwargs)
+ tokenizer.save_pretrained(tiny_model_path)
# copy tokenizer.model
if not os.path.isdir(model_name_or_path):
from auto_round.utils import download_hf_model
diff --git a/test/test_cuda/backends/test_marlin_backend.py b/test/test_cuda/backends/test_marlin_backend.py
index 4d82e9058..ff00315d4 100644
--- a/test/test_cuda/backends/test_marlin_backend.py
+++ b/test/test_cuda/backends/test_marlin_backend.py
@@ -183,7 +183,7 @@ class TestAutoRoundMarlinBackend:
quantization_config = AutoRoundConfig(backend="gptqmodel:awq_marlin")
model = AutoModelForCausalLM.from_pretrained(
- self.save_dir, torch_dtype="auto", device_map="auto", quantization_config=quantization_config
+ self.save_dir, torch_dtype="auto", device_map="cuda:0", quantization_config=quantization_config
)
tokenizer = AutoTokenizer.from_pretrained(self.save_dir)
@@ -215,7 +215,7 @@ class TestAutoRoundMarlinBackend:
quantization_config = AutoRoundConfig(backend="gptqmodel:awq_marlin")
model = AutoModelForCausalLM.from_pretrained(
- self.save_dir, torch_dtype="auto", device_map="auto", quantization_config=quantization_config
+ self.save_dir, torch_dtype="auto", device_map="cuda:0", quantization_config=quantization_config
)
tokenizer = AutoTokenizer.from_pretrained(self.save_dir)
diff --git a/test/test_cuda/export/test_auto_round_format.py b/test/test_cuda/export/test_auto_round_format.py
index 02dd40b1c..ba9862712 100644
--- a/test/test_cuda/export/test_auto_round_format.py
+++ b/test/test_cuda/export/test_auto_round_format.py
@@ -96,7 +96,7 @@ class TestAutoRound:
quantization_config = AutoRoundConfig(backend="auto")
model = AutoModelForCausalLM.from_pretrained(
- self.save_dir, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config
+ self.save_dir, torch_dtype=torch.float16, device_map="cuda:0", quantization_config=quantization_config
)
tokenizer = AutoTokenizer.from_pretrained(self.save_dir)
@@ -105,7 +105,7 @@ class TestAutoRound:
torch.cuda.empty_cache()
model = AutoModelForCausalLM.from_pretrained(
- self.save_dir, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config
+ self.save_dir, torch_dtype=torch.bfloat16, device_map="cuda:0", quantization_config=quantization_config
)
tokenizer = AutoTokenizer.from_pretrained(self.save_dir)
@@ -117,7 +117,7 @@ class TestAutoRound:
model_name = get_model_path("OPEA/Meta-Llama-3.1-8B-Instruct-int4-sym-inc")
quantization_config = AutoRoundConfig(backend="tritonv2")
model = AutoModelForCausalLM.from_pretrained(
- model_name, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config
+ model_name, torch_dtype=torch.bfloat16, device_map="cuda:0", quantization_config=quantization_config
)
tokenizer = AutoTokenizer.from_pretrained(model_name)
eval_generated_prompt(model, tokenizer)
@@ -227,4 +227,4 @@ class TestAutoRound:
assert compressed_model.config.quantization_config["quant_method"] == "fp8"
assert compressed_model.config.quantization_config["weight_block_size"] == (128, 128)
if is_cuda_support_fp8():
- eval_generated_prompt(quantized_model_path, device="cuda")
+ eval_generated_prompt(quantized_model_path, device="cuda:0")
diff --git a/test/test_cuda/export/test_gguf_format.py b/test/test_cuda/export/test_gguf_format.py
index 832233226..557c9b6bc 100644
--- a/test/test_cuda/export/test_gguf_format.py
+++ b/test/test_cuda/export/test_gguf_format.py
@@ -145,7 +145,9 @@ class TestAutoRound:
from ...helpers import save_tiny_model
model_name = "google/gemma-3-4b-it"
- tiny_model_path = save_tiny_model(model_name, "tiny_model_path", num_layers=3, is_mllm=True, use_fast=False)
+ tiny_model_path = save_tiny_model(
+ model_name, "tiny_model_path", num_layers=3, is_mllm=True, use_fast=False, use_config=True
+ )
# Needs tokenizer.model for gguf
# New transformers won't download it even with use_fast=False
file_path = hf_hub_download(repo_id=model_name, filename="tokenizer.model", local_dir=tiny_model_path)
diff --git a/test/test_cuda/models/test_diffusion.py b/test/test_cuda/models/test_diffusion.py
index 2f2a1b940..11ea7da54 100644
--- a/test/test_cuda/models/test_diffusion.py
+++ b/test/test_cuda/models/test_diffusion.py
@@ -15,7 +15,6 @@ from ...helpers import get_captions_dataset_path, get_model_path, transformers_v
class TestAutoRound:
- model_name = get_model_path("black-forest-labs/FLUX.1-dev")
@pytest.fixture(autouse=True)
def _save_dir(self, tmp_path):
@@ -28,11 +27,11 @@ class TestAutoRound:
yield
shutil.rmtree("runs", ignore_errors=True)
- def test_diffusion_rtn(self):
+ def test_diffusion_rtn(self, tiny_flux_model_path):
from diffusers import AutoPipelineForText2Image
## load the model
- pipe = AutoPipelineForText2Image.from_pretrained(self.model_name)
+ pipe = AutoPipelineForText2Image.from_pretrained(tiny_flux_model_path)
# build tiny model for testing since the full model is too large to quantize and evaluate in CI
pipe.transformer.transformer_blocks = pipe.transformer.transformer_blocks[:2]
pipe.transformer.single_transformer_blocks = pipe.transformer.single_transformer_blocks[:2]
@@ -52,11 +51,11 @@ class TestAutoRound:
@pytest.mark.skip_ci(reason="Tuning will OOM in CI; Only tiny model is suggested") # skip this test in CI
@require_optimum
- def test_diffusion_tune(self):
+ def test_diffusion_tune(self, tiny_flux_model_path):
from diffusers import AutoPipelineForText2Image
## load the model
- pipe = AutoPipelineForText2Image.from_pretrained(self.model_name)
+ pipe = AutoPipelineForText2Image.from_pretrained(tiny_flux_model_path)
model = pipe.transformer
# build tiny model for testing since the full model is too large to quantize and evaluate in CI
pipe.transformer.transformer_blocks = pipe.transformer.transformer_blocks[:2]
diff --git a/test/test_cuda/requirements_vlm.txt b/test/test_cuda/requirements_vlm.txt
index 46e89dc96..16438f26b 100644
--- a/test/test_cuda/requirements_vlm.txt
+++ b/test/test_cuda/requirements_vlm.txt
@@ -1,11 +1,10 @@
# git+https://github.com/haotian-liu/[email protected] --no-deps
# pip install git+https://github.com/deepseek-ai/DeepSeek-VL2.git --no-deps
# pip install -v git+https://github.com/casper-hansen/[email protected] --no-build-isolation
-# pip install flash-attn==2.7.4.post1 --no-build-isolation
+# pip install flash-attn==2.8.3 --no-build-isolation
bitsandbytes
einops
flash-attn
-intel-extension-for-transformers
lm-eval>=0.4.2,<0.5
optimum
pandas