PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Update torch to 2.11.0 in CI (#1606)
Sun, Xuehaoae7661ba26f0
diff --git a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.shindex 398a7da4c..0fe6f4440 100644--- a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh+++ b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh@@ -62,27 +62,15 @@ function run_unit_test() {# install unit test dependenciesecho "##[group]set up UT env..."cd "${BUILD_SOURCESDIRECTORY}" || exit 1- uv pip install torch==2.10.0 torchvision- uv pip install git+https://github.com/casper-hansen/AutoAWQ.git --no-build-isolation-- # install gptqmodel- CUDA_VER=$(python -c 'import torch; print(f"cu{torch.version.cuda.replace(".", "")}")')- PY_VER=$(python -c 'import sys; print(f"cp{sys.version_info.major}{sys.version_info.minor}")')- TORCH_VER="torch2.10"- WHEEL="gptqmodel-5.7.0-${CUDA_VER}${TORCH_VER}-${PY_VER}-${PY_VER}-linux_x86_64.whl"- URL="https://pkgs.dev.azure.com/lpot-inc/b7121868-d73a-4794-90c1-23135f974d09/_packaging/4728fbab-e069-4cbd-bcca-d35f4d42256b/pypi/download/gptqmodel/5.7/${WHEEL}"- wget -q "$URL" -O "$WHEEL" || { echo "Download failed. Check CUDA/PyTorch/Python versions match (cu126/cu128/cu130, torch2.10, cp310-cp313)"; exit 1; }- mv "$WHEEL" "${WHEEL/-${CUDA_VER}${TORCH_VER}-/+${CUDA_VER}.${TORCH_VER}-}"- uv pip install "./${WHEEL/-${CUDA_VER}${TORCH_VER}-/+${CUDA_VER}.${TORCH_VER}-}" --no-build-isolation- rm -f "./${WHEEL/-${CUDA_VER}${TORCH_VER}-/+${CUDA_VER}.${TORCH_VER}-}"-- uv pip install gptqmodel --extra-index-url https://pkgs.dev.azure.com/lpot-inc/neural-compressor/_packaging/gptqmodel-wheels/pypi/simple/- uv pip install -r https://raw.githubusercontent.com/ModelCloud/GPTQModel/refs/tags/v5.7.0/requirements.txt+ uv pip install https://github.com/XuehaoSun/GPTQModel/releases/download/v5.8.0/gptqmodel-5.8.0+cu128torch2.11-cp312-cp312-linux_x86_64.whl+ uv pip install -r https://raw.githubusercontent.com/ModelCloud/GPTQModel/refs/tags/v5.8.0/requirements.txtuv pip install https://github.com/XuehaoSun/llama-cpp-python/releases/download/v0.3.16/llama_cpp_python-0.3.16-cp312-cp312-linux_x86_64.whluv pip install 'git+https://github.com/ggml-org/llama.cpp.git#subdirectory=gguf-py'uv pip install -r test/test_cuda/requirements.txtuv pip install -r test/test_cuda/requirements_diffusion.txtuv pip install -U transformers+ uv pip uninstall torch torchvision+ uv pip install torch==2.11.0 torchvision --index-url https://download.pytorch.org/whl/cu128uv pip install .echo "##[endgroup]"@@ -126,8 +114,9 @@ function run_unit_test_llmc() {uv pip install -U pytest-cov pytest-htmlBUILD_TYPE="nightly" uv pip install -r test/test_cuda/requirements_llmc.txt --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-matchuv pip install .- echo "##[endgroup]"uv pip list+ echo "##[endgroup]"+cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coverage"@@ -151,9 +140,9 @@ function run_unit_test_sglang() {uv pip install -U pytest-cov pytest-htmluv pip install -r test/test_cuda/requirements_sglang.txtuv pip install .+ uv pip listecho "##[endgroup]"- uv pip listcd "${BUILD_SOURCESDIRECTORY}/test" || exit 1export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coverage"@@ -174,11 +163,11 @@ function run_unit_test_vllm() {rm -rf /root/.venvuv venv --python=3.12 /root/.venvuv pip install -U pytest-cov pytest-html- uv pip install -r test/test_cuda/requirements_vllm.txt+ uv pip install -r test/test_cuda/requirements_vllm.txt --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-matchuv pip install .+ uv pip listecho "##[endgroup]"- uv pip listcd "${BUILD_SOURCESDIRECTORY}/test" || exit 1export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coverage"@@ -197,16 +186,14 @@ function main() {setup_environmentif [ "${test_case}" == "vlm" ]; thenrun_unit_test_vlm- elif [ "${test_case}" == "llmc" ]; then- run_unit_test_llmc- elif [ "${test_case}" == "sglang" ]; then+ elif [ "${test_case}" == "specific" ]; thenrun_unit_test_sglang- elif [ "${test_case}" == "vllm" ]; then+ run_unit_test_llmcrun_unit_test_vllmelif [ "${test_case}" == "all" ]; thenrun_unit_testelse- echo "##[error]Invalid test case specified: ${test_case}. Please use 'vlm', 'llmc', 'sglang', 'vllm', or 'all'."+ echo "##[error]Invalid test case specified: ${test_case}. Please use 'vlm', 'specific', or 'all'."exit 1ficheck_storage_usagediff --git a/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py b/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.pyindex 1959433cd..fcdfc8ab0 100644--- a/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py+++ b/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py@@ -13,6 +13,36 @@ TARGET_GPUS = ["NVIDIA L40S","NVIDIA RTX PRO 6000 Blackwell Server Edition",]+DATA_CENTER_IDS = [+ "AP-JP-1",+ "CA-MTL-1",+ "CA-MTL-2",+ "CA-MTL-3",+ "EU-CZ-1",+ "EU-FR-1",+ "EU-NL-1",+ "EU-RO-1",+ "EU-SE-1",+ "EUR-IS-1",+ "EUR-IS-2",+ "EUR-IS-3",+ "EUR-NO-1",+ "OC-AU-1",+ "US-CA-2",+ "US-DE-1",+ "US-GA-1",+ "US-GA-2",+ "US-IL-1",+ "US-KS-2",+ "US-KS-3",+ "US-NC-1",+ "US-TX-1",+ "US-TX-3",+ "US-TX-4",+ "US-WA-1",+]+DATA_CENTER_BAN_LIST = ["EUR-IS-2", "US-IL-1"]+DATA_CENTER_SELECT_LIST = [dc for dc in DATA_CENTER_IDS if dc not in DATA_CENTER_BAN_LIST]REQUIRED_COUNT = 1IMAGES_NAME = "xuehaosu/azure-agent:v0.1"@@ -46,7 +76,17 @@ def check_gpu_count(token):}}""" % ids_string- variables = {"input": {"gpuCount": 1, "secureCloud": True, "minMemoryInGb": 0, "minVcpuCount": 0}}++ datacenter_id_string = ",".join(DATA_CENTER_SELECT_LIST)+ variables = {+ "input": {+ "gpuCount": 1,+ "secureCloud": True,+ "minMemoryInGb": 0,+ "minVcpuCount": 0,+ "dataCenterId": datacenter_id_string,+ }+ }try:response = requests.post(@@ -94,15 +134,27 @@ def check_gpu_count(token):def run_create_pod(api_key, payload):url = "https://rest.runpod.io/v1/pods"headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}- response = requests.post(url, json=payload, headers=headers)+ max_retries = 3++ for attempt in range(max_retries + 1):+ response = requests.post(url, json=payload, headers=headers)++ if response.status_code >= 500 or response.status_code == 400:+ if attempt < max_retries:+ print(f"⚠️ {response.status_code} Error, Retrying in 60 seconds ({attempt + 1}/{max_retries})...")+ time.sleep(60)+ continue+ else:+ print(f"❌ {response.status_code} Error, Reached maximum retry attempts ({max_retries}), giving up.")+ response.raise_for_status()- response.raise_for_status()- result = response.json()- if "errors" in result:- print("❌ Errors:")- print(json.dumps(result["errors"], indent=2))- sys.exit(1)- return result+ result = response.json()+ if "errors" in result:+ print("❌ Errors:")+ print(json.dumps(result["errors"], indent=2))+ sys.exit(1)++ return resultdef create_pod(args):@@ -125,6 +177,8 @@ def create_pod(args):payload = {"cloudType": "SECURE","containerDiskInGb": args.container_disk_size,+ "dataCenterIds": DATA_CENTER_SELECT_LIST,+ "dataCenterPriority": "availability","env": env_dict,"gpuCount": args.gpu_count,"gpuTypeIds": [gpu_type],@@ -186,11 +240,23 @@ def terminate_pod(args):url = f"https://rest.runpod.io/v1/pods/{pod_id}"headers = {"Authorization": f"Bearer {args.api_key}"}- response = requests.delete(url, headers=headers)- response.raise_for_status()max_tries = 30+ for attempt in range(max_tries + 1):+ response = requests.delete(url, headers=headers)+ if response.status_code >= 500 or response.status_code == 400:+ if attempt < max_tries:+ print(f"⚠️ {response.status_code} Error, Retrying in 5 seconds ({attempt + 1}/{max_tries})...")+ time.sleep(5)+ continue+ else:+ print(f"❌ {response.status_code} Error, Reached maximum retry attempts ({max_tries}), giving up.")+ else:+ break++ response.raise_for_status()+for i in range(max_tries): # Wait up to 5 minutes for terminationpod_id = get_pod_id(args)if pod_id:diff --git a/.azure-pipelines/scripts/ut/run_ut_cuda.sh b/.azure-pipelines/scripts/ut/run_ut_cuda.shindex b11e17abd..1bdc63f5b 100644--- a/.azure-pipelines/scripts/ut/run_ut_cuda.sh+++ b/.azure-pipelines/scripts/ut/run_ut_cuda.sh@@ -2,7 +2,7 @@set -xeCONDA_ENV_NAME="unittest_cuda"-PYTHON_VERSION="3.10"+PYTHON_VERSION="3.12"REPO_PATH=$(git rev-parse --show-toplevel)LOG_DIR=${REPO_PATH}/ut_log_dirSUMMARY_LOG=${LOG_DIR}/results_summary.log@@ -91,15 +91,16 @@ function run_unit_test() {cd ${REPO_PATH}/testrm -rf .coverage* *.xml *.html- uv pip install torch==2.10.0 torchvision- uv pip install -v git+https://github.com/casper-hansen/AutoAWQ.git --no-build-isolation+ uv pip install torch==2.11.0 torchvision --index-url https://download.pytorch.org/whl/cu128uv pip install gptqmodel --no-build-isolationuv pip install -r https://raw.githubusercontent.com/ModelCloud/GPTQModel/refs/heads/main/requirements.txt- CMAKE_ARGS="-DGGML_CUDA=on -DLLAVA_BUILD=off" uv pip install llama-cpp-python+ uv pip install https://github.com/XuehaoSun/llama-cpp-python/releases/download/v0.3.16/llama_cpp_python-0.3.16-cp312-cp312-linux_x86_64.whluv pip install 'git+https://github.com/ggml-org/llama.cpp.git#subdirectory=gguf-py'uv pip install -r test_cuda/requirements.txtuv pip install -r test_cuda/requirements_diffusion.txtuv pip install -U transformers+ uv pip uninstall torch torchvision+ uv pip install torch==2.11.0 torchvision --index-url https://download.pytorch.org/whl/cu128cd ${REPO_PATH} && uv pip install . && cd ${REPO_PATH}/testpip list > ${LOG_DIR}/ut_pip_list.txt@@ -132,15 +133,12 @@ function run_unit_test_vlm() {cd ${REPO_PATH}/testrm -rf .coverage* *.xml *.html- uv pip install torch==2.10.0 torchvision+ uv pip install torch==2.11.0 torchvision --index-url https://download.pytorch.org/whl/cu128+ uv pip install https://github.com/XuehaoSun/GPTQModel/releases/download/v5.8.0/gptqmodel-5.8.0+cu128torch2.11-cp312-cp312-linux_x86_64.whluv pip install git+https://github.com/haotian-liu/[email protected] --no-deps- local site_path=$(python -c "import site; print(site.getsitepackages()[0])")- # reference https://github.com/haotian-liu/LLaVA/issues/1448#issuecomment-2119845242- sed -i '/inputs\[.*image_sizes.*\] = image_sizes/a\ inputs.pop("cache_position")' ${site_path}/llava/model/language_model/llava_llama.py- uv pip install git+https://github.com/deepseek-ai/DeepSeek-VL2.git timm attrdict --no-depsuv pip install -v git+https://github.com/casper-hansen/[email protected] --no-build-isolation- uv pip install flash-attn==2.7.4.post1 --no-build-isolation- uv pip install -r test_cuda/requirements_vlm.txt+ uv pip install flash-attn==2.8.3 --no-build-isolation+ uv pip install -r test_cuda/requirements_vlm.txt --extra-index-url https://download.pytorch.org/whl/cu128cd ${REPO_PATH} && uv pip install . && cd ${REPO_PATH}/testpip list > ${LOG_DIR}/vlm_ut_pip_list.txt@@ -173,7 +171,7 @@ function run_unit_test_llmc() {cd ${REPO_PATH}/testrm -rf .coverage* *.xml *.html- BUILD_TYPE="nightly" uv pip install -r test_cuda/requirements_llmc.txt+ BUILD_TYPE="nightly" uv pip install -r test_cuda/requirements_llmc.txt --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-matchcd ${REPO_PATH} && uv pip install . && cd ${REPO_PATH}/testpip list > ${LOG_DIR}/llmc_ut_pip_list.txt@@ -234,10 +232,11 @@ function run_unit_test_sglang() {function run_unit_test_vllm() {# install unit test dependenciescreate_conda_env+ unset LD_LIBRARY_PATHcd ${REPO_PATH}/testrm -rf .coverage* *.xml *.html- uv pip install -r test_cuda/requirements_vllm.txt+ uv pip install -r test_cuda/requirements_vllm.txt --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-matchcd ${REPO_PATH} && uv pip install . && cd ${REPO_PATH}/testpip list > ${LOG_DIR}/vllm_ut_pip_list.txtdiff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.ymlindex f8fd5c826..e7508a7b7 100644--- a/.azure-pipelines/template/ut-template.yml+++ b/.azure-pipelines/template/ut-template.yml@@ -46,8 +46,8 @@ steps:- script: |if [ "${{ parameters.utScriptFileName }}" == "run_ut_xpu" ];thendocker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round \- && uv pip install torch==2.10.0 torchvision --index-url https://download.pytorch.org/whl/xpu \- && uv pip install torch==2.10.0 auto-round-lib \+ && uv pip install torch==2.11.0 torchvision --index-url https://download.pytorch.org/whl/xpu \+ && uv pip install torch==2.11.0 auto-round-lib \&& uv pip install -r requirements.txt \&& uv pip install -r test/test_ark/requirements.txt \&& uv pip install -r test/test_xpu/requirements.txt \@@ -55,8 +55,8 @@ steps:&& uv pip list"elsedocker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round \- && uv pip install torch==2.10.0 torchvision --index-url https://download.pytorch.org/whl/cpu \- && uv pip install torch==2.10.0 auto-round-lib \+ && uv pip install torch==2.11.0 torchvision --index-url https://download.pytorch.org/whl/cpu \+ && uv pip install torch==2.11.0 auto-round-lib \&& uv pip install -r requirements.txt \&& uv pip install -r requirements-cpu.txt \&& uv pip install -r test/test_cpu/requirements.txt \diff --git a/.azure-pipelines/unit-test-cuda.yml b/.azure-pipelines/unit-test-cuda.ymlindex 8c0a77341..b3f7f354d 100644--- a/.azure-pipelines/unit-test-cuda.yml+++ b/.azure-pipelines/unit-test-cuda.yml@@ -138,11 +138,7 @@ stages:if [ ${{ pair.value.PART }} -eq 0 ]; thenexport UV_NO_CACHE=0bash .azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh \- --test_case=sglang- bash .azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh \- --test_case=llmc- bash .azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh \- --test_case=vllm+ --test_case=specificelsebash .azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh \--test_case=all \diff --git a/test/fixtures.py b/test/fixtures.pyindex 147c0663c..fe74030f6 100644--- a/test/fixtures.py+++ b/test/fixtures.py@@ -12,6 +12,7 @@ from auto_round.utils import is_transformers_version_greater_or_equal_5_4_0from .helpers import (DataLoader,deepseek_v2_name_or_path,+ flux_name_or_path,gemma_name_or_path,get_model_path,get_tiny_model,@@ -70,7 +71,9 @@ def tiny_phi2_model_path():def tiny_deepseek_v2_model_path():model_name_or_path = deepseek_v2_name_or_pathtiny_model_path = "./tmp/tiny_deepseek_v2_model_path"- tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2, trust_remote_code=False)+ tiny_model_path = save_tiny_model(+ model_name_or_path, tiny_model_path, num_layers=2, trust_remote_code=False, use_config=True+ )yield tiny_model_pathshutil.rmtree(tiny_model_path, ignore_errors=True)@@ -105,6 +108,21 @@ def tiny_fp8_qwen_model_path():shutil.rmtree(tiny_model_path, ignore_errors=True)[email protected](scope="session")+def tiny_flux_model_path():+ model_name_or_path = flux_name_or_path+ tiny_model_path = "./tmp/tiny_flux_model_path"+ tiny_model_path = save_tiny_model(+ model_name_or_path,+ tiny_model_path,+ num_layers=1,+ is_diffusion=True,+ from_config=True,+ )+ yield tiny_model_path+ shutil.rmtree(tiny_model_path, ignore_errors=True)++@pytest.fixture(scope="session")def tiny_untied_qwen_model_path():model_name_or_path = qwen_name_or_pathdiff --git a/test/helpers.py b/test/helpers.pyindex f7beaf51f..d65bdb8c3 100644--- a/test/helpers.py+++ b/test/helpers.py@@ -9,7 +9,7 @@ import transformersfrom packaging import versionfrom auto_round.eval.evaluation import simple_evaluate, simple_evaluate_user_model-from auto_round.utils import detect_device, get_attr, llm_load_model, mllm_load_model, set_attr+from auto_round.utils import detect_device, diffusion_load_model, get_attr, llm_load_model, mllm_load_model, set_attrtransformers_version = version.parse(transformers.__version__)@@ -157,6 +157,7 @@ qwen_2_5_vl_name_or_path = get_model_path("Qwen/Qwen2.5-VL-3B-Instruct")gemma_name_or_path = get_model_path("benzart/gemma-2b-it-fine-tuning-for-code-test")qwen2_5_omni_name_or_path = get_model_path("Qwen/Qwen2.5-Omni-3B")qwen3_omni_name_or_path = get_model_path("Qwen/Qwen3-Omni-30B-A3B-Instruct")+flux_name_or_path = get_model_path("black-forest-labs/FLUX.1-dev")# Slice model into tiny model for speedup@@ -167,18 +168,35 @@ def _reduce_config_layers(config, num_layers, num_experts=None):thinker_config, talker_config, etc.) commonly found in multi-modal andMoE architectures."""- n_block_keys = ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth", "encoder_layers"]+ n_block_keys = [+ "n_layers",+ "num_hidden_layers",+ "n_layer",+ "num_layers",+ "depth",+ "encoder_layers",+ "num_single_layers",+ "num_decoder_layers",+ ]n_expert_keys = ["num_experts", "num_local_experts"]def _apply(cfg):for key in n_block_keys:- if hasattr(cfg, key):+ if isinstance(cfg, dict) and key in cfg:+ cfg[key] = num_layers+ elif hasattr(cfg, key):setattr(cfg, key, num_layers)- if hasattr(cfg, "layer_types"):++ if isinstance(cfg, dict) and "layer_types" in cfg:+ cfg["layer_types"] = cfg["layer_types"][:num_layers]+ elif hasattr(cfg, "layer_types"):cfg.layer_types = cfg.layer_types[:num_layers]+if num_experts is not None:for key in n_expert_keys:- if hasattr(cfg, key):+ if isinstance(cfg, dict) and key in cfg:+ cfg[key] = num_experts+ elif hasattr(cfg, key):setattr(cfg, key, num_experts)# Top-level config@@ -202,7 +220,9 @@ def _reduce_config_layers(config, num_layers, num_experts=None):_apply(inner_cfg)-def get_tiny_model(model_name_or_path, num_layers=2, num_experts=None, is_mllm=False, from_config=False, **kwargs):+def get_tiny_model(+ model_name_or_path, num_layers=2, num_experts=None, is_mllm=False, from_config=False, is_diffusion=False, **kwargs+):"""Generate a tiny model by slicing layers from the original model.Args:@@ -220,29 +240,81 @@ def get_tiny_model(model_name_or_path, num_layers=2, num_experts=None, is_mllm=Fif from_config:# ---- lightweight path: config-only, random weights ----- trust_remote_code = kwargs.pop("trust_remote_code", True)- config = transformers.AutoConfig.from_pretrained(model_name_or_path, trust_remote_code=trust_remote_code)- # Special cases- if config.model_type == "qwen3_omni_moe":- config.initializer_range = 0.02 # Default initializer range for weight initialization-- _reduce_config_layers(config, num_layers, num_experts)-- # Pick the right model class- base_lib = transformers- architectures = getattr(config, "architectures", [None])[0]- if (- is_mllm- and architectures.endswith("Model")- and hasattr(base_lib, n := architectures.replace("Model", "ForConditionalGeneration"))- ):- model_cls = getattr(base_lib, n)- elif hasattr(base_lib, architectures):- model_cls = getattr(base_lib, architectures)+ if is_diffusion:+ import importlib+ import json++ from diffusers import AutoPipelineForText2Image+ from huggingface_hub import snapshot_download++ diffusers_module = importlib.import_module("diffusers")+ transformers_module = importlib.import_module("transformers")++ existing = False+ if os.path.exists(model_name_or_path):+ local_dir = model_name_or_path+ existing = True+ else:+ local_dir = snapshot_download(+ repo_id=model_name_or_path, ignore_patterns=["*.safetensors", "*.safetensors.index.json"]+ )++ def _get_module(cls_name, mod_name, folder_name):+ if cls_name == "diffusers":+ with open(os.path.join(local_dir, folder_name, "config.json"), "r", encoding="utf-8") as f:+ config = json.load(f)+ _reduce_config_layers(config, num_layers, num_experts)+ return getattr(getattr(diffusers_module, mod_name), "from_config")(config)+ else:+ config = transformers.AutoConfig.from_pretrained(+ os.path.join(local_dir, folder_name, "config.json")+ )+ _reduce_config_layers(config, num_layers, num_experts)+ return getattr(getattr(transformers_module, mod_name), "_from_config")(config)++ with open(os.path.join(local_dir, "model_index.json"), "r", encoding="utf-8") as f:+ model_index = json.load(f)++ if not existing:+ for k, v in model_index.items():+ if k in ["scheduler", "tokenizer", "tokenizer_2"]:+ continue+ if isinstance(v, list) and v[0] in ["diffusers", "transformers"]:+ module = _get_module(v[0], v[1], k)+ module.save_pretrained(os.path.join(local_dir, k))+ model = AutoPipelineForText2Image.from_pretrained(local_dir)+ else:+ model = AutoPipelineForText2Image.from_pretrained(local_dir)+ for k, v in model_index.items():+ if (+ k not in ["scheduler", "tokenizer", "tokenizer_2"]+ and isinstance(v, list)+ and v[0] in ["diffusers", "transformers"]+ ):+ _reduce_config_layers(getattr(model, k).config, num_layers, num_experts)else:- model_cls = transformers.AutoModelForCausalLM # default to causal LM if we can't find a better match- model = model_cls._from_config(config)- model = model.eval()+ trust_remote_code = kwargs.pop("trust_remote_code", True)+ config = transformers.AutoConfig.from_pretrained(model_name_or_path, trust_remote_code=trust_remote_code)+ # Special cases, for transformers == 5.4.0+ if config.model_type == "qwen3_omni_moe":+ config.initializer_range = 0.02 # Default initializer range for weight initialization+ _reduce_config_layers(config, num_layers, num_experts)++ # Pick the right model class+ base_lib = transformers+ architectures = getattr(config, "architectures", [None])[0]+ if (+ is_mllm+ and architectures.endswith("Model")+ and hasattr(base_lib, n := architectures.replace("Model", "ForConditionalGeneration"))+ ):+ model_cls = getattr(base_lib, n)+ elif hasattr(base_lib, architectures):+ model_cls = getattr(base_lib, architectures)+ else:+ model_cls = transformers.AutoModelForCausalLM # default to causal LM if we can't find a better match+ model = model_cls._from_config(config)+ model = model.eval()return model# ---- original path: load pretrained weights then slice ----@@ -267,6 +339,15 @@ def get_tiny_model(model_name_or_path, num_layers=2, num_experts=None, is_mllm=Fmodel.config.num_hidden_layers = num_layerselif hasattr(model.config.vision_config, "depth"): # qwen vlmodel.config.vision_config.depth = num_layers+ elif is_diffusion:+ pipe, model = diffusion_load_model(model_name_or_path, **kwargs)+ if (+ hasattr(model, "config")+ and hasattr(model.config, "num_layers")+ and hasattr(model.config, "num_single_layers")+ ):+ model.config.num_layers = num_layers+ model.config.num_single_layers = num_layerselse:model, tokenizer = llm_load_model(model_name_or_path, **kwargs)@@ -286,6 +367,7 @@ def save_tiny_model(is_mllm=False,force_untie=False,from_config=False,+ is_diffusion=False,**kwargs,):"""Generate a tiny model and save to the specified path.@@ -307,6 +389,7 @@ def save_tiny_model(num_experts=num_experts,is_mllm=is_mllm,from_config=from_config,+ is_diffusion=is_diffusion,**kwargs,)if force_untie:@@ -315,11 +398,13 @@ def save_tiny_model(for key in model._tied_weights_keys:weight = get_attr(model, key)set_attr(model, key, copy.deepcopy(weight))- tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path, **kwargs)test_path = os.path.dirname(__file__)tiny_model_path = os.path.join(test_path, tiny_model_path.removeprefix("./"))+model.save_pretrained(tiny_model_path)- tokenizer.save_pretrained(tiny_model_path)+ if not is_diffusion:+ tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path, **kwargs)+ tokenizer.save_pretrained(tiny_model_path)# copy tokenizer.modelif not os.path.isdir(model_name_or_path):from auto_round.utils import download_hf_modeldiff --git a/test/test_cuda/backends/test_marlin_backend.py b/test/test_cuda/backends/test_marlin_backend.pyindex 4d82e9058..ff00315d4 100644--- a/test/test_cuda/backends/test_marlin_backend.py+++ b/test/test_cuda/backends/test_marlin_backend.py@@ -183,7 +183,7 @@ class TestAutoRoundMarlinBackend:quantization_config = AutoRoundConfig(backend="gptqmodel:awq_marlin")model = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype="auto", device_map="auto", quantization_config=quantization_config+ self.save_dir, torch_dtype="auto", device_map="cuda:0", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_dir)@@ -215,7 +215,7 @@ class TestAutoRoundMarlinBackend:quantization_config = AutoRoundConfig(backend="gptqmodel:awq_marlin")model = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype="auto", device_map="auto", quantization_config=quantization_config+ self.save_dir, torch_dtype="auto", device_map="cuda:0", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_dir)diff --git a/test/test_cuda/export/test_auto_round_format.py b/test/test_cuda/export/test_auto_round_format.pyindex 02dd40b1c..ba9862712 100644--- a/test/test_cuda/export/test_auto_round_format.py+++ b/test/test_cuda/export/test_auto_round_format.py@@ -96,7 +96,7 @@ class TestAutoRound:quantization_config = AutoRoundConfig(backend="auto")model = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config+ self.save_dir, torch_dtype=torch.float16, device_map="cuda:0", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_dir)@@ -105,7 +105,7 @@ class TestAutoRound:torch.cuda.empty_cache()model = AutoModelForCausalLM.from_pretrained(- self.save_dir, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config+ self.save_dir, torch_dtype=torch.bfloat16, device_map="cuda:0", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(self.save_dir)@@ -117,7 +117,7 @@ class TestAutoRound:model_name = get_model_path("OPEA/Meta-Llama-3.1-8B-Instruct-int4-sym-inc")quantization_config = AutoRoundConfig(backend="tritonv2")model = AutoModelForCausalLM.from_pretrained(- model_name, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config+ model_name, torch_dtype=torch.bfloat16, device_map="cuda:0", quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(model_name)eval_generated_prompt(model, tokenizer)@@ -227,4 +227,4 @@ class TestAutoRound:assert compressed_model.config.quantization_config["quant_method"] == "fp8"assert compressed_model.config.quantization_config["weight_block_size"] == (128, 128)if is_cuda_support_fp8():- eval_generated_prompt(quantized_model_path, device="cuda")+ eval_generated_prompt(quantized_model_path, device="cuda:0")diff --git a/test/test_cuda/export/test_gguf_format.py b/test/test_cuda/export/test_gguf_format.pyindex 832233226..557c9b6bc 100644--- a/test/test_cuda/export/test_gguf_format.py+++ b/test/test_cuda/export/test_gguf_format.py@@ -145,7 +145,9 @@ class TestAutoRound:from ...helpers import save_tiny_modelmodel_name = "google/gemma-3-4b-it"- tiny_model_path = save_tiny_model(model_name, "tiny_model_path", num_layers=3, is_mllm=True, use_fast=False)+ tiny_model_path = save_tiny_model(+ model_name, "tiny_model_path", num_layers=3, is_mllm=True, use_fast=False, use_config=True+ )# Needs tokenizer.model for gguf# New transformers won't download it even with use_fast=Falsefile_path = hf_hub_download(repo_id=model_name, filename="tokenizer.model", local_dir=tiny_model_path)diff --git a/test/test_cuda/models/test_diffusion.py b/test/test_cuda/models/test_diffusion.pyindex 2f2a1b940..11ea7da54 100644--- a/test/test_cuda/models/test_diffusion.py+++ b/test/test_cuda/models/test_diffusion.py@@ -15,7 +15,6 @@ from ...helpers import get_captions_dataset_path, get_model_path, transformers_vclass TestAutoRound:- model_name = get_model_path("black-forest-labs/FLUX.1-dev")@pytest.fixture(autouse=True)def _save_dir(self, tmp_path):@@ -28,11 +27,11 @@ class TestAutoRound:yieldshutil.rmtree("runs", ignore_errors=True)- def test_diffusion_rtn(self):+ def test_diffusion_rtn(self, tiny_flux_model_path):from diffusers import AutoPipelineForText2Image## load the model- pipe = AutoPipelineForText2Image.from_pretrained(self.model_name)+ pipe = AutoPipelineForText2Image.from_pretrained(tiny_flux_model_path)# build tiny model for testing since the full model is too large to quantize and evaluate in CIpipe.transformer.transformer_blocks = pipe.transformer.transformer_blocks[:2]pipe.transformer.single_transformer_blocks = pipe.transformer.single_transformer_blocks[:2]@@ -52,11 +51,11 @@ class TestAutoRound:@pytest.mark.skip_ci(reason="Tuning will OOM in CI; Only tiny model is suggested") # skip this test in CI@require_optimum- def test_diffusion_tune(self):+ def test_diffusion_tune(self, tiny_flux_model_path):from diffusers import AutoPipelineForText2Image## load the model- pipe = AutoPipelineForText2Image.from_pretrained(self.model_name)+ pipe = AutoPipelineForText2Image.from_pretrained(tiny_flux_model_path)model = pipe.transformer# build tiny model for testing since the full model is too large to quantize and evaluate in CIpipe.transformer.transformer_blocks = pipe.transformer.transformer_blocks[:2]diff --git a/test/test_cuda/requirements_vlm.txt b/test/test_cuda/requirements_vlm.txtindex 46e89dc96..16438f26b 100644--- a/test/test_cuda/requirements_vlm.txt+++ b/test/test_cuda/requirements_vlm.txt@@ -1,11 +1,10 @@# git+https://github.com/haotian-liu/[email protected] --no-deps# pip install git+https://github.com/deepseek-ai/DeepSeek-VL2.git --no-deps# pip install -v git+https://github.com/casper-hansen/[email protected] --no-build-isolation-# pip install flash-attn==2.7.4.post1 --no-build-isolation+# pip install flash-attn==2.8.3 --no-build-isolationbitsandbyteseinopsflash-attn-intel-extension-for-transformerslm-eval>=0.4.2,<0.5optimumpandas