Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Optimize CUDA CI and Code Scan workflows (#1770)

Sun, Xuehaoa46ad817c82b
diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml
index bc1cd951f..cb97a6e7b 100644
--- a/.azure-pipelines/code-scan.yml
+++ b/.azure-pipelines/code-scan.yml
@@ -24,19 +24,6 @@ variables:
CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog"
stages:
-
- - stage: BanditCodeScan
- displayName: Bandit Code Scan
- dependsOn: []
- jobs:
- - job: Bandit
- displayName: Bandit
- steps:
- - template: template/code-scan-template.yml
- parameters:
- codeScanFileName: "bandit"
- uploadPath: "bandit.log"
-
- stage: PylintCodeScan
displayName: Pylint Code Scan
dependsOn: []
diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh
deleted file mode 100644
index 07c1efc92..000000000
--- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-
-for var in "$@"
-do
- case $var in
- --scan_module=*)
- scan_module=$(echo $var |cut -f2 -d=)
- ;;
- esac
-done
-
-source /auto-round/.azure-pipelines/scripts/change_color.sh
-RESET="echo -en \\E[0m \\n" # close color
-
-log_dir="/auto-round/.azure-pipelines/scripts/codeScan/scanLog"
-mkdir -p $log_dir
-
-python -m bandit -r -lll -iii "/auto-round/${scan_module}" >$log_dir/bandit.log
-exit_code=$?
-
-$BOLD_YELLOW && echo " ----------------- Current bandit cmd start --------------------------" && $RESET
-echo "python -m bandit -r -lll -iii /auto-round/${scan_module} > $log_dir/bandit.log"
-$BOLD_YELLOW && echo " ----------------- Current bandit cmd end --------------------------" && $RESET
-
-$BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------"
-cat $log_dir/bandit.log
-$BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET
-
-if [ ${exit_code} -ne 0 ]; then
- $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Bandit error details." && $RESET
- exit 1
-fi
-$BOLD_PURPLE && echo "Congratulations, Bandit check passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET
-exit 0
diff --git a/.azure-pipelines/scripts/cuda_unit_test/monitor_gpu.py b/.azure-pipelines/scripts/cuda_unit_test/monitor_gpu.py
new file mode 100644
index 000000000..607c6b0b2
--- /dev/null
+++ b/.azure-pipelines/scripts/cuda_unit_test/monitor_gpu.py
@@ -0,0 +1,170 @@
+# /// script
+# requires-python = ">=3.8"
+# dependencies = [
+# "nvitop",
+# "plotille",
+# ]
+# ///
+
+import math
+import os
+import sys
+import time
+
+import plotille
+from nvitop import Device
+
+SIGNAL_FILE = "stop_monitor.flag"
+DATA_FILE = "gpu_metrics.csv"
+
+
+def run_daemon():
+ """Background daemon: streams metrics to a local file in real time."""
+ if os.path.exists(SIGNAL_FILE):
+ os.remove(SIGNAL_FILE)
+ if os.path.exists(DATA_FILE):
+ os.remove(DATA_FILE)
+
+ print("GPU Monitor Daemon started.")
+
+ try:
+ device = Device(0) # Monitor GPU 0; adjust if multiple GPUs are present
+ except Exception as e:
+ print(f"NVML error: {e}")
+ sys.exit(0)
+
+ start_time = time.time()
+ print("Daemon is running. Streaming metrics to CSV every 5 seconds...")
+
+ with open(DATA_FILE, "w") as f:
+ f.write("elapsed_sec,gpu_util_pct,mem_used_gb\n")
+
+ while not os.path.exists(SIGNAL_FILE):
+ try:
+ elapsed_sec = int(time.time() - start_time)
+ util = device.gpu_utilization()
+ util_val = util if util is not None else 0
+ mem_bytes = device.memory_used()
+ mem_gb = round((mem_bytes / (1024**3)), 2) if mem_bytes is not None else 0.0
+
+ f.write(f"{elapsed_sec},{util_val},{mem_gb}\n")
+ f.flush()
+ except Exception:
+ pass
+
+ time.sleep(5.0)
+
+ print("Stop signal received. Daemon exiting.")
+
+
+def stop_and_plot():
+ """Stop monitoring and plot charts with clean axis limits."""
+ with open(SIGNAL_FILE, "w") as f:
+ f.write("STOP")
+
+ time.sleep(2)
+
+ if not os.path.exists(DATA_FILE):
+ print("Error: Could not find GPU metrics data file.")
+ sys.exit(1)
+
+ timestamps_sec = []
+ gpu_util = []
+ mem_gb = []
+
+ with open(DATA_FILE, "r") as f:
+ lines = f.readlines()[1:] # skip header
+ for line in lines:
+ parts = line.strip().split(",")
+ if len(parts) == 3:
+ timestamps_sec.append(int(parts[0]))
+ gpu_util.append(int(parts[1]))
+ mem_gb.append(float(parts[2]))
+
+ if not timestamps_sec:
+ print("No valid data to plot.")
+ return
+
+ # --- Dynamic time unit selection ---
+ max_sec = max(timestamps_sec) if timestamps_sec else 1
+ if max_sec < 300:
+ x_data = timestamps_sec
+ x_label = "Time (Seconds)"
+ elif max_sec < 7200:
+ x_data = [round(t / 60.0, 2) for t in timestamps_sec]
+ x_label = "Time (Minutes)"
+ else:
+ x_data = [round(t / 3600.0, 2) for t in timestamps_sec]
+ x_label = "Time (Hours)"
+
+ # --- Compute clean axis upper limits ---
+ SCALE = 11
+
+ max_x = max(x_data) if x_data and max(x_data) > 0 else 1
+ max_mem = max(mem_gb) if mem_gb and max(mem_gb) > 0 else 1
+
+ x_lim = math.ceil(max_x / 8.0) * 8
+ y_lim_mem = math.ceil(max_mem / 10.0) * SCALE
+
+ # ==========================================
+ # Workaround for Boundary Clipping:
+ # Nudge values that sit exactly on the axis limit slightly inward
+ # so plotille does not clip them at the canvas edge.
+ # ==========================================
+ safe_x_data = [min(x, x_lim - 0.0001) for x in x_data]
+ safe_gpu_util = gpu_util
+ safe_mem_gb = [min(m, y_lim_mem - 0.001) for m in mem_gb]
+
+ print("\n" + "=" * 35 + " GPU Utilization (%) " + "=" * 35)
+ try:
+ print(
+ plotille.plot(
+ safe_x_data,
+ safe_gpu_util,
+ height=SCALE,
+ width=80,
+ X_label=x_label,
+ Y_label="GPU Util (%)",
+ x_min=0,
+ x_max=x_lim,
+ y_min=0,
+ y_max=SCALE * 10,
+ interp="linear",
+ )
+ )
+ except Exception as e:
+ print(f"Failed to plot GPU Utilization: {e}")
+
+ print("\n" + "=" * 35 + " GPU Memory Used (GB) " + "=" * 35)
+ try:
+ print(
+ plotille.plot(
+ safe_x_data,
+ safe_mem_gb,
+ height=SCALE,
+ width=80,
+ X_label=x_label,
+ Y_label="Memory (GB)",
+ x_min=0,
+ x_max=x_lim,
+ y_min=0,
+ y_max=y_lim_mem,
+ interp="linear",
+ )
+ )
+ except Exception as e:
+ print(f"Failed to plot GPU Memory: {e}")
+
+
+if __name__ == "__main__":
+ if len(sys.argv) < 2:
+ print("Usage: python monitor_gpu.py [daemon|stop]")
+ sys.exit(1)
+
+ command = sys.argv[1]
+ if command == "daemon":
+ run_daemon()
+ elif command == "stop":
+ stop_and_plot()
+ else:
+ print("Unknown command.")
diff --git a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
index 415635f52..11cacae23 100644
--- a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
+++ b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
@@ -62,8 +62,7 @@ function run_unit_test() {
# install unit test dependencies
echo "##[group]set up UT env..."
cd "${BUILD_SOURCESDIRECTORY}" || exit 1
- uv pip install https://github.com/ModelCloud/GPTQModel/releases/download/v6.0.3/gptqmodel-6.0.3+cu128torch2.11-cp312-cp312-linux_x86_64.whl
- uv pip install -r https://raw.githubusercontent.com/ModelCloud/GPTQModel/refs/tags/v6.0.3/requirements.txt
+ uv pip install torch==2.11.0 torchvision --index-url https://download.pytorch.org/whl/cu128
uv pip install https://github.com/XuehaoSun/llama-cpp-python/releases/download/v0.3.16/llama_cpp_python-0.3.16-cp312-cp312-linux_x86_64.whl
uv pip install 'git+https://github.com/ggml-org/llama.cpp.git#subdirectory=gguf-py'
uv pip install -r test/test_cuda/requirements.txt
@@ -112,7 +111,10 @@ function run_unit_test_llmc() {
rm -rf /root/.venv
uv venv --python=3.12 /root/.venv
uv pip install -U pytest-cov pytest-html
- BUILD_TYPE="nightly" uv pip install -r test/test_cuda/requirements_llmc.txt --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match
+ BUILD_TYPE="nightly" uv pip install \
+ -r test/test_cuda/requirements_llmc.txt \
+ --extra-index-url https://download.pytorch.org/whl/cu130 \
+ --index-strategy unsafe-best-match
uv pip install .
uv pip list
echo "##[endgroup]"
@@ -138,7 +140,10 @@ function run_unit_test_sglang() {
rm -rf /root/.venv
uv venv --python=3.12 /root/.venv
uv pip install -U pytest-cov pytest-html
- uv pip install -r test/test_cuda/requirements_sglang.txt --prerelease=allow
+ uv pip install -r test/test_cuda/requirements_sglang.txt \
+ --prerelease=allow \
+ --extra-index-url https://download.pytorch.org/whl/cu130 \
+ --index-strategy unsafe-best-match
uv pip install .
uv pip list
echo "##[endgroup]"
@@ -163,9 +168,9 @@ function run_unit_test_vllm() {
rm -rf /root/.venv
uv venv --python=3.12 /root/.venv
uv pip install -U pytest-cov pytest-html
+ vllm_version=$(curl -s https://api.github.com/repos/vllm-project/vllm/releases/latest | jq -r .tag_name | sed 's/^v//')
uv pip install -r test/test_cuda/requirements_vllm.txt \
- --extra-index-url https://wheels.vllm.ai/0.20.0/cu129 \
- --extra-index-url https://download.pytorch.org/whl/cu128 \
+ --extra-index-url https://download.pytorch.org/whl/cu130 \
--index-strategy unsafe-best-match
uv pip install .
uv pip list
diff --git a/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py b/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py
index 26faa1868..21aa866c7 100644
--- a/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py
+++ b/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py
@@ -11,6 +11,7 @@ TARGET_GPUS = [
"NVIDIA RTX 6000 Ada Generation",
"NVIDIA L40S",
"NVIDIA RTX PRO 6000 Blackwell Server Edition",
+ "NVIDIA RTX PRO 6000 Blackwell Workstation Edition",
]
DATA_CENTER_IDS = [
"AP-JP-1",
@@ -40,10 +41,11 @@ DATA_CENTER_IDS = [
"US-TX-4",
"US-WA-1",
]
-DATA_CENTER_BAN_LIST = ["EUR-IS-2", "US-IL-1"]
+DATA_CENTER_BAN_LIST = ["EUR-IS-1", "EUR-IS-2", "US-IL-1"]
DATA_CENTER_SELECT_LIST = [dc for dc in DATA_CENTER_IDS if dc not in DATA_CENTER_BAN_LIST]
REQUIRED_COUNT = 1
IMAGES_NAME = "xuehaosu/azure-agent:v0.1"
+CUDA_VERSIONS = ["13.0", "12.9", "12.8", "12.7", "12.6", "12.5", "12.4", "12.3", "12.2", "12.1", "12.0", "11.8"]
def run_create_pod(api_key, payload):
@@ -89,6 +91,7 @@ def create_pod(args):
env_dict = {kv.split("=", 1)[0]: kv.split("=", 1)[1] for kv in args.env}
payload = {
+ "allowedCudaVersions": [args.cuda_version] if (args.cuda_version and args.cuda_version in CUDA_VERSIONS) else CUDA_VERSIONS,
"cloudType": "SECURE",
"containerDiskInGb": args.container_disk_size,
"dataCenterIds": DATA_CENTER_SELECT_LIST,
@@ -195,6 +198,7 @@ def main():
parser.add_argument("--gpu_count", type=int, default=1)
parser.add_argument("--container_disk_size", type=int, default=50)
parser.add_argument("--part", type=int, default=0)
+ parser.add_argument("--cuda_version", help="CUDA version for the pod")
parser.add_argument("--env", nargs="*", help="Environment variables in KEY=VALUE format")
args = parser.parse_args()
diff --git a/.azure-pipelines/scripts/ut/run_ut_cuda.sh b/.azure-pipelines/scripts/ut/run_ut_cuda.sh
index ce24916e3..e5fa7a245 100644
--- a/.azure-pipelines/scripts/ut/run_ut_cuda.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_cuda.sh
@@ -20,9 +20,9 @@ function create_conda_env() {
# create conda env
source activate base > /dev/null 2>&1
if conda info --envs | grep -q "^$CONDA_ENV_NAME\s"; then conda remove -n ${CONDA_ENV_NAME} --all -y; fi
- conda create -n ${CONDA_ENV_NAME} python=${PYTHON_VERSION} setuptools -y
+ conda create --quiet -n ${CONDA_ENV_NAME} python=${PYTHON_VERSION} setuptools -y
source activate ${CONDA_ENV_NAME} > /dev/null 2>&1
- conda install -c conda-forge git gxx=11.2.0 gcc=11.2.0 gdb sysroot_linux-64 libgcc uv -y
+ conda install --quiet -c conda-forge git gxx=11.2.0 gcc=11.2.0 gdb sysroot_linux-64 libgcc uv -y
export LD_PRELOAD=${CONDA_PREFIX}/lib/libstdc++.so.6
# install AutoRound
@@ -92,8 +92,6 @@ function run_unit_test() {
rm -rf .coverage* *.xml *.html
uv pip install torch==2.11.0 torchvision --index-url https://download.pytorch.org/whl/cu128
- uv pip install gptqmodel --no-build-isolation
- uv pip install -r https://raw.githubusercontent.com/ModelCloud/GPTQModel/refs/heads/main/requirements.txt
uv pip install https://github.com/XuehaoSun/llama-cpp-python/releases/download/v0.3.16/llama_cpp_python-0.3.16-cp312-cp312-linux_x86_64.whl
uv pip install 'git+https://github.com/ggml-org/llama.cpp.git#subdirectory=gguf-py'
uv pip install -r test_cuda/requirements.txt
@@ -134,7 +132,6 @@ function run_unit_test_vlm() {
rm -rf .coverage* *.xml *.html
uv pip install torch==2.11.0 torchvision --index-url https://download.pytorch.org/whl/cu128
- uv pip install https://github.com/XuehaoSun/GPTQModel/releases/download/v5.8.0/gptqmodel-5.8.0+cu128torch2.11-cp312-cp312-linux_x86_64.whl
uv pip install git+https://github.com/haotian-liu/[email protected] --no-deps
uv pip install -v git+https://github.com/casper-hansen/[email protected] --no-build-isolation
uv pip install flash-attn==2.8.3 --no-build-isolation
@@ -236,8 +233,9 @@ function run_unit_test_vllm() {
cd ${REPO_PATH}/test
rm -rf .coverage* *.xml *.html
+ vllm_version=$(curl -s https://api.github.com/repos/vllm-project/vllm/releases/latest | jq -r .tag_name | sed 's/^v//')
uv pip install -r test_cuda/requirements_vllm.txt \
- --extra-index-url https://wheels.vllm.ai/0.20.0/cu129 \
+ --extra-index-url https://wheels.vllm.ai/${vllm_version}/cu129 \
--extra-index-url https://download.pytorch.org/whl/cu128 \
--index-strategy unsafe-best-match
cd ${REPO_PATH} && uv pip install . && cd ${REPO_PATH}/test
diff --git a/.azure-pipelines/unit-test-cuda.yml b/.azure-pipelines/unit-test-cuda.yml
index b3f7f354d..410c2c8f5 100644
--- a/.azure-pipelines/unit-test-cuda.yml
+++ b/.azure-pipelines/unit-test-cuda.yml
@@ -71,12 +71,18 @@ stages:
script: |
export PYTHONUNBUFFERED=1
cd ${BUILD_SOURCESDIRECTORY}
+ if [ ${{ pair.value.PART }} -eq 0 ]; then
+ CUDA_VERSION="13.0"
+ else
+ CUDA_VERSION="any"
+ fi
python .azure-pipelines/scripts/cuda_unit_test/runpod_manager.py \
--action create \
--api_key $(RUNPOD_API_KEY) \
--name "ci-$(Build.BuildId)-${{ pair.value.PART }}" \
--container_disk_size "100" \
--part ${{ pair.value.PART }} \
+ --cuda_version "${CUDA_VERSION}" \
--env AZP_URL=$(System.CollectionUri) AZP_TOKEN=$(AZP_DEVOPS_PAT) AZP_POOL=$(POOL_NAME) AZP_AGENT_NAME="$(AGENT_NAME)-part${{ pair.value.PART }}"
- task: Bash@3
@@ -125,6 +131,8 @@ stages:
echo "Running on GPU Agent: $(Agent.Name)"
nvidia-smi
lscpu
+ cd .azure-pipelines/scripts/cuda_unit_test
+ nohup uv run monitor_gpu.py daemon > monitor.log 2>&1 &
displayName: "Verify GPU Access"
- script: |
@@ -135,8 +143,8 @@ stages:
displayName: "Install Python Dependencies"
- script: |
+ export UV_NO_CACHE=0
if [ ${{ pair.value.PART }} -eq 0 ]; then
- export UV_NO_CACHE=0
bash .azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh \
--test_case=specific
else
@@ -153,6 +161,12 @@ stages:
artifact: PART_${{ pair.value.PART }}_logs
publishLocation: "pipeline"
+ - script: |
+ cd .azure-pipelines/scripts/cuda_unit_test
+ uv run monitor_gpu.py stop
+ condition: always()
+ displayName: "GPU Monitor"
+
- stage: Cleanup_Verification_${{ pair.key }}
displayName: "Verify Pod Cleanup ${{ pair.value.PART }}"
dependsOn:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0d16fb6c5..f93b689f4 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -76,4 +76,15 @@ repos:
rev: v3.14.2
hooks:
- id: markdown-link-check
- args: [-q, -c .azure-pipelines/scripts/codeScan/markdown-link-check/markdown-link-check.json]
\ No newline at end of file
+ args: [-q, -c .azure-pipelines/scripts/codeScan/markdown-link-check/markdown-link-check.json]
+
+ - repo: https://github.com/PyCQA/bandit
+ rev: "1.9.4"
+ hooks:
+ - id: bandit
+ files: |
+ (?x)^(
+ auto_round/.*\.py|
+ auto_round_extension/.*\.py
+ )$
+ args: [-r, -lll, -iii, auto_round/, auto_round_extension/]
diff --git a/test/test_cuda/requirements.txt b/test/test_cuda/requirements.txt
index b85b06535..7270797e1 100644
--- a/test/test_cuda/requirements.txt
+++ b/test/test_cuda/requirements.txt
@@ -1,7 +1,5 @@
-# pip install -v git+https://github.com/casper-hansen/AutoAWQ.git --no-build-isolation
-# auto-gptq
einops
-# pip install -v git+https://github.com/ModelCloud/GPTQModel.git --no-build-isolation
+gptqmodel
# pip install git+https://github.com/ggml-org/llama.cpp.git#subdirectory=gguf-py
lm_eval >= 0.4.10 # for transformers >= 5.0.0
optimum
diff --git a/test/test_cuda/requirements_sglang.txt b/test/test_cuda/requirements_sglang.txt
index 604b6c44c..cfb861600 100644
--- a/test/test_cuda/requirements_sglang.txt
+++ b/test/test_cuda/requirements_sglang.txt
@@ -1 +1 @@
-sglang >= 0.5.10
\ No newline at end of file
+sglang < 0.5.11 # due to not support cu128
\ No newline at end of file
diff --git a/test/test_cuda/requirements_vlm.txt b/test/test_cuda/requirements_vlm.txt
index 16438f26b..f9d195112 100644
--- a/test/test_cuda/requirements_vlm.txt
+++ b/test/test_cuda/requirements_vlm.txt
@@ -5,6 +5,7 @@
bitsandbytes
einops
flash-attn
+gptqmodel
lm-eval>=0.4.2,<0.5
optimum
pandas