LLM inference in C/C++
C++ ggml
snapdragon: ci updates to use new run script (#28293)
Max Krasnyanskyd30500b83b0f
diff --git a/scripts/snapdragon/build.py b/scripts/snapdragon/build.pyindex 5e9fab3d08..b36e6361b3 100755--- a/scripts/snapdragon/build.py+++ b/scripts/snapdragon/build.py@@ -226,6 +226,10 @@ def main():if res.returncode != 0:logger.error("ADB push failed.")sys.exit(res.returncode)++ chmod_cmd = adb_cmd + ["shell", f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]+ logger.info(f"+ {' '.join(chmod_cmd)}")+ subprocess.run(chmod_cmd)logger.info("ADB push completed successfully!")elif target_type == "linux":@@ -249,6 +253,10 @@ def main():if res.returncode != 0:logger.error("SSH/SCP deploy failed.")sys.exit(res.returncode)++ chmod_cmd = ["ssh", ssh_host, f"chmod -R 755 {target_dir}/bin 2>/dev/null || true"]+ logger.info(f"+ {' '.join(chmod_cmd)}")+ subprocess.run(chmod_cmd)logger.info("SSH/SCP deploy completed successfully!")elif target_type == "windows":diff --git a/scripts/snapdragon/qdc/run_qdc_jobs.py b/scripts/snapdragon/qdc/run_qdc_jobs.pyindex f1b0453eec..4ccf39dd29 100644--- a/scripts/snapdragon/qdc/run_qdc_jobs.py+++ b/scripts/snapdragon/qdc/run_qdc_jobs.py@@ -35,7 +35,6 @@ import shutilimport sysimport tempfileimport time-import urllib.requestimport xml.etree.ElementTree as ETfrom dataclasses import dataclass, fieldfrom pathlib import Path@@ -104,15 +103,7 @@ _NON_TERMINAL_STATE_VALUES = {s.value for s in NON_TERMINAL_STATES}_RUN_BENCH = _TESTS_DIR / "run_bench_tests_posix.py"_RUN_BACKEND_OPS = _TESTS_DIR / "run_backend_ops_posix.py"_REQUIREMENTS = _SCRIPTS_DIR / "requirements.txt"-_UPSTREAM_ADB_SCRIPTS = (- "https://raw.githubusercontent.com/ggml-org/llama.cpp/master/scripts/snapdragon/adb"-)-_ADB_SCRIPT_NAMES = [- "run-bench.sh",- "run-cli.sh",- "run-completion.sh",- "run-tool.sh",-]+_RUN_PY = _SCRIPTS_DIR.parent / "run.py"# --- Linux (BASH) assets ------------------------------------------------------_RUN_LINUX_TEMPLATE = _TESTS_DIR / "linux" / "run_linux.sh"@@ -147,7 +138,7 @@ def _build_android_artifact(Zip structure:llama_cpp_bundle/ installed package (adb pushed to /data/local/tmp/)- run-{bench,cli,completion,tool}.sh upstream adb wrappers (patched)+ run.py Snapdragon runnertests/utils.py shared adb helpersconftest.py Appium pytest fixtures@@ -159,21 +150,9 @@ def _build_android_artifact(bundle_dir = stage_dir / "llama_cpp_bundle"shutil.copytree(pkg_dir, bundle_dir)- # Download upstream adb scripts so they land at /qdc/appium/ on the QDC- # runner. They wrap `adb shell` internally. Patch in `chmod +x bin/* lib/*`- # right after `cd $basedir` so device binaries are executable.- for name in _ADB_SCRIPT_NAMES:- url = f"{_UPSTREAM_ADB_SCRIPTS}/{name}"- dest = stage_dir / name- log.info("Downloading %s", url)- urllib.request.urlretrieve(url, str(dest))- content = dest.read_text()- content = content.replace(- "cd $basedir;",- "cd $basedir; chmod +x bin/* lib/* 2>/dev/null;",- )- dest.write_text(content)- dest.chmod(0o755)+ dest = stage_dir / "run.py"+ shutil.copy(_RUN_PY, dest)+ dest.chmod(0o755)tests_dir = stage_dir / "tests"tests_dir.mkdir()diff --git a/scripts/snapdragon/qdc/tests/linux/run_linux.sh b/scripts/snapdragon/qdc/tests/linux/run_linux.shindex a6abf8ec30..11083f5213 100644--- a/scripts/snapdragon/qdc/tests/linux/run_linux.sh+++ b/scripts/snapdragon/qdc/tests/linux/run_linux.sh@@ -124,9 +124,9 @@ note_timeout_if_triggered() {completion_extra_args() {case "$1" in- cpu) echo "--device none --ctx-size 128 -no-cnv -n 32 --seed 42 --batch-size 128" ;;- gpu) echo "--device GPUOpenCL --ctx-size 128 -no-cnv -n 32 --seed 42 --ubatch-size 512" ;;- npu) echo "--device HTP0 --ctx-size 128 -no-cnv -n 32 --seed 42 --ubatch-size 1024" ;;+ cpu) echo "--device none --ctx-size 2048 -no-cnv -n 32 --seed 42" ;;+ gpu) echo "--device GPUOpenCL --ctx-size 2048 -no-cnv -n 32 --seed 42" ;;+ npu) echo "--device HTP0 --ctx-size 2048 -no-cnv -n 32 --seed 42 --ubatch-size 1024" ;;esac}@@ -161,12 +161,14 @@ run_bench_case() {local ndev=${parts[0]} device=${parts[1]}local log_suffix=$(backend_log_name "$name")local log="$LOG_DIR/llama_bench_${log_suffix}.log"+ local ubatch_arg=""+ [ "$name" = "npu" ] && ubatch_arg="--ubatch-size 1024"echo "=== [bench:$name] llama-bench --device $device (NDEV=$ndev) ==="timeout 600 env GGML_HEXAGON_NDEV=$ndev ./bin/llama-bench \-m "$MODEL_PATH" \--device "$device" \-ngl 99 \- --batch-size 128 \+ $ubatch_arg \-t 4 \-p 128 \-n 32 \diff --git a/scripts/snapdragon/qdc/tests/run_backend_ops_posix.py b/scripts/snapdragon/qdc/tests/run_backend_ops_posix.pyindex 355bf6c6a5..f2f870f131 100644--- a/scripts/snapdragon/qdc/tests/run_backend_ops_posix.py+++ b/scripts/snapdragon/qdc/tests/run_backend_ops_posix.py@@ -14,7 +14,7 @@ import pytestfrom utils import (BIN_PATH,push_bundle_if_needed,- run_script,+ run_snapdragon,write_qdc_log,)@@ -31,11 +31,8 @@ def test_backend_ops_htp0(type_a):else:pattern = f"type_a={type_a}"- quoted_pattern = f'"{pattern}"' if type_a == "q4_0" else pattern- result = run_script(- "run-tool.sh",- extra_env={"HB": "0"},- extra_args=["test-backend-ops", "-b", "HTP0", "-o", "MUL_MAT", "-p", quoted_pattern],+ result = run_snapdragon(+ ["test-backend-ops", "-b", "HTP0", "-o", "MUL_MAT", "-p", pattern],)write_qdc_log(f"backend_ops_{type_a}.log", result.stdout or "")assert result.returncode == 0, (diff --git a/scripts/snapdragon/qdc/tests/run_bench_tests_posix.py b/scripts/snapdragon/qdc/tests/run_bench_tests_posix.pyindex f42227c9f6..f1c9377e69 100644--- a/scripts/snapdragon/qdc/tests/run_bench_tests_posix.py+++ b/scripts/snapdragon/qdc/tests/run_bench_tests_posix.py@@ -1,8 +1,8 @@"""On-device bench and completion test runner for llama.cpp (CPU, GPU, NPU backends).-On Android: calls upstream run-*.sh scripts from llama.cpp/scripts/snapdragon/adb/-on the QDC runner host (scripts wrap commands in ``adb shell`` internally).+On Android: calls scripts/snapdragon/run.py on the QDC runner host+(script wraps commands in adb shell internally).On Linux: runs llama-bench directly via run_linux.sh (BASH framework).@@ -19,11 +19,10 @@ import pytestfrom utils import (BIN_PATH,MODEL_DEVICE_PATH,- MODEL_NAME,PROMPT_DIR,push_bundle_if_needed,run_adb_command,- run_script,+ run_snapdragon,write_qdc_log,)@@ -52,12 +51,18 @@ def install(driver):],)def test_llama_completion(device):- result = run_script(- "run-completion.sh",- extra_env={"D": device, "M": MODEL_NAME},- extra_args=["--batch-size", "128", "-n", "128", "--seed", "42",- "-f", f"{PROMPT_DIR}/bench_prompt.txt"],- )+ args = [+ "llama-completion",+ "-m", MODEL_DEVICE_PATH,+ "-f", f"{PROMPT_DIR}/bench_prompt.txt",+ "-no-cnv",+ "--ctx-size", "8192",+ "-n", "128",+ "--seed", "42",+ ]+ if device == "HTP0":+ args += ["--ubatch-size", "1024"]+ result = run_snapdragon(args, device=device)write_qdc_log(f"llama_completion_{device}.log", result.stdout or "")assert result.returncode == 0, (f"llama-completion {device} failed (exit {result.returncode})"@@ -76,11 +81,16 @@ _DEVICE_LOG_NAME = {"none": "cpu", "GPUOpenCL": "gpu", "HTP0": "htp"}],)def test_llama_bench(device):- result = run_script(- "run-bench.sh",- extra_env={"D": device, "M": MODEL_NAME},- extra_args=["--batch-size", "128", "-p", "128", "-n", "32"],- )+ args = [+ "llama-bench",+ "-m", MODEL_DEVICE_PATH,+ "-ngl", "99",+ "-p", "128",+ "-n", "32",+ ]+ if device == "HTP0":+ args += ["--ubatch-size", "1024"]+ result = run_snapdragon(args, device=device)write_qdc_log(f"llama_bench_{_DEVICE_LOG_NAME[device]}.log", result.stdout or "")assert result.returncode == 0, (f"llama-bench {device} failed (exit {result.returncode})"diff --git a/scripts/snapdragon/qdc/tests/utils.py b/scripts/snapdragon/qdc/tests/utils.pyindex fad6a92329..7a02420c5d 100644--- a/scripts/snapdragon/qdc/tests/utils.py+++ b/scripts/snapdragon/qdc/tests/utils.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport loggingimport osimport subprocess+import sysimport tempfilefrom appium.options.common import AppiumOptions@@ -93,17 +94,25 @@ def run_adb_command(cmd: str, *, check: bool = True) -> subprocess.CompletedProcreturn result-def run_script(- script: str,+def run_snapdragon(+ cmd_args: list[str],+ *,+ device: str | None = None,+ extra_run_args: list[str] | None = None,extra_env: dict[str, str] | None = None,- extra_args: list[str] | None = None,) -> subprocess.CompletedProcess:- """Run an upstream shell script from /qdc/appium/ on the QDC runner host."""+ """Run a tool via scripts/snapdragon/run.py targeting android."""env = os.environ.copy()env["GGML_HEXAGON_EXPERIMENTAL"] = "1"if extra_env:env.update(extra_env)- cmd = [f"{SCRIPTS_DIR}/{script}"] + (extra_args or [])+ cmd = [sys.executable, f"{SCRIPTS_DIR}/run.py", "--target", "android"]+ if device is not None:+ cmd.extend(["-d", device])+ if extra_run_args:+ cmd.extend(extra_run_args)+ cmd.append("--")+ cmd.extend(cmd_args)result = subprocess.run(cmd, env=env,text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,diff --git a/scripts/snapdragon/run.py b/scripts/snapdragon/run.pyindex d639b16b82..81eecd2e0c 100755--- a/scripts/snapdragon/run.py+++ b/scripts/snapdragon/run.py@@ -146,7 +146,22 @@ def main():env_vars[env_name] = os.environ[env_name]# Resolve and filter devices (HTP vs OpenCL)- devices_val = args.devices if args.devices is not None else "HTP0"+ device_in_cmd = None+ for i, arg in enumerate(cmd_args):+ if arg == "--device" and i + 1 < len(cmd_args):+ device_in_cmd = cmd_args[i + 1]+ break+ elif arg.startswith("--device="):+ device_in_cmd = arg.split("=", 1)[1]+ break++ if args.devices is not None:+ devices_val = args.devices+ elif device_in_cmd is not None:+ devices_val = device_in_cmd+ else:+ devices_val = "HTP0"+if devices_val.isdigit():hex_devices = devices_valcl_device = ""@@ -154,7 +169,12 @@ def main():parts = [p.strip() for p in devices_val.split(",")]# Any device containing "htp" is Hexagon, rest is OpenCLhex_parts = [p for p in parts if "htp" in p.lower()]- cl_parts = [p for p in parts if "htp" not in p.lower()]+ cl_parts = [+ p for p in parts+ if "htp" not in p.lower()+ and p.lower() not in ("none", "cpu")+ and not p.lower().startswith("gpuopencl")+ ]hex_devices = ",".join(hex_parts)cl_device = ",".join(cl_parts)@@ -316,11 +336,17 @@ def main():if basename in ("llama-cli", "llama-completion", "llama-server"):if "-ngl" not in cmd_args and "--n-gpu-layers" not in cmd_args:cmd_args += ["-ngl", "99"]- if "--ubatch-size" not in cmd_args and "-ub" not in cmd_args:- cmd_args += ["--ubatch-size", "1024"]if "-fa" not in cmd_args and "--flash-attn" not in cmd_args:cmd_args += ["-fa", "on"]+ # Use ubatch-size 1024 for hexagon backend (HTP devices)+ if hex_devices and basename in ("llama-cli", "llama-completion", "llama-server", "llama-bench"):+ if "--ubatch-size" not in cmd_args and "-ub" not in cmd_args:+ cmd_args += ["--ubatch-size", "1024"]+ elif basename in ("llama-cli", "llama-completion", "llama-server"):+ if "--ubatch-size" not in cmd_args and "-ub" not in cmd_args:+ cmd_args += ["--ubatch-size", "1024"]+if basename in ("llama-cli", "llama-completion", "llama-server", "llama-bench"):if "-t" not in cmd_args and "--threads" not in cmd_args:cmd_args += ["-t", "6"]