LLM inference in C/C++
C++ ggml
sycl : update guide Q&A and script for device setting (#26442)
Neo Zhangc1f41098988a
diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.mdindex 814e541e1a..73f89a7063 100644--- a/docs/backend/SYCL.md+++ b/docs/backend/SYCL.md@@ -449,6 +449,8 @@ Oruse 1 SYCL GPUs: [0] with Max compute units:512```+User can use the device management in [docs/multi-gpu.md](https://github.com/ggml-org/llama.cpp/blob/master/docs/multi-gpu.md), like parameter `--device SYCL0,SYCL1` to assign one or more devices.+## Windows### Install GPU driver@@ -763,6 +765,7 @@ Oruse 1 SYCL GPUs: [0] with Max compute units:512```+User can use the device management in [docs/multi-gpu.md](https://github.com/ggml-org/llama.cpp/blob/master/docs/multi-gpu.md), like parameter `--device SYCL0,SYCL1` to assign one or more devices.## Environment Variable@@ -895,6 +898,45 @@ Pass these via `CXXFLAGS` or add a one-off `#define` to enable a flag on the sposet UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1```+- When I set `SYCL_CACHE_PERSISTENT=1` in running time, I meet crash.++ `SYCL_CACHE_PERSISTENT=1` is not recommended by llama.cpp SYCL backend.+ When cache is enabled, SYCL runtime will try to cache and reuse JIT-compiled binaries.++ We find some AI will tell user this cmd to speed up SYCL backend. It only speeds up the startup to skip the JIT process, instead of running speed.++ It will bring negative impact when the SYCL binary file is changed frequently in your running environment. The new & old codes mix will lead to crash.++ Compare to the benefit, it has brought more failed cases.+ If you are not familiar with the SYCL compiler principle of JIT and AOT, please don't use it.++ To restore, you need to remove the local cache: `~/.cache/libsycl_cache/` and execute `unset SYCL_CACHE_PERSISTENT` in running time.++- How to use iGPU and dGPU in same time?++ 1. Detect the devices in your running time.+ ```+ source /opt/intel/oneapi/setvars.sh+ ./build/bin/llama-server --list-devices++ or+ ./build/bin/llama-cli --list-devices+ ./build/bin/llama-bench --list-devices+ ./build/bin/llama-completion --list-devices++ Available devices:+ SYCL0: Intel(R) Arc(TM) A770 Graphics (15473 MiB, 15473 MiB free)+ SYCL1: Intel(R) UHD Graphics 770 (59675 MiB, 44986 MiB free)+ ```++ The dGPU will be in the head of this list and iGPU will be the end.+ If not all GPUs are listed, please check the env var: ONEAPI_DEVICE_SELECTOR and unset it.++ 2. Set the iGPU and dGPU++ Set the iGPU and dGPU by `./build/bin/llama-server --device SYCL0,SYCL1,SYCLxxx`.++### **GitHub contribution**:Please add the `[SYCL]` prefix/tag in issues/PRs titles to help the SYCL contributors to check/address them without delay.diff --git a/examples/sycl/start-svr.sh b/examples/sycl/start-svr.shindex ce31ec51d2..49177ba2dc 100755--- a/examples/sycl/start-svr.sh+++ b/examples/sycl/start-svr.sh@@ -12,6 +12,7 @@ This script processes files with specified options.Options:-h, --help Display this help message and exit.+ -d, --device <value> Set SYCL devices (default: SYCL0).-c, --context <value> Set context length. Bigger need more memory.-p, --promote <value> Prompt to start generation with.-m, --model <value> Full model file path.@@ -41,10 +42,16 @@ MODEL_FILE=../models/Qwen3.5-4B-Q4_0.ggufNGL=99CONTEXT=4096GGML_SYCL_DEVICE=-1+SYCL_DEVICES="SYCL0"SPLIT_MODE=layerLOG_VERBOSE=3while [[ $# -gt 0 ]]; docase "$1" in+ -d|--device)+ SYCL_DEVICES="$2"+ shift+ shift+ ;;-c|--context)CONTEXT=$2# Shift twice to consume both the option flag and its value@@ -95,8 +102,6 @@ while [[ $# -gt 0 ]]; doesacdone--source /opt/intel/oneapi/setvars.sh#export GGML_SYCL_DEBUG=1@@ -107,17 +112,19 @@ source /opt/intel/oneapi/setvars.shexport UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1echo "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=${UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS}"+echo "ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR}"++if [ $GGML_SYCL_DEVICE -ne -1 ]; thenecho "Use $GGML_SYCL_DEVICE as main GPU"#use signle GPU onlyGPUS_SETTING="-mg $GGML_SYCL_DEVICE -sm ${SPLIT_MODE}"- echo "ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR}"else- echo "Use all Intel GPUs, including iGPU & dGPU"+ echo "Use Intel GPUs: ${SYCL_DEVICES}"GPUS_SETTING="-sm ${SPLIT_MODE}"- fi+fi-echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --mmap --host 0.0.0.0 --port 8000"-ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --mmap --host 0.0.0.0 --port 8000+echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap --host 0.0.0.0 --port 8000"+ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap --host 0.0.0.0 --port 8000diff --git a/examples/sycl/test.sh b/examples/sycl/test.shindex 116047cd2e..b9498f49b7 100755--- a/examples/sycl/test.sh+++ b/examples/sycl/test.sh@@ -12,6 +12,7 @@ This script processes files with specified options.Options:-h, --help Display this help message and exit.+ -d, --device <value> Set SYCL devices (default: SYCL0).-c, --context <value> Set context length. Bigger need more memory.-p, --promote <value> Prompt to start generation with.-m, --model <value> Full model file path.@@ -42,10 +43,16 @@ MODEL_FILE=../models/llama-2-7b.Q4_0.ggufNGL=99CONTEXT=4096GGML_SYCL_DEVICE=-1+SYCL_DEVICES="SYCL0"SPLIT_MODE=layerLOG_VERBOSE=3while [[ $# -gt 0 ]]; docase "$1" in+ -d|--device)+ SYCL_DEVICES="$2"+ shift+ shift+ ;;-c|--context)CONTEXT=$2# Shift twice to consume both the option flag and its value@@ -115,16 +122,17 @@ source /opt/intel/oneapi/setvars.shexport UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1echo "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=${UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS}"+echo "ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR}"+if [ $GGML_SYCL_DEVICE -ne -1 ]; thenecho "Use $GGML_SYCL_DEVICE as main GPU"#use signle GPU onlyGPUS_SETTING="-mg $GGML_SYCL_DEVICE -sm ${SPLIT_MODE}"- echo "ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR}"else- echo "Use all Intel GPUs, including iGPU & dGPU"+ echo "Use Intel GPUs: ${SYCL_DEVICES}"GPUS_SETTING="-sm ${SPLIT_MODE}"fi-echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --mmap "-ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --mmap+echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap "+ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmapdiff --git a/examples/sycl/win-start-svr.bat b/examples/sycl/win-start-svr.batindex 13b5159e00..8077105893 100644--- a/examples/sycl/win-start-svr.bat+++ b/examples/sycl/win-start-svr.bat@@ -13,6 +13,7 @@ set "MODEL_FILE=..\models\Qwen3.5-4B-Q4_0.gguf"set "NGL=99"set "CONTEXT=4096"set "GGML_SYCL_DEVICE=-1"+set "SYCL_DEVICES=SYCL0"set "SPLIT_MODE=layer"set "LOG_VERBOSE=3"@@ -36,6 +37,21 @@ if /I "%~1"=="--context" (goto parse_args)+if /I "%~1"=="-d" (+ if "%~2"=="" goto missing_value+ set "SYCL_DEVICES=%~2"+ shift+ shift+ goto parse_args+)+if /I "%~1"=="--device" (+ if "%~2"=="" goto missing_value+ set "SYCL_DEVICES=%~2"+ shift+ shift+ goto parse_args+)+if /I "%~1"=="-m" (if "%~2"=="" goto missing_valueset "MODEL_FILE=%~2"@@ -130,6 +146,7 @@ echo This script processes files with specified options.echo.echo Options:echo -h, --help Display this help message and exit.+echo -d, --device ^<value^> Set SYCL devices (default: SYCL0).echo -c, --context ^<value^> Set context length. Bigger need more memory.echo -m, --model ^<value^> Full model file path.echo -mg,--main-gpu ^<value^> Set main GPU ID (0 - n) for single GPU mode.@@ -160,19 +177,20 @@ REM Support malloc device memory more than 4GB.set "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1"echo UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=%UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS%+echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR%+if not "%GGML_SYCL_DEVICE%"=="-1" (echo Use %GGML_SYCL_DEVICE% as main GPUREM Use single GPU only.set "GPUS_SETTING=-mg %GGML_SYCL_DEVICE% -sm %SPLIT_MODE%"- echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR%-) else (- echo Use all Intel GPUs, including iGPU ^& dGPU+ ) else (+ echo Use Intel GPUs: %SYCL_DEVICES%set "GPUS_SETTING=-sm %SPLIT_MODE%")-echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap --host 0.0.0.0 --port 8000+echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device %SYCL_DEVICES% --mmap --host 0.0.0.0 --port 8000set "ZES_ENABLE_SYSMAN=1"-%BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap --host 0.0.0.0 --port 8000+%BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device "%SYCL_DEVICES%" --mmap --host 0.0.0.0 --port 8000endlocaldiff --git a/examples/sycl/win-test.bat b/examples/sycl/win-test.batindex 39640908b0..cc6da44138 100644--- a/examples/sycl/win-test.bat+++ b/examples/sycl/win-test.bat@@ -19,6 +19,7 @@ set "MODEL_FILE=..\models\llama-2-7b.Q4_0.gguf"set "NGL=99"set "CONTEXT=4096"set "GGML_SYCL_DEVICE=-1"+set "SYCL_DEVICES=SYCL0"set "SPLIT_MODE=layer"set "LOG_VERBOSE=3"@@ -42,6 +43,21 @@ if /I "%~1"=="--context" (goto parse_args)+if /I "%~1"=="-d" (+ if "%~2"=="" goto missing_value+ set "SYCL_DEVICES=%~2"+ shift+ shift+ goto parse_args+)+if /I "%~1"=="--device" (+ if "%~2"=="" goto missing_value+ set "SYCL_DEVICES=%~2"+ shift+ shift+ goto parse_args+)+if /I "%~1"=="-p" (if "%~2"=="" goto missing_valueset "INPUT_PROMPT=%~2"@@ -151,6 +167,7 @@ echo This script processes files with specified options.echo.echo Options:echo -h, --help Display this help message and exit.+echo -d, --device ^<value^> Set SYCL devices (default: SYCL0).echo -c, --context ^<value^> Set context length. Bigger need more memory.echo -p, --promote ^<value^> Prompt to start generation with.echo -m, --model ^<value^> Full model file path.@@ -182,19 +199,21 @@ REM Support malloc device memory more than 4GB.set "UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1"echo UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=%UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS%+echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR%+if not "%GGML_SYCL_DEVICE%"=="-1" (echo Use %GGML_SYCL_DEVICE% as main GPUREM Use single GPU only.set "GPUS_SETTING=-mg %GGML_SYCL_DEVICE% -sm %SPLIT_MODE%"- echo ONEAPI_DEVICE_SELECTOR=%ONEAPI_DEVICE_SELECTOR%-) else (- echo Use all Intel GPUs, including iGPU ^& dGPU+ )+else (+ echo Use Intel GPUs: %SYCL_DEVICES%set "GPUS_SETTING=-sm %SPLIT_MODE%")-echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m %MODEL_FILE% -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap+echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m %MODEL_FILE% -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device %SYCL_DEVICES% --mmapset "ZES_ENABLE_SYSMAN=1"-%BIN_FILE% -m "%MODEL_FILE%" -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --mmap+%BIN_FILE% -m "%MODEL_FILE%" -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device "%SYCL_DEVICES%" --mmapendlocal