Skip to content

Distribute and run LLMs with a single file.

Updated ggml build scripts with -DGGML_CUDA_USE_GRAPHS / -DGGML_HIP_GRAPHS=1 (#1051)

Davide Eynard42ff11b31491
diff --git a/llamafile/cuda.bat b/llamafile/cuda.bat
index 6c773259..c7364109 100644
--- a/llamafile/cuda.bat
+++ b/llamafile/cuda.bat
@@ -188,6 +188,7 @@ set "COMMON_FLAGS=%COMMON_FLAGS% --std=c++17"
set "COMMON_FLAGS=%COMMON_FLAGS% -Xcompiler="/nologo /EHsc /O2 /GR /MT /std:c++17 /Zc:preprocessor""
set "COMMON_FLAGS=%COMMON_FLAGS% -diag-suppress 177 -diag-suppress 221 -diag-suppress 550"
set "COMMON_FLAGS=%COMMON_FLAGS% -DNDEBUG -DGGML_BUILD=1 -DGGML_SHARED=1 -DGGML_BACKEND_SHARED=1 -DGGML_BACKEND_BUILD=1 -DGGML_MULTIPLATFORM"
+set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_USE_GRAPHS"
set "COMMON_FLAGS=%COMMON_FLAGS% %BLAS_DEFINE%"
if "%NO_IQ_QUANTS%"=="1" set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_NO_IQ_QUANTS"
if "%FA_ALL_QUANTS%"=="1" set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_FA_ALL_QUANTS"
diff --git a/llamafile/cuda.sh b/llamafile/cuda.sh
index 3349b738..ff35f1c5 100755
--- a/llamafile/cuda.sh
+++ b/llamafile/cuda.sh
@@ -245,6 +245,7 @@ COMMON_FLAGS="\
-DGGML_BUILD=1 \
-DGGML_SHARED=1 \
-DGGML_MULTIPLATFORM \
+ -DGGML_CUDA_USE_GRAPHS \
$BLAS_DEFINE"
if [ "$NO_IQ_QUANTS" = "1" ]; then
diff --git a/llamafile/cuda_parallel.bat b/llamafile/cuda_parallel.bat
index 104fbfdc..dbf17440 100644
--- a/llamafile/cuda_parallel.bat
+++ b/llamafile/cuda_parallel.bat
@@ -211,6 +211,7 @@ set "COMMON_FLAGS=%COMMON_FLAGS% --std=c++17"
set "COMMON_FLAGS=%COMMON_FLAGS% -Xcompiler="/nologo /EHsc /O2 /GR /MT /std:c++17 /Zc:preprocessor""
set "COMMON_FLAGS=%COMMON_FLAGS% -diag-suppress 177 -diag-suppress 221 -diag-suppress 550"
set "COMMON_FLAGS=%COMMON_FLAGS% -DNDEBUG -DGGML_BUILD=1 -DGGML_SHARED=1 -DGGML_BACKEND_SHARED=1 -DGGML_BACKEND_BUILD=1 -DGGML_MULTIPLATFORM"
+set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_USE_GRAPHS"
set "COMMON_FLAGS=%COMMON_FLAGS% %BLAS_DEFINE%"
if "%NO_IQ_QUANTS%"=="1" set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_NO_IQ_QUANTS"
if "%FA_ALL_QUANTS%"=="1" set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_FA_ALL_QUANTS"
diff --git a/llamafile/rocm.bat b/llamafile/rocm.bat
index 448f3764..6101c063 100644
--- a/llamafile/rocm.bat
+++ b/llamafile/rocm.bat
@@ -132,6 +132,7 @@ set "COMMON_FLAGS=%COMMON_FLAGS% -Wno-ignored-attributes -Wno-nested-anon-types"
set "COMMON_FLAGS=%COMMON_FLAGS% -I%BUILD_DIR% -I%GGML_INC_DIR% -I%GGML_SRC_DIR% -I%GGML_CUDA_DIR%"
set "COMMON_FLAGS=%COMMON_FLAGS% -I"%HIP_PATH%\include""
set "COMMON_FLAGS=%COMMON_FLAGS% -DNDEBUG -DGGML_BUILD=1 -DGGML_SHARED=1 -DGGML_BACKEND_SHARED=1 -DGGML_BACKEND_BUILD=1 -DGGML_MULTIPLATFORM"
+set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_HIP_GRAPHS=1"
set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_USE_HIP=1 -DGGML_USE_TINYBLAS=1 -DGGML_HIP_NO_VMM=1 -D__HIP_PLATFORM_AMD__"
:: --offload-compress shrinks the fat binary (mirrors rocm.sh / #995); requires a
:: recent ROCm HIP SDK -- drop this flag if your clang++ rejects it
diff --git a/llamafile/rocm.sh b/llamafile/rocm.sh
index 14d83238..77740375 100755
--- a/llamafile/rocm.sh
+++ b/llamafile/rocm.sh
@@ -121,6 +121,7 @@ COMMON_FLAGS="\
-DGGML_SHARED=1 \
-DGGML_MULTIPLATFORM \
-DGGML_USE_HIP=1 \
+ -DGGML_HIP_GRAPHS=1 \
-DGGML_HIP_NO_VMM=1 \
-DGGML_USE_TINYBLAS=1 \
-Wno-return-type \
diff --git a/llamafile/rocm_parallel.bat b/llamafile/rocm_parallel.bat
index a18052ef..3c9c855b 100644
--- a/llamafile/rocm_parallel.bat
+++ b/llamafile/rocm_parallel.bat
@@ -154,6 +154,7 @@ set "COMMON_FLAGS=%COMMON_FLAGS% -Wno-ignored-attributes -Wno-nested-anon-types"
set "COMMON_FLAGS=%COMMON_FLAGS% -I%BUILD_DIR% -I%GGML_INC_DIR% -I%GGML_SRC_DIR% -I%GGML_CUDA_DIR%"
set "COMMON_FLAGS=%COMMON_FLAGS% -I"%HIP_PATH%\include""
set "COMMON_FLAGS=%COMMON_FLAGS% -DNDEBUG -DGGML_BUILD=1 -DGGML_SHARED=1 -DGGML_BACKEND_SHARED=1 -DGGML_BACKEND_BUILD=1 -DGGML_MULTIPLATFORM"
+set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_HIP_GRAPHS=1"
set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_USE_HIP=1 -DGGML_USE_TINYBLAS=1 -DGGML_HIP_NO_VMM=1 -D__HIP_PLATFORM_AMD__"
:: --offload-compress shrinks the fat binary (mirrors rocm.sh / #995); requires a
:: recent ROCm HIP SDK -- drop this flag if your clang++ rejects it
diff --git a/tests/integration/tests/test_gpu.py b/tests/integration/tests/test_gpu.py
index 08a072e3..2749fa7e 100644
--- a/tests/integration/tests/test_gpu.py
+++ b/tests/integration/tests/test_gpu.py
@@ -2,6 +2,7 @@
import os
import platform
+import re
import subprocess
import tempfile
@@ -203,6 +204,94 @@ class TestGPUAcceleration:
if os.path.exists(log_path):
os.unlink(log_path)
+ def test_gpu_cuda_graphs(self, executable, model, available_gpu, server_port, timeouts):
+ """NVIDIA: the CUDA module must be built with CUDA graphs and use them.
+
+ ggml keeps its CUDA-graph code behind the compile-time define
+ GGML_CUDA_USE_GRAPHS. A ggml-cuda.so built without it submits every
+ kernel of every decode step individually and generates 3-18% slower
+ at batch size 1 (measured on an L40S; the cost is per layer per token,
+ so small models lose most). Two checks:
+
+ * the backend feature list in the server's system_info line carries
+ ``USE_GRAPHS = 1`` -- the compile-time property of the module;
+ * after a couple of decode steps ggml logs ``CUDA graph warmup
+ complete``, i.e. graphs actually engaged for this model. ggml may
+ legitimately refuse them (pre-Volta GPU, unsupported node type); it
+ then logs ``disabling CUDA graphs due to ...`` and the runtime check
+ is skipped rather than failed.
+
+ The module's own log lines go to stderr, not to --log-file (they are
+ written by the ggml logger inside the DSO), so stderr is captured too.
+ """
+ if available_gpu != "nvidia":
+ pytest.skip("CUDA graphs are a property of the NVIDIA module")
+
+ runner = LlamafileRunner(
+ executable=executable,
+ model=model,
+ gpu=available_gpu,
+ )
+
+ paths = []
+ for suffix in (".log", ".stderr"):
+ with tempfile.NamedTemporaryFile(
+ mode="w", suffix=suffix, delete=False
+ ) as f:
+ paths.append(f.name)
+ log_path, stderr_path = paths
+
+ proc = runner.start_server(port=server_port, log_file=log_path,
+ stderr_file=stderr_path,
+ extra_args=["--verbose"])
+ try:
+ ready = LlamafileRunner.wait_for_server(
+ server_port, timeout=timeouts.server_ready, proc=proc
+ )
+ assert ready
+
+ # Graph warm-up needs at least two decode steps whose graph
+ # properties did not change; two short requests give plenty.
+ for _ in range(2):
+ response = LlamafileRunner.chat_completion(
+ port=server_port,
+ messages=[{"role": "user", "content": GREETING_PROMPT}],
+ timeout=timeouts.http_request,
+ max_tokens=32,
+ )
+ assert response["choices"]
+ finally:
+ proc.terminate()
+ proc.wait()
+ log_output = LlamafileRunner.read_log_file(log_path)
+ stderr_output = LlamafileRunner.read_log_file(stderr_path)
+ for path in paths:
+ if os.path.exists(path):
+ os.unlink(path)
+
+ gpu_info = check_gpu_in_output(log_output)
+ assert gpu_info["gpu_used"], (
+ f"No GPU offloading detected. Devices: {gpu_info['devices']}"
+ )
+
+ system_info = [ln for ln in log_output.splitlines() if "system_info" in ln]
+ assert system_info, "no system_info line in the server log"
+ assert "CUDA : " in system_info[-1], (
+ f"CUDA backend not listed in system_info: {system_info[-1]}"
+ )
+ assert "USE_GRAPHS = 1" in system_info[-1], (
+ "ggml-cuda module was built without GGML_CUDA_USE_GRAPHS "
+ f"(see llamafile/cuda.sh): {system_info[-1]}"
+ )
+
+ disabled = re.search(r"disabling CUDA graphs due to ([^\n]*)", stderr_output)
+ if disabled:
+ pytest.skip(f"ggml disabled CUDA graphs at runtime: {disabled.group(1).strip()}")
+ assert "CUDA graph warmup complete" in stderr_output, (
+ "CUDA graphs never engaged during decode (no 'CUDA graph warmup "
+ "complete' on stderr)"
+ )
+
@pytest.mark.cpu
class TestCPUExecution:
diff --git a/tests/integration/utils/llamafile.py b/tests/integration/utils/llamafile.py
index d4508882..28a10942 100644
--- a/tests/integration/utils/llamafile.py
+++ b/tests/integration/utils/llamafile.py
@@ -310,6 +310,7 @@ class LlamafileRunner:
thinking: bool | None = None,
extra_args: list[str] | None = None,
log_file: str | None = None,
+ stderr_file: str | None = None,
) -> subprocess.Popen:
"""Start llamafile in server mode.
@@ -321,6 +322,10 @@ class LlamafileRunner:
extra_args: Additional command-line arguments
log_file: If provided, adds --log-file flag. Caller should read
the file after terminating the process.
+ stderr_file: If provided, the server's stderr is written to this
+ file. Messages logged by a GPU backend module
+ (ggml-cuda.so, ...) are emitted by the module's own ggml
+ logger and reach stderr, not --log-file.
Returns:
Popen process handle (caller must terminate)
@@ -341,12 +346,17 @@ class LlamafileRunner:
args.extend(extra_args)
logger.info("Starting server: %s", " ".join(args))
- return subprocess.Popen(
- args,
- stdout=subprocess.DEVNULL,
- stderr=subprocess.DEVNULL,
- text=True,
- )
+ stderr = open(stderr_file, "w") if stderr_file else subprocess.DEVNULL
+ try:
+ return subprocess.Popen(
+ args,
+ stdout=subprocess.DEVNULL,
+ stderr=stderr,
+ text=True,
+ )
+ finally:
+ if stderr_file:
+ stderr.close() # the child holds its own copy of the fd
def start_combined(
self,