Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Updated ggml build scripts with -DGGML_CUDA_USE_GRAPHS / -DGGML_HIP_GRAPHS=1 (#1051)
Davide Eynard42ff11b31491
diff --git a/llamafile/cuda.bat b/llamafile/cuda.batindex 6c773259..c7364109 100644--- a/llamafile/cuda.bat+++ b/llamafile/cuda.bat@@ -188,6 +188,7 @@ set "COMMON_FLAGS=%COMMON_FLAGS% --std=c++17"set "COMMON_FLAGS=%COMMON_FLAGS% -Xcompiler="/nologo /EHsc /O2 /GR /MT /std:c++17 /Zc:preprocessor""set "COMMON_FLAGS=%COMMON_FLAGS% -diag-suppress 177 -diag-suppress 221 -diag-suppress 550"set "COMMON_FLAGS=%COMMON_FLAGS% -DNDEBUG -DGGML_BUILD=1 -DGGML_SHARED=1 -DGGML_BACKEND_SHARED=1 -DGGML_BACKEND_BUILD=1 -DGGML_MULTIPLATFORM"+set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_USE_GRAPHS"set "COMMON_FLAGS=%COMMON_FLAGS% %BLAS_DEFINE%"if "%NO_IQ_QUANTS%"=="1" set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_NO_IQ_QUANTS"if "%FA_ALL_QUANTS%"=="1" set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_FA_ALL_QUANTS"diff --git a/llamafile/cuda.sh b/llamafile/cuda.shindex 3349b738..ff35f1c5 100755--- a/llamafile/cuda.sh+++ b/llamafile/cuda.sh@@ -245,6 +245,7 @@ COMMON_FLAGS="\-DGGML_BUILD=1 \-DGGML_SHARED=1 \-DGGML_MULTIPLATFORM \+ -DGGML_CUDA_USE_GRAPHS \$BLAS_DEFINE"if [ "$NO_IQ_QUANTS" = "1" ]; thendiff --git a/llamafile/cuda_parallel.bat b/llamafile/cuda_parallel.batindex 104fbfdc..dbf17440 100644--- a/llamafile/cuda_parallel.bat+++ b/llamafile/cuda_parallel.bat@@ -211,6 +211,7 @@ set "COMMON_FLAGS=%COMMON_FLAGS% --std=c++17"set "COMMON_FLAGS=%COMMON_FLAGS% -Xcompiler="/nologo /EHsc /O2 /GR /MT /std:c++17 /Zc:preprocessor""set "COMMON_FLAGS=%COMMON_FLAGS% -diag-suppress 177 -diag-suppress 221 -diag-suppress 550"set "COMMON_FLAGS=%COMMON_FLAGS% -DNDEBUG -DGGML_BUILD=1 -DGGML_SHARED=1 -DGGML_BACKEND_SHARED=1 -DGGML_BACKEND_BUILD=1 -DGGML_MULTIPLATFORM"+set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_USE_GRAPHS"set "COMMON_FLAGS=%COMMON_FLAGS% %BLAS_DEFINE%"if "%NO_IQ_QUANTS%"=="1" set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_NO_IQ_QUANTS"if "%FA_ALL_QUANTS%"=="1" set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_CUDA_FA_ALL_QUANTS"diff --git a/llamafile/rocm.bat b/llamafile/rocm.batindex 448f3764..6101c063 100644--- a/llamafile/rocm.bat+++ b/llamafile/rocm.bat@@ -132,6 +132,7 @@ set "COMMON_FLAGS=%COMMON_FLAGS% -Wno-ignored-attributes -Wno-nested-anon-types"set "COMMON_FLAGS=%COMMON_FLAGS% -I%BUILD_DIR% -I%GGML_INC_DIR% -I%GGML_SRC_DIR% -I%GGML_CUDA_DIR%"set "COMMON_FLAGS=%COMMON_FLAGS% -I"%HIP_PATH%\include""set "COMMON_FLAGS=%COMMON_FLAGS% -DNDEBUG -DGGML_BUILD=1 -DGGML_SHARED=1 -DGGML_BACKEND_SHARED=1 -DGGML_BACKEND_BUILD=1 -DGGML_MULTIPLATFORM"+set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_HIP_GRAPHS=1"set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_USE_HIP=1 -DGGML_USE_TINYBLAS=1 -DGGML_HIP_NO_VMM=1 -D__HIP_PLATFORM_AMD__":: --offload-compress shrinks the fat binary (mirrors rocm.sh / #995); requires a:: recent ROCm HIP SDK -- drop this flag if your clang++ rejects itdiff --git a/llamafile/rocm.sh b/llamafile/rocm.shindex 14d83238..77740375 100755--- a/llamafile/rocm.sh+++ b/llamafile/rocm.sh@@ -121,6 +121,7 @@ COMMON_FLAGS="\-DGGML_SHARED=1 \-DGGML_MULTIPLATFORM \-DGGML_USE_HIP=1 \+ -DGGML_HIP_GRAPHS=1 \-DGGML_HIP_NO_VMM=1 \-DGGML_USE_TINYBLAS=1 \-Wno-return-type \diff --git a/llamafile/rocm_parallel.bat b/llamafile/rocm_parallel.batindex a18052ef..3c9c855b 100644--- a/llamafile/rocm_parallel.bat+++ b/llamafile/rocm_parallel.bat@@ -154,6 +154,7 @@ set "COMMON_FLAGS=%COMMON_FLAGS% -Wno-ignored-attributes -Wno-nested-anon-types"set "COMMON_FLAGS=%COMMON_FLAGS% -I%BUILD_DIR% -I%GGML_INC_DIR% -I%GGML_SRC_DIR% -I%GGML_CUDA_DIR%"set "COMMON_FLAGS=%COMMON_FLAGS% -I"%HIP_PATH%\include""set "COMMON_FLAGS=%COMMON_FLAGS% -DNDEBUG -DGGML_BUILD=1 -DGGML_SHARED=1 -DGGML_BACKEND_SHARED=1 -DGGML_BACKEND_BUILD=1 -DGGML_MULTIPLATFORM"+set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_HIP_GRAPHS=1"set "COMMON_FLAGS=%COMMON_FLAGS% -DGGML_USE_HIP=1 -DGGML_USE_TINYBLAS=1 -DGGML_HIP_NO_VMM=1 -D__HIP_PLATFORM_AMD__":: --offload-compress shrinks the fat binary (mirrors rocm.sh / #995); requires a:: recent ROCm HIP SDK -- drop this flag if your clang++ rejects itdiff --git a/tests/integration/tests/test_gpu.py b/tests/integration/tests/test_gpu.pyindex 08a072e3..2749fa7e 100644--- a/tests/integration/tests/test_gpu.py+++ b/tests/integration/tests/test_gpu.py@@ -2,6 +2,7 @@import osimport platform+import reimport subprocessimport tempfile@@ -203,6 +204,94 @@ class TestGPUAcceleration:if os.path.exists(log_path):os.unlink(log_path)+ def test_gpu_cuda_graphs(self, executable, model, available_gpu, server_port, timeouts):+ """NVIDIA: the CUDA module must be built with CUDA graphs and use them.++ ggml keeps its CUDA-graph code behind the compile-time define+ GGML_CUDA_USE_GRAPHS. A ggml-cuda.so built without it submits every+ kernel of every decode step individually and generates 3-18% slower+ at batch size 1 (measured on an L40S; the cost is per layer per token,+ so small models lose most). Two checks:++ * the backend feature list in the server's system_info line carries+ ``USE_GRAPHS = 1`` -- the compile-time property of the module;+ * after a couple of decode steps ggml logs ``CUDA graph warmup+ complete``, i.e. graphs actually engaged for this model. ggml may+ legitimately refuse them (pre-Volta GPU, unsupported node type); it+ then logs ``disabling CUDA graphs due to ...`` and the runtime check+ is skipped rather than failed.++ The module's own log lines go to stderr, not to --log-file (they are+ written by the ggml logger inside the DSO), so stderr is captured too.+ """+ if available_gpu != "nvidia":+ pytest.skip("CUDA graphs are a property of the NVIDIA module")++ runner = LlamafileRunner(+ executable=executable,+ model=model,+ gpu=available_gpu,+ )++ paths = []+ for suffix in (".log", ".stderr"):+ with tempfile.NamedTemporaryFile(+ mode="w", suffix=suffix, delete=False+ ) as f:+ paths.append(f.name)+ log_path, stderr_path = paths++ proc = runner.start_server(port=server_port, log_file=log_path,+ stderr_file=stderr_path,+ extra_args=["--verbose"])+ try:+ ready = LlamafileRunner.wait_for_server(+ server_port, timeout=timeouts.server_ready, proc=proc+ )+ assert ready++ # Graph warm-up needs at least two decode steps whose graph+ # properties did not change; two short requests give plenty.+ for _ in range(2):+ response = LlamafileRunner.chat_completion(+ port=server_port,+ messages=[{"role": "user", "content": GREETING_PROMPT}],+ timeout=timeouts.http_request,+ max_tokens=32,+ )+ assert response["choices"]+ finally:+ proc.terminate()+ proc.wait()+ log_output = LlamafileRunner.read_log_file(log_path)+ stderr_output = LlamafileRunner.read_log_file(stderr_path)+ for path in paths:+ if os.path.exists(path):+ os.unlink(path)++ gpu_info = check_gpu_in_output(log_output)+ assert gpu_info["gpu_used"], (+ f"No GPU offloading detected. Devices: {gpu_info['devices']}"+ )++ system_info = [ln for ln in log_output.splitlines() if "system_info" in ln]+ assert system_info, "no system_info line in the server log"+ assert "CUDA : " in system_info[-1], (+ f"CUDA backend not listed in system_info: {system_info[-1]}"+ )+ assert "USE_GRAPHS = 1" in system_info[-1], (+ "ggml-cuda module was built without GGML_CUDA_USE_GRAPHS "+ f"(see llamafile/cuda.sh): {system_info[-1]}"+ )++ disabled = re.search(r"disabling CUDA graphs due to ([^\n]*)", stderr_output)+ if disabled:+ pytest.skip(f"ggml disabled CUDA graphs at runtime: {disabled.group(1).strip()}")+ assert "CUDA graph warmup complete" in stderr_output, (+ "CUDA graphs never engaged during decode (no 'CUDA graph warmup "+ "complete' on stderr)"+ )+@pytest.mark.cpuclass TestCPUExecution:diff --git a/tests/integration/utils/llamafile.py b/tests/integration/utils/llamafile.pyindex d4508882..28a10942 100644--- a/tests/integration/utils/llamafile.py+++ b/tests/integration/utils/llamafile.py@@ -310,6 +310,7 @@ class LlamafileRunner:thinking: bool | None = None,extra_args: list[str] | None = None,log_file: str | None = None,+ stderr_file: str | None = None,) -> subprocess.Popen:"""Start llamafile in server mode.@@ -321,6 +322,10 @@ class LlamafileRunner:extra_args: Additional command-line argumentslog_file: If provided, adds --log-file flag. Caller should readthe file after terminating the process.+ stderr_file: If provided, the server's stderr is written to this+ file. Messages logged by a GPU backend module+ (ggml-cuda.so, ...) are emitted by the module's own ggml+ logger and reach stderr, not --log-file.Returns:Popen process handle (caller must terminate)@@ -341,12 +346,17 @@ class LlamafileRunner:args.extend(extra_args)logger.info("Starting server: %s", " ".join(args))- return subprocess.Popen(- args,- stdout=subprocess.DEVNULL,- stderr=subprocess.DEVNULL,- text=True,- )+ stderr = open(stderr_file, "w") if stderr_file else subprocess.DEVNULL+ try:+ return subprocess.Popen(+ args,+ stdout=subprocess.DEVNULL,+ stderr=stderr,+ text=True,+ )+ finally:+ if stderr_file:+ stderr.close() # the child holds its own copy of the fddef start_combined(self,