Skip to content

Distribute and run LLMs with a single file.

Fix uncaught SIGSEGV when GPU init fails, restore CPU fallback (#988) (#989)

Davide Eynard6e28ad272afa
diff --git a/diffusionfile/BUILD.mk b/diffusionfile/BUILD.mk
index ab199e86..3cbdb0d1 100644
--- a/diffusionfile/BUILD.mk
+++ b/diffusionfile/BUILD.mk
@@ -76,9 +76,7 @@ DIFFUSIONFILE_CPPFLAGS := \
DIFFUSIONFILE_LLAMAFILE_OBJS := \
o/$(MODE)/llamafile/llamafile.o \
- o/$(MODE)/llamafile/metal.o \
- o/$(MODE)/llamafile/cuda.o \
- o/$(MODE)/llamafile/vulkan.o \
+ o/$(MODE)/llamafile/gpu.a \
o/$(MODE)/llamafile/zip.o \
o/$(MODE)/llamafile/check_cpu.o
diff --git a/docs/skills/llamafile/SKILL.md b/docs/skills/llamafile/SKILL.md
index 7532ed00..d1990f5f 100644
--- a/docs/skills/llamafile/SKILL.md
+++ b/docs/skills/llamafile/SKILL.md
@@ -1,7 +1,7 @@
---
name: llamafile
description: This skill should be used when the user asks to "build llamafile", "rebuild llamafile", "run llamafile", "run llamafile tests", "debug llamafile", "set up llamafile", "update patches", "fix patch conflict", "update llama.cpp", "pull latest llama.cpp", "sync upstream llama.cpp", "reset submodules", "write a test for llamafile", "how does llamafile work", "llamafile architecture", or needs guidance on the llamafile build system, patch workflow, submodule integration, cosmocc toolchain, or development practices.
-version: 0.1.3
+version: 0.1.4
---
# Llamafile Development Guide
@@ -120,6 +120,10 @@ Outputs: `o/$(MODE)/package/file.o`
Binaries include both x86_64 and aarch64 code paths with runtime CPU feature detection (AVX, AVX2, AVX-512, ARM NEON).
+### GPU Backend Loaders
+
+Dynamically-loaded backends that export the ggml C ABI — CUDA, ROCm, Vulkan — all go through the shared probe core in `llamafile/gpu_backend.c`. Each is just a `GpuBackendDesc` + a link thunk; the core does load → log-suppress → **device-count gate** (reject 0-device DSOs so AUTO falls back) → register, with a SIGSEGV/SIGABRT crash guard around the foreign probe call (driver init can fault across the cosmo/ms_abi boundary — issue #988). Metal stays separate by design (runtime-compiled, no ms_abi split, no device gate). When adding/changing a backend: route it through the core, keep the gate, and add a case to `tests/gpu_backend_test.cpp`. A more detailed design doc lives separately.
+
## Main Executables
After building, find binaries in `o/$(MODE)/`:
diff --git a/llama.cpp.patches/llamafile-files/BUILD.mk b/llama.cpp.patches/llamafile-files/BUILD.mk
index 15500421..84e02c33 100644
--- a/llama.cpp.patches/llamafile-files/BUILD.mk
+++ b/llama.cpp.patches/llamafile-files/BUILD.mk
@@ -426,9 +426,7 @@ UI_GEN_OBJ := $(UI_CPP_GEN:%.cpp=%.cpp.o)
# llamafile objects are used to add dynamic GPU support (Metal, CUDA, ROCm, Vulkan)
TOOL_LLAMAFILE_OBJS := \
o/$(MODE)/llamafile/llamafile.o \
- o/$(MODE)/llamafile/metal.o \
- o/$(MODE)/llamafile/cuda.o \
- o/$(MODE)/llamafile/vulkan.o \
+ o/$(MODE)/llamafile/gpu.a \
o/$(MODE)/llamafile/zip.o
# Server objects depend on the llamafile bridge header and on the
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index 00d48f21..da2ab857 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -121,6 +121,7 @@ LLAMAFILE_HIGHLIGHT_SRCS := \
LLAMAFILE_SRCS_C := \
llamafile/bestline.c \
llamafile/cuda.c \
+ llamafile/gpu_backend.c \
llamafile/llamafile.c \
llamafile/metal.c \
llamafile/vulkan.c \
@@ -334,6 +335,24 @@ o/$(MODE)/llamafile/%.o: llamafile/%.c
@mkdir -p $(@D)
$(CC) $(CFLAGS) $(LLAMAFILE_CPPFLAGS) -c -o $@ $<
+# ==============================================================================
+# GPU backend archive
+# ==============================================================================
+# Single linkable unit grouping the runtime GPU loaders (CUDA/ROCm, Vulkan,
+# Metal) and their shared probe core. The non-llamafile executables
+# (whisperfile, diffusionfile, the llama.cpp tools) pull these in via
+# llamafile_has_gpu(); linking the archive instead of listing each object means
+# adding a GPU backend source does not require editing every consumer's
+# BUILD.mk. Tidiness only: every consumer references llamafile_has_gpu(), so all
+# members are pulled and the build output is unchanged.
+LLAMAFILE_GPU_OBJS := \
+ o/$(MODE)/llamafile/cuda.o \
+ o/$(MODE)/llamafile/gpu_backend.o \
+ o/$(MODE)/llamafile/metal.o \
+ o/$(MODE)/llamafile/vulkan.o
+
+o/$(MODE)/llamafile/gpu.a: $(LLAMAFILE_GPU_OBJS)
+
o/$(MODE)/llamafile/%.o: llamafile/%.cpp
@mkdir -p $(@D)
$(CXX) $(CXXFLAGS) $(LLAMAFILE_CPPFLAGS) -c -o $@ $<
diff --git a/llamafile/cuda.c b/llamafile/cuda.c
index e5dc4f27..6c4779fc 100644
--- a/llamafile/cuda.c
+++ b/llamafile/cuda.c
@@ -23,187 +23,60 @@
// At runtime on Linux/Windows with NVIDIA or AMD GPU:
// 1. Try to load pre-built DSO from /zip/ggml-cuda.so (bundled)
// 2. Or try to load from ~/.llamafile/ (pre-compiled)
-// 3. Or compile at runtime if nvcc/hipcc is available
-// 4. Load the DSO with cosmo_dlopen() and register the CUDA backend
+// 3. Load the DSO with cosmo_dlopen() and register the CUDA backend
+//
+// The load/probe/register mechanics live in gpu_backend.c, shared with the
+// Vulkan backend so all of them behave identically. CUDA and ROCm use the same
+// exported symbols and the same DSO-loading machinery, so they share one
+// GpuBackend; which descriptor loaded records whether it's NVIDIA or AMD.
//
+#include "gpu_backend.h"
#include "llamafile.h"
#include <cosmo.h>
-#include <dlfcn.h>
-#include <errno.h>
-#include <limits.h>
-#include <pthread.h>
-#include <stdatomic.h>
#include <stdbool.h>
+#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/wait.h>
-#include <unistd.h>
-
-// Forward declarations for ggml backend types
-typedef struct ggml_backend * ggml_backend_t;
-typedef struct ggml_backend_reg * ggml_backend_reg_t;
-
-// Function to register a backend with ggml (from ggml-backend.h)
-extern void ggml_backend_register(ggml_backend_reg_t reg);
-
-// Log callback type (must match ggml_log_callback from ggml.h)
-typedef void (*llamafile_log_callback)(int level, const char *text, void *user_data);
-
-// CUDA backend state
-//
-// On Windows the DSO exports functions with ms_abi calling convention,
-// but the cosmocc host uses System V ABI. We store each dlsym'd pointer
-// in a union so the correct ABI variant is called at each call site,
-// following the same pattern used in localscore/nvml.cpp.
-static struct CudaBackend {
- bool supported;
- bool is_amd; // true if this is ROCm/AMD, false if NVIDIA
- atomic_uint once;
- void *lib_handle;
-
- // Function pointers for CUDA backend
- union {
- ggml_backend_t (*default_abi)(int device);
- ggml_backend_t (__attribute__((__ms_abi__)) *windows_abi)(int device);
- } backend_init;
-
- union {
- ggml_backend_reg_t (*default_abi)(void);
- ggml_backend_reg_t (__attribute__((__ms_abi__)) *windows_abi)(void);
- } backend_reg;
-
- union {
- int (*default_abi)(void);
- int (__attribute__((__ms_abi__)) *windows_abi)(void);
- } get_device_count;
-
- union {
- void (*default_abi)(int device, char *description, size_t description_size);
- void (__attribute__((__ms_abi__)) *windows_abi)(int device, char *description, size_t description_size);
- } get_device_description;
-
- // Logging control
- union {
- void (*default_abi)(llamafile_log_callback log_callback, void *user_data);
- void (__attribute__((__ms_abi__)) *windows_abi)(llamafile_log_callback log_callback, void *user_data);
- } log_set;
-} g_cuda;
-
-static bool LinkCuda(const char *dso) {
- // Load dynamic shared object using Cosmopolitan's dlopen
- void *lib = cosmo_dlopen(dso, RTLD_LAZY);
- if (!lib) {
- char *err = cosmo_dlerror();
- llamafile_info("cuda", "failed to load library %s: %s",
- dso, err ? err : "unknown error");
- return false;
- }
-
- // Import functions into the correct ABI union member
- bool ok = true;
- void *sym;
-
- sym = cosmo_dlsym(lib, "ggml_backend_cuda_init");
- ok &= (sym != NULL);
- if (IsWindows())
- *(void **)(&g_cuda.backend_init.windows_abi) = sym;
- else
- *(void **)(&g_cuda.backend_init.default_abi) = sym;
-
- sym = cosmo_dlsym(lib, "ggml_backend_cuda_reg");
- ok &= (sym != NULL);
- if (IsWindows())
- *(void **)(&g_cuda.backend_reg.windows_abi) = sym;
- else
- *(void **)(&g_cuda.backend_reg.default_abi) = sym;
-
- // Required: TryGpuBackend uses this to reject 0-device DSOs
- sym = cosmo_dlsym(lib, "ggml_backend_cuda_get_device_count");
- ok &= (sym != NULL);
- if (IsWindows())
- *(void **)(&g_cuda.get_device_count.windows_abi) = sym;
- else
- *(void **)(&g_cuda.get_device_count.default_abi) = sym;
-
- // Optional - don't fail if not found
- sym = cosmo_dlsym(lib, "ggml_backend_cuda_get_device_description");
- if (IsWindows())
- *(void **)(&g_cuda.get_device_description.windows_abi) = sym;
- else
- *(void **)(&g_cuda.get_device_description.default_abi) = sym;
-
- // Import logging control (optional)
- sym = cosmo_dlsym(lib, "ggml_log_set");
- if (IsWindows())
- *(void **)(&g_cuda.log_set.windows_abi) = sym;
- else
- *(void **)(&g_cuda.log_set.default_abi) = sym;
-
- if (!ok) {
- char *err = cosmo_dlerror();
- llamafile_info("cuda", "could not import all symbols from %s: %s",
- dso, err ? err : "unknown error");
- memset(&g_cuda.backend_init, 0, sizeof(g_cuda.backend_init));
- memset(&g_cuda.backend_reg, 0, sizeof(g_cuda.backend_reg));
- memset(&g_cuda.get_device_count, 0, sizeof(g_cuda.get_device_count));
- memset(&g_cuda.get_device_description, 0, sizeof(g_cuda.get_device_description));
- memset(&g_cuda.log_set, 0, sizeof(g_cuda.log_set));
- cosmo_dlclose(lib);
- return false;
- }
- g_cuda.lib_handle = lib;
- return true;
+// CUDA and ROCm share symbol names but differ in display name, so they are two
+// descriptors over one GpuBackend. The loaded descriptor tells NVIDIA vs AMD.
+static GpuBackend g_cuda;
+
+static const GpuBackendDesc CUDA_DESC = {
+ .tag = "cuda",
+ .name = "CUDA",
+ .init = "ggml_backend_cuda_init",
+ .reg = "ggml_backend_cuda_reg",
+ .get_device_count = "ggml_backend_cuda_get_device_count",
+ .get_device_description = "ggml_backend_cuda_get_device_description",
+};
+
+static const GpuBackendDesc ROCM_DESC = {
+ .tag = "cuda",
+ .name = "ROCm",
+ .init = "ggml_backend_cuda_init",
+ .reg = "ggml_backend_cuda_reg",
+ .get_device_count = "ggml_backend_cuda_get_device_count",
+ .get_device_description = "ggml_backend_cuda_get_device_description",
+};
+
+// Thin llamafile_link_dso_fn thunks over the shared linker, one per identity.
+static bool LinkCudaNvidia(const char *dso) {
+ return gpu_backend_link(&g_cuda, dso, &CUDA_DESC);
}
-static void UnlinkCuda(void) {
- if (g_cuda.lib_handle) {
- cosmo_dlclose(g_cuda.lib_handle);
- g_cuda.lib_handle = NULL;
- }
- memset(&g_cuda.backend_init, 0, sizeof(g_cuda.backend_init));
- memset(&g_cuda.backend_reg, 0, sizeof(g_cuda.backend_reg));
- memset(&g_cuda.get_device_count, 0, sizeof(g_cuda.get_device_count));
- memset(&g_cuda.get_device_description, 0, sizeof(g_cuda.get_device_description));
- memset(&g_cuda.log_set, 0, sizeof(g_cuda.log_set));
+static bool LinkCudaAmd(const char *dso) {
+ return gpu_backend_link(&g_cuda, dso, &ROCM_DESC);
}
-static bool TryGpuBackend(const char *dso, bool is_amd) {
- if (!llamafile_try_load_prebuilt_dso(dso, "cuda", LinkCuda))
- return false;
-
- // Suppress the DSO's ggml logging before we touch any function that
- // triggers ggml_cuda_init() (e.g. get_device_count). Without this, a
- // failed init on the wrong backend would print a confusing error to
- // stderr even when --verbose is not set.
- if (!FLAG_verbose && (g_cuda.log_set.default_abi || g_cuda.log_set.windows_abi)) {
- if (IsWindows())
- g_cuda.log_set.windows_abi(llamafile_log_callback_null, NULL);
- else
- g_cuda.log_set.default_abi(llamafile_log_callback_null, NULL);
- }
-
- // Verify the backend has at least one device before committing. The DSO
- // loads fine even when no compatible hardware is present, so we must
- // probe device count to avoid registering a 0-device backend (which
- // would then prevent fallback to other GPU backends in AUTO mode).
- int count;
- if (IsWindows())
- count = g_cuda.get_device_count.windows_abi();
- else
- count = g_cuda.get_device_count.default_abi();
- if (count <= 0) {
- llamafile_info("cuda", "%s library loaded but no devices detected; trying next backend",
- is_amd ? "ROCm" : "CUDA");
- UnlinkCuda();
+// Load a prebuilt DSO and commit to it only if it exposes a usable device. The
+// DSO loads fine even with no compatible hardware, so gpu_backend_probe()
+// rejects 0-device backends and lets AUTO mode fall through to the next one.
+static bool TryGpuBackend(const char *dso, llamafile_link_dso_fn link_fn) {
+ if (!llamafile_try_load_prebuilt_dso(dso, "cuda", link_fn))
return false;
- }
-
- g_cuda.is_amd = is_amd;
- return true;
+ return gpu_backend_probe(&g_cuda);
}
static bool ImportCudaImpl(void) {
@@ -232,12 +105,12 @@ static bool ImportCudaImpl(void) {
// In AUTO mode, prefer CUDA over ROCm: it covers the common NVIDIA case
// and lets ROCm be the fallback when CUDA is absent or has no devices.
if (FLAG_gpu == LLAMAFILE_GPU_NVIDIA || FLAG_gpu == LLAMAFILE_GPU_AUTO) {
- if (TryGpuBackend(cuda_dso, false))
+ if (TryGpuBackend(cuda_dso, LinkCudaNvidia))
goto RegisterBackend;
}
if (FLAG_gpu == LLAMAFILE_GPU_AMD || FLAG_gpu == LLAMAFILE_GPU_AUTO) {
- if (TryGpuBackend(rocm_dso, true))
+ if (TryGpuBackend(rocm_dso, LinkCudaAmd))
goto RegisterBackend;
}
@@ -249,20 +122,7 @@ static bool ImportCudaImpl(void) {
return false;
RegisterBackend:
- // Register the CUDA backend with GGML
- if (g_cuda.backend_reg.default_abi || g_cuda.backend_reg.windows_abi) {
- ggml_backend_reg_t reg;
- if (IsWindows())
- reg = g_cuda.backend_reg.windows_abi();
- else
- reg = g_cuda.backend_reg.default_abi();
- if (reg) {
- ggml_backend_register(reg);
- llamafile_info("cuda", "%s backend registered with GGML",
- g_cuda.is_amd ? "ROCm" : "CUDA");
- }
- }
-
+ gpu_backend_register(&g_cuda);
return true;
}
@@ -270,12 +130,8 @@ static void ImportCuda(void) {
if (ImportCudaImpl()) {
g_cuda.supported = true;
llamafile_info("cuda", "%s GPU support successfully loaded",
- g_cuda.is_amd ? "AMD ROCm" : "NVIDIA CUDA");
- int count;
- if (IsWindows())
- count = g_cuda.get_device_count.windows_abi();
- else
- count = g_cuda.get_device_count.default_abi();
+ g_cuda.desc == &ROCM_DESC ? "AMD ROCm" : "NVIDIA CUDA");
+ int count = gpu_call_device_count(g_cuda.get_device_count);
llamafile_info("cuda", "found %d GPU device(s)", count);
} else if (FLAG_gpu == LLAMAFILE_GPU_NVIDIA || FLAG_gpu == LLAMAFILE_GPU_AMD) {
fprintf(stderr, "fatal error: support for --gpu %s was explicitly requested, "
@@ -286,12 +142,12 @@ static void ImportCuda(void) {
bool llamafile_has_cuda(void) {
cosmo_once(&g_cuda.once, ImportCuda);
- return g_cuda.supported && !g_cuda.is_amd;
+ return g_cuda.supported && g_cuda.desc != &ROCM_DESC;
}
bool llamafile_has_amd_gpu(void) {
cosmo_once(&g_cuda.once, ImportCuda);
- return g_cuda.supported && g_cuda.is_amd;
+ return g_cuda.supported && g_cuda.desc == &ROCM_DESC;
}
// Wrapper functions for dynamically loaded CUDA backend
@@ -299,21 +155,20 @@ bool llamafile_has_amd_gpu(void) {
ggml_backend_t ggml_backend_cuda_init(int device) {
if (!llamafile_has_cuda() && !llamafile_has_amd_gpu())
return NULL;
- if (!g_cuda.backend_init.default_abi && !g_cuda.backend_init.windows_abi)
+ void *fp = g_cuda.backend_init;
+ if (!fp)
return NULL;
+ // backend_init has a CUDA-specific signature (int device), so it is called
+ // here rather than through a shared gpu_call_* helper.
if (IsWindows())
- return g_cuda.backend_init.windows_abi(device);
- return g_cuda.backend_init.default_abi(device);
+ return ((ggml_backend_t(__attribute__((__ms_abi__)) *)(int))fp)(device);
+ return ((ggml_backend_t(*)(int))fp)(device);
}
int ggml_backend_cuda_get_device_count(void) {
if (!llamafile_has_cuda() && !llamafile_has_amd_gpu())
return 0;
- if (!g_cuda.get_device_count.default_abi && !g_cuda.get_device_count.windows_abi)
- return 0;
- if (IsWindows())
- return g_cuda.get_device_count.windows_abi();
- return g_cuda.get_device_count.default_abi();
+ return gpu_call_device_count(g_cuda.get_device_count);
}
void ggml_backend_cuda_get_device_description(int device, char *description, size_t description_size) {
@@ -322,24 +177,16 @@ void ggml_backend_cuda_get_device_description(int device, char *description, siz
description[0] = '\0';
return;
}
- if (!g_cuda.get_device_description.default_abi && !g_cuda.get_device_description.windows_abi) {
+ if (!g_cuda.get_device_description) {
if (description_size > 0)
snprintf(description, description_size, "GPU %d", device);
return;
}
- if (IsWindows())
- g_cuda.get_device_description.windows_abi(device, description, description_size);
- else
- g_cuda.get_device_description.default_abi(device, description, description_size);
+ gpu_call_get_description(g_cuda.get_device_description, device, description, description_size);
}
void llamafile_cuda_log_set(llamafile_log_callback log_callback, void *user_data) {
if (!llamafile_has_cuda() && !llamafile_has_amd_gpu())
return;
- if (g_cuda.log_set.default_abi || g_cuda.log_set.windows_abi) {
- if (IsWindows())
- g_cuda.log_set.windows_abi(log_callback, user_data);
- else
- g_cuda.log_set.default_abi(log_callback, user_data);
- }
+ gpu_call_log_set(g_cuda.log_set, log_callback, user_data);
}
diff --git a/llamafile/gpu_backend.c b/llamafile/gpu_backend.c
new file mode 100644
index 00000000..68c76240
--- /dev/null
+++ b/llamafile/gpu_backend.c
@@ -0,0 +1,250 @@
+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2026 Mozilla.ai
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "gpu_backend.h"
+#include "llamafile.h"
+#include <cosmo.h>
+#include <dlfcn.h>
+#include <setjmp.h>
+#include <signal.h>
+#include <string.h>
+
+// Register a backend with ggml (from ggml-backend.h).
+extern void ggml_backend_register(ggml_backend_reg_t reg);
+
+// =============================================================================
+// ABI-correct call helpers
+//
+// On Windows the DSO exports functions with the ms_abi calling convention,
+// while the cosmocc host uses System V. We keep the IsWindows() branch in one
+// place so each call site (and each backend) does the right thing identically.
+// =============================================================================
+
+ggml_backend_reg_t gpu_call_reg(void *fp) {
+ if (!fp)
+ return NULL;
+ if (IsWindows())
+ return ((ggml_backend_reg_t(__attribute__((__ms_abi__)) *)(void))fp)();
+ return ((ggml_backend_reg_t(*)(void))fp)();
+}
+
+int gpu_call_device_count(void *fp) {
+ if (!fp)
+ return 0;
+ if (IsWindows())
+ return ((int(__attribute__((__ms_abi__)) *)(void))fp)();
+ return ((int(*)(void))fp)();
+}
+
+void gpu_call_get_description(void *fp, int device, char *buf, size_t n) {
+ if (!fp)
+ return;
+ if (IsWindows())
+ ((void(__attribute__((__ms_abi__)) *)(int, char *, size_t))fp)(device, buf, n);
+ else
+ ((void(*)(int, char *, size_t))fp)(device, buf, n);
+}
+
+void gpu_call_log_set(void *fp, llamafile_log_callback cb, void *user_data) {
+ if (!fp)
+ return;
+ if (IsWindows())
+ ((void(__attribute__((__ms_abi__)) *)(llamafile_log_callback, void *))fp)(cb, user_data);
+ else
+ ((void(*)(llamafile_log_callback, void *))fp)(cb, user_data);
+}
+
+// =============================================================================
+// Load / unload
+// =============================================================================
+
+void gpu_backend_unlink(GpuBackend *b) {
+ if (b->lib_handle) {
+ cosmo_dlclose(b->lib_handle);
+ b->lib_handle = NULL;
+ }
+ b->backend_init = NULL;
+ b->backend_reg = NULL;
+ b->get_device_count = NULL;
+ b->get_device_description = NULL;
+ b->log_set = NULL;
+}
+
+bool gpu_backend_link(GpuBackend *b, const char *dso, const GpuBackendDesc *desc) {
+ b->desc = desc;
+
+ void *lib = cosmo_dlopen(dso, RTLD_LAZY);
+ if (!lib) {
+ char *err = cosmo_dlerror();
+ llamafile_info(desc->tag, "failed to load library %s: %s", dso,
+ err ? err : "unknown error");
+ return false;
+ }
+
+ // Required symbols: the backend is unusable without them. get_device_count
+ // is required because gpu_backend_probe() relies on it to reject 0-device
+ // DSOs (which is what lets AUTO mode fall through to the next backend).
+ bool ok = true;
+ b->backend_init = cosmo_dlsym(lib, desc->init);
+ ok &= (b->backend_init != NULL);
+ b->backend_reg = cosmo_dlsym(lib, desc->reg);
+ ok &= (b->backend_reg != NULL);
+ b->get_device_count = cosmo_dlsym(lib, desc->get_device_count);
+ ok &= (b->get_device_count != NULL);
+
+ // Optional symbols: degrade gracefully if absent.
+ b->get_device_description =
+ desc->get_device_description ? cosmo_dlsym(lib, desc->get_device_description) : NULL;
+ b->log_set = cosmo_dlsym(lib, "ggml_log_set");
+
+ if (!ok) {
+ char *err = cosmo_dlerror();
+ llamafile_info(desc->tag, "could not import all symbols from %s: %s", dso,
+ err ? err : "unknown error");
+ cosmo_dlclose(lib);
+ b->lib_handle = NULL;
+ b->backend_init = NULL;
+ b->backend_reg = NULL;
+ b->get_device_count = NULL;
+ b->get_device_description = NULL;
+ b->log_set = NULL;
+ return false;
+ }
+
+ b->lib_handle = lib;
+ return true;
+}
+
+// =============================================================================
+// Crash guard
+//
+// Some backends' get_device_count triggers full driver/instance initialisation
+// inside the DSO (e.g. ggml's ggml_vk_instance_init), which can throw a C++
+// exception on a broken / unsupported driver. That exception does NOT unwind
+// across the cosmo_dlopen/ms_abi boundary: even the DSO's own try/catch is
+// bypassed and cosmo surfaces it as an uncaught SIGSEGV (issue #988). C++
+// `catch` on our side can't help for the same reason. So we install a
+// temporary signal guard around the foreign call and siglongjmp back on a
+// fault, converting the crash into a clean "backend unavailable" result.
+//
+// This runs during one-time GPU init (under cosmo_once), before any worker
+// threads exist, so a single static jmp_buf is safe.
+// =============================================================================
+
+static sigjmp_buf g_gpu_guard_jmp;
+static volatile sig_atomic_t g_gpu_guard_active;
+
+static void gpu_guard_handler(int sig) {
+ if (g_gpu_guard_active) {
+ g_gpu_guard_active = 0;
+ siglongjmp(g_gpu_guard_jmp, sig);
+ }
+ // Fault outside a guarded call: restore default disposition and re-raise so
+ // a genuine crash is not silently swallowed.
+ signal(sig, SIG_DFL);
+ raise(sig);
+}
+
+// Run fn(arg) with SIGSEGV/SIGABRT/SIGILL/SIGBUS/SIGFPE trapped. Returns true if
+// it completed normally, false if it faulted (in which case the DSO is left for
+// the caller to unlink and abandon).
+static bool gpu_run_guarded(void (*fn)(void *), void *arg) {
+ static const int kSignals[] = {SIGSEGV, SIGABRT, SIGILL, SIGBUS, SIGFPE};
+ struct sigaction sa, old[sizeof(kSignals) / sizeof(kSignals[0])];
+ memset(&sa, 0, sizeof(sa));
+ sa.sa_handler = gpu_guard_handler;
+ sigemptyset(&sa.sa_mask);
+ sa.sa_flags = SA_NODEFER;
+ for (int i = 0; i < (int)(sizeof(kSignals) / sizeof(kSignals[0])); ++i)
+ sigaction(kSignals[i], &sa, &old[i]);
+
+ bool ok;
+ // SA_NODEFER (set above) leaves the trapped signal unblocked while the
+ // handler runs, so a re-fault in the tiny window before siglongjmp is not
+ // dropped. savesigs=1 makes siglongjmp restore the signal mask captured
+ // here, keeping the mask correct after we jump out of the fault.
+ if (sigsetjmp(g_gpu_guard_jmp, 1) == 0) {
+ g_gpu_guard_active = 1;
+ fn(arg);
+ g_gpu_guard_active = 0;
+ ok = true;
+ } else {
+ g_gpu_guard_active = 0;
+ ok = false;
+ }
+
+ for (int i = 0; i < (int)(sizeof(kSignals) / sizeof(kSignals[0])); ++i)
+ sigaction(kSignals[i], &old[i], NULL);
+ return ok;
+}
+
+// =============================================================================
+// Probe (device-count gate) and register
+// =============================================================================
+
+struct gpu_device_count_call {
+ void *fp;
+ int count;
+};
+
+static void gpu_device_count_thunk(void *p) {
+ struct gpu_device_count_call *c = (struct gpu_device_count_call *)p;
+ c->count = gpu_call_device_count(c->fp);
+}
+
+bool gpu_backend_probe(GpuBackend *b) {
+ // Suppress the DSO's ggml logging before touching any function that
+ // triggers device enumeration inside the DSO. Without this, a failed
+ // probe on the wrong backend prints confusing errors even without
+ // --verbose.
+ if (!FLAG_verbose)
+ gpu_call_log_set(b->log_set, llamafile_log_callback_null, NULL);
+
+ // The DSO loads fine even when no compatible hardware is present, so probe
+ // the device count before committing. The call goes through the crash guard
+ // because for some backends it triggers driver init that can fault across
+ // the DSO boundary (see the Crash guard section / issue #988). A fault is
+ // treated exactly like "no usable device": unlink and let AUTO mode fall
+ // through to the next backend and ultimately to CPU.
+ struct gpu_device_count_call call = {b->get_device_count, 0};
+ if (!gpu_run_guarded(gpu_device_count_thunk, &call)) {
+ llamafile_info(b->desc->tag, "%s crashed during device probe; trying next backend",
+ b->desc->name);
+ gpu_backend_unlink(b);
+ return false;
+ }
+
+ if (call.count <= 0) {
+ llamafile_info(b->desc->tag,
+ "%s library loaded but no devices detected; trying next backend",
+ b->desc->name);
+ gpu_backend_unlink(b);
+ return false;
+ }
+ return true;
+}
+
+void gpu_backend_register(GpuBackend *b) {
+ // No crash guard here: register() runs only after gpu_backend_probe()
+ // succeeded, so the DSO's driver/instance init already completed without
+ // faulting and reg() just returns the (cached) registry.
+ ggml_backend_reg_t reg = gpu_call_reg(b->backend_reg);
+ if (reg) {
+ ggml_backend_register(reg);
+ llamafile_info(b->desc->tag, "%s backend registered with GGML", b->desc->name);
+ }
+}
diff --git a/llamafile/gpu_backend.h b/llamafile/gpu_backend.h
new file mode 100644
index 00000000..3624e286
--- /dev/null
+++ b/llamafile/gpu_backend.h
@@ -0,0 +1,119 @@
+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2026 Mozilla.ai
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef LLAMAFILE_GPU_BACKEND_H_
+#define LLAMAFILE_GPU_BACKEND_H_
+#include <stdatomic.h>
+#include <stdbool.h>
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Shared probing logic for dynamically-loaded GPU backends that export the
+// ggml C ABI and, on Windows, use the ms_abi calling convention.
+//
+// Used by the prebuilt-DSO backends: CUDA, ROCm (cuda.c) and Vulkan
+// (vulkan.c). Metal is intentionally NOT built on top of this: it is
+// compiled at runtime, is macOS/AArch64-only (no ms_abi/sysv split) and has
+// no notion of selecting among multiple devices, so it does not share this
+// pattern.
+//
+// The point of centralising this is consistency: every backend here goes
+// through the same load -> suppress-logs -> device-count gate -> register
+// path, so a fix or a behavioural change applies to all of them at once.
+// Vulkan was missing the device-count gate, which is what made a 0-device /
+// failed Vulkan init block fallback to CPU instead of being skipped (#988).
+
+typedef struct ggml_backend * ggml_backend_t;
+typedef struct ggml_backend_reg * ggml_backend_reg_t;
+
+// Must match ggml_log_callback from ggml.h.
+typedef void (*llamafile_log_callback)(int level, const char *text, void *user_data);
+
+// Static identity of a backend: its log/display names plus the C symbols its
+// DSO exports. The tag and name live here (not at each call site) so the
+// shared probe/register routines read them straight off the backend. CUDA and
+// ROCm share symbol strings but are two distinct descriptors because their
+// display name differs. The log-control symbol is always "ggml_log_set" and is
+// resolved unconditionally.
+typedef struct GpuBackendDesc {
+ const char *tag; // lowercase log channel: "cuda" / "vulkan"
+ const char *name; // display word: "CUDA" / "ROCm" / "Vulkan"
+ const char *init; // e.g. "ggml_backend_cuda_init" (required)
+ const char *reg; // e.g. "ggml_backend_cuda_reg" (required)
+ const char *get_device_count; // required: the device-count gate needs it
+ const char *get_device_description; // optional
+} GpuBackendDesc;
+
+// Runtime state for one loaded backend DSO.
+//
+// `desc` is the backend's identity, set by gpu_backend_link(); the shared
+// routines read tag/name from it. Symbols are stored as raw pointers and
+// invoked through the gpu_call_* helpers, which apply the correct calling
+// convention (ms_abi on Windows, sysv elsewhere). backend_init has a
+// backend-specific signature (CUDA takes int, Vulkan takes size_t), so it is
+// stored here but cast and called by each backend's own typed wrapper rather
+// than through a shared helper.
+typedef struct GpuBackend {
+ const GpuBackendDesc *desc;
+ bool supported;
+ atomic_uint once;
+ void *lib_handle;
+ void *backend_init;
+ void *backend_reg;
+ void *get_device_count;
+ void *get_device_description;
+ void *log_set;
+} GpuBackend;
+
+// dlopen `dso` and resolve `desc`'s symbols into `b`, recording `desc` as the
+// backend's identity. On any failure (load error or a missing required symbol)
+// it logs on desc->tag, frees what it took, and returns false. Designed to be
+// wrapped by a tiny per-backend llamafile_link_dso_fn thunk so it can be passed
+// to llamafile_try_load_prebuilt_dso().
+bool gpu_backend_link(GpuBackend *b, const char *dso, const GpuBackendDesc *desc);
+
+// dlclose the DSO (if any) and zero every pointer in `b` (keeping `desc`).
+void gpu_backend_unlink(GpuBackend *b);
+
+// Run immediately after a successful link, before registering: suppress the
+// DSO's own ggml logging unless --verbose, then require get_device_count() > 0.
+// If the backend reports no usable devices it logs, on desc->tag,
+// "<name> library loaded but no devices detected; trying next backend"
+// calls gpu_backend_unlink() and returns false, so an AUTO-mode caller falls
+// through to the next backend and ultimately to CPU. Returns true if the
+// backend has at least one device and should be registered.
+bool gpu_backend_probe(GpuBackend *b);
+
+// Call backend_reg() and, if it returns a registry, hand it to
+// ggml_backend_register(), logging "<name> backend registered with GGML". Safe
+// to call only after gpu_backend_probe() returned true.
+void gpu_backend_register(GpuBackend *b);
+
+// ABI-correct invocation helpers (ms_abi on Windows, sysv elsewhere). Each
+// tolerates a NULL pointer the way the old per-backend wrappers did.
+ggml_backend_reg_t gpu_call_reg(void *fp);
+int gpu_call_device_count(void *fp);
+void gpu_call_get_description(void *fp, int device, char *buf, size_t n);
+void gpu_call_log_set(void *fp, llamafile_log_callback cb, void *user_data);
+
+#ifdef __cplusplus
+}
+#endif
+#endif // LLAMAFILE_GPU_BACKEND_H_
diff --git a/llamafile/version.h b/llamafile/version.h
index e75724cb..aeb76619 100644
--- a/llamafile/version.h
+++ b/llamafile/version.h
@@ -20,7 +20,7 @@
#define LLAMAFILE_MAJOR 0
#define LLAMAFILE_MINOR 10
-#define LLAMAFILE_PATCH 2
+#define LLAMAFILE_PATCH 3
#define LLAMAFILE_VERSION \
(100000000 * LLAMAFILE_MAJOR + 1000000 * LLAMAFILE_MINOR + LLAMAFILE_PATCH)
diff --git a/llamafile/vulkan.c b/llamafile/vulkan.c
index 91d4a2fc..5ac19111 100644
--- a/llamafile/vulkan.c
+++ b/llamafile/vulkan.c
@@ -24,134 +24,33 @@
// 2. Or try to load from ~/.llamafile/ (pre-compiled)
// 3. Load the DSO with cosmo_dlopen() and register the Vulkan backend
//
+// The load/probe/register mechanics live in gpu_backend.c, shared with the
+// CUDA/ROCm backends so all three behave identically.
+//
+#include "gpu_backend.h"
#include "llamafile.h"
#include <cosmo.h>
-#include <dlfcn.h>
-#include <errno.h>
-#include <limits.h>
-#include <pthread.h>
-#include <stdatomic.h>
#include <stdbool.h>
+#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <unistd.h>
-
-// Forward declarations for ggml backend types
-typedef struct ggml_backend * ggml_backend_t;
-typedef struct ggml_backend_reg * ggml_backend_reg_t;
-
-// Function to register a backend with ggml (from ggml-backend.h)
-extern void ggml_backend_register(ggml_backend_reg_t reg);
-
-// Log callback type (must match ggml_log_callback from ggml.h)
-typedef void (*llamafile_log_callback)(int level, const char *text, void *user_data);
-
-// Vulkan backend state
-//
-// On Windows the DSO exports functions with ms_abi calling convention,
-// but the cosmocc host uses System V ABI. We store each dlsym'd pointer
-// in a union so the correct ABI variant is called at each call site,
-// following the same pattern used in cuda.c and localscore/nvml.cpp.
-static struct VulkanBackend {
- bool supported;
- atomic_uint once;
- void *lib_handle;
-
- // Function pointers for Vulkan backend
- union {
- ggml_backend_t (*default_abi)(size_t device);
- ggml_backend_t (__attribute__((__ms_abi__)) *windows_abi)(size_t device);
- } backend_init;
-
- union {
- ggml_backend_reg_t (*default_abi)(void);
- ggml_backend_reg_t (__attribute__((__ms_abi__)) *windows_abi)(void);
- } backend_reg;
-
- union {
- int (*default_abi)(void);
- int (__attribute__((__ms_abi__)) *windows_abi)(void);
- } get_device_count;
- union {
- void (*default_abi)(int device, char *description, size_t description_size);
- void (__attribute__((__ms_abi__)) *windows_abi)(int device, char *description, size_t description_size);
- } get_device_description;
+// Vulkan backend state and identity (log/display names + exported symbols).
+static GpuBackend g_vulkan;
- // Logging control
- union {
- void (*default_abi)(llamafile_log_callback log_callback, void *user_data);
- void (__attribute__((__ms_abi__)) *windows_abi)(llamafile_log_callback log_callback, void *user_data);
- } log_set;
-} g_vulkan;
+static const GpuBackendDesc VULKAN_DESC = {
+ .tag = "vulkan",
+ .name = "Vulkan",
+ .init = "ggml_backend_vk_init",
+ .reg = "ggml_backend_vk_reg",
+ .get_device_count = "ggml_backend_vk_get_device_count",
+ .get_device_description = "ggml_backend_vk_get_device_description",
+};
+// Thin llamafile_link_dso_fn thunk over the shared linker.
static bool LinkVulkan(const char *dso) {
- // Load dynamic shared object using Cosmopolitan's dlopen
- void *lib = cosmo_dlopen(dso, RTLD_LAZY);
- if (!lib) {
- char *err = cosmo_dlerror();
- llamafile_info("vulkan", "failed to load library %s: %s",
- dso, err ? err : "unknown error");
- return false;
- }
-
- // Import functions into the correct ABI union member
- bool ok = true;
- void *sym;
-
- sym = cosmo_dlsym(lib, "ggml_backend_vk_init");
- ok &= (sym != NULL);
- if (IsWindows())
- *(void **)(&g_vulkan.backend_init.windows_abi) = sym;
- else
- *(void **)(&g_vulkan.backend_init.default_abi) = sym;
-
- sym = cosmo_dlsym(lib, "ggml_backend_vk_reg");
- ok &= (sym != NULL);
- if (IsWindows())
- *(void **)(&g_vulkan.backend_reg.windows_abi) = sym;
- else
- *(void **)(&g_vulkan.backend_reg.default_abi) = sym;
-
- // Optional - don't fail if not found
- sym = cosmo_dlsym(lib, "ggml_backend_vk_get_device_count");
- if (IsWindows())
- *(void **)(&g_vulkan.get_device_count.windows_abi) = sym;
- else
- *(void **)(&g_vulkan.get_device_count.default_abi) = sym;
-
- // Optional - don't fail if not found
- sym = cosmo_dlsym(lib, "ggml_backend_vk_get_device_description");
- if (IsWindows())
- *(void **)(&g_vulkan.get_device_description.windows_abi) = sym;
- else
- *(void **)(&g_vulkan.get_device_description.default_abi) = sym;
-
- // Import logging control (optional)
- sym = cosmo_dlsym(lib, "ggml_log_set");
- if (IsWindows())
- *(void **)(&g_vulkan.log_set.windows_abi) = sym;
- else
- *(void **)(&g_vulkan.log_set.default_abi) = sym;
-
- if (!ok) {
- char *err = cosmo_dlerror();
- llamafile_info("vulkan", "could not import all symbols from %s: %s",
- dso, err ? err : "unknown error");
- memset(&g_vulkan.backend_init, 0, sizeof(g_vulkan.backend_init));
- memset(&g_vulkan.backend_reg, 0, sizeof(g_vulkan.backend_reg));
- memset(&g_vulkan.get_device_count, 0, sizeof(g_vulkan.get_device_count));
- memset(&g_vulkan.get_device_description, 0, sizeof(g_vulkan.get_device_description));
- memset(&g_vulkan.log_set, 0, sizeof(g_vulkan.log_set));
- cosmo_dlclose(lib);
- return false;
- }
-
- g_vulkan.lib_handle = lib;
- return true;
+ return gpu_backend_link(&g_vulkan, dso, &VULKAN_DESC);
}
static bool ImportVulkanImpl(void) {
@@ -181,29 +80,14 @@ static bool ImportVulkanImpl(void) {
return false;
}
- // Suppress DSO's ggml logging before backend registration, which triggers
- // device enumeration inside the DSO. Without this, Vulkan device messages
- // appear even when --verbose is not set.
- if (!FLAG_verbose && (g_vulkan.log_set.default_abi || g_vulkan.log_set.windows_abi)) {
- if (IsWindows())
- g_vulkan.log_set.windows_abi(llamafile_log_callback_null, NULL);
- else
- g_vulkan.log_set.default_abi(llamafile_log_callback_null, NULL);
- }
-
- // Register the Vulkan backend with GGML
- if (g_vulkan.backend_reg.default_abi || g_vulkan.backend_reg.windows_abi) {
- ggml_backend_reg_t reg;
- if (IsWindows())
- reg = g_vulkan.backend_reg.windows_abi();
- else
- reg = g_vulkan.backend_reg.default_abi();
- if (reg) {
- ggml_backend_register(reg);
- llamafile_info("vulkan", "Vulkan backend registered with GGML");
- }
- }
+ // Gate on device count before registering (issue #988). The DSO loads even
+ // when Vulkan has no usable device; without this check we'd register a
+ // 0-device backend and block fallback to CPU. gpu_backend_probe() unlinks
+ // and returns false in that case so AUTO mode moves on.
+ if (!gpu_backend_probe(&g_vulkan))
+ return false;
+ gpu_backend_register(&g_vulkan);
return true;
}
@@ -211,14 +95,8 @@ static void ImportVulkan(void) {
if (ImportVulkanImpl()) {
g_vulkan.supported = true;
llamafile_info("vulkan", "Vulkan GPU support successfully loaded");
- if (g_vulkan.get_device_count.default_abi || g_vulkan.get_device_count.windows_abi) {
- int count;
- if (IsWindows())
- count = g_vulkan.get_device_count.windows_abi();
- else
- count = g_vulkan.get_device_count.default_abi();
- llamafile_info("vulkan", "found %d GPU device(s)", count);
- }
+ int count = gpu_call_device_count(g_vulkan.get_device_count);
+ llamafile_info("vulkan", "found %d GPU device(s)", count);
} else if (FLAG_gpu == LLAMAFILE_GPU_VULKAN) {
fprintf(stderr, "fatal error: support for --gpu vulkan was explicitly requested, "
"but it wasn't available\n");
@@ -236,21 +114,20 @@ bool llamafile_has_vulkan(void) {
ggml_backend_t ggml_backend_vk_init(size_t device) {
if (!llamafile_has_vulkan())
return NULL;
- if (!g_vulkan.backend_init.default_abi && !g_vulkan.backend_init.windows_abi)
+ void *fp = g_vulkan.backend_init;
+ if (!fp)
return NULL;
+ // backend_init has a Vulkan-specific signature (size_t device), so it is
+ // called here rather than through a shared gpu_call_* helper.
if (IsWindows())
- return g_vulkan.backend_init.windows_abi(device);
- return g_vulkan.backend_init.default_abi(device);
+ return ((ggml_backend_t(__attribute__((__ms_abi__)) *)(size_t))fp)(device);
+ return ((ggml_backend_t(*)(size_t))fp)(device);
}
int ggml_backend_vk_get_device_count(void) {
if (!llamafile_has_vulkan())
return 0;
- if (!g_vulkan.get_device_count.default_abi && !g_vulkan.get_device_count.windows_abi)
- return 0;
- if (IsWindows())
- return g_vulkan.get_device_count.windows_abi();
- return g_vulkan.get_device_count.default_abi();
+ return gpu_call_device_count(g_vulkan.get_device_count);
}
void ggml_backend_vk_get_device_description(int device, char *description, size_t description_size) {
@@ -259,24 +136,16 @@ void ggml_backend_vk_get_device_description(int device, char *description, size_
description[0] = '\0';
return;
}
- if (!g_vulkan.get_device_description.default_abi && !g_vulkan.get_device_description.windows_abi) {
+ if (!g_vulkan.get_device_description) {
if (description_size > 0)
snprintf(description, description_size, "Vulkan GPU %d", device);
return;
}
- if (IsWindows())
- g_vulkan.get_device_description.windows_abi(device, description, description_size);
- else
- g_vulkan.get_device_description.default_abi(device, description, description_size);
+ gpu_call_get_description(g_vulkan.get_device_description, device, description, description_size);
}
void llamafile_vulkan_log_set(llamafile_log_callback log_callback, void *user_data) {
if (!llamafile_has_vulkan())
return;
- if (g_vulkan.log_set.default_abi || g_vulkan.log_set.windows_abi) {
- if (IsWindows())
- g_vulkan.log_set.windows_abi(log_callback, user_data);
- else
- g_vulkan.log_set.default_abi(log_callback, user_data);
- }
+ gpu_call_log_set(g_vulkan.log_set, log_callback, user_data);
}
diff --git a/tests/BUILD.mk b/tests/BUILD.mk
index 9ef384ea..5bec12d3 100644
--- a/tests/BUILD.mk
+++ b/tests/BUILD.mk
@@ -98,6 +98,28 @@ o/$(MODE)/tests/fa_helpers_test: \
@mkdir -p $(@D)
$(CXX) $(LDFLAGS) -fopenmp -o $@ $^ $(LDLIBS)
+# ==============================================================================
+# Test: gpu_backend_test (issue #988 device-count gate / fallback)
+# ==============================================================================
+#
+# Exercises the shared GPU backend probe core. The test injects stub entry
+# points into a GpuBackend and provides its own doubles for gpu_backend.c's
+# externs, so it links against gpu_backend.o alone — no DSO, no GPU, no
+# llamafile.o/llama.cpp.a.
+
+GPU_BACKEND_TEST_DEPS := \
+ o/$(MODE)/llamafile/gpu_backend.o
+
+o/$(MODE)/tests/gpu_backend_test.o: tests/gpu_backend_test.cpp
+ @mkdir -p $(@D)
+ $(CXX) $(CXXFLAGS) $(TESTS_CPPFLAGS) -c -o $@ $<
+
+o/$(MODE)/tests/gpu_backend_test: \
+ o/$(MODE)/tests/gpu_backend_test.o \
+ $(GPU_BACKEND_TEST_DEPS)
+ @mkdir -p $(@D)
+ $(CXX) $(LDFLAGS) -o $@ $^ $(LDLIBS)
+
# ==============================================================================
# Phony targets
# ==============================================================================
@@ -105,4 +127,5 @@ o/$(MODE)/tests/fa_helpers_test: \
.PHONY: o/$(MODE)/tests
o/$(MODE)/tests: \
o/$(MODE)/tests/extract_data_uris_test.runs \
- o/$(MODE)/tests/fa_helpers_test.runs
+ o/$(MODE)/tests/fa_helpers_test.runs \
+ o/$(MODE)/tests/gpu_backend_test.runs
diff --git a/tests/gpu_backend_test.cpp b/tests/gpu_backend_test.cpp
new file mode 100644
index 00000000..4fdf7f4b
--- /dev/null
+++ b/tests/gpu_backend_test.cpp
@@ -0,0 +1,282 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2026 Mozilla.ai
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// Unit tests for the shared GPU backend probe core (llamafile/gpu_backend.c).
+//
+// The device-count gate exercised here is the fix for issue #988: a backend
+// DSO that loads but exposes no usable device must be rejected (and unlinked)
+// so AUTO mode can fall through to the next backend and ultimately to CPU,
+// instead of registering a dead backend and crashing / blocking fallback.
+//
+// Because GpuBackend stores its DSO entry points as plain pointers, we can
+// drive gpu_backend_probe()/gpu_backend_register() with injected stub
+// functions and never touch a real DSO or GPU. The handful of externs
+// gpu_backend.c needs (FLAG_verbose, llamafile_info, llamafile_log_callback_null,
+// ggml_backend_register) are provided here as test doubles so the test links
+// against gpu_backend.o alone.
+//
+
+#include "gpu_backend.h"
+#include <signal.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// ---------------------------------------------------------------------------
+// Test doubles for gpu_backend.c's external dependencies
+// ---------------------------------------------------------------------------
+
+extern "C" {
+
+int FLAG_verbose;
+
+void llamafile_info(const char *, const char *, ...) {
+ // swallow log output during tests
+}
+
+void llamafile_log_callback_null(int, const char *, void *) {}
+
+// Records the last registry handed to ggml so register tests can assert on it.
+static ggml_backend_reg_t g_last_registered = (ggml_backend_reg_t)-1;
+void ggml_backend_register(ggml_backend_reg_t reg) {
+ g_last_registered = reg;
+}
+
+} // extern "C"
+
+// ---------------------------------------------------------------------------
+// Injected backend entry points (stubs)
+// ---------------------------------------------------------------------------
+
+static int dc_two(void) { return 2; }
+static int dc_zero(void) { return 0; }
+static int dc_negative(void) { return -1; }
+
+// Stand-ins for a backend whose get_device_count faults during driver init:
+// a SIGSEGV (the cosmo-translated form of the #988 C++ exception) and a
+// SIGABRT (the GGML_ASSERT -> abort() path).
+static int dc_segv(void) {
+ raise(SIGSEGV);
+ return 0;
+}
+static int dc_abort(void) {
+ abort();
+ return 0;
+}
+
+static int g_log_set_calls;
+static void log_set_spy(llamafile_log_callback, void *) { ++g_log_set_calls; }
+
+static ggml_backend_reg_t g_fake_reg = (ggml_backend_reg_t)0x1234;
+static ggml_backend_reg_t reg_ok(void) { return g_fake_reg; }
+static ggml_backend_reg_t reg_null(void) { return NULL; }
+
+static const GpuBackendDesc TEST_DESC = {
+ /*.tag*/ "test",
+ /*.name*/ "Test",
+ /*.init*/ "x_init",
+ /*.reg*/ "x_reg",
+ /*.get_device_count*/ "x_get_device_count",
+ /*.get_device_description*/ "x_get_device_description",
+};
+
+// A non-NULL sentinel for symbol pointers, so we can assert unlink() cleared
+// them. lib_handle stays NULL so unlink() doesn't call cosmo_dlclose() on it.
+static void *const SENTINEL = (void *)0xABCD;
+
+// Fills `b` in place (GpuBackend holds an atomic_uint, so it is not copyable
+// in C++ and cannot be returned by value).
+static void init_linked_backend(GpuBackend *b, void *get_device_count, void *log_set) {
+ memset(b, 0, sizeof(*b));
+ b->desc = &TEST_DESC;
+ b->lib_handle = NULL; // pretend "loaded" without a real dlopen handle
+ b->backend_init = SENTINEL;
+ b->backend_reg = SENTINEL;
+ b->get_device_count = get_device_count;
+ b->get_device_description = SENTINEL;
+ b->log_set = log_set;
+}
+
+static bool is_unlinked(const GpuBackend *b) {
+ return b->lib_handle == NULL && b->backend_init == NULL && b->backend_reg == NULL &&
+ b->get_device_count == NULL && b->get_device_description == NULL &&
+ b->log_set == NULL;
+}
+
+// ---------------------------------------------------------------------------
+// Tiny test harness
+// ---------------------------------------------------------------------------
+
+static int g_failures;
+static int g_checks;
+
+#define CHECK(cond) \
+ do { \
+ ++g_checks; \
+ if (!(cond)) { \
+ ++g_failures; \
+ fprintf(stderr, "FAIL: %s:%d: %s\n", __FILE__, __LINE__, #cond); \
+ } \
+ } while (0)
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+// A backend with >0 devices is accepted and left intact for registration.
+static void test_probe_accepts_when_devices_present(void) {
+ FLAG_verbose = 1; // skip suppression branch
+ GpuBackend b;
+ init_linked_backend(&b, (void *)dc_two, (void *)log_set_spy);
+ CHECK(gpu_backend_probe(&b) == true);
+ CHECK(b.backend_init == SENTINEL); // still linked
+ CHECK(b.get_device_count == (void *)dc_two);
+}
+
+// The #988 regression guard: a loaded-but-0-device backend is rejected AND
+// unlinked, so the caller falls through to the next backend / CPU.
+static void test_probe_rejects_and_unlinks_when_zero_devices(void) {
+ FLAG_verbose = 1;
+ GpuBackend b;
+ init_linked_backend(&b, (void *)dc_zero, (void *)log_set_spy);
+ CHECK(gpu_backend_probe(&b) == false);
+ CHECK(is_unlinked(&b));
+ CHECK(b.desc == &TEST_DESC); // identity is preserved across unlink
+}
+
+// A negative device count is treated the same as zero.
+static void test_probe_rejects_on_negative_count(void) {
+ FLAG_verbose = 1;
+ GpuBackend b;
+ init_linked_backend(&b, (void *)dc_negative, (void *)log_set_spy);
+ CHECK(gpu_backend_probe(&b) == false);
+ CHECK(is_unlinked(&b));
+}
+
+// A missing get_device_count symbol means we can't confirm any device, so the
+// backend is rejected rather than optimistically registered.
+static void test_probe_rejects_when_device_count_symbol_absent(void) {
+ FLAG_verbose = 1;
+ GpuBackend b;
+ init_linked_backend(&b, /*get_device_count=*/NULL, (void *)log_set_spy);
+ CHECK(gpu_backend_probe(&b) == false);
+ CHECK(is_unlinked(&b));
+}
+
+// Without --verbose, the DSO's logging is suppressed before device enumeration;
+// with --verbose it is left alone.
+static void test_probe_log_suppression_respects_verbose(void) {
+ g_log_set_calls = 0;
+ FLAG_verbose = 0;
+ GpuBackend quiet;
+ init_linked_backend(&quiet, (void *)dc_two, (void *)log_set_spy);
+ gpu_backend_probe(&quiet);
+ CHECK(g_log_set_calls == 1); // suppression installed
+
+ g_log_set_calls = 0;
+ FLAG_verbose = 1;
+ GpuBackend loud;
+ init_linked_backend(&loud, (void *)dc_two, (void *)log_set_spy);
+ gpu_backend_probe(&loud);
+ CHECK(g_log_set_calls == 0); // left alone
+}
+
+// The crash boundary (issue #988): a get_device_count that faults (SIGSEGV)
+// must be caught and turned into a clean rejection + unlink, so AUTO mode falls
+// through to the next backend / CPU instead of the process dying.
+static void test_probe_survives_segv_in_device_count(void) {
+ FLAG_verbose = 1;
+ GpuBackend b;
+ init_linked_backend(&b, (void *)dc_segv, (void *)log_set_spy);
+ CHECK(gpu_backend_probe(&b) == false);
+ CHECK(is_unlinked(&b));
+}
+
+// Same for a GGML_ASSERT-style abort() (SIGABRT) during probing.
+static void test_probe_survives_abort_in_device_count(void) {
+ FLAG_verbose = 1;
+ GpuBackend b;
+ init_linked_backend(&b, (void *)dc_abort, (void *)log_set_spy);
+ CHECK(gpu_backend_probe(&b) == false);
+ CHECK(is_unlinked(&b));
+}
+
+// After a guarded fault, normal probing must still work (handlers restored, no
+// lingering signal mask / disposition).
+static void test_probe_works_after_guarded_fault(void) {
+ FLAG_verbose = 1;
+ GpuBackend crashed;
+ init_linked_backend(&crashed, (void *)dc_segv, (void *)log_set_spy);
+ gpu_backend_probe(&crashed); // faults, handled
+
+ GpuBackend good;
+ init_linked_backend(&good, (void *)dc_two, (void *)log_set_spy);
+ CHECK(gpu_backend_probe(&good) == true);
+}
+
+// register() forwards a real registry to ggml, and skips a NULL one.
+static void test_register_forwards_non_null_reg(void) {
+ g_last_registered = (ggml_backend_reg_t)-1;
+ GpuBackend b;
+ init_linked_backend(&b, (void *)dc_two, (void *)log_set_spy);
+ b.backend_reg = (void *)reg_ok;
+ gpu_backend_register(&b);
+ CHECK(g_last_registered == g_fake_reg);
+}
+
+static void test_register_skips_null_reg(void) {
+ g_last_registered = (ggml_backend_reg_t)-1;
+ GpuBackend b;
+ init_linked_backend(&b, (void *)dc_two, (void *)log_set_spy);
+ b.backend_reg = (void *)reg_null;
+ gpu_backend_register(&b);
+ CHECK(g_last_registered == (ggml_backend_reg_t)-1); // never called
+}
+
+// The ABI call helpers tolerate NULL pointers like the old per-backend wrappers.
+static void test_call_helpers_tolerate_null(void) {
+ char buf[32] = "untouched";
+ CHECK(gpu_call_reg(NULL) == NULL);
+ CHECK(gpu_call_device_count(NULL) == 0);
+ gpu_call_get_description(NULL, 0, buf, sizeof(buf)); // must not crash/modify
+ CHECK(strcmp(buf, "untouched") == 0);
+ gpu_call_log_set(NULL, llamafile_log_callback_null, NULL); // must not crash
+ CHECK(gpu_call_device_count((void *)dc_two) == 2); // and dispatches when set
+}
+
+int main(void) {
+ test_probe_accepts_when_devices_present();
+ test_probe_rejects_and_unlinks_when_zero_devices();
+ test_probe_rejects_on_negative_count();
+ test_probe_rejects_when_device_count_symbol_absent();
+ test_probe_log_suppression_respects_verbose();
+ test_probe_survives_segv_in_device_count();
+ test_probe_survives_abort_in_device_count();
+ test_probe_works_after_guarded_fault();
+ test_register_forwards_non_null_reg();
+ test_register_skips_null_reg();
+ test_call_helpers_tolerate_null();
+
+ if (g_failures) {
+ fprintf(stderr, "gpu_backend_test: %d/%d checks FAILED\n", g_failures, g_checks);
+ return 1;
+ }
+ printf("gpu_backend_test: all %d checks PASSED\n", g_checks);
+ return 0;
+}
diff --git a/whisperfile/BUILD.mk b/whisperfile/BUILD.mk
index d6a39747..95ba9068 100644
--- a/whisperfile/BUILD.mk
+++ b/whisperfile/BUILD.mk
@@ -100,9 +100,7 @@ WHISPERFILE_CPPFLAGS := \
WHISPERFILE_LLAMAFILE_OBJS := \
o/$(MODE)/llamafile/llamafile.o \
- o/$(MODE)/llamafile/metal.o \
- o/$(MODE)/llamafile/cuda.o \
- o/$(MODE)/llamafile/vulkan.o \
+ o/$(MODE)/llamafile/gpu.a \
o/$(MODE)/llamafile/zip.o \
o/$(MODE)/llamafile/check_cpu.o