Skip to content

Distribute and run LLMs with a single file.

Get CUDA working on Linux x86-64

Justine Tunney7b6b0d872ffb
diff --git a/Makefile b/Makefile
index dad2569a..8cf1b9ff 100644
--- a/Makefile
+++ b/Makefile
@@ -13,5 +13,10 @@ include build/rules.mk
include llama.cpp/BUILD.mk
+# the root package is `o//` by default
+# building a package also builds its sub-packages
+.PHONY: o/$(MODE)/
+o/$(MODE)/: o/$(MODE)/llama.cpp
+
include build/deps.mk
include build/tags.mk
diff --git a/build/config.mk b/build/config.mk
index a9978b62..c8cca517 100644
--- a/build/config.mk
+++ b/build/config.mk
@@ -15,12 +15,19 @@ CPPFLAGS += -iquote.
TMPDIR = o//tmp
IGNORE := $(shell mkdir -p $(TMPDIR))
+# apple still distributes a 17 year old version of gnu make
+ifeq ($(MAKE_VERSION), 3.81)
+$(error please use bin/make from cosmocc.zip rather than old xcode make)
+endif
+
+# let `make m=foo` be shorthand for `make MODE=foo`
ifneq ($(m),)
ifeq ($(MODE),)
MODE := $(m)
endif
endif
+# make build more deterministic
LC_ALL = C
SOURCE_DATE_EPOCH = 0
export MODE
@@ -28,11 +35,9 @@ export TMPDIR
export LC_ALL
export SOURCE_DATE_EPOCH
+# `make` runs `make all` by default
.PHONY: all
all: o/$(MODE)/
-.PHONY: o/$(MODE)/
-o/$(MODE)/: o/$(MODE)/llama.cpp
-
.PHONY: clean
clean:; rm -rf o
diff --git a/llama.cpp/common.cc b/llama.cpp/common.cc
index b5d19c7c..aeb6be10 100644
--- a/llama.cpp/common.cc
+++ b/llama.cpp/common.cc
@@ -143,6 +143,7 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {
bool invalid_param = false;
std::string arg;
+ bool passed_gpu_flags = false;
const std::string arg_prefix = "--";
llama_sampling_params & sparams = params.sparams;
@@ -514,43 +515,32 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {
} else if (arg == "--mlock") {
params.use_mlock = true;
} else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") {
+ passed_gpu_flags = true;
if (++i >= argc) {
invalid_param = true;
break;
}
- if (ggml_metal_supported() || ggml_cuda_supported()) {
- params.n_gpu_layers = std::stoi(argv[i]);
- } else {
- fprintf(stderr, "warning: GPU offload not supported on this platform, --n-gpu-layers option will be ignored\n");
- fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n");
- }
+ params.n_gpu_layers = std::stoi(argv[i]);
} else if (arg == "--gpu-layers-draft" || arg == "-ngld" || arg == "--n-gpu-layers-draft") {
+ passed_gpu_flags = true;
if (++i >= argc) {
invalid_param = true;
break;
}
- if (ggml_metal_supported() || ggml_cuda_supported()) {
- params.n_gpu_layers_draft = std::stoi(argv[i]);
- } else {
- fprintf(stderr, "warning: GPU offload not supported on this platform, --n-gpu-layers-draft option will be ignored\n");
- fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n");
- }
+ params.n_gpu_layers_draft = std::stoi(argv[i]);
} else if (arg == "--main-gpu" || arg == "-mg") {
+ passed_gpu_flags = true;
if (++i >= argc) {
invalid_param = true;
break;
}
- if (ggml_cuda_supported()) {
- params.main_gpu = std::stoi(argv[i]);
- } else {
- fprintf(stderr, "warning: cuBLAS GPU couldn't be supported on this system. It is not possible to set a main GPU.\n");
- }
+ params.main_gpu = std::stoi(argv[i]);
} else if (arg == "--tensor-split" || arg == "-ts") {
+ passed_gpu_flags = true;
if (++i >= argc) {
invalid_param = true;
break;
}
- if (ggml_cuda_supported()) {
std::string arg_next = argv[i];
// split string by , and /
@@ -566,15 +556,9 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {
params.tensor_split[i] = 0.0f;
}
}
- } else {
- fprintf(stderr, "warning: cuBLAS GPU couldn't be supported on this system. It is not possible to set a tensor split.\n");
- }
} else if (arg == "--no-mul-mat-q" || arg == "-nommq") {
- if (ggml_cuda_supported()) {
- params.mul_mat_q = false;
- } else {
- fprintf(stderr, "warning: cuBLAS GPU couldn't be supported on this system. Disabling mul_mat_q kernels has no effect.\n");
- }
+ passed_gpu_flags = true;
+ params.mul_mat_q = false;
} else if (arg == "--no-mmap") {
params.use_mmap = false;
} else if (arg == "--numa") {
@@ -740,6 +724,23 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {
}
}
+ if (passed_gpu_flags) {
+ // user is tuning their gpu
+ if (!ggml_metal_supported() && !ggml_cuda_supported()) {
+ fprintf(stderr, "warning: GPU offload not supported on this platform; GPU related options will be ignored\n");
+ fprintf(stderr, "warning: you might need to install xcode (macos) or cuda (windows, linux, etc.) check the output above to see why support wasn't linked\n");
+ }
+ } else {
+ // no gpu flags were passed
+ if (ggml_metal_supported()) {
+ // apple metal gpu doesn't require explicit flags to enable
+ } else {
+ // avoid the >1 second cuda startup latency if cuda isn't being used
+ fprintf(stderr, "protip: pass the --n-gpu-layers N flag to link NVIDIA cuBLAS support\n");
+ ggml_cuda_disable();
+ }
+ }
+
return true;
}
diff --git a/llama.cpp/cuda.c b/llama.cpp/cuda.c
index b27ae6e9..cfa33d33 100644
--- a/llama.cpp/cuda.c
+++ b/llama.cpp/cuda.c
@@ -10,6 +10,7 @@
#include <unistd.h>
#include <stdlib.h>
#include <limits.h>
+#include <signal.h>
#include <pthread.h>
#include <sys/wait.h>
#include <sys/stat.h>
@@ -20,16 +21,33 @@ __static_yoink("llama.cpp/ggml.h");
__static_yoink("llama.cpp/ggml-cuda.h");
__static_yoink("llama.cpp/ggml-cuda.cu");
+#define NVCC_LIBS "-lcublas"
+
+#define NVCC_FLAGS "--shared", \
+ "--forward-unknown-to-host-compiler", \
+ "-use_fast_math", \
+ "-arch=compute_87", \
+ "--compiler-options", "-fPIC -O3 -march=native -mtune=native", \
+ "-DNDEBUG", \
+ "-DGGML_BUILD=1", \
+ "-DGGML_SHARED=1", \
+ "-DGGML_CUDA_DMMV_X=32", \
+ "-DGGML_CUDA_MMV_Y=1", \
+ "-DK_QUANTS_PER_ITERATION=2", \
+ "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128", \
+ "-DGGML_USE_CUBLAS"
+
static const struct Source {
const char *zip;
const char *name;
} srcs[] = {
{"/zip/llama.cpp/ggml.h", "ggml.h"},
- {"/zip/llama.cpp/ggml-impl.h", "ggml-cuda.h"},
+ {"/zip/llama.cpp/ggml-cuda.h", "ggml-cuda.h"},
{"/zip/llama.cpp/ggml-cuda.cu", "ggml-cuda.cu"}, // must come last
};
static struct Cuda {
+ bool disabled;
bool supported;
atomic_uint once;
typeof(ggml_init_cublas) *init;
@@ -71,19 +89,98 @@ static const char *GetDsoExtension(void) {
}
}
-static bool CompileNativeCuda(char dso[static PATH_MAX]) {
+static bool FileExists(const char *path) {
+ struct stat st;
+ return !stat(path, &st);
+}
- // find path of nvidia compiler
- char nvcc[PATH_MAX];
+static bool IsExecutable(const char *path) {
+ struct stat st;
+ return !stat(path, &st) &&
+ (st.st_mode & 0111) &&
+ !S_ISDIR(st.st_mode);
+}
+
+static bool CreateTempPath(const char *path, char tmp[static PATH_MAX]) {
+ int fd;
+ strlcpy(tmp, path, PATH_MAX);
+ strlcat(tmp, ".XXXXXX", PATH_MAX);
+ if ((fd = mkstemp(tmp)) != -1) {
+ close(fd);
+ return true;
+ } else {
+ perror(tmp);
+ return false;
+ }
+}
+
+static bool Compile(const char *src,
+ const char *tmp,
+ const char *out,
+ char *args[]) {
+ int pid, ws;
+ errno_t err = posix_spawnp(&pid, args[0], NULL, NULL, args, environ);
+ if (err) {
+ perror(args[0]);
+ unlink(tmp);
+ return false;
+ }
+ while (waitpid(pid, &ws, 0) == -1) {
+ if (errno != EINTR) {
+ perror(args[0]);
+ unlink(tmp);
+ return false;
+ }
+ }
+ if (ws) {
+ tinyprint(2, args[0], ": returned nonzero exit status\n", NULL);
+ unlink(tmp);
+ return false;
+ }
+ if (rename(tmp, out)) {
+ perror(out);
+ unlink(tmp);
+ return false;
+ }
+ return true;
+}
+
+// finds nvidia compiler
+//
+// 1. $CUDA_PATH/bin/nvcc
+// 2. nvcc on $PATH environ
+// 3. /opt/cuda/bin/nvcc
+// 4. /usr/local/cuda/bin/nvcc
+//
+// set $CUDA_PATH to empty string to disable cuda
+static bool GetNvccPath(char path[static PATH_MAX]) {
const char *cuda_path;
- nvcc[0] = 0;
if ((cuda_path = getenv("CUDA_PATH"))) {
- strlcat(nvcc, cuda_path, sizeof(nvcc));
- strlcat(nvcc, "/bin/", sizeof(nvcc));
+ if (!*cuda_path) return false;
+ strlcpy(path, cuda_path, PATH_MAX);
+ strlcat(path, "/bin/", PATH_MAX);
+ } else if (commandv(IsWindows() ? "nvcc.exe" : "nvcc", path, PATH_MAX)) {
+ return true;
+ } else if (FileExists("/opt/cuda")) {
+ strlcpy(path, "/opt/cuda/bin/", PATH_MAX);
+ } else {
+ strlcpy(path, "/usr/local/cuda/bin/", PATH_MAX);
}
- strlcat(nvcc, "nvcc", sizeof(nvcc));
+ strlcat(path, "nvcc", PATH_MAX);
if (IsWindows()) {
- strlcat(nvcc, ".exe", sizeof(nvcc));
+ strlcat(path, ".exe", PATH_MAX);
+ }
+ return IsExecutable(path);
+}
+
+static bool CompileNativeCuda(char dso[static PATH_MAX]) {
+
+ // find full path of nvidia compiler
+ char nvcc[PATH_MAX];
+ if (!GetNvccPath(nvcc)) {
+ tinyprint(2, "warning: couldn't find nvcc (nvidia c compiler) "
+ "try setting $CUDA_PATH if it's installed\n", NULL);
+ return false;
}
// extract source code
@@ -96,63 +193,53 @@ static bool CompileNativeCuda(char dso[static PATH_MAX]) {
return false;
}
strlcat(src, srcs[i].name, sizeof(src));
- if (ggml_is_newer_than(srcs[i].zip, src)) {
- needs_rebuild = true;
- if (!ggml_extract(srcs[i].zip, src)) {
+ switch (ggml_is_file_newer_than(srcs[i].zip, src)) {
+ case -1:
return false;
- }
+ case false:
+ break;
+ case true:
+ needs_rebuild = true;
+ if (!ggml_extract(srcs[i].zip, src)) {
+ return false;
+ }
+ break;
+ default:
+ __builtin_unreachable();
}
}
- // compile dynamic shared object
+ // check if dso is already compiled
ggml_get_app_dir(dso, PATH_MAX);
strlcat(dso, "ggml-cuda.", PATH_MAX);
strlcat(dso, GetDsoExtension(), PATH_MAX);
- if (needs_rebuild || ggml_is_newer_than(src, dso)) {
- tinyprint(2, "building ggml-cuda with nvcc...\n", NULL);
- int fd;
- char tmpdso[PATH_MAX];
- strlcpy(tmpdso, dso, sizeof(tmpdso));
- strlcat(tmpdso, ".XXXXXX", sizeof(tmpdso));
- if ((fd = mkstemp(tmpdso)) != -1) {
- close(fd);
- } else {
- perror(tmpdso);
- return false;
- }
- char *args[] = {
- nvcc,
- "--shared",
- "-arch=native",
- "-DGGML_BUILD=1",
- "-DGGML_SHARED=1",
- "-o", tmpdso,
- src,
- "-lcublas",
- NULL,
- };
- int pid, ws;
- errno_t err = posix_spawnp(&pid, nvcc, NULL, NULL, args, environ);
- if (err) {
- perror(nvcc);
- return false;
- }
- while (waitpid(pid, &ws, 0) == -1) {
- if (errno != EINTR) {
- perror(nvcc);
+ if (!needs_rebuild) {
+ switch (ggml_is_file_newer_than(src, dso)) {
+ case -1:
return false;
- }
- }
- if (ws) {
- tinyprint(2, nvcc, ": returned nonzero exit status\n", NULL);
- return false;
- }
- if (rename(tmpdso, dso)) {
- perror(dso);
- return false;
+ case false:
+ return true;
+ case true:
+ break;
+ default:
+ __builtin_unreachable();
}
}
+ // create temporary output path for atomicity
+ char tmpdso[PATH_MAX];
+ if (!CreateTempPath(dso, tmpdso)) {
+ return false;
+ }
+
+ // try building dso with host nvidia microarchitecture
+ tinyprint(2, "building ggml-cuda with nvcc...\n", NULL);
+ if (!Compile(src, tmpdso, dso, (char *[]){
+ nvcc, NVCC_FLAGS, "-o", tmpdso, src, NVCC_LIBS, NULL})) {
+ return false;
+ }
+
+ // we're good!
return true;
}
@@ -166,7 +253,6 @@ static bool ImportCudaImpl(void) {
// runtime link dynamic shared object
void *lib;
- tinyprint(2, "OPENING: ", dso, "\n", NULL);
lib = cosmo_dlopen(dso, RTLD_LAZY);
if (!lib) {
tinyprint(2, Dlerror(), ": failed to load library\n", NULL);
@@ -205,6 +291,9 @@ static bool ImportCudaImpl(void) {
}
static void ImportCuda(void) {
+ if (ggml_cuda.disabled) {
+ return;
+ }
if (ImportCudaImpl()) {
ggml_cuda.supported = true;
tinyprint(2, "NVIDIA cuBLAS GPU supported locked and loaded\n", NULL);
@@ -213,6 +302,10 @@ static void ImportCuda(void) {
}
}
+void ggml_cuda_disable(void) {
+ ggml_cuda.disabled = true;
+}
+
bool ggml_cuda_supported(void) {
cosmo_once(&ggml_cuda.once, ImportCuda);
return ggml_cuda.supported;
diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cu
index 4d6a9691..0630ea15 100644
--- a/llama.cpp/ggml-cuda.cu
+++ b/llama.cpp/ggml-cuda.cu
@@ -4134,7 +4134,7 @@ template <bool need_check> static __global__ void
(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);
#else
(void) vec_dot_q8_0_q8_1_mul_mat;
- assert(false);
+ assert((__CUDA_ARCH__, 0));
#endif // __CUDA_ARCH__ >= CC_VOLTA
}
diff --git a/llama.cpp/ggml-cuda.h b/llama.cpp/ggml-cuda.h
index 0d9e7641..0cb41da4 100644
--- a/llama.cpp/ggml-cuda.h
+++ b/llama.cpp/ggml-cuda.h
@@ -18,6 +18,7 @@ extern "C" {
// Always success. To check if CUDA is actually loaded, use `ggml_cublas_loaded`.
GGML_API void ggml_init_cublas(void);
+GGML_API void ggml_cuda_disable(void);
GGML_API bool ggml_cuda_supported(void);
// Returns `true` if there are available CUDA devices and cublas loads successfully; otherwise, it returns `false`.
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 4289d19c..1d079513 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -14322,14 +14322,12 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
return;
}
-#ifdef GGML_USE_CUBLAS
bool skip_cpu = ggml_cuda_compute_forward(params, tensor);
if (skip_cpu) {
return;
}
GGML_ASSERT(tensor->src[0] == NULL || tensor->src[0]->backend == GGML_BACKEND_CPU);
GGML_ASSERT(tensor->src[1] == NULL || tensor->src[1]->backend == GGML_BACKEND_CPU);
-#endif // GGML_USE_CUBLAS
switch (tensor->op) {
case GGML_OP_DUP:
diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.h
index 749bd2c7..16cda397 100644
--- a/llama.cpp/ggml.h
+++ b/llama.cpp/ggml.h
@@ -2160,7 +2160,7 @@ extern "C" {
GGML_API void ggml_get_app_dir(char *path, size_t size);
GGML_API void ggml_schlep(const void *data, size_t size);
GGML_API bool ggml_extract(const char *zip, const char *to);
- GGML_API bool ggml_is_newer_than(const char *path, const char *other);
+ GGML_API int ggml_is_file_newer_than(const char *path, const char *other);
#ifdef __cplusplus
}
diff --git a/llama.cpp/metal.c b/llama.cpp/metal.c
index bc08bab9..5a973a13 100644
--- a/llama.cpp/metal.c
+++ b/llama.cpp/metal.c
@@ -69,19 +69,42 @@ static bool ImportMetalImpl(void) {
return false;
}
strlcat(src, srcs[i].name, sizeof(src));
- if (ggml_is_newer_than(srcs[i].zip, src)) {
- needs_rebuild = true;
- if (!ggml_extract(srcs[i].zip, src)) {
+ switch (ggml_is_file_newer_than(srcs[i].zip, src)) {
+ case -1:
return false;
- }
+ case 0:
+ break;
+ case 1:
+ needs_rebuild = true;
+ if (!ggml_extract(srcs[i].zip, src)) {
+ return false;
+ }
+ break;
+ default:
+ __builtin_unreachable();
}
}
- // compile dynamic shared object
+ // determine if we need to build
char dso[PATH_MAX];
ggml_get_app_dir(dso, PATH_MAX);
strlcat(dso, "ggml-metal.dylib", sizeof(dso));
- if (needs_rebuild || ggml_is_newer_than(src, dso)) {
+ if (!needs_rebuild) {
+ switch (ggml_is_file_newer_than(src, dso)) {
+ case -1:
+ return false;
+ case 0:
+ return true;
+ case 1:
+ needs_rebuild = true;
+ break;
+ default:
+ __builtin_unreachable();
+ }
+ }
+
+ // compile dynamic shared object
+ if (needs_rebuild) {
tinyprint(2, "building ggml-metal.dylib with xcode...\n", NULL);
int fd;
char tmpdso[PATH_MAX];
diff --git a/llama.cpp/schlep.c b/llama.cpp/schlep.c
index 9492964d..ed7ae769 100644
--- a/llama.cpp/schlep.c
+++ b/llama.cpp/schlep.c
@@ -24,9 +24,8 @@ static char Peek(volatile const char *ptr) {
char (*pPeek)(volatile const char *) = Peek;
static void FormatPercent(char sbuf[static 8], double x) {
- x *= 100000.5;
- int n = x;
char *p = sbuf;
+ int n = x * 100000.5;
if (n >= 100000) *p++ = '0' + n / 100000 % 10;
if (n >= 10000) *p++ = '0' + n / 10000 % 10;
*p++ = '0' + n / 1000 % 10;