Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Get CUDA working on Linux x86-64
Justine Tunney7b6b0d872ffb
diff --git a/Makefile b/Makefileindex dad2569a..8cf1b9ff 100644--- a/Makefile+++ b/Makefile@@ -13,5 +13,10 @@ include build/rules.mkinclude llama.cpp/BUILD.mk+# the root package is `o//` by default+# building a package also builds its sub-packages+.PHONY: o/$(MODE)/+o/$(MODE)/: o/$(MODE)/llama.cpp+include build/deps.mkinclude build/tags.mkdiff --git a/build/config.mk b/build/config.mkindex a9978b62..c8cca517 100644--- a/build/config.mk+++ b/build/config.mk@@ -15,12 +15,19 @@ CPPFLAGS += -iquote.TMPDIR = o//tmpIGNORE := $(shell mkdir -p $(TMPDIR))+# apple still distributes a 17 year old version of gnu make+ifeq ($(MAKE_VERSION), 3.81)+$(error please use bin/make from cosmocc.zip rather than old xcode make)+endif++# let `make m=foo` be shorthand for `make MODE=foo`ifneq ($(m),)ifeq ($(MODE),)MODE := $(m)endifendif+# make build more deterministicLC_ALL = CSOURCE_DATE_EPOCH = 0export MODE@@ -28,11 +35,9 @@ export TMPDIRexport LC_ALLexport SOURCE_DATE_EPOCH+# `make` runs `make all` by default.PHONY: allall: o/$(MODE)/-.PHONY: o/$(MODE)/-o/$(MODE)/: o/$(MODE)/llama.cpp-.PHONY: cleanclean:; rm -rf odiff --git a/llama.cpp/common.cc b/llama.cpp/common.ccindex b5d19c7c..aeb6be10 100644--- a/llama.cpp/common.cc+++ b/llama.cpp/common.cc@@ -143,6 +143,7 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {bool invalid_param = false;std::string arg;+ bool passed_gpu_flags = false;const std::string arg_prefix = "--";llama_sampling_params & sparams = params.sparams;@@ -514,43 +515,32 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {} else if (arg == "--mlock") {params.use_mlock = true;} else if (arg == "--gpu-layers" || arg == "-ngl" || arg == "--n-gpu-layers") {+ passed_gpu_flags = true;if (++i >= argc) {invalid_param = true;break;}- if (ggml_metal_supported() || ggml_cuda_supported()) {- params.n_gpu_layers = std::stoi(argv[i]);- } else {- fprintf(stderr, "warning: GPU offload not supported on this platform, --n-gpu-layers option will be ignored\n");- fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n");- }+ params.n_gpu_layers = std::stoi(argv[i]);} else if (arg == "--gpu-layers-draft" || arg == "-ngld" || arg == "--n-gpu-layers-draft") {+ passed_gpu_flags = true;if (++i >= argc) {invalid_param = true;break;}- if (ggml_metal_supported() || ggml_cuda_supported()) {- params.n_gpu_layers_draft = std::stoi(argv[i]);- } else {- fprintf(stderr, "warning: GPU offload not supported on this platform, --n-gpu-layers-draft option will be ignored\n");- fprintf(stderr, "warning: see main README.md for information on enabling GPU BLAS support\n");- }+ params.n_gpu_layers_draft = std::stoi(argv[i]);} else if (arg == "--main-gpu" || arg == "-mg") {+ passed_gpu_flags = true;if (++i >= argc) {invalid_param = true;break;}- if (ggml_cuda_supported()) {- params.main_gpu = std::stoi(argv[i]);- } else {- fprintf(stderr, "warning: cuBLAS GPU couldn't be supported on this system. It is not possible to set a main GPU.\n");- }+ params.main_gpu = std::stoi(argv[i]);} else if (arg == "--tensor-split" || arg == "-ts") {+ passed_gpu_flags = true;if (++i >= argc) {invalid_param = true;break;}- if (ggml_cuda_supported()) {std::string arg_next = argv[i];// split string by , and /@@ -566,15 +556,9 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {params.tensor_split[i] = 0.0f;}}- } else {- fprintf(stderr, "warning: cuBLAS GPU couldn't be supported on this system. It is not possible to set a tensor split.\n");- }} else if (arg == "--no-mul-mat-q" || arg == "-nommq") {- if (ggml_cuda_supported()) {- params.mul_mat_q = false;- } else {- fprintf(stderr, "warning: cuBLAS GPU couldn't be supported on this system. Disabling mul_mat_q kernels has no effect.\n");- }+ passed_gpu_flags = true;+ params.mul_mat_q = false;} else if (arg == "--no-mmap") {params.use_mmap = false;} else if (arg == "--numa") {@@ -740,6 +724,23 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {}}+ if (passed_gpu_flags) {+ // user is tuning their gpu+ if (!ggml_metal_supported() && !ggml_cuda_supported()) {+ fprintf(stderr, "warning: GPU offload not supported on this platform; GPU related options will be ignored\n");+ fprintf(stderr, "warning: you might need to install xcode (macos) or cuda (windows, linux, etc.) check the output above to see why support wasn't linked\n");+ }+ } else {+ // no gpu flags were passed+ if (ggml_metal_supported()) {+ // apple metal gpu doesn't require explicit flags to enable+ } else {+ // avoid the >1 second cuda startup latency if cuda isn't being used+ fprintf(stderr, "protip: pass the --n-gpu-layers N flag to link NVIDIA cuBLAS support\n");+ ggml_cuda_disable();+ }+ }+return true;}diff --git a/llama.cpp/cuda.c b/llama.cpp/cuda.cindex b27ae6e9..cfa33d33 100644--- a/llama.cpp/cuda.c+++ b/llama.cpp/cuda.c@@ -10,6 +10,7 @@#include <unistd.h>#include <stdlib.h>#include <limits.h>+#include <signal.h>#include <pthread.h>#include <sys/wait.h>#include <sys/stat.h>@@ -20,16 +21,33 @@ __static_yoink("llama.cpp/ggml.h");__static_yoink("llama.cpp/ggml-cuda.h");__static_yoink("llama.cpp/ggml-cuda.cu");+#define NVCC_LIBS "-lcublas"++#define NVCC_FLAGS "--shared", \+ "--forward-unknown-to-host-compiler", \+ "-use_fast_math", \+ "-arch=compute_87", \+ "--compiler-options", "-fPIC -O3 -march=native -mtune=native", \+ "-DNDEBUG", \+ "-DGGML_BUILD=1", \+ "-DGGML_SHARED=1", \+ "-DGGML_CUDA_DMMV_X=32", \+ "-DGGML_CUDA_MMV_Y=1", \+ "-DK_QUANTS_PER_ITERATION=2", \+ "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128", \+ "-DGGML_USE_CUBLAS"+static const struct Source {const char *zip;const char *name;} srcs[] = {{"/zip/llama.cpp/ggml.h", "ggml.h"},- {"/zip/llama.cpp/ggml-impl.h", "ggml-cuda.h"},+ {"/zip/llama.cpp/ggml-cuda.h", "ggml-cuda.h"},{"/zip/llama.cpp/ggml-cuda.cu", "ggml-cuda.cu"}, // must come last};static struct Cuda {+ bool disabled;bool supported;atomic_uint once;typeof(ggml_init_cublas) *init;@@ -71,19 +89,98 @@ static const char *GetDsoExtension(void) {}}-static bool CompileNativeCuda(char dso[static PATH_MAX]) {+static bool FileExists(const char *path) {+ struct stat st;+ return !stat(path, &st);+}- // find path of nvidia compiler- char nvcc[PATH_MAX];+static bool IsExecutable(const char *path) {+ struct stat st;+ return !stat(path, &st) &&+ (st.st_mode & 0111) &&+ !S_ISDIR(st.st_mode);+}++static bool CreateTempPath(const char *path, char tmp[static PATH_MAX]) {+ int fd;+ strlcpy(tmp, path, PATH_MAX);+ strlcat(tmp, ".XXXXXX", PATH_MAX);+ if ((fd = mkstemp(tmp)) != -1) {+ close(fd);+ return true;+ } else {+ perror(tmp);+ return false;+ }+}++static bool Compile(const char *src,+ const char *tmp,+ const char *out,+ char *args[]) {+ int pid, ws;+ errno_t err = posix_spawnp(&pid, args[0], NULL, NULL, args, environ);+ if (err) {+ perror(args[0]);+ unlink(tmp);+ return false;+ }+ while (waitpid(pid, &ws, 0) == -1) {+ if (errno != EINTR) {+ perror(args[0]);+ unlink(tmp);+ return false;+ }+ }+ if (ws) {+ tinyprint(2, args[0], ": returned nonzero exit status\n", NULL);+ unlink(tmp);+ return false;+ }+ if (rename(tmp, out)) {+ perror(out);+ unlink(tmp);+ return false;+ }+ return true;+}++// finds nvidia compiler+//+// 1. $CUDA_PATH/bin/nvcc+// 2. nvcc on $PATH environ+// 3. /opt/cuda/bin/nvcc+// 4. /usr/local/cuda/bin/nvcc+//+// set $CUDA_PATH to empty string to disable cuda+static bool GetNvccPath(char path[static PATH_MAX]) {const char *cuda_path;- nvcc[0] = 0;if ((cuda_path = getenv("CUDA_PATH"))) {- strlcat(nvcc, cuda_path, sizeof(nvcc));- strlcat(nvcc, "/bin/", sizeof(nvcc));+ if (!*cuda_path) return false;+ strlcpy(path, cuda_path, PATH_MAX);+ strlcat(path, "/bin/", PATH_MAX);+ } else if (commandv(IsWindows() ? "nvcc.exe" : "nvcc", path, PATH_MAX)) {+ return true;+ } else if (FileExists("/opt/cuda")) {+ strlcpy(path, "/opt/cuda/bin/", PATH_MAX);+ } else {+ strlcpy(path, "/usr/local/cuda/bin/", PATH_MAX);}- strlcat(nvcc, "nvcc", sizeof(nvcc));+ strlcat(path, "nvcc", PATH_MAX);if (IsWindows()) {- strlcat(nvcc, ".exe", sizeof(nvcc));+ strlcat(path, ".exe", PATH_MAX);+ }+ return IsExecutable(path);+}++static bool CompileNativeCuda(char dso[static PATH_MAX]) {++ // find full path of nvidia compiler+ char nvcc[PATH_MAX];+ if (!GetNvccPath(nvcc)) {+ tinyprint(2, "warning: couldn't find nvcc (nvidia c compiler) "+ "try setting $CUDA_PATH if it's installed\n", NULL);+ return false;}// extract source code@@ -96,63 +193,53 @@ static bool CompileNativeCuda(char dso[static PATH_MAX]) {return false;}strlcat(src, srcs[i].name, sizeof(src));- if (ggml_is_newer_than(srcs[i].zip, src)) {- needs_rebuild = true;- if (!ggml_extract(srcs[i].zip, src)) {+ switch (ggml_is_file_newer_than(srcs[i].zip, src)) {+ case -1:return false;- }+ case false:+ break;+ case true:+ needs_rebuild = true;+ if (!ggml_extract(srcs[i].zip, src)) {+ return false;+ }+ break;+ default:+ __builtin_unreachable();}}- // compile dynamic shared object+ // check if dso is already compiledggml_get_app_dir(dso, PATH_MAX);strlcat(dso, "ggml-cuda.", PATH_MAX);strlcat(dso, GetDsoExtension(), PATH_MAX);- if (needs_rebuild || ggml_is_newer_than(src, dso)) {- tinyprint(2, "building ggml-cuda with nvcc...\n", NULL);- int fd;- char tmpdso[PATH_MAX];- strlcpy(tmpdso, dso, sizeof(tmpdso));- strlcat(tmpdso, ".XXXXXX", sizeof(tmpdso));- if ((fd = mkstemp(tmpdso)) != -1) {- close(fd);- } else {- perror(tmpdso);- return false;- }- char *args[] = {- nvcc,- "--shared",- "-arch=native",- "-DGGML_BUILD=1",- "-DGGML_SHARED=1",- "-o", tmpdso,- src,- "-lcublas",- NULL,- };- int pid, ws;- errno_t err = posix_spawnp(&pid, nvcc, NULL, NULL, args, environ);- if (err) {- perror(nvcc);- return false;- }- while (waitpid(pid, &ws, 0) == -1) {- if (errno != EINTR) {- perror(nvcc);+ if (!needs_rebuild) {+ switch (ggml_is_file_newer_than(src, dso)) {+ case -1:return false;- }- }- if (ws) {- tinyprint(2, nvcc, ": returned nonzero exit status\n", NULL);- return false;- }- if (rename(tmpdso, dso)) {- perror(dso);- return false;+ case false:+ return true;+ case true:+ break;+ default:+ __builtin_unreachable();}}+ // create temporary output path for atomicity+ char tmpdso[PATH_MAX];+ if (!CreateTempPath(dso, tmpdso)) {+ return false;+ }++ // try building dso with host nvidia microarchitecture+ tinyprint(2, "building ggml-cuda with nvcc...\n", NULL);+ if (!Compile(src, tmpdso, dso, (char *[]){+ nvcc, NVCC_FLAGS, "-o", tmpdso, src, NVCC_LIBS, NULL})) {+ return false;+ }++ // we're good!return true;}@@ -166,7 +253,6 @@ static bool ImportCudaImpl(void) {// runtime link dynamic shared objectvoid *lib;- tinyprint(2, "OPENING: ", dso, "\n", NULL);lib = cosmo_dlopen(dso, RTLD_LAZY);if (!lib) {tinyprint(2, Dlerror(), ": failed to load library\n", NULL);@@ -205,6 +291,9 @@ static bool ImportCudaImpl(void) {}static void ImportCuda(void) {+ if (ggml_cuda.disabled) {+ return;+ }if (ImportCudaImpl()) {ggml_cuda.supported = true;tinyprint(2, "NVIDIA cuBLAS GPU supported locked and loaded\n", NULL);@@ -213,6 +302,10 @@ static void ImportCuda(void) {}}+void ggml_cuda_disable(void) {+ ggml_cuda.disabled = true;+}+bool ggml_cuda_supported(void) {cosmo_once(&ggml_cuda.once, ImportCuda);return ggml_cuda.supported;diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cuindex 4d6a9691..0630ea15 100644--- a/llama.cpp/ggml-cuda.cu+++ b/llama.cpp/ggml-cuda.cu@@ -4134,7 +4134,7 @@ template <bool need_check> static __global__ void(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q8_0_q8_1_mul_mat;- assert(false);+ assert((__CUDA_ARCH__, 0));#endif // __CUDA_ARCH__ >= CC_VOLTA}diff --git a/llama.cpp/ggml-cuda.h b/llama.cpp/ggml-cuda.hindex 0d9e7641..0cb41da4 100644--- a/llama.cpp/ggml-cuda.h+++ b/llama.cpp/ggml-cuda.h@@ -18,6 +18,7 @@ extern "C" {// Always success. To check if CUDA is actually loaded, use `ggml_cublas_loaded`.GGML_API void ggml_init_cublas(void);+GGML_API void ggml_cuda_disable(void);GGML_API bool ggml_cuda_supported(void);// Returns `true` if there are available CUDA devices and cublas loads successfully; otherwise, it returns `false`.diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 4289d19c..1d079513 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -14322,14 +14322,12 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggmreturn;}-#ifdef GGML_USE_CUBLASbool skip_cpu = ggml_cuda_compute_forward(params, tensor);if (skip_cpu) {return;}GGML_ASSERT(tensor->src[0] == NULL || tensor->src[0]->backend == GGML_BACKEND_CPU);GGML_ASSERT(tensor->src[1] == NULL || tensor->src[1]->backend == GGML_BACKEND_CPU);-#endif // GGML_USE_CUBLASswitch (tensor->op) {case GGML_OP_DUP:diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex 749bd2c7..16cda397 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -2160,7 +2160,7 @@ extern "C" {GGML_API void ggml_get_app_dir(char *path, size_t size);GGML_API void ggml_schlep(const void *data, size_t size);GGML_API bool ggml_extract(const char *zip, const char *to);- GGML_API bool ggml_is_newer_than(const char *path, const char *other);+ GGML_API int ggml_is_file_newer_than(const char *path, const char *other);#ifdef __cplusplus}diff --git a/llama.cpp/metal.c b/llama.cpp/metal.cindex bc08bab9..5a973a13 100644--- a/llama.cpp/metal.c+++ b/llama.cpp/metal.c@@ -69,19 +69,42 @@ static bool ImportMetalImpl(void) {return false;}strlcat(src, srcs[i].name, sizeof(src));- if (ggml_is_newer_than(srcs[i].zip, src)) {- needs_rebuild = true;- if (!ggml_extract(srcs[i].zip, src)) {+ switch (ggml_is_file_newer_than(srcs[i].zip, src)) {+ case -1:return false;- }+ case 0:+ break;+ case 1:+ needs_rebuild = true;+ if (!ggml_extract(srcs[i].zip, src)) {+ return false;+ }+ break;+ default:+ __builtin_unreachable();}}- // compile dynamic shared object+ // determine if we need to buildchar dso[PATH_MAX];ggml_get_app_dir(dso, PATH_MAX);strlcat(dso, "ggml-metal.dylib", sizeof(dso));- if (needs_rebuild || ggml_is_newer_than(src, dso)) {+ if (!needs_rebuild) {+ switch (ggml_is_file_newer_than(src, dso)) {+ case -1:+ return false;+ case 0:+ return true;+ case 1:+ needs_rebuild = true;+ break;+ default:+ __builtin_unreachable();+ }+ }++ // compile dynamic shared object+ if (needs_rebuild) {tinyprint(2, "building ggml-metal.dylib with xcode...\n", NULL);int fd;char tmpdso[PATH_MAX];diff --git a/llama.cpp/schlep.c b/llama.cpp/schlep.cindex 9492964d..ed7ae769 100644--- a/llama.cpp/schlep.c+++ b/llama.cpp/schlep.c@@ -24,9 +24,8 @@ static char Peek(volatile const char *ptr) {char (*pPeek)(volatile const char *) = Peek;static void FormatPercent(char sbuf[static 8], double x) {- x *= 100000.5;- int n = x;char *p = sbuf;+ int n = x * 100000.5;if (n >= 100000) *p++ = '0' + n / 100000 % 10;if (n >= 10000) *p++ = '0' + n / 10000 % 10;*p++ = '0' + n / 1000 % 10;