Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Make it possible for CUDA to extract prebuilt DSOs
Justine Tunney2849b08470cd
diff --git a/.gitattributes b/.gitattributesnew file mode 100644index 00000000..c44c4567--- /dev/null+++ b/.gitattributes@@ -0,0 +1,2 @@+# -*- conf -*-+*.dll binarydiff --git a/llama.cpp/tinyblas.cu b/llama.cpp/tinyblas.cuindex f7a3d6c8..ca39d9e7 100644--- a/llama.cpp/tinyblas.cu+++ b/llama.cpp/tinyblas.cu@@ -2,7 +2,7 @@#define READ(A, trans, ld, i, j) \(((trans) == CUBLAS_OP_N) ? (A)[(i) + (j) * (ld)] : (A)[(j) + (i) * (ld)])-#define READ16(...) __half2float(READ(__VA_ARGS__))+#define READ16(A, trans, ld, i, j) __half2float(READ(A, trans, ld, i, j))static __device__ __forceinline__ void matmul(int m, int n, int k,const half *A, int lda,diff --git a/llamafile/cuda.bat b/llamafile/cuda.batnew file mode 100644index 00000000..dce31599--- /dev/null+++ b/llamafile/cuda.bat@@ -0,0 +1,17 @@+call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"++nvcc -arch=all ^+ --shared ^+ --forward-unknown-to-host-compiler ^+ -Xcompiler="/nologo /EHsc /O2 /GR /MT" ^+ -use_fast_math ^+ -DNDEBUG ^+ -DGGML_BUILD=1 ^+ -DGGML_SHARED=1 ^+ -DGGML_CUDA_DMMV_X=32 ^+ -DGGML_CUDA_MMV_Y=1 ^+ -DK_QUANTS_PER_ITERATION=2 ^+ -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 ^+ -DGGML_USE_TINYBLAS ^+ -o ggml-cuda.dll ^+ ggml-cuda.cudiff --git a/llamafile/cuda.c b/llamafile/cuda.cindex 3f0acfe7..9d50962b 100644--- a/llamafile/cuda.c+++ b/llamafile/cuda.c@@ -34,15 +34,13 @@__static_yoink("llama.cpp/ggml.h");__static_yoink("llamafile/compcap.cu");__static_yoink("llamafile/llamafile.h");-__static_yoink("llama.cpp/tinyblas.h");-__static_yoink("llama.cpp/tinyblas.cu");__static_yoink("llama.cpp/ggml-cuda.h");__static_yoink("llama.cpp/ggml-cuda.cu");+__static_yoink("llama.cpp/ggml-cuda.dll");-#define NVCC_LIBS_CUBLAS "-lcublas",-#define NVCC_LIBS_TINYBLAS+#define NVCC_LIBS "-lcublas"-#define NVCC_FLAGS_BASE "--shared", \+#define NVCC_FLAGS "--shared", \"--forward-unknown-to-host-compiler", \"-use_fast_math", \"--compiler-options", "-fPIC -O3 -march=native -mtune=native", \@@ -52,20 +50,8 @@ __static_yoink("llama.cpp/ggml-cuda.cu");"-DGGML_CUDA_DMMV_X=32", \"-DGGML_CUDA_MMV_Y=1", \"-DK_QUANTS_PER_ITERATION=2", \- "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128"--#define NVCC_FLAGS_CUBLAS NVCC_FLAGS_BASE, "-DGGML_USE_CUBLAS"-#define NVCC_FLAGS_TINYBLAS NVCC_FLAGS_BASE, "-DGGML_USE_TINYBLAS"--// change this to TINYBLAS to build against tinyblas.-#define BLAS CUBLAS--// remove this when adding runtime detection of cublas.-#define CONCAT_(A, B) A ## B-#define CONCAT(A, B) CONCAT_(A, B)-#define NVCC_FLAGS CONCAT(NVCC_FLAGS_, BLAS)-#define NVCC_LIBS CONCAT(NVCC_LIBS_, BLAS)-#define NVCC_LIBS_NULL NVCC_LIBS NULL+ "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128", \+ "-DGGML_USE_CUBLAS"static const struct Source {const char *zip;@@ -75,8 +61,6 @@ static const struct Source {{"/zip/llamafile/compcap.cu", "compcap.cu"},{"/zip/llamafile/llamafile.h", "llamafile.h"},{"/zip/llama.cpp/ggml-cuda.h", "ggml-cuda.h"},- {"/zip/llama.cpp/tinyblas.h", "tinyblas.h"},- {"/zip/llama.cpp/tinyblas.cu", "tinyblas.cu"},{"/zip/llama.cpp/ggml-cuda.cu", "ggml-cuda.cu"}, // must come last};@@ -373,7 +357,7 @@ static bool CompileNativeCuda(char dso[static PATH_MAX]) {tinyprint(2, "building ggml-cuda with nvcc -arch=native...\n", NULL);if (Compile(src, tmpdso, dso, (char *[]){nvcc, "-arch=native", NVCC_FLAGS, "-o", tmpdso,- src, NVCC_LIBS_NULL })) {+ src, NVCC_LIBS, NULL})) {return true;}@@ -385,7 +369,7 @@ static bool CompileNativeCuda(char dso[static PATH_MAX]) {tinyprint(2, "building ggml-cuda with nvcc ", archflag, "...\n", NULL);if (Compile(src, tmpdso, dso, (char *[]){nvcc, archflag, NVCC_FLAGS, "-o", tmpdso,- src, NVCC_LIBS_NULL })) {+ src, NVCC_LIBS, NULL})) {return true;}@@ -393,19 +377,39 @@ static bool CompileNativeCuda(char dso[static PATH_MAX]) {return false;}-static bool ImportCudaImpl(void) {+static bool ExtractCudaDso(char dso[static PATH_MAX]) {- // No dynamic linking support on OpenBSD yet.- if (IsOpenbsd()) {+ // see if prebuilt dso is bundled in zip assets+ char zip[80];+ strlcpy(zip, "/zip/llama.cpp/ggml-cuda.", PATH_MAX);+ strlcat(zip, GetDsoExtension(), PATH_MAX);+ if (!FileExists(zip)) {return false;}- // get native cuda dll if possible- char dso[PATH_MAX];- if (!CompileNativeCuda(dso)) {- return false;+ // get destination path+ llamafile_get_app_dir(dso, PATH_MAX);+ strlcat(dso, "ggml-cuda.", PATH_MAX);+ strlcat(dso, GetDsoExtension(), PATH_MAX);++ // check if we need to extract again+ switch (llamafile_is_file_newer_than(zip, dso)) {+ case -1:+ return false;+ case false:+ return true;+ case true:+ break;+ default:+ __builtin_unreachable();}+ // extract prebuilt dso+ return llamafile_extract(zip, dso);+}++static bool LinkCudaDso(const char *dso) {+// runtime link dynamic shared objectvoid *lib;lib = cosmo_dlopen(dso, RTLD_LAZY);@@ -445,6 +449,28 @@ static bool ImportCudaImpl(void) {return true;}+static bool ImportCudaImpl(void) {++ // No dynamic linking support on OpenBSD yet.+ if (IsOpenbsd()) {+ return false;+ }++ // try building cuda code from source using cublas+ char dso[PATH_MAX];+ if (CompileNativeCuda(dso)) {+ return LinkCudaDso(dso);+ }++ // try using a prebuilt dso+ if (ExtractCudaDso(dso)) {+ return LinkCudaDso(dso);+ }++ // too bad+ return false;+}+static void ImportCuda(void) {if (!ggml_cuda.disabled && ImportCudaImpl()) {ggml_cuda.supported = true;diff --git a/llamafile/cuda.sh b/llamafile/cuda.shnew file mode 100755index 00000000..22a3809d--- /dev/null+++ b/llamafile/cuda.sh@@ -0,0 +1,18 @@+#!/bin/sh+set -x+HOST=${1:?HOST}++ssh $HOST mkdir -p lfbuild+scp llama.cpp/ggml-cuda.cu \+ llama.cpp/ggml-cuda.h \+ llama.cpp/tinyblas.cu \+ llama.cpp/tinyblas.h \+ llama.cpp/ggml.h \+ llamafile/llamafile.h \+ llamafile/cuda.bat \+ $HOST:lfbuild/++# TODO(jart): make windows tools able to do automation somehow+# ssh $HOST 'cd lfbuild; C:/WINDOWS/system32/cmd.exe /k cuda.bat'+# mkdir -p o//llama.cpp+# scp $HOST:lfbuild/ggml-cuda.dll o//llama.cpp/