Skip to content

Distribute and run LLMs with a single file.

Make it possible for CUDA to extract prebuilt DSOs

Justine Tunney2849b08470cd
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 00000000..c44c4567
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,2 @@
+# -*- conf -*-
+*.dll binary
diff --git a/llama.cpp/tinyblas.cu b/llama.cpp/tinyblas.cu
index f7a3d6c8..ca39d9e7 100644
--- a/llama.cpp/tinyblas.cu
+++ b/llama.cpp/tinyblas.cu
@@ -2,7 +2,7 @@
#define READ(A, trans, ld, i, j) \
(((trans) == CUBLAS_OP_N) ? (A)[(i) + (j) * (ld)] : (A)[(j) + (i) * (ld)])
-#define READ16(...) __half2float(READ(__VA_ARGS__))
+#define READ16(A, trans, ld, i, j) __half2float(READ(A, trans, ld, i, j))
static __device__ __forceinline__ void matmul(int m, int n, int k,
const half *A, int lda,
diff --git a/llamafile/cuda.bat b/llamafile/cuda.bat
new file mode 100644
index 00000000..dce31599
--- /dev/null
+++ b/llamafile/cuda.bat
@@ -0,0 +1,17 @@
+call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
+
+nvcc -arch=all ^
+ --shared ^
+ --forward-unknown-to-host-compiler ^
+ -Xcompiler="/nologo /EHsc /O2 /GR /MT" ^
+ -use_fast_math ^
+ -DNDEBUG ^
+ -DGGML_BUILD=1 ^
+ -DGGML_SHARED=1 ^
+ -DGGML_CUDA_DMMV_X=32 ^
+ -DGGML_CUDA_MMV_Y=1 ^
+ -DK_QUANTS_PER_ITERATION=2 ^
+ -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 ^
+ -DGGML_USE_TINYBLAS ^
+ -o ggml-cuda.dll ^
+ ggml-cuda.cu
diff --git a/llamafile/cuda.c b/llamafile/cuda.c
index 3f0acfe7..9d50962b 100644
--- a/llamafile/cuda.c
+++ b/llamafile/cuda.c
@@ -34,15 +34,13 @@
__static_yoink("llama.cpp/ggml.h");
__static_yoink("llamafile/compcap.cu");
__static_yoink("llamafile/llamafile.h");
-__static_yoink("llama.cpp/tinyblas.h");
-__static_yoink("llama.cpp/tinyblas.cu");
__static_yoink("llama.cpp/ggml-cuda.h");
__static_yoink("llama.cpp/ggml-cuda.cu");
+__static_yoink("llama.cpp/ggml-cuda.dll");
-#define NVCC_LIBS_CUBLAS "-lcublas",
-#define NVCC_LIBS_TINYBLAS
+#define NVCC_LIBS "-lcublas"
-#define NVCC_FLAGS_BASE "--shared", \
+#define NVCC_FLAGS "--shared", \
"--forward-unknown-to-host-compiler", \
"-use_fast_math", \
"--compiler-options", "-fPIC -O3 -march=native -mtune=native", \
@@ -52,20 +50,8 @@ __static_yoink("llama.cpp/ggml-cuda.cu");
"-DGGML_CUDA_DMMV_X=32", \
"-DGGML_CUDA_MMV_Y=1", \
"-DK_QUANTS_PER_ITERATION=2", \
- "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128"
-
-#define NVCC_FLAGS_CUBLAS NVCC_FLAGS_BASE, "-DGGML_USE_CUBLAS"
-#define NVCC_FLAGS_TINYBLAS NVCC_FLAGS_BASE, "-DGGML_USE_TINYBLAS"
-
-// change this to TINYBLAS to build against tinyblas.
-#define BLAS CUBLAS
-
-// remove this when adding runtime detection of cublas.
-#define CONCAT_(A, B) A ## B
-#define CONCAT(A, B) CONCAT_(A, B)
-#define NVCC_FLAGS CONCAT(NVCC_FLAGS_, BLAS)
-#define NVCC_LIBS CONCAT(NVCC_LIBS_, BLAS)
-#define NVCC_LIBS_NULL NVCC_LIBS NULL
+ "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128", \
+ "-DGGML_USE_CUBLAS"
static const struct Source {
const char *zip;
@@ -75,8 +61,6 @@ static const struct Source {
{"/zip/llamafile/compcap.cu", "compcap.cu"},
{"/zip/llamafile/llamafile.h", "llamafile.h"},
{"/zip/llama.cpp/ggml-cuda.h", "ggml-cuda.h"},
- {"/zip/llama.cpp/tinyblas.h", "tinyblas.h"},
- {"/zip/llama.cpp/tinyblas.cu", "tinyblas.cu"},
{"/zip/llama.cpp/ggml-cuda.cu", "ggml-cuda.cu"}, // must come last
};
@@ -373,7 +357,7 @@ static bool CompileNativeCuda(char dso[static PATH_MAX]) {
tinyprint(2, "building ggml-cuda with nvcc -arch=native...\n", NULL);
if (Compile(src, tmpdso, dso, (char *[]){
nvcc, "-arch=native", NVCC_FLAGS, "-o", tmpdso,
- src, NVCC_LIBS_NULL })) {
+ src, NVCC_LIBS, NULL})) {
return true;
}
@@ -385,7 +369,7 @@ static bool CompileNativeCuda(char dso[static PATH_MAX]) {
tinyprint(2, "building ggml-cuda with nvcc ", archflag, "...\n", NULL);
if (Compile(src, tmpdso, dso, (char *[]){
nvcc, archflag, NVCC_FLAGS, "-o", tmpdso,
- src, NVCC_LIBS_NULL })) {
+ src, NVCC_LIBS, NULL})) {
return true;
}
@@ -393,19 +377,39 @@ static bool CompileNativeCuda(char dso[static PATH_MAX]) {
return false;
}
-static bool ImportCudaImpl(void) {
+static bool ExtractCudaDso(char dso[static PATH_MAX]) {
- // No dynamic linking support on OpenBSD yet.
- if (IsOpenbsd()) {
+ // see if prebuilt dso is bundled in zip assets
+ char zip[80];
+ strlcpy(zip, "/zip/llama.cpp/ggml-cuda.", PATH_MAX);
+ strlcat(zip, GetDsoExtension(), PATH_MAX);
+ if (!FileExists(zip)) {
return false;
}
- // get native cuda dll if possible
- char dso[PATH_MAX];
- if (!CompileNativeCuda(dso)) {
- return false;
+ // get destination path
+ llamafile_get_app_dir(dso, PATH_MAX);
+ strlcat(dso, "ggml-cuda.", PATH_MAX);
+ strlcat(dso, GetDsoExtension(), PATH_MAX);
+
+ // check if we need to extract again
+ switch (llamafile_is_file_newer_than(zip, dso)) {
+ case -1:
+ return false;
+ case false:
+ return true;
+ case true:
+ break;
+ default:
+ __builtin_unreachable();
}
+ // extract prebuilt dso
+ return llamafile_extract(zip, dso);
+}
+
+static bool LinkCudaDso(const char *dso) {
+
// runtime link dynamic shared object
void *lib;
lib = cosmo_dlopen(dso, RTLD_LAZY);
@@ -445,6 +449,28 @@ static bool ImportCudaImpl(void) {
return true;
}
+static bool ImportCudaImpl(void) {
+
+ // No dynamic linking support on OpenBSD yet.
+ if (IsOpenbsd()) {
+ return false;
+ }
+
+ // try building cuda code from source using cublas
+ char dso[PATH_MAX];
+ if (CompileNativeCuda(dso)) {
+ return LinkCudaDso(dso);
+ }
+
+ // try using a prebuilt dso
+ if (ExtractCudaDso(dso)) {
+ return LinkCudaDso(dso);
+ }
+
+ // too bad
+ return false;
+}
+
static void ImportCuda(void) {
if (!ggml_cuda.disabled && ImportCudaImpl()) {
ggml_cuda.supported = true;
diff --git a/llamafile/cuda.sh b/llamafile/cuda.sh
new file mode 100755
index 00000000..22a3809d
--- /dev/null
+++ b/llamafile/cuda.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+set -x
+HOST=${1:?HOST}
+
+ssh $HOST mkdir -p lfbuild
+scp llama.cpp/ggml-cuda.cu \
+ llama.cpp/ggml-cuda.h \
+ llama.cpp/tinyblas.cu \
+ llama.cpp/tinyblas.h \
+ llama.cpp/ggml.h \
+ llamafile/llamafile.h \
+ llamafile/cuda.bat \
+ $HOST:lfbuild/
+
+# TODO(jart): make windows tools able to do automation somehow
+# ssh $HOST 'cd lfbuild; C:/WINDOWS/system32/cmd.exe /k cuda.bat'
+# mkdir -p o//llama.cpp
+# scp $HOST:lfbuild/ggml-cuda.dll o//llama.cpp/