Skip to content

Distribute and run LLMs with a single file.

Fix tinyBLAS support

Justine Tunneydfd380461f8a
diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cu
index f31917ab..66c550ab 100644
--- a/llama.cpp/ggml-cuda.cu
+++ b/llama.cpp/ggml-cuda.cu
@@ -8338,7 +8338,7 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, const
// there is no broadcast and src0, src1 are contiguous across dims 2, 3
// use cublasGemmStridedBatchedEx
CUBLAS_CHECK(
- cublasGemmStridedBatchedEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N,
+ cublasGemmStridedBatchedEx(CUBLAS_HANDLE(g_main_device), CUBLAS_OP_T, CUBLAS_OP_N,
ne01, ne11, ne10,
&alpha_f16, (const char *) src0_as_f16, CUDA_R_16F, nb01/sizeof(half), src0->nb[2]/sizeof(half), // strideA
(const char *) src1_as_f16, CUDA_R_16F, nb11/sizeof(float), src1->nb[2]/sizeof(float), // strideB
@@ -8372,7 +8372,7 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, const
CUDA_CHECK(cudaGetLastError());
CUBLAS_CHECK(
- cublasGemmBatchedEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N,
+ cublasGemmBatchedEx(CUBLAS_HANDLE(g_main_device), CUBLAS_OP_T, CUBLAS_OP_N,
ne01, ne11, ne10,
&alpha_f16, (const void **) (ptrs_src + 0*ne23), CUDA_R_16F, nb01/sizeof(half),
(const void **) (ptrs_src + 1*ne23), CUDA_R_16F, nb11/sizeof(float),
diff --git a/llamafile/cuda.c b/llamafile/cuda.c
index 5082f51c..ce0d80ed 100644
--- a/llamafile/cuda.c
+++ b/llamafile/cuda.c
@@ -33,6 +33,8 @@
__static_yoink("llama.cpp/ggml.h");
__static_yoink("llamafile/compcap.cu");
+__static_yoink("llama.cpp/tinyblas.h");
+__static_yoink("llama.cpp/tinyblas.cu");
__static_yoink("llama.cpp/ggml-impl.h");
__static_yoink("llamafile/llamafile.h");
__static_yoink("llama.cpp/ggml-cuda.h");
@@ -63,6 +65,8 @@ static const struct Source {
{"/zip/llama.cpp/ggml.h", "ggml.h"},
{"/zip/llamafile/compcap.cu", "compcap.cu"},
{"/zip/llamafile/llamafile.h", "llamafile.h"},
+ {"/zip/llama.cpp/tinyblas.h", "tinyblas.h"},
+ {"/zip/llama.cpp/tinyblas.cu", "tinyblas.cu"},
{"/zip/llama.cpp/ggml-impl.h", "ggml-impl.h"},
{"/zip/llama.cpp/ggml-cuda.h", "ggml-cuda.h"},
{"/zip/llama.cpp/ggml-alloc.h", "ggml-alloc.h"},
@@ -473,6 +477,8 @@ static bool ImportCudaImpl(void) {
return LinkCudaDso(dso);
}
+ if (1) exit(1);
+
// try using a prebuilt dso
if (ExtractCudaDso(dso)) {
return LinkCudaDso(dso);