Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Fix tinyBLAS support
Justine Tunneydfd380461f8a
diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cuindex f31917ab..66c550ab 100644--- a/llama.cpp/ggml-cuda.cu+++ b/llama.cpp/ggml-cuda.cu@@ -8338,7 +8338,7 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, const// there is no broadcast and src0, src1 are contiguous across dims 2, 3// use cublasGemmStridedBatchedExCUBLAS_CHECK(- cublasGemmStridedBatchedEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N,+ cublasGemmStridedBatchedEx(CUBLAS_HANDLE(g_main_device), CUBLAS_OP_T, CUBLAS_OP_N,ne01, ne11, ne10,&alpha_f16, (const char *) src0_as_f16, CUDA_R_16F, nb01/sizeof(half), src0->nb[2]/sizeof(half), // strideA(const char *) src1_as_f16, CUDA_R_16F, nb11/sizeof(float), src1->nb[2]/sizeof(float), // strideB@@ -8372,7 +8372,7 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, constCUDA_CHECK(cudaGetLastError());CUBLAS_CHECK(- cublasGemmBatchedEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N,+ cublasGemmBatchedEx(CUBLAS_HANDLE(g_main_device), CUBLAS_OP_T, CUBLAS_OP_N,ne01, ne11, ne10,&alpha_f16, (const void **) (ptrs_src + 0*ne23), CUDA_R_16F, nb01/sizeof(half),(const void **) (ptrs_src + 1*ne23), CUDA_R_16F, nb11/sizeof(float),diff --git a/llamafile/cuda.c b/llamafile/cuda.cindex 5082f51c..ce0d80ed 100644--- a/llamafile/cuda.c+++ b/llamafile/cuda.c@@ -33,6 +33,8 @@__static_yoink("llama.cpp/ggml.h");__static_yoink("llamafile/compcap.cu");+__static_yoink("llama.cpp/tinyblas.h");+__static_yoink("llama.cpp/tinyblas.cu");__static_yoink("llama.cpp/ggml-impl.h");__static_yoink("llamafile/llamafile.h");__static_yoink("llama.cpp/ggml-cuda.h");@@ -63,6 +65,8 @@ static const struct Source {{"/zip/llama.cpp/ggml.h", "ggml.h"},{"/zip/llamafile/compcap.cu", "compcap.cu"},{"/zip/llamafile/llamafile.h", "llamafile.h"},+ {"/zip/llama.cpp/tinyblas.h", "tinyblas.h"},+ {"/zip/llama.cpp/tinyblas.cu", "tinyblas.cu"},{"/zip/llama.cpp/ggml-impl.h", "ggml-impl.h"},{"/zip/llama.cpp/ggml-cuda.h", "ggml-cuda.h"},{"/zip/llama.cpp/ggml-alloc.h", "ggml-alloc.h"},@@ -473,6 +477,8 @@ static bool ImportCudaImpl(void) {return LinkCudaDso(dso);}+ if (1) exit(1);+// try using a prebuilt dsoif (ExtractCudaDso(dso)) {return LinkCudaDso(dso);