Skip to content

Distribute and run LLMs with a single file.

Support CUDA without cuBLAS (#82)

Jōshin72e1c72591ae
diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cu
index 63487ce8..40497c49 100644
--- a/llama.cpp/ggml-cuda.cu
+++ b/llama.cpp/ggml-cuda.cu
@@ -10,6 +10,7 @@
#include <assert.h>
#if defined(GGML_USE_HIPBLAS)
+
#include <hip/hip_runtime.h>
#include <hipblas/hipblas.h>
#include <hip/hip_fp16.h>
@@ -75,11 +76,32 @@
#define cudaStreamWaitEvent(stream, event, flags) hipStreamWaitEvent(stream, event, flags)
#define cudaStream_t hipStream_t
#define cudaSuccess hipSuccess
+
+#elif defined(GGML_USE_TINYBLAS)
+
+#include "tinyblas.cu"
+#define cublasSgemm tinyblasSgemm
+#define cublasGemmEx tinyblasGemmEx
+#define cublasGemmBatchedEx tinyblasGemmBatchedEx
+#define cublasGemmStridedBatchedEx tinyblasGemmStridedBatchedEx
+
#else
+
#include <cuda_runtime.h>
#include <cublas_v2.h>
#include <cuda_fp16.h>
-#endif // defined(GGML_USE_HIPBLAS)
+
+#endif // defined(GGML_USE_HIPBLAS) || defined(GGML_USE_TINYBLAS)
+
+#if defined(GGML_USE_TINYBLAS)
+#define CUBLAS_ENTRY() cudaStream_t _ggml_stream = nullptr
+#define CUBLAS_SET_STREAM(_, stream) do _ggml_stream = (stream); while (0)
+#define CUBLAS_HANDLE(_) _ggml_stream
+#else
+#define CUBLAS_ENTRY()
+#define CUBLAS_SET_STREAM(id, stream) CUBLAS_CHECK(cublasSetStream(g_cublas_handles[id], stream))
+#define CUBLAS_HANDLE(id) g_cublas_handles[id]
+#endif
#include "ggml-cuda.h"
#include "ggml.h"
@@ -421,7 +443,7 @@ static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");
} \
} while (0)
-#if CUDART_VERSION >= 12000
+#if CUDART_VERSION >= 12000 && !defined(GGML_USE_TINYBLAS)
#define CUBLAS_CHECK(err) \
do { \
cublasStatus_t err_ = (err); \
@@ -446,7 +468,7 @@ static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");
exit(1); \
} \
} while (0)
-#endif // CUDART_VERSION >= 11
+#endif // CUDART_VERSION >= 11 && !defined(GGML_USE_TINYBLAS)
#if CUDART_VERSION >= 11100
#define GGML_CUDA_ASSUME(x) __builtin_assume(x)
@@ -724,7 +746,9 @@ static void * g_scratch_buffer = nullptr;
static size_t g_scratch_size = 0; // disabled by default
static size_t g_scratch_offset = 0;
+#if !defined(GGML_USE_TINYBLAS)
static cublasHandle_t g_cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr};
+#endif // !defined(GGML_USE_TINYBLAS)
static __global__ void add_f32(const float * x, const float * y, float * dst, const int kx, const int ky) {
const int i = blockDim.x*blockIdx.x + threadIdx.x;
@@ -6091,9 +6115,11 @@ void ggml_init_cublas() {
CUDA_CHECK(cudaStreamCreateWithFlags(&g_cudaStreams[id][is], cudaStreamNonBlocking));
}
+#if !defined(GGML_USE_TINYBLAS)
// create cublas handle
CUBLAS_CHECK(cublasCreate(&g_cublas_handles[id]));
CUBLAS_CHECK(cublasSetMathMode(g_cublas_handles[id], CUBLAS_TF32_TENSOR_OP_MATH));
+#endif // !defined(GGML_USE_TINYBLAS)
}
// configure logging to stdout
@@ -6662,6 +6688,7 @@ inline void ggml_cuda_op_mul_mat_cublas(
const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
const int64_t src1_padded_row_size, const cudaStream_t & stream) {
+ CUBLAS_ENTRY();
GGML_ASSERT(src0_dd_i != nullptr);
GGML_ASSERT(src1_ddf_i != nullptr);
GGML_ASSERT(dst_dd_i != nullptr);
@@ -6712,9 +6739,9 @@ inline void ggml_cuda_op_mul_mat_cublas(
const half alpha_f16 = 1.0f;
const half beta_f16 = 0.0f;
- CUBLAS_CHECK(cublasSetStream(g_cublas_handles[id], stream));
+ CUBLAS_SET_STREAM(id, stream);
CUBLAS_CHECK(
- cublasGemmEx(g_cublas_handles[id], CUBLAS_OP_T, CUBLAS_OP_N,
+ cublasGemmEx(CUBLAS_HANDLE(id), CUBLAS_OP_T, CUBLAS_OP_N,
row_diff, src1_ncols, ne10,
&alpha_f16, src0_ptr, CUDA_R_16F, ne00,
src1_ptr, CUDA_R_16F, ne10,
@@ -6750,9 +6777,9 @@ inline void ggml_cuda_op_mul_mat_cublas(
const float alpha = 1.0f;
const float beta = 0.0f;
- CUBLAS_CHECK(cublasSetStream(g_cublas_handles[id], stream));
+ CUBLAS_SET_STREAM(id, stream);
CUBLAS_CHECK(
- cublasSgemm(g_cublas_handles[id], CUBLAS_OP_T, CUBLAS_OP_N,
+ cublasSgemm(CUBLAS_HANDLE(id), CUBLAS_OP_T, CUBLAS_OP_N,
row_diff, src1_ncols, ne10,
&alpha, src0_ddf_i, ne00,
src1_ddf_i, ne10,
@@ -7512,6 +7539,7 @@ __global__ void k_compute_batched_ptrs(
}
static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+ CUBLAS_ENTRY();
GGML_ASSERT(!ggml_is_transposed(src0));
GGML_ASSERT(!ggml_is_transposed(src1));
@@ -7545,7 +7573,7 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, const
int id;
CUDA_CHECK(cudaGetDevice(&id));
- CUBLAS_CHECK(cublasSetStream(g_cublas_handles[id], main_stream));
+ CUBLAS_SET_STREAM(id, main_stream);
ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;
void * src0_ddq = src0_extra->data_device[g_main_device];
@@ -7587,7 +7615,7 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, const
int i02 = i12 / r2;
CUBLAS_CHECK(
- cublasGemmEx(g_cublas_handles[id], CUBLAS_OP_T, CUBLAS_OP_N,
+ cublasGemmEx(CUBLAS_HANDLE(id), CUBLAS_OP_T, CUBLAS_OP_N,
ne01, ne11, ne10,
&alpha_f16, (const char *) src0_as_f16 + i02*src0->nb[2] + i03*src0->nb[3] , CUDA_R_16F, nb01/sizeof(half),
(const char *) src1_as_f16 + i12*src1->nb[2]/2 + i13*src1->nb[3]/2, CUDA_R_16F, nb11/sizeof(float),
@@ -7602,7 +7630,7 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, const
// there is no broadcast and src0, src1 are contiguous across dims 2, 3
// use cublasGemmStridedBatchedEx
CUBLAS_CHECK(
- cublasGemmStridedBatchedEx(g_cublas_handles[id], CUBLAS_OP_T, CUBLAS_OP_N,
+ cublasGemmStridedBatchedEx(CUBLAS_HANDLE(id), CUBLAS_OP_T, CUBLAS_OP_N,
ne01, ne11, ne10,
&alpha_f16, (const char *) src0_as_f16, CUDA_R_16F, nb01/sizeof(half), src0->nb[2]/sizeof(half), // strideA
(const char *) src1_as_f16, CUDA_R_16F, nb11/sizeof(float), src1->nb[2]/sizeof(float), // strideB
@@ -7636,7 +7664,7 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, const
CUDA_CHECK(cudaGetLastError());
CUBLAS_CHECK(
- cublasGemmBatchedEx(g_cublas_handles[id], CUBLAS_OP_T, CUBLAS_OP_N,
+ cublasGemmBatchedEx(CUBLAS_HANDLE(id), CUBLAS_OP_T, CUBLAS_OP_N,
ne01, ne11, ne10,
&alpha_f16, (const void **) (ptrs_src + 0*ne23), CUDA_R_16F, nb01/sizeof(half),
(const void **) (ptrs_src + 1*ne23), CUDA_R_16F, nb11/sizeof(float),
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 9480a5bc..138daf86 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -16183,7 +16183,7 @@ struct ggml_cplan ggml_graph_plan(struct ggml_cgraph * cgraph, int n_threads) {
size_t cur = 0;
const enum ggml_type vec_dot_type = type_traits[node->src[0]->type].vec_dot_type;
-#if defined(GGML_USE_CUBLAS)
+#if defined(GGML_USE_CUBLAS) || defined(GGML_USE_TINYBLAS)
if (ggml_cuda_can_mul_mat(node->src[0], node->src[1], node)) {
n_tasks = 1; // TODO: this actually is doing nothing
// the threads are still spinning
@@ -19607,7 +19607,7 @@ int ggml_cpu_has_wasm_simd(void) {
}
int ggml_cpu_has_blas(void) {
-#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) || defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST)
+#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS) || defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST) || defined(GGML_USE_TINYBLAS)
return 1;
#else
return 0;
diff --git a/llama.cpp/tinyblas.cu b/llama.cpp/tinyblas.cu
new file mode 100644
index 00000000..f7a3d6c8
--- /dev/null
+++ b/llama.cpp/tinyblas.cu
@@ -0,0 +1,230 @@
+#include "tinyblas.h"
+
+#define READ(A, trans, ld, i, j) \
+ (((trans) == CUBLAS_OP_N) ? (A)[(i) + (j) * (ld)] : (A)[(j) + (i) * (ld)])
+#define READ16(...) __half2float(READ(__VA_ARGS__))
+
+static __device__ __forceinline__ void matmul(int m, int n, int k,
+ const half *A, int lda,
+ const half *B, int ldb,
+ half *C, int ldc) {
+ for (int i = 0; i < m; ++i) {
+ for (int j = 0; j < n; ++j) {
+ float sum = 0.0;
+ half *cptr = C + i + j * ldc;
+ for (int l = 0; l < k; ++l) {
+ sum += READ16(A, CUBLAS_OP_T, lda, i, l) *
+ READ16(B, CUBLAS_OP_N, ldb, l, j);
+ }
+ *cptr = __float2half(sum);
+ }
+ }
+}
+
+static __global__ void wrap_matmul(int m, int n, int k, const half *A, int lda,
+ const half *B, int ldb, half *C, int ldc) {
+ matmul(m, n, k, A, lda, B, ldb, C, ldc);
+}
+
+static __global__ void matmul32(int m, int n, int k, const float *A, int lda,
+ const float *B, int ldb, float *C, int ldc) {
+ for (int i = 0; i < m; ++i) {
+ for (int j = 0; j < n; ++j) {
+ float sum = 0.0;
+ float *cptr = C + i + j * ldc;
+ for (int l = 0; l < k; ++l) {
+ sum += READ(A, CUBLAS_OP_T, lda, i, l) *
+ READ(B, CUBLAS_OP_N, ldb, l, j);
+ }
+ *cptr = sum;
+ }
+ }
+}
+
+static bool check_args(cublasOperation_t transa, cublasOperation_t transb,
+ const void *pAlpha, cudaDataType_t Atype,
+ cudaDataType_t Btype, const void *pBeta,
+ cudaDataType_t Ctype, cublasComputeType_t computeType) {
+ return transa == CUBLAS_OP_T && transb == CUBLAS_OP_N &&
+ Atype == CUDA_R_16F && Btype == CUDA_R_16F && Ctype == CUDA_R_16F &&
+ computeType == CUBLAS_COMPUTE_16F &&
+ __half2float(*(half *)pAlpha) == 1.0f &&
+ __half2float(*(half *)pBeta) == 0.0f;
+}
+
+cublasStatus_t tinyblasSgemm(cudaStream_t stream,
+ cublasOperation_t transa,
+ cublasOperation_t transb,
+ int m, int n, int k,
+ const float *alpha,
+ const float *A, int lda,
+ const float *B, int ldb,
+ const float *beta,
+ float *C, int ldc) {
+ if (transa != CUBLAS_OP_T || transb != CUBLAS_OP_N ||
+ *alpha != 1.0f || *beta != 0.0f) {
+ return CUBLAS_STATUS_NOT_SUPPORTED;
+ }
+ matmul32<<<1, 1, 0, stream>>>(m, n, k, A, lda, B, ldb, C, ldc);
+ return CUBLAS_STATUS_SUCCESS;
+}
+
+// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmex
+
+cublasStatus_t tinyblasGemmEx(cudaStream_t stream,
+ cublasOperation_t transa,
+ cublasOperation_t transb,
+ int m,
+ int n,
+ int k,
+ const void *alpha,
+ const void *A,
+ cudaDataType_t Atype,
+ int lda,
+ const void *B,
+ cudaDataType_t Btype,
+ int ldb,
+ const void *beta,
+ void *C,
+ cudaDataType_t Ctype,
+ int ldc,
+ cublasComputeType_t computeType,
+ cublasGemmAlgo_t algo) {
+ if (!check_args(transa, transb, alpha, Atype, Btype, beta, Ctype,
+ computeType)) {
+ return CUBLAS_STATUS_NOT_SUPPORTED;
+ }
+
+ wrap_matmul<<<1, 1, 0, stream>>>(
+ m, n, k, (const half*)A, lda, (const half *)B, ldb, (half *)C, ldc);
+ return CUBLAS_STATUS_SUCCESS;
+}
+
+// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmbatchedex
+
+static __global__ void tinyblasGBE_entry(int m, int n, int k,
+ const half *const Aarray[],
+ int lda,
+ const half *const Barray[],
+ int ldb,
+ half *const Carray[],
+ int ldc,
+ int batchCount) {
+ int x = blockIdx.x * blockDim.x + threadIdx.x;
+ int jump = blockDim.x * gridDim.x;
+
+ for (; x < batchCount; x += jump) {
+ matmul(m, n, k, Aarray[x], lda, Barray[x], ldb, Carray[x], ldc);
+ }
+}
+
+cublasStatus_t tinyblasGemmBatchedEx(cudaStream_t stream,
+ cublasOperation_t transa,
+ cublasOperation_t transb,
+ int m,
+ int n,
+ int k,
+ const void *alpha,
+ const void *const Aarray[],
+ cudaDataType_t Atype,
+ int lda,
+ const void *const Barray[],
+ cudaDataType_t Btype,
+ int ldb,
+ const void *beta,
+ void *const Carray[],
+ cudaDataType_t Ctype,
+ int ldc,
+ int batchCount,
+ cublasComputeType_t computeType,
+ cublasGemmAlgo_t algo) {
+ if (!check_args(transa, transb, alpha, Atype, Btype, beta, Ctype,
+ computeType)) {
+ return CUBLAS_STATUS_NOT_SUPPORTED;
+ }
+
+ // https://developer.nvidia.com/blog/cuda-pro-tip-write-flexible-kernels-grid-stride-loops/
+ int numSMs, devId;
+ cudaGetDevice(&devId);
+ cudaDeviceGetAttribute(&numSMs, cudaDevAttrMultiProcessorCount, devId);
+ int maxblocks = 16 * numSMs;
+ int maxthreads = 128;
+
+ tinyblasGBE_entry<<<maxblocks, maxthreads, 0, stream>>>(
+ m, n, k, (const half **)Aarray, lda, (const half **)Barray, ldb,
+ (half **)Carray, ldc, batchCount);
+ return CUBLAS_STATUS_SUCCESS;
+}
+
+// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmstridedbatchedex
+
+#define STRIDE0(A, i, stride) ((A) + (i) * (stride))
+#define STRIDE(A, type, i, stride) \
+ ((type) == CUDA_R_16F \
+ ? (void *)STRIDE0((half *)(A), (i), (stride)) \
+ : (void *)STRIDE0((float *)(A), (i), (stride)))
+#define STRIDE_CONST(A, type, i, stride) \
+ ((type) == CUDA_R_16F \
+ ? (const void *)STRIDE0((const half *)(A), (i), (stride)) \
+ : (const void *)STRIDE0((const float *)(A), (i), (stride)))
+
+static __global__ void tinyblasGSBE_entry(int m, int n, int k,
+ const half *A,
+ int lda,
+ long long int strideA,
+ const half *B,
+ int ldb,
+ long long int strideB,
+ half *C,
+ int ldc,
+ long long int strideC,
+ int batchCount) {
+ int x = blockIdx.x * blockDim.x + threadIdx.x;
+ int jump = blockDim.x * gridDim.x;
+
+ for (; x < batchCount; x += jump) {
+ matmul(m, n, k, A + x * strideA, lda, B + x * strideB, ldb, C + x * strideC,
+ ldc);
+ }
+}
+
+cublasStatus_t tinyblasGemmStridedBatchedEx(cudaStream_t stream,
+ cublasOperation_t transa,
+ cublasOperation_t transb,
+ int m, int n, int k,
+ const void *pAlpha,
+ const void *A,
+ cudaDataType_t Atype,
+ int lda,
+ long long int strideA,
+ const void *B,
+ cudaDataType_t Btype,
+ int ldb,
+ long long int strideB,
+ const void *pBeta,
+ void *C,
+ cudaDataType_t Ctype,
+ int ldc,
+ long long int strideC,
+ int batchCount,
+ cublasComputeType_t computeType,
+ cublasGemmAlgo_t algo) {
+ if (!check_args(transa, transb, pAlpha, Atype, Btype, pBeta, Ctype,
+ computeType)) {
+ return CUBLAS_STATUS_NOT_SUPPORTED;
+ }
+
+ // https://developer.nvidia.com/blog/cuda-pro-tip-write-flexible-kernels-grid-stride-loops/
+ int numSMs, devId;
+ cudaGetDevice(&devId);
+ cudaDeviceGetAttribute(&numSMs, cudaDevAttrMultiProcessorCount, devId);
+ int maxblocks = 16 * numSMs;
+ int maxthreads = 128;
+
+ // call the entry function
+ tinyblasGSBE_entry<<<maxblocks, maxthreads, 0, stream>>>(
+ m, n, k, (const half*)A, lda, strideA, (const half*)B, ldb, strideB,
+ (half *)C, ldc, strideC, batchCount);
+
+ return CUBLAS_STATUS_SUCCESS;
+}
diff --git a/llama.cpp/tinyblas.h b/llama.cpp/tinyblas.h
new file mode 100644
index 00000000..3e31e3fb
--- /dev/null
+++ b/llama.cpp/tinyblas.h
@@ -0,0 +1,98 @@
+#pragma once
+
+#include <cuda_runtime.h>
+#include <cuda_fp16.h>
+
+enum cublasOperation_t {
+ CUBLAS_OP_N = 111,
+ CUBLAS_OP_T = 112,
+ CUBLAS_OP_C = 113,
+};
+
+enum cublasStatus_t {
+ CUBLAS_STATUS_SUCCESS = 0,
+ CUBLAS_STATUS_NOT_SUPPORTED = 7,
+};
+
+enum cublasComputeType_t {
+ CUBLAS_COMPUTE_16F = 150,
+};
+
+enum cublasGemmAlgo_t {
+ CUBLAS_GEMM_DEFAULT_TENSOR_OP = 160,
+};
+
+#define cublasHandle_t cudaStream_t
+
+cublasStatus_t tinyblasSgemm(cublasHandle_t handle,
+ cublasOperation_t transa,
+ cublasOperation_t transb,
+ int m, int n, int k,
+ const float *alpha,
+ const float *A, int lda,
+ const float *B, int ldb,
+ const float *beta,
+ float *C, int ldc);
+
+cublasStatus_t tinyblasGemmEx(cublasHandle_t handle,
+ cublasOperation_t transa,
+ cublasOperation_t transb,
+ int m,
+ int n,
+ int k,
+ const void *alpha,
+ const void *A,
+ cudaDataType_t Atype,
+ int lda,
+ const void *B,
+ cudaDataType_t Btype,
+ int ldb,
+ const void *beta,
+ void *C,
+ cudaDataType_t Ctype,
+ int ldc,
+ cublasComputeType_t computeType,
+ cublasGemmAlgo_t algo);
+
+cublasStatus_t tinyblasGemmBatchedEx(cublasHandle_t handle,
+ cublasOperation_t transa,
+ cublasOperation_t transb,
+ int m,
+ int n,
+ int k,
+ const void *alpha,
+ const void *const Aarray[],
+ cudaDataType_t Atype,
+ int lda,
+ const void *const Barray[],
+ cudaDataType_t Btype,
+ int ldb,
+ const void *beta,
+ void *const Carray[],
+ cudaDataType_t Ctype,
+ int ldc,
+ int batchCount,
+ cublasComputeType_t computeType,
+ cublasGemmAlgo_t algo);
+
+cublasStatus_t tinyblasGemmStridedBatchedEx(cublasHandle_t handle,
+ cublasOperation_t transa,
+ cublasOperation_t transb,
+ int m, int n, int k,
+ const void *pAlpha,
+ const void *A,
+ cudaDataType_t Atype,
+ int lda,
+ long long int strideA,
+ const void *B,
+ cudaDataType_t Btype,
+ int ldb,
+ long long int strideB,
+ const void *pBeta,
+ void *C,
+ cudaDataType_t Ctype,
+ int ldc,
+ long long int strideC,
+ int batchCount,
+ cublasComputeType_t computeType,
+ cublasGemmAlgo_t algo);
diff --git a/llamafile/cuda.c b/llamafile/cuda.c
index ee3371cf..3f0acfe7 100644
--- a/llamafile/cuda.c
+++ b/llamafile/cuda.c
@@ -34,12 +34,15 @@
__static_yoink("llama.cpp/ggml.h");
__static_yoink("llamafile/compcap.cu");
__static_yoink("llamafile/llamafile.h");
+__static_yoink("llama.cpp/tinyblas.h");
+__static_yoink("llama.cpp/tinyblas.cu");
__static_yoink("llama.cpp/ggml-cuda.h");
__static_yoink("llama.cpp/ggml-cuda.cu");
-#define NVCC_LIBS "-lcublas"
+#define NVCC_LIBS_CUBLAS "-lcublas",
+#define NVCC_LIBS_TINYBLAS
-#define NVCC_FLAGS "--shared", \
+#define NVCC_FLAGS_BASE "--shared", \
"--forward-unknown-to-host-compiler", \
"-use_fast_math", \
"--compiler-options", "-fPIC -O3 -march=native -mtune=native", \
@@ -49,8 +52,20 @@ __static_yoink("llama.cpp/ggml-cuda.cu");
"-DGGML_CUDA_DMMV_X=32", \
"-DGGML_CUDA_MMV_Y=1", \
"-DK_QUANTS_PER_ITERATION=2", \
- "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128", \
- "-DGGML_USE_CUBLAS"
+ "-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128"
+
+#define NVCC_FLAGS_CUBLAS NVCC_FLAGS_BASE, "-DGGML_USE_CUBLAS"
+#define NVCC_FLAGS_TINYBLAS NVCC_FLAGS_BASE, "-DGGML_USE_TINYBLAS"
+
+// change this to TINYBLAS to build against tinyblas.
+#define BLAS CUBLAS
+
+// remove this when adding runtime detection of cublas.
+#define CONCAT_(A, B) A ## B
+#define CONCAT(A, B) CONCAT_(A, B)
+#define NVCC_FLAGS CONCAT(NVCC_FLAGS_, BLAS)
+#define NVCC_LIBS CONCAT(NVCC_LIBS_, BLAS)
+#define NVCC_LIBS_NULL NVCC_LIBS NULL
static const struct Source {
const char *zip;
@@ -60,6 +75,8 @@ static const struct Source {
{"/zip/llamafile/compcap.cu", "compcap.cu"},
{"/zip/llamafile/llamafile.h", "llamafile.h"},
{"/zip/llama.cpp/ggml-cuda.h", "ggml-cuda.h"},
+ {"/zip/llama.cpp/tinyblas.h", "tinyblas.h"},
+ {"/zip/llama.cpp/tinyblas.cu", "tinyblas.cu"},
{"/zip/llama.cpp/ggml-cuda.cu", "ggml-cuda.cu"}, // must come last
};
@@ -356,7 +373,7 @@ static bool CompileNativeCuda(char dso[static PATH_MAX]) {
tinyprint(2, "building ggml-cuda with nvcc -arch=native...\n", NULL);
if (Compile(src, tmpdso, dso, (char *[]){
nvcc, "-arch=native", NVCC_FLAGS, "-o", tmpdso,
- src, NVCC_LIBS, NULL})) {
+ src, NVCC_LIBS_NULL })) {
return true;
}
@@ -368,7 +385,7 @@ static bool CompileNativeCuda(char dso[static PATH_MAX]) {
tinyprint(2, "building ggml-cuda with nvcc ", archflag, "...\n", NULL);
if (Compile(src, tmpdso, dso, (char *[]){
nvcc, archflag, NVCC_FLAGS, "-o", tmpdso,
- src, NVCC_LIBS, NULL})) {
+ src, NVCC_LIBS_NULL })) {
return true;
}