Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Fixup commit for 84490a7bca53 (#265)
Gautham725eca9fb963
diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cuindex d73c9bfa..336e9ccd 100644--- a/llamafile/tinyblas.cu+++ b/llamafile/tinyblas.cu@@ -17,7 +17,7 @@#include "tinyblas.h"-#define READ(A, trans, ld, i, j) \+#define READ(A, trans, ld, i, j) \(((trans) == TINYBLAS_OP_N) ? (A)[(i) + (j) * (ld)] : (A)[(j) + (i) * (ld)])#define READ16(A, trans, ld, i, j) __half2float(READ(A, trans, ld, i, j))@@ -33,35 +33,33 @@ __device__ __forceinline__ DST typechange(SRC x) {}}-template <>-__device__ __forceinline__ float typechange(half x) {+template <> __device__ __forceinline__ float typechange(half x) {return __half2float(x);}-template <>-__device__ __forceinline__ half typechange(float x) {+template <> __device__ __forceinline__ half typechange(float x) {return __float2half(x);}template <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST>-static __device__ void matmul_block2d(tinyblasOperation_t transa, //- tinyblasOperation_t transb, //- int m, int n, int k, //- int x, int y, //- const DST alpha, //- const SRC *A, int lda, //- const SRC *B, int ldb, //- const DST beta, //+static __device__ void matmul_block2d(tinyblasOperation_t transa, //+ tinyblasOperation_t transb, //+ int m, int n, int k, //+ int x, int y, //+ const DST alpha, //+ const SRC *A, int lda, //+ const SRC *B, int ldb, //+ const DST beta, //DST *C, int ldc) {static_assert((BM % TM == 0) && (BN % TN == 0),"can't divide work for threads");- static_assert(BK == ((BM * BN) / (TM * TN)), // optional+ static_assert(BK == ((BM * BN) / (TM * TN)), // optional"threads can't load memory properly");static_assert((BM * BN) <= (BM * BK) + (BK * BN),"didn't allocate enough shared mem for threads");const int ii0 = threadIdx.x / (BN / TN); /* {0, ..., (BM/TM) - 1} */const int ii1 = threadIdx.x % (BN / TN); /* {0, ..., (BN/TN) - 1} */- extern __shared__ float svals[]; // shared across all threads in a block+ extern __shared__ float svals[]; // shared across all threads in a blockSRC *As = (SRC *)svals;SRC *Bs = (SRC *)svals + BM * BK;@@ -71,7 +69,8 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, //int i, h, j, l, blob;// within each block// we first zero out Cs- for (j = 0; j < TM * TN; ++j) Cs[j] = 0;+ for (j = 0; j < TM * TN; ++j)+ Cs[j] = 0;for (blob = 0; blob < k; blob += BK) {for (i = threadIdx.x; i < BK; i += blockDim.x) {@@ -97,8 +96,10 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, //// We matmul the blobs, basically Cs += matmul(As, Bs)for (l = 0; l < BK; ++l) {- for (j = 0; j < TM; ++j) At[j] = As[(l * BM) + ii0 * TM + j];- for (h = 0; h < TN; ++h) Bt[h] = Bs[(l * BN) + ii1 * TN + h];+ for (j = 0; j < TM; ++j)+ At[j] = As[(l * BM) + ii0 * TM + j];+ for (h = 0; h < TN; ++h)+ Bt[h] = Bs[(l * BN) + ii1 * TN + h];for (j = 0; j < TM; ++j) {for (h = 0; h < TN; ++h) {Cs[j * TN + h] += At[j] * Bt[h];@@ -115,21 +116,21 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, //for (j = 0; j < TM && x + j < m; ++j) {for (l = 0; l < TN && y + l < n; ++l) {*(C + (x + j) + (y + l) * ldc) =- alpha * typechange<SRC, DST>(Cs[j * TN + l]) +- beta * (*(C + (x + j) + (y + l) * ldc));+ alpha * typechange<SRC, DST>(Cs[j * TN + l]);+ // + beta * (*(C + (x + j) + (y + l) * ldc));}}__syncthreads();}template <int BM, int BN, int BK, int TM, int TN>-static __global__ void tinyblasS_entry(tinyblasOperation_t transa, //- tinyblasOperation_t transb, //- int m, int n, int k, //- const float alpha, //- const float *A, int lda, //- const float *B, int ldb, //- const float beta, //+static __global__ void tinyblasS_entry(tinyblasOperation_t transa, //+ tinyblasOperation_t transb, //+ int m, int n, int k, //+ const float alpha, //+ const float *A, int lda, //+ const float *B, int ldb, //+ const float beta, //float *C, int ldc) {int x = blockIdx.x * BM;const int jump1 = gridDim.x * BM;@@ -148,21 +149,22 @@ static __global__ void tinyblasS_entry(tinyblasOperation_t transa, //}template <int BM, int BN, int BK, int TM, int TN>-static void tinyblasS_wrapper(tinyblasHandle_t stream, //- tinyblasOperation_t transa, //- tinyblasOperation_t transb, //- int m, int n, int k, //- const float alpha, //- const float *A, int lda, //- const float *B, int ldb, //- const float beta, //+static void tinyblasS_wrapper(tinyblasHandle_t stream, //+ tinyblasOperation_t transa, //+ tinyblasOperation_t transb, //+ int m, int n, int k, //+ const float alpha, //+ const float *A, int lda, //+ const float *B, int ldb, //+ const float beta, //float *C, int ldc) {dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 1);int maxthreads = ((BM * BN) / (TM * TN));tinyblasS_entry<BM, BN, BK, TM, TN><<<maxblocks, maxthreads, (sizeof(float) * (BM * BK + BK * BN)),- stream>>>(transa, transb, m, n, k, A, lda, B, ldb, C, ldc);+ stream>>>(transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C,+ ldc);}tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t stream,@@ -187,13 +189,13 @@ static inline bool check_args(cudaDataType_t Atype, cudaDataType_t Btype,// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmextemplate <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST>-static __global__ void tinyblasGE_entry(tinyblasOperation_t transa, //- tinyblasOperation_t transb, //- int m, int n, int k, //- const DST alpha, //- const SRC *A, int lda, //- const SRC *B, int ldb, //- const DST beta, //+static __global__ void tinyblasGE_entry(tinyblasOperation_t transa, //+ tinyblasOperation_t transb, //+ int m, int n, int k, //+ const DST alpha, //+ const SRC *A, int lda, //+ const SRC *B, int ldb, //+ const DST beta, //DST *C, int ldc) {int x = blockIdx.x * BM;const int jump1 = gridDim.x * BM;@@ -204,22 +206,22 @@ static __global__ void tinyblasGE_entry(tinyblasOperation_t transa, //for (x = blockIdx.x * BM; x < m; x += jump1) {for (y = blockIdx.y * BN; y < n; y += jump2) {matmul_block2d<BM, BN, BK, TM, TN, SRC, DST>(- transa, transb, m, n, k, x, y, //- alpha, A, lda, B, ldb, //+ transa, transb, m, n, k, x, y, //+ alpha, A, lda, B, ldb, //beta, C, ldc);}}}template <int BM, int BN, int BK, int TM, int TN, typename SRC>-static void tinyblasGE_wrapper(tinyblasHandle_t stream, //- tinyblasOperation_t transa, //- tinyblasOperation_t transb, //- int m, int n, int k, //- const float alpha, //- const SRC *A, int lda, //- const SRC *B, int ldb, //- const float beta, //+static void tinyblasGE_wrapper(tinyblasHandle_t stream, //+ tinyblasOperation_t transa, //+ tinyblasOperation_t transb, //+ int m, int n, int k, //+ const float alpha, //+ const SRC *A, int lda, //+ const SRC *B, int ldb, //+ const float beta, //void *C, cudaDataType_t Ctype, int ldc) {dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 1);int maxthreads = ((BM * BN) / (TM * TN));@@ -237,32 +239,30 @@ static void tinyblasGE_wrapper(tinyblasHandle_t stream, //}}-tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t stream,- tinyblasOperation_t transa,- tinyblasOperation_t transb, int m, int n, int k,- const void *alpha, const void *A,- cudaDataType_t Atype, int lda, const void *B,- cudaDataType_t Btype, int ldb, const void *beta,- void *C, cudaDataType_t Ctype, int ldc,- tinyblasComputeType_t computeType,- tinyblasGemmAlgo_t algo) {+tinyblasStatus_t+tinyblasGemmEx(tinyblasHandle_t stream, tinyblasOperation_t transa,+ tinyblasOperation_t transb, int m, int n, int k,+ const void *alpha, const void *A, cudaDataType_t Atype, int lda,+ const void *B, cudaDataType_t Btype, int ldb, const void *beta,+ void *C, cudaDataType_t Ctype, int ldc,+ tinyblasComputeType_t computeType, tinyblasGemmAlgo_t algo) {if (!check_args(Atype, Btype, Ctype, computeType)) {return TINYBLAS_STATUS_NOT_SUPPORTED;}float alpha0, beta0;- if (computeType == CUBLAS_COMPUTE_32F) {+ if (computeType == TINYBLAS_COMPUTE_32F) {alpha0 = *(float *)alpha;beta0 = *(float *)beta;- } else /* computeType == CUBLAS_COMPUTE_16F */ {+ } else /* computeType == TINYBLAS_COMPUTE_16F */ {alpha0 = __half2float(*(half *)alpha);beta0 = __half2float(*(half *)beta);}tinyblasGE_wrapper<48, 32, 64, 3, 8, half>(stream, transa, transb, m, n, k,alpha0, (const half *)A, lda,- (const half *)B, ldb, //+ (const half *)B, ldb, //beta0, C, Ctype, ldc);return TINYBLAS_STATUS_SUCCESS;}@@ -270,13 +270,13 @@ tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t stream,// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmbatchedextemplate <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST>-static __global__ void tinyblasGBE_entry(tinyblasOperation_t transa, //- tinyblasOperation_t transb, //- int m, int n, int k, //- const DST alpha, //- const SRC *const Aarray[], int lda, //- const SRC *const Barray[], int ldb, //- const DST beta, //+static __global__ void tinyblasGBE_entry(tinyblasOperation_t transa, //+ tinyblasOperation_t transb, //+ int m, int n, int k, //+ const DST alpha, //+ const SRC *const Aarray[], int lda, //+ const SRC *const Barray[], int ldb, //+ const DST beta, //void *const Carray[], int ldc,int batchCount) {int x = blockIdx.x * BM;@@ -300,26 +300,26 @@ static __global__ void tinyblasGBE_entry(tinyblasOperation_t transa, //template <int BM, int BN, int BK, int TM, int TN, typename SRC>static void tinyblasGBE_wrapper(tinyblasHandle_t stream,- tinyblasOperation_t transa, //- tinyblasOperation_t transb, //- int m, int n, int k, //- const float alpha, //- const SRC *const Aarray[], int lda, //- const SRC *const Barray[], int ldb, //- const float beta, //+ tinyblasOperation_t transa, //+ tinyblasOperation_t transb, //+ int m, int n, int k, //+ const float alpha, //+ const SRC *const Aarray[], int lda, //+ const SRC *const Barray[], int ldb, //+ const float beta, //void *const Carray[], cudaDataType_t Ctype,int ldc, int batchCount) {dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 32);int maxthreads = ((BM * BN) / (TM * TN));if (Ctype == CUDA_R_16F) {- tinyblasGBE_entry<BM, BN, BK, TM, TN, SRC>+ tinyblasGBE_entry<BM, BN, BK, TM, TN, SRC, half><<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)),stream>>>(transa, transb, m, n, k, __float2half(alpha), Aarray,lda, Barray, ldb, __float2half(beta), Carray, ldc,batchCount);} else {- tinyblasGBE_entry<BM, BN, BK, TM, TN, float>+ tinyblasGBE_entry<BM, BN, BK, TM, TN, SRC, float><<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)),stream>>>(transa, transb, m, n, k, alpha, Aarray, lda, Barray,ldb, beta, Carray, ldc, batchCount);@@ -327,12 +327,12 @@ static void tinyblasGBE_wrapper(tinyblasHandle_t stream,}tinyblasStatus_t tinyblasGemmBatchedEx(- tinyblasHandle_t stream, //- tinyblasOperation_t transa, tinyblasOperation_t transb, //- int m, int n, int k, const void *alpha, //- const void *const Aarray[], cudaDataType_t Atype, int lda, //- const void *const Barray[], cudaDataType_t Btype, int ldb, //- const void *beta, void *const Carray[], cudaDataType_t Ctype, int ldc, //+ tinyblasHandle_t stream, //+ tinyblasOperation_t transa, tinyblasOperation_t transb, //+ int m, int n, int k, const void *alpha, //+ const void *const Aarray[], cudaDataType_t Atype, int lda, //+ const void *const Barray[], cudaDataType_t Btype, int ldb, //+ const void *beta, void *const Carray[], cudaDataType_t Ctype, int ldc, //int batchCount, tinyblasComputeType_t computeType,tinyblasGemmAlgo_t algo) {if (!check_args(Atype, Btype, Ctype, computeType)) {@@ -340,31 +340,33 @@ tinyblasStatus_t tinyblasGemmBatchedEx(}float alpha0, beta0;- if (computeType == CUBLAS_COMPUTE_32F) {+ if (computeType == TINYBLAS_COMPUTE_32F) {alpha0 = *(float *)alpha;beta0 = *(float *)beta;- } else /* computeType == CUBLAS_COMPUTE_16F */ {+ } else /* computeType == TINYBLAS_COMPUTE_16F */ {alpha0 = __half2float(*(half *)alpha);beta0 = __half2float(*(half *)beta);}tinyblasGBE_wrapper<48, 32, 64, 3, 8, half>(- stream, transa, transb, m, n, k, (const half **)Aarray, lda,- (const half **)Barray, ldb, Carray, Ctype, ldc, batchCount);+ stream, transa, transb, m, n, k, alpha0, //+ (const half **)Aarray, lda, //+ (const half **)Barray, ldb, //+ beta0, Carray, Ctype, ldc, batchCount);return TINYBLAS_STATUS_SUCCESS;}// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmstridedbatchedextemplate <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST>-static __global__ void tinyblasGSBE_entry(- tinyblasOperation_t transa, tinyblasOperation_t transb, //- int m, int n, int k, //- const DST alpha, //- const SRC *A, int lda, long long int strideA, //- const SRC *B, int ldb, long long int strideB, //- const DST beta, //- void *C, int ldc, long long int strideC, //- int batchCount) {+static __global__ void+tinyblasGSBE_entry(tinyblasOperation_t transa, tinyblasOperation_t transb, //+ int m, int n, int k, //+ const DST alpha, //+ const SRC *A, int lda, long long int strideA, //+ const SRC *B, int ldb, long long int strideB, //+ const DST beta, //+ void *C, int ldc, long long int strideC, //+ int batchCount) {int x = blockIdx.x * BM;const int jump1 = gridDim.x * BM;int y = blockIdx.y * BN;@@ -377,12 +379,12 @@ static __global__ void tinyblasGSBE_entry(for (x = blockIdx.x * BM; x < m; x += jump1) {for (y = blockIdx.y * BN; y < n; y += jump2) {matmul_block2d<BM, BN, BK, TM, TN, SRC, DST>(- transa, transb, //- m, n, k, x, y, //- alpha, //- A + z * strideA, lda, //- B + z * strideB, ldb, //- beta, //+ transa, transb, //+ m, n, k, x, y, //+ alpha, //+ A + z * strideA, lda, //+ B + z * strideB, ldb, //+ beta, //((DST *)C + z * strideC), ldc);}}@@ -390,17 +392,17 @@ static __global__ void tinyblasGSBE_entry(}template <int BM, int BN, int BK, int TM, int TN, typename SRC>-static void tinyblasGSBE_wrapper(tinyblasHandle_t stream, //- tinyblasOperation_t transa, //- tinyblasOperation_t transb, //- int m, int n, int k, //- const float alpha, //- const SRC *A, int lda, //- long long int strideA, //- const SRC *B, int ldb, //- long long int strideB, //- const float beta, //- void *C, cudaDataType_t Ctype, int ldc, //+static void tinyblasGSBE_wrapper(tinyblasHandle_t stream, //+ tinyblasOperation_t transa, //+ tinyblasOperation_t transb, //+ int m, int n, int k, //+ const float alpha, //+ const SRC *A, int lda, //+ long long int strideA, //+ const SRC *B, int ldb, //+ long long int strideB, //+ const float beta, //+ void *C, cudaDataType_t Ctype, int ldc, //long long int strideC, int batchCount) {dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 32);int maxthreads = ((BM * BN) / (TM * TN));@@ -408,24 +410,24 @@ static void tinyblasGSBE_wrapper(tinyblasHandle_t stream, //if (Ctype == CUDA_R_16F) {tinyblasGSBE_entry<BM, BN, BK, TM, TN, SRC, half><<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)),- stream>>>(transa, transb, //- m, n, k, //- __float2half(alpha), //- A, lda, strideA, //- B, ldb, strideB, //- __half2float(beta), //- C, ldc, strideC, //+ stream>>>(transa, transb, //+ m, n, k, //+ __float2half(alpha), //+ A, lda, strideA, //+ B, ldb, strideB, //+ __half2float(beta), //+ C, ldc, strideC, //batchCount);} else {tinyblasGSBE_entry<BM, BN, BK, TM, TN, SRC, float><<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)),- stream>>>(transa, transb, //- m, n, k, //- alpha, //- A, lda, strideA, //- B, ldb, strideB, //- beta, //- C, ldc, strideC, //+ stream>>>(transa, transb, //+ m, n, k, //+ alpha, //+ A, lda, strideA, //+ B, ldb, strideB, //+ beta, //+ C, ldc, strideC, //batchCount);}}@@ -443,12 +445,12 @@ tinyblasStatus_t tinyblasGemmStridedBatchedEx(}float alpha0, beta0;- if (computeType == CUBLAS_COMPUTE_32F) {- alpha0 = *(float *)alpha;- beta0 = *(float *)beta;- } else /* computeType == CUBLAS_COMPUTE_16F */ {- alpha0 = __half2float(*(half *)alpha);- beta0 = __half2float(*(half *)beta);+ if (computeType == TINYBLAS_COMPUTE_32F) {+ alpha0 = *(float *)pAlpha;+ beta0 = *(float *)pBeta;+ } else /* computeType == TINYBLAS_COMPUTE_16F */ {+ alpha0 = __half2float(*(half *)pAlpha);+ beta0 = __half2float(*(half *)pBeta);}tinyblasGSBE_wrapper<32, 4, 64, 1, 2, half>(@@ -461,11 +463,11 @@ tinyblasStatus_t tinyblasGemmStridedBatchedEx(const char *tinyblasGetStatusString(tinyblasStatus_t err) {switch (err) {- case TINYBLAS_STATUS_SUCCESS:- return "TINYBLAS_STATUS_SUCCESS";- case TINYBLAS_STATUS_NOT_SUPPORTED:- return "TINYBLAS_STATUS_NOT_SUPPORTED";- default:- return "unknown tinyblas error";+ case TINYBLAS_STATUS_SUCCESS:+ return "TINYBLAS_STATUS_SUCCESS";+ case TINYBLAS_STATUS_NOT_SUPPORTED:+ return "TINYBLAS_STATUS_NOT_SUPPORTED";+ default:+ return "unknown tinyblas error";}}