Skip to content

Distribute and run LLMs with a single file.

Fixup commit for 84490a7bca53 (#265)

Gautham725eca9fb963
diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cu
index d73c9bfa..336e9ccd 100644
--- a/llamafile/tinyblas.cu
+++ b/llamafile/tinyblas.cu
@@ -17,7 +17,7 @@
#include "tinyblas.h"
-#define READ(A, trans, ld, i, j) \
+#define READ(A, trans, ld, i, j) \
(((trans) == TINYBLAS_OP_N) ? (A)[(i) + (j) * (ld)] : (A)[(j) + (i) * (ld)])
#define READ16(A, trans, ld, i, j) __half2float(READ(A, trans, ld, i, j))
@@ -33,35 +33,33 @@ __device__ __forceinline__ DST typechange(SRC x) {
}
}
-template <>
-__device__ __forceinline__ float typechange(half x) {
+template <> __device__ __forceinline__ float typechange(half x) {
return __half2float(x);
}
-template <>
-__device__ __forceinline__ half typechange(float x) {
+template <> __device__ __forceinline__ half typechange(float x) {
return __float2half(x);
}
template <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST>
-static __device__ void matmul_block2d(tinyblasOperation_t transa, //
- tinyblasOperation_t transb, //
- int m, int n, int k, //
- int x, int y, //
- const DST alpha, //
- const SRC *A, int lda, //
- const SRC *B, int ldb, //
- const DST beta, //
+static __device__ void matmul_block2d(tinyblasOperation_t transa, //
+ tinyblasOperation_t transb, //
+ int m, int n, int k, //
+ int x, int y, //
+ const DST alpha, //
+ const SRC *A, int lda, //
+ const SRC *B, int ldb, //
+ const DST beta, //
DST *C, int ldc) {
static_assert((BM % TM == 0) && (BN % TN == 0),
"can't divide work for threads");
- static_assert(BK == ((BM * BN) / (TM * TN)), // optional
+ static_assert(BK == ((BM * BN) / (TM * TN)), // optional
"threads can't load memory properly");
static_assert((BM * BN) <= (BM * BK) + (BK * BN),
"didn't allocate enough shared mem for threads");
const int ii0 = threadIdx.x / (BN / TN); /* {0, ..., (BM/TM) - 1} */
const int ii1 = threadIdx.x % (BN / TN); /* {0, ..., (BN/TN) - 1} */
- extern __shared__ float svals[]; // shared across all threads in a block
+ extern __shared__ float svals[]; // shared across all threads in a block
SRC *As = (SRC *)svals;
SRC *Bs = (SRC *)svals + BM * BK;
@@ -71,7 +69,8 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, //
int i, h, j, l, blob;
// within each block
// we first zero out Cs
- for (j = 0; j < TM * TN; ++j) Cs[j] = 0;
+ for (j = 0; j < TM * TN; ++j)
+ Cs[j] = 0;
for (blob = 0; blob < k; blob += BK) {
for (i = threadIdx.x; i < BK; i += blockDim.x) {
@@ -97,8 +96,10 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, //
// We matmul the blobs, basically Cs += matmul(As, Bs)
for (l = 0; l < BK; ++l) {
- for (j = 0; j < TM; ++j) At[j] = As[(l * BM) + ii0 * TM + j];
- for (h = 0; h < TN; ++h) Bt[h] = Bs[(l * BN) + ii1 * TN + h];
+ for (j = 0; j < TM; ++j)
+ At[j] = As[(l * BM) + ii0 * TM + j];
+ for (h = 0; h < TN; ++h)
+ Bt[h] = Bs[(l * BN) + ii1 * TN + h];
for (j = 0; j < TM; ++j) {
for (h = 0; h < TN; ++h) {
Cs[j * TN + h] += At[j] * Bt[h];
@@ -115,21 +116,21 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, //
for (j = 0; j < TM && x + j < m; ++j) {
for (l = 0; l < TN && y + l < n; ++l) {
*(C + (x + j) + (y + l) * ldc) =
- alpha * typechange<SRC, DST>(Cs[j * TN + l]) +
- beta * (*(C + (x + j) + (y + l) * ldc));
+ alpha * typechange<SRC, DST>(Cs[j * TN + l]);
+ // + beta * (*(C + (x + j) + (y + l) * ldc));
}
}
__syncthreads();
}
template <int BM, int BN, int BK, int TM, int TN>
-static __global__ void tinyblasS_entry(tinyblasOperation_t transa, //
- tinyblasOperation_t transb, //
- int m, int n, int k, //
- const float alpha, //
- const float *A, int lda, //
- const float *B, int ldb, //
- const float beta, //
+static __global__ void tinyblasS_entry(tinyblasOperation_t transa, //
+ tinyblasOperation_t transb, //
+ int m, int n, int k, //
+ const float alpha, //
+ const float *A, int lda, //
+ const float *B, int ldb, //
+ const float beta, //
float *C, int ldc) {
int x = blockIdx.x * BM;
const int jump1 = gridDim.x * BM;
@@ -148,21 +149,22 @@ static __global__ void tinyblasS_entry(tinyblasOperation_t transa, //
}
template <int BM, int BN, int BK, int TM, int TN>
-static void tinyblasS_wrapper(tinyblasHandle_t stream, //
- tinyblasOperation_t transa, //
- tinyblasOperation_t transb, //
- int m, int n, int k, //
- const float alpha, //
- const float *A, int lda, //
- const float *B, int ldb, //
- const float beta, //
+static void tinyblasS_wrapper(tinyblasHandle_t stream, //
+ tinyblasOperation_t transa, //
+ tinyblasOperation_t transb, //
+ int m, int n, int k, //
+ const float alpha, //
+ const float *A, int lda, //
+ const float *B, int ldb, //
+ const float beta, //
float *C, int ldc) {
dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 1);
int maxthreads = ((BM * BN) / (TM * TN));
tinyblasS_entry<BM, BN, BK, TM, TN>
<<<maxblocks, maxthreads, (sizeof(float) * (BM * BK + BK * BN)),
- stream>>>(transa, transb, m, n, k, A, lda, B, ldb, C, ldc);
+ stream>>>(transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C,
+ ldc);
}
tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t stream,
@@ -187,13 +189,13 @@ static inline bool check_args(cudaDataType_t Atype, cudaDataType_t Btype,
// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmex
template <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST>
-static __global__ void tinyblasGE_entry(tinyblasOperation_t transa, //
- tinyblasOperation_t transb, //
- int m, int n, int k, //
- const DST alpha, //
- const SRC *A, int lda, //
- const SRC *B, int ldb, //
- const DST beta, //
+static __global__ void tinyblasGE_entry(tinyblasOperation_t transa, //
+ tinyblasOperation_t transb, //
+ int m, int n, int k, //
+ const DST alpha, //
+ const SRC *A, int lda, //
+ const SRC *B, int ldb, //
+ const DST beta, //
DST *C, int ldc) {
int x = blockIdx.x * BM;
const int jump1 = gridDim.x * BM;
@@ -204,22 +206,22 @@ static __global__ void tinyblasGE_entry(tinyblasOperation_t transa, //
for (x = blockIdx.x * BM; x < m; x += jump1) {
for (y = blockIdx.y * BN; y < n; y += jump2) {
matmul_block2d<BM, BN, BK, TM, TN, SRC, DST>(
- transa, transb, m, n, k, x, y, //
- alpha, A, lda, B, ldb, //
+ transa, transb, m, n, k, x, y, //
+ alpha, A, lda, B, ldb, //
beta, C, ldc);
}
}
}
template <int BM, int BN, int BK, int TM, int TN, typename SRC>
-static void tinyblasGE_wrapper(tinyblasHandle_t stream, //
- tinyblasOperation_t transa, //
- tinyblasOperation_t transb, //
- int m, int n, int k, //
- const float alpha, //
- const SRC *A, int lda, //
- const SRC *B, int ldb, //
- const float beta, //
+static void tinyblasGE_wrapper(tinyblasHandle_t stream, //
+ tinyblasOperation_t transa, //
+ tinyblasOperation_t transb, //
+ int m, int n, int k, //
+ const float alpha, //
+ const SRC *A, int lda, //
+ const SRC *B, int ldb, //
+ const float beta, //
void *C, cudaDataType_t Ctype, int ldc) {
dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 1);
int maxthreads = ((BM * BN) / (TM * TN));
@@ -237,32 +239,30 @@ static void tinyblasGE_wrapper(tinyblasHandle_t stream, //
}
}
-tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t stream,
- tinyblasOperation_t transa,
- tinyblasOperation_t transb, int m, int n, int k,
- const void *alpha, const void *A,
- cudaDataType_t Atype, int lda, const void *B,
- cudaDataType_t Btype, int ldb, const void *beta,
- void *C, cudaDataType_t Ctype, int ldc,
- tinyblasComputeType_t computeType,
- tinyblasGemmAlgo_t algo) {
+tinyblasStatus_t
+tinyblasGemmEx(tinyblasHandle_t stream, tinyblasOperation_t transa,
+ tinyblasOperation_t transb, int m, int n, int k,
+ const void *alpha, const void *A, cudaDataType_t Atype, int lda,
+ const void *B, cudaDataType_t Btype, int ldb, const void *beta,
+ void *C, cudaDataType_t Ctype, int ldc,
+ tinyblasComputeType_t computeType, tinyblasGemmAlgo_t algo) {
if (!check_args(Atype, Btype, Ctype, computeType)) {
return TINYBLAS_STATUS_NOT_SUPPORTED;
}
float alpha0, beta0;
- if (computeType == CUBLAS_COMPUTE_32F) {
+ if (computeType == TINYBLAS_COMPUTE_32F) {
alpha0 = *(float *)alpha;
beta0 = *(float *)beta;
- } else /* computeType == CUBLAS_COMPUTE_16F */ {
+ } else /* computeType == TINYBLAS_COMPUTE_16F */ {
alpha0 = __half2float(*(half *)alpha);
beta0 = __half2float(*(half *)beta);
}
tinyblasGE_wrapper<48, 32, 64, 3, 8, half>(stream, transa, transb, m, n, k,
alpha0, (const half *)A, lda,
- (const half *)B, ldb, //
+ (const half *)B, ldb, //
beta0, C, Ctype, ldc);
return TINYBLAS_STATUS_SUCCESS;
}
@@ -270,13 +270,13 @@ tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t stream,
// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmbatchedex
template <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST>
-static __global__ void tinyblasGBE_entry(tinyblasOperation_t transa, //
- tinyblasOperation_t transb, //
- int m, int n, int k, //
- const DST alpha, //
- const SRC *const Aarray[], int lda, //
- const SRC *const Barray[], int ldb, //
- const DST beta, //
+static __global__ void tinyblasGBE_entry(tinyblasOperation_t transa, //
+ tinyblasOperation_t transb, //
+ int m, int n, int k, //
+ const DST alpha, //
+ const SRC *const Aarray[], int lda, //
+ const SRC *const Barray[], int ldb, //
+ const DST beta, //
void *const Carray[], int ldc,
int batchCount) {
int x = blockIdx.x * BM;
@@ -300,26 +300,26 @@ static __global__ void tinyblasGBE_entry(tinyblasOperation_t transa, //
template <int BM, int BN, int BK, int TM, int TN, typename SRC>
static void tinyblasGBE_wrapper(tinyblasHandle_t stream,
- tinyblasOperation_t transa, //
- tinyblasOperation_t transb, //
- int m, int n, int k, //
- const float alpha, //
- const SRC *const Aarray[], int lda, //
- const SRC *const Barray[], int ldb, //
- const float beta, //
+ tinyblasOperation_t transa, //
+ tinyblasOperation_t transb, //
+ int m, int n, int k, //
+ const float alpha, //
+ const SRC *const Aarray[], int lda, //
+ const SRC *const Barray[], int ldb, //
+ const float beta, //
void *const Carray[], cudaDataType_t Ctype,
int ldc, int batchCount) {
dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 32);
int maxthreads = ((BM * BN) / (TM * TN));
if (Ctype == CUDA_R_16F) {
- tinyblasGBE_entry<BM, BN, BK, TM, TN, SRC>
+ tinyblasGBE_entry<BM, BN, BK, TM, TN, SRC, half>
<<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)),
stream>>>(transa, transb, m, n, k, __float2half(alpha), Aarray,
lda, Barray, ldb, __float2half(beta), Carray, ldc,
batchCount);
} else {
- tinyblasGBE_entry<BM, BN, BK, TM, TN, float>
+ tinyblasGBE_entry<BM, BN, BK, TM, TN, SRC, float>
<<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)),
stream>>>(transa, transb, m, n, k, alpha, Aarray, lda, Barray,
ldb, beta, Carray, ldc, batchCount);
@@ -327,12 +327,12 @@ static void tinyblasGBE_wrapper(tinyblasHandle_t stream,
}
tinyblasStatus_t tinyblasGemmBatchedEx(
- tinyblasHandle_t stream, //
- tinyblasOperation_t transa, tinyblasOperation_t transb, //
- int m, int n, int k, const void *alpha, //
- const void *const Aarray[], cudaDataType_t Atype, int lda, //
- const void *const Barray[], cudaDataType_t Btype, int ldb, //
- const void *beta, void *const Carray[], cudaDataType_t Ctype, int ldc, //
+ tinyblasHandle_t stream, //
+ tinyblasOperation_t transa, tinyblasOperation_t transb, //
+ int m, int n, int k, const void *alpha, //
+ const void *const Aarray[], cudaDataType_t Atype, int lda, //
+ const void *const Barray[], cudaDataType_t Btype, int ldb, //
+ const void *beta, void *const Carray[], cudaDataType_t Ctype, int ldc, //
int batchCount, tinyblasComputeType_t computeType,
tinyblasGemmAlgo_t algo) {
if (!check_args(Atype, Btype, Ctype, computeType)) {
@@ -340,31 +340,33 @@ tinyblasStatus_t tinyblasGemmBatchedEx(
}
float alpha0, beta0;
- if (computeType == CUBLAS_COMPUTE_32F) {
+ if (computeType == TINYBLAS_COMPUTE_32F) {
alpha0 = *(float *)alpha;
beta0 = *(float *)beta;
- } else /* computeType == CUBLAS_COMPUTE_16F */ {
+ } else /* computeType == TINYBLAS_COMPUTE_16F */ {
alpha0 = __half2float(*(half *)alpha);
beta0 = __half2float(*(half *)beta);
}
tinyblasGBE_wrapper<48, 32, 64, 3, 8, half>(
- stream, transa, transb, m, n, k, (const half **)Aarray, lda,
- (const half **)Barray, ldb, Carray, Ctype, ldc, batchCount);
+ stream, transa, transb, m, n, k, alpha0, //
+ (const half **)Aarray, lda, //
+ (const half **)Barray, ldb, //
+ beta0, Carray, Ctype, ldc, batchCount);
return TINYBLAS_STATUS_SUCCESS;
}
// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmstridedbatchedex
template <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST>
-static __global__ void tinyblasGSBE_entry(
- tinyblasOperation_t transa, tinyblasOperation_t transb, //
- int m, int n, int k, //
- const DST alpha, //
- const SRC *A, int lda, long long int strideA, //
- const SRC *B, int ldb, long long int strideB, //
- const DST beta, //
- void *C, int ldc, long long int strideC, //
- int batchCount) {
+static __global__ void
+tinyblasGSBE_entry(tinyblasOperation_t transa, tinyblasOperation_t transb, //
+ int m, int n, int k, //
+ const DST alpha, //
+ const SRC *A, int lda, long long int strideA, //
+ const SRC *B, int ldb, long long int strideB, //
+ const DST beta, //
+ void *C, int ldc, long long int strideC, //
+ int batchCount) {
int x = blockIdx.x * BM;
const int jump1 = gridDim.x * BM;
int y = blockIdx.y * BN;
@@ -377,12 +379,12 @@ static __global__ void tinyblasGSBE_entry(
for (x = blockIdx.x * BM; x < m; x += jump1) {
for (y = blockIdx.y * BN; y < n; y += jump2) {
matmul_block2d<BM, BN, BK, TM, TN, SRC, DST>(
- transa, transb, //
- m, n, k, x, y, //
- alpha, //
- A + z * strideA, lda, //
- B + z * strideB, ldb, //
- beta, //
+ transa, transb, //
+ m, n, k, x, y, //
+ alpha, //
+ A + z * strideA, lda, //
+ B + z * strideB, ldb, //
+ beta, //
((DST *)C + z * strideC), ldc);
}
}
@@ -390,17 +392,17 @@ static __global__ void tinyblasGSBE_entry(
}
template <int BM, int BN, int BK, int TM, int TN, typename SRC>
-static void tinyblasGSBE_wrapper(tinyblasHandle_t stream, //
- tinyblasOperation_t transa, //
- tinyblasOperation_t transb, //
- int m, int n, int k, //
- const float alpha, //
- const SRC *A, int lda, //
- long long int strideA, //
- const SRC *B, int ldb, //
- long long int strideB, //
- const float beta, //
- void *C, cudaDataType_t Ctype, int ldc, //
+static void tinyblasGSBE_wrapper(tinyblasHandle_t stream, //
+ tinyblasOperation_t transa, //
+ tinyblasOperation_t transb, //
+ int m, int n, int k, //
+ const float alpha, //
+ const SRC *A, int lda, //
+ long long int strideA, //
+ const SRC *B, int ldb, //
+ long long int strideB, //
+ const float beta, //
+ void *C, cudaDataType_t Ctype, int ldc, //
long long int strideC, int batchCount) {
dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 32);
int maxthreads = ((BM * BN) / (TM * TN));
@@ -408,24 +410,24 @@ static void tinyblasGSBE_wrapper(tinyblasHandle_t stream, //
if (Ctype == CUDA_R_16F) {
tinyblasGSBE_entry<BM, BN, BK, TM, TN, SRC, half>
<<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)),
- stream>>>(transa, transb, //
- m, n, k, //
- __float2half(alpha), //
- A, lda, strideA, //
- B, ldb, strideB, //
- __half2float(beta), //
- C, ldc, strideC, //
+ stream>>>(transa, transb, //
+ m, n, k, //
+ __float2half(alpha), //
+ A, lda, strideA, //
+ B, ldb, strideB, //
+ __half2float(beta), //
+ C, ldc, strideC, //
batchCount);
} else {
tinyblasGSBE_entry<BM, BN, BK, TM, TN, SRC, float>
<<<maxblocks, maxthreads, (sizeof(SRC) * (BM * BK + BK * BN)),
- stream>>>(transa, transb, //
- m, n, k, //
- alpha, //
- A, lda, strideA, //
- B, ldb, strideB, //
- beta, //
- C, ldc, strideC, //
+ stream>>>(transa, transb, //
+ m, n, k, //
+ alpha, //
+ A, lda, strideA, //
+ B, ldb, strideB, //
+ beta, //
+ C, ldc, strideC, //
batchCount);
}
}
@@ -443,12 +445,12 @@ tinyblasStatus_t tinyblasGemmStridedBatchedEx(
}
float alpha0, beta0;
- if (computeType == CUBLAS_COMPUTE_32F) {
- alpha0 = *(float *)alpha;
- beta0 = *(float *)beta;
- } else /* computeType == CUBLAS_COMPUTE_16F */ {
- alpha0 = __half2float(*(half *)alpha);
- beta0 = __half2float(*(half *)beta);
+ if (computeType == TINYBLAS_COMPUTE_32F) {
+ alpha0 = *(float *)pAlpha;
+ beta0 = *(float *)pBeta;
+ } else /* computeType == TINYBLAS_COMPUTE_16F */ {
+ alpha0 = __half2float(*(half *)pAlpha);
+ beta0 = __half2float(*(half *)pBeta);
}
tinyblasGSBE_wrapper<32, 4, 64, 1, 2, half>(
@@ -461,11 +463,11 @@ tinyblasStatus_t tinyblasGemmStridedBatchedEx(
const char *tinyblasGetStatusString(tinyblasStatus_t err) {
switch (err) {
- case TINYBLAS_STATUS_SUCCESS:
- return "TINYBLAS_STATUS_SUCCESS";
- case TINYBLAS_STATUS_NOT_SUPPORTED:
- return "TINYBLAS_STATUS_NOT_SUPPORTED";
- default:
- return "unknown tinyblas error";
+ case TINYBLAS_STATUS_SUCCESS:
+ return "TINYBLAS_STATUS_SUCCESS";
+ case TINYBLAS_STATUS_NOT_SUPPORTED:
+ return "TINYBLAS_STATUS_NOT_SUPPORTED";
+ default:
+ return "unknown tinyblas error";
}
}