Skip to content

Distribute and run LLMs with a single file.

Change BM/BN/BK to template parameters (#203)

Gautham4892494c7b8b
diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cu
index 1f951ae3..19779f64 100644
--- a/llamafile/tinyblas.cu
+++ b/llamafile/tinyblas.cu
@@ -21,11 +21,9 @@
(((trans) == TINYBLAS_OP_N) ? (A)[(i) + (j) * (ld)] : (A)[(j) + (i) * (ld)])
#define READ16(A, trans, ld, i, j) __half2float(READ(A, trans, ld, i, j))
-#define BM 48
-#define BN 12
-#define BK 48
#define CEIL_DIV(M, N) (((M) + (N)-1) / (N))
+template<int BM, int BN, int BK>
static __device__ void matmul32_block2d(int m, int n, int k, int x, int y,
const float *A, int lda, float *As,
const float *B, int ldb, float *Bs,
@@ -84,6 +82,7 @@ static __device__ void matmul32_block2d(int m, int n, int k, int x, int y,
__syncthreads();
}
+template<int BM, int BN, int BK>
static __global__ void tinyblasS_entry(int m, int n, int k,
const float *A, int lda,
const float *B, int ldb,
@@ -102,10 +101,10 @@ static __global__ void tinyblasS_entry(int m, int n, int k,
// each thread handles a sub-row of size BN
for (x = blockIdx.x * BM; x < m; x += jump1) {
for (y = blockIdx.y * BN; y < n; y += jump2) {
- matmul32_block2d(m, n, k, x, y, //
- A, lda, As, //
- B, ldb, Bs, //
- C, ldc, Cs);
+ matmul32_block2d<BM, BN, BK>(m, n, k, x, y, //
+ A, lda, As, //
+ B, ldb, Bs, //
+ C, ldc, Cs);
}
}
}
@@ -128,6 +127,18 @@ static bool check_args(tinyblasOperation_t transa, tinyblasOperation_t transb,
*(float *)pBeta == 0.0f)));
}
+template <int BM, int BN, int BK>
+static void tinyblasS_wrapper(tinyblasHandle_t stream, int m, int n, int k,
+ const float *A, int lda, const float *B, int ldb,
+ float *C, int ldc) {
+ dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 1);
+ int maxthreads = BK;
+
+ tinyblasS_entry<BM, BN, BK>
+ <<<maxblocks, maxthreads, (sizeof(float) * (BM * BK + BK * BN)),
+ stream>>>(m, n, k, A, lda, B, ldb, C, ldc);
+}
+
tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t stream,
tinyblasOperation_t transa,
tinyblasOperation_t transb,
@@ -142,15 +153,11 @@ tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t stream,
return TINYBLAS_STATUS_NOT_SUPPORTED;
}
- dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 1);
- int maxthreads = BK;
-
- tinyblasS_entry<<<maxblocks, maxthreads,
- (sizeof(float) * (BM * BK + BK * BN)), stream>>>(
- m, n, k, A, lda, B, ldb, C, ldc);
+ tinyblasS_wrapper<48, 12, 48>(stream, m, n, k, A, lda, B, ldb, C, ldc);
return TINYBLAS_STATUS_SUCCESS;
}
+template<int BM, int BN, int BK>
static __device__ void matmul_block2d(int m, int n, int k, int x, int y,
const half *A, int lda, float *As,
const half *B, int ldb, float *Bs,
@@ -216,6 +223,7 @@ static __device__ void matmul_block2d(int m, int n, int k, int x, int y,
}
// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmex
+template<int BM, int BN, int BK>
static __global__ void tinyblasGE_entry(int m, int n, int k, const half *A,
int lda, const half *B, int ldb,
void *C, cudaDataType_t Ctype,
@@ -234,14 +242,26 @@ static __global__ void tinyblasGE_entry(int m, int n, int k, const half *A,
// each thread handles a sub-row of size BN
for (x = blockIdx.x * BM; x < m; x += jump1) {
for (y = blockIdx.y * BN; y < n; y += jump2) {
- matmul_block2d(m, n, k, x, y, //
- A, lda, As, //
- B, ldb, Bs, //
- C, Ctype, ldc, Cs);
+ matmul_block2d<BM, BN, BK>(m, n, k, x, y, //
+ A, lda, As, //
+ B, ldb, Bs, //
+ C, Ctype, ldc, Cs);
}
}
}
+template <int BM, int BN, int BK>
+static void tinyblasGE_wrapper(tinyblasHandle_t stream, int m, int n, int k,
+ const half *A, int lda, const half *B, int ldb,
+ void *C, cudaDataType_t Ctype, int ldc) {
+ dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 1);
+ int maxthreads = BK;
+
+ tinyblasGE_entry<BM, BN, BK>
+ <<<maxblocks, maxthreads, (sizeof(float) * (BM * BK + BK * BN)),
+ stream>>>(m, n, k, A, lda, B, ldb, C, Ctype, ldc);
+}
+
tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t stream,
tinyblasOperation_t transa,
tinyblasOperation_t transb,
@@ -266,17 +286,14 @@ tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t stream,
return TINYBLAS_STATUS_NOT_SUPPORTED;
}
- dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 1);
- int maxthreads = BK;
-
- tinyblasGE_entry<<<maxblocks, maxthreads,
- (sizeof(float) * (BM * BK + BK * BN)), stream>>>(
- m, n, k, (const half *)A, lda, (const half *)B, ldb, C, Ctype, ldc);
+ tinyblasGE_wrapper<48, 12, 48>(stream, m, n, k, (const half *)A, lda,
+ (const half *)B, ldb, C, Ctype, ldc);
return TINYBLAS_STATUS_SUCCESS;
}
// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmbatchedex
+template<int BM, int BN, int BK>
static __global__ void tinyblasGBE_entry(int m, int n, int k,
const half *const Aarray[], int lda,
const half *const Barray[], int ldb,
@@ -300,15 +317,30 @@ static __global__ void tinyblasGBE_entry(int m, int n, int k,
for (z = blockIdx.z; z < batchCount; z += jump3) {
for (x = blockIdx.x * BM; x < m; x += jump1) {
for (y = blockIdx.y * BN; y < n; y += jump2) {
- matmul_block2d(m, n, k, x, y, //
- Aarray[z], lda, As, //
- Barray[z], ldb, Bs, //
- Carray[z], Ctype, ldc, Cs);
+ matmul_block2d<BM, BN, BK>(m, n, k, x, y, //
+ Aarray[z], lda, As, //
+ Barray[z], ldb, Bs, //
+ Carray[z], Ctype, ldc, Cs);
}
}
}
}
+template<int BM, int BN, int BK>
+static void tinyblasGBE_wrapper(tinyblasHandle_t stream, int m, int n, int k,
+ const half *const Aarray[], int lda,
+ const half *const Barray[], int ldb,
+ void *const Carray[], cudaDataType_t Ctype,
+ int ldc, int batchCount) {
+ dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 32);
+ int maxthreads = BK;
+
+ tinyblasGBE_entry<BM, BN, BK>
+ <<<maxblocks, maxthreads, (sizeof(float) * (BM * BK + BK * BN)),
+ stream>>>(m, n, k, Aarray, lda, Barray,
+ ldb, Carray, Ctype, ldc, batchCount);
+}
+
tinyblasStatus_t tinyblasGemmBatchedEx(tinyblasHandle_t stream,
tinyblasOperation_t transa,
tinyblasOperation_t transb,
@@ -334,88 +366,14 @@ tinyblasStatus_t tinyblasGemmBatchedEx(tinyblasHandle_t stream,
return TINYBLAS_STATUS_NOT_SUPPORTED;
}
- dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 32);
- int maxthreads = BK;
-
- tinyblasGBE_entry<<<maxblocks, maxthreads,
- (sizeof(float) * (BM * BK + BK * BN)), stream>>>(
- m, n, k, (const half **)Aarray, lda, (const half **)Barray, ldb,
- Carray, Ctype, ldc, batchCount);
+ tinyblasGBE_wrapper<48, 12, 48>(stream, m, n, k, (const half **)Aarray, lda,
+ (const half **)Barray, ldb, Carray, Ctype,
+ ldc, batchCount);
return TINYBLAS_STATUS_SUCCESS;
}
// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmstridedbatchedex
-#undef BM
-#undef BN
-#undef BK
-#define BM 64
-#define BN 4
-#define BK 64
-
-static __device__ void matmul_block2d_sb(int m, int n, int k, int x, int y,
- const half *A, int lda, float *As,
- const half *B, int ldb, float *Bs,
- void *C, cudaDataType_t Ctype, int ldc,
- float *Cs) {
- assert(blockDim.x == BK);
- static_assert(BK == BM, "");
- static_assert(BN <= BM, "");
- const int i = threadIdx.x;
- int j, l, blob;
- // within each block
- // we first zero out Cs
- for (j = 0; j < BN; ++j) Cs[j] = 0;
-
- for (blob = 0; blob < k; blob += BK) {
- if (i < BK) {
- if ((blob + i) < k) {
- // we copy into As from A
- for (j = 0; j < BM && x + j < m; ++j) {
- As[(j * BK) + i] =
- READ16(A, TINYBLAS_OP_T, lda, x + j, blob + i);
- }
- for (; j < BM; ++j) As[(j * BK) + i] = 0;
- // we copy into Bs from B
- for (j = 0; j < BN && y + j < n; ++j) {
- Bs[(i * BN) + j] =
- READ16(B, TINYBLAS_OP_N, ldb, blob + i, y + j);
- }
- for (; j < BN; ++j) Bs[(i * BN) + j] = 0;
- } else { // UNLIKELY
- for (j = 0; j < BM; ++j) As[(j * BK) + i] = 0;
- for (j = 0; j < BN; ++j) Bs[(i * BN) + j] = 0;
- }
- }
- __syncthreads();
-
- // We matmul the blobs, basically Cs += matmul(As, Bs)
- for (j = 0; j < BN; ++j) {
- for (l = 0; l < BK; ++l) {
- Cs[j] += As[(i * BK) + l] * Bs[(l * BN) + j];
- }
- }
- __syncthreads();
- }
-
- for (j = 0; j < BN; ++j) {
- As[(i*BN) + j] = Cs[j];
- }
-
- // We write Cs out into C
- if (y + i < n && i < BN) {
- if (Ctype == CUDA_R_16F) {
- for (j = 0; j < BM && x + j < m; ++j) {
- *((half *)C + (x + j) + (y + i) * ldc) = __float2half(As[j*BN + i]);
- }
- } else {
- for (j = 0; j < BM && x + j < m; ++j) {
- *((float *)C + (x + j) + (y + i) * ldc) = As[j*BN + i];
- }
- }
- }
- __syncthreads();
-}
-
+template<int BM, int BN, int BK>
static __global__ void tinyblasGSBE_entry(int m, int n, int k,
const half *A,
int lda,
@@ -445,18 +403,37 @@ static __global__ void tinyblasGSBE_entry(int m, int n, int k,
for (z = blockIdx.z; z < batchCount; z += jump3) {
for (x = blockIdx.x * BM; x < m; x += jump1) {
for (y = blockIdx.y * BN; y < n; y += jump2) {
- matmul_block2d_sb(m, n, k, x, y, //
- A + z * strideA, lda, As, //
- B + z * strideB, ldb, Bs, //
- (Ctype == CUDA_R_16F
- ? (void *)((half *)C + z * strideC)
- : (void *)((float *)C + z * strideC)),
- Ctype, ldc, Cs);
+ matmul_block2d<BM, BN, BK>(
+ m, n, k, x, y, //
+ A + z * strideA, lda, As, //
+ B + z * strideB, ldb, Bs, //
+ (Ctype == CUDA_R_16F ? (void *)((half *)C + z * strideC)
+ : (void *)((float *)C + z * strideC)),
+ Ctype, ldc, Cs);
}
}
}
}
+template <int BM, int BN, int BK>
+static void tinyblasGSBE_wrapper(tinyblasHandle_t stream, int m, int n, int k,
+ const half *A, int lda, long long int strideA,
+ const half *B, int ldb, long long int strideB,
+ void *C, cudaDataType_t Ctype, int ldc,
+ long long int strideC, int batchCount) {
+ // call the entry function
+ dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 32);
+ int maxthreads = BK;
+
+ tinyblasGSBE_entry<BM, BN, BK>
+ <<<maxblocks, maxthreads, (sizeof(float) * (BM * BK + BK * BN)),
+ stream>>>(m, n, k, //
+ A, lda, strideA, //
+ B, ldb, strideB, //
+ C, Ctype, ldc, strideC, //
+ batchCount);
+}
+
tinyblasStatus_t tinyblasGemmStridedBatchedEx(tinyblasHandle_t stream,
tinyblasOperation_t transa,
tinyblasOperation_t transb,
@@ -483,14 +460,9 @@ tinyblasStatus_t tinyblasGemmStridedBatchedEx(tinyblasHandle_t stream,
return TINYBLAS_STATUS_NOT_SUPPORTED;
}
- // call the entry function
- dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 32);
- int maxthreads = BK;
-
- tinyblasGSBE_entry<<<maxblocks, maxthreads,
- (sizeof(float) * (BM * BK + BK * BN)), stream>>>(
- m, n, k, (const half*)A, lda, strideA, (const half*)B, ldb, strideB,
- C, Ctype, ldc, strideC, batchCount);
+ tinyblasGSBE_wrapper<64, 4, 64>(stream, m, n, k, (const half *)A, lda, strideA,
+ (const half *)B, ldb, strideB, C, Ctype,
+ ldc, strideC, batchCount);
return TINYBLAS_STATUS_SUCCESS;
}