Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Have tinyBLAS conform to project style
Justine Tunney875c8a6cd5b8
diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cuindex dc127399..1b35d766 100644--- a/llamafile/tinyblas.cu+++ b/llamafile/tinyblas.cu@@ -1,32 +1,49 @@+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2023 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#include "tinyblas.h"-#define READ(A, trans, ld, i, j) \- (((trans) == CUBLAS_OP_N) ? (A)[(i) + (j) * (ld)] : (A)[(j) + (i) * (ld)])+#define READ(A, trans, ld, i, j) \+ (((trans) == CUBLAS_OP_N) ? (A)[(i) + (j) * (ld)] : (A)[(j) + (i) * (ld)])#define READ16(A, trans, ld, i, j) __half2float(READ(A, trans, ld, i, j))static __device__ __forceinline__ void matmul_single(int m, int n, int k,- int i, int j,- const half *A, int lda,- const half *B, int ldb,- half *C, int ldc) {- float sum = 0.0f;- half *cptr = C + i + j * ldc;- for (int l = 0; l < k; ++l) {- sum += READ16(A, CUBLAS_OP_T, lda, i, l) *- READ16(B, CUBLAS_OP_N, ldb, l, j);- }- *cptr = __float2half(sum);+ int i, int j,+ const half *A, int lda,+ const half *B, int ldb,+ half *C, int ldc) {+ float sum = 0.0f;+ half *cptr = C + i + j * ldc;+ for (int l = 0; l < k; ++l) {+ sum += READ16(A, CUBLAS_OP_T, lda, i, l) *+ READ16(B, CUBLAS_OP_N, ldb, l, j);+ }+ *cptr = __float2half(sum);}static __device__ __forceinline__ void matmul_single32(int m, int n, int k, int i, int j,- const float *A, int lda,- const float *B, int ldb,- float *C, int ldc) {+ const float *A, int lda,+ const float *B, int ldb,+ float *C, int ldc) {float sum = 0.0f;float *cptr = C + i + j * ldc;for (int l = 0; l < k; ++l) {- sum += READ16(A, CUBLAS_OP_T, lda, i, l) *- READ16(B, CUBLAS_OP_N, ldb, l, j);+ sum += READ16(A, CUBLAS_OP_T, lda, i, l) *+ READ16(B, CUBLAS_OP_N, ldb, l, j);}*cptr = sum;}@@ -35,52 +52,52 @@ static bool check_args(cublasOperation_t transa, cublasOperation_t transb,const void *pAlpha, cudaDataType_t Atype,cudaDataType_t Btype, const void *pBeta,cudaDataType_t Ctype, cublasComputeType_t computeType) {- return transa == CUBLAS_OP_T && transb == CUBLAS_OP_N &&- Atype == CUDA_R_16F && Btype == CUDA_R_16F && Ctype == CUDA_R_16F &&- computeType == CUBLAS_COMPUTE_16F &&- __half2float(*(half *)pAlpha) == 1.0f &&- __half2float(*(half *)pBeta) == 0.0f;+ return transa == CUBLAS_OP_T && transb == CUBLAS_OP_N &&+ Atype == CUDA_R_16F && Btype == CUDA_R_16F && Ctype == CUDA_R_16F &&+ computeType == CUBLAS_COMPUTE_16F &&+ __half2float(*(half *)pAlpha) == 1.0f &&+ __half2float(*(half *)pBeta) == 0.0f;}static __global__ void tinyblasS_entry(int m, int n, int k,const float *A, int lda,const float *B, int ldb,float *C, int ldc) {- int x = blockIdx.x * blockDim.x + threadIdx.x;- int jump = blockDim.x * gridDim.x;- int y = threadIdx.y;- int jump2 = blockDim.y;+ int x = blockIdx.x * blockDim.x + threadIdx.x;+ int jump = blockDim.x * gridDim.x;+ int y = threadIdx.y;+ int jump2 = blockDim.y;- for (; x < m; x += jump) {- for (y=threadIdx.y; y < n; y += jump2) {- matmul_single32(m, n, k, x, y, A, lda, B, ldb, C, ldc);+ for (; x < m; x += jump) {+ for (y=threadIdx.y; y < n; y += jump2) {+ matmul_single32(m, n, k, x, y, A, lda, B, ldb, C, ldc);+ }}- }}cublasStatus_t tinyblasSgemm(cudaStream_t stream,cublasOperation_t transa,cublasOperation_t transb,int m, int n, int k,- const float *alpha,- const float *A, int lda,- const float *B, int ldb,- const float *beta,- float *C, int ldc) {- if (transa != CUBLAS_OP_T || transb != CUBLAS_OP_N ||- *alpha != 1.0f || *beta != 0.0f) {- return CUBLAS_STATUS_NOT_SUPPORTED;- }+ const float *alpha,+ const float *A, int lda,+ const float *B, int ldb,+ const float *beta,+ float *C, int ldc) {+ if (transa != CUBLAS_OP_T || transb != CUBLAS_OP_N ||+ *alpha != 1.0f || *beta != 0.0f) {+ return CUBLAS_STATUS_NOT_SUPPORTED;+ }- int numSMs, devId;- cudaGetDevice(&devId);- cudaDeviceGetAttribute(&numSMs, cudaDevAttrMultiProcessorCount, devId);- int maxblocks = 16 * numSMs;- dim3 maxthreads(16, 64, 1);+ int numSMs, devId;+ cudaGetDevice(&devId);+ cudaDeviceGetAttribute(&numSMs, cudaDevAttrMultiProcessorCount, devId);+ int maxblocks = 16 * numSMs;+ dim3 maxthreads(16, 64, 1);- tinyblasS_entry<<<maxblocks, maxthreads, 0, stream>>>(- m, n, k, A, lda, B, ldb, C, ldc);- return CUBLAS_STATUS_SUCCESS;+ tinyblasS_entry<<<maxblocks, maxthreads, 0, stream>>>(+ m, n, k, A, lda, B, ldb, C, ldc);+ return CUBLAS_STATUS_SUCCESS;}// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmex@@ -89,17 +106,17 @@ static __global__ void tinyblasGE_entry(int m, int n, int k,const half *A, int lda,const half *B, int ldb,half *C, int ldc) {- int x = blockIdx.x * blockDim.x + threadIdx.x;- int jump = blockDim.x * gridDim.x;- int y = threadIdx.y;- int jump2 = blockDim.y;+ int x = blockIdx.x * blockDim.x + threadIdx.x;+ int jump = blockDim.x * gridDim.x;+ int y = threadIdx.y;+ int jump2 = blockDim.y;- for (; x < m; x += jump) {- for (; y < n; y += jump2) {- matmul_single(m, n, k, x, y, A, lda, B, ldb, C, ldc);+ for (; x < m; x += jump) {+ for (; y < n; y += jump2) {+ matmul_single(m, n, k, x, y, A, lda, B, ldb, C, ldc);+ }}- }}cublasStatus_t tinyblasGemmEx(cudaStream_t stream,@@ -121,46 +138,43 @@ cublasStatus_t tinyblasGemmEx(cudaStream_t stream,int ldc,cublasComputeType_t computeType,cublasGemmAlgo_t algo) {- if (!check_args(transa, transb, alpha, Atype, Btype, beta, Ctype,- computeType)) {- return CUBLAS_STATUS_NOT_SUPPORTED;- }+ if (!check_args(transa, transb, alpha, Atype, Btype, beta, Ctype,+ computeType)) {+ return CUBLAS_STATUS_NOT_SUPPORTED;+ }- int numSMs, devId;- cudaGetDevice(&devId);- cudaDeviceGetAttribute(&numSMs, cudaDevAttrMultiProcessorCount, devId);- int maxblocks = 16 * numSMs;- dim3 maxthreads(16, 64, 1);+ int numSMs, devId;+ cudaGetDevice(&devId);+ cudaDeviceGetAttribute(&numSMs, cudaDevAttrMultiProcessorCount, devId);+ int maxblocks = 16 * numSMs;+ dim3 maxthreads(16, 64, 1);- tinyblasGE_entry<<<maxblocks, maxthreads, 0, stream>>>(- m, n, k, (const half*)A, lda, (const half *)B, ldb, (half *)C, ldc);- return CUBLAS_STATUS_SUCCESS;+ tinyblasGE_entry<<<maxblocks, maxthreads, 0, stream>>>(+ m, n, k, (const half*)A, lda, (const half *)B, ldb, (half *)C, ldc);+ return CUBLAS_STATUS_SUCCESS;}// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmbatchedexstatic __global__ void tinyblasGBE_entry(int m, int n, int k,- const half *const Aarray[],- int lda,- const half *const Barray[],- int ldb,- half *const Carray[],- int ldc,+ const half *const Aarray[], int lda,+ const half *const Barray[], int ldb,+ half *const Carray[], int ldc,int batchCount) {- int x = blockIdx.x * blockDim.x + threadIdx.x;- int y = blockIdx.y * blockDim.y + threadIdx.y;- int z = threadIdx.z;- int jump = blockDim.x * gridDim.x;- int jump2 = blockDim.y * gridDim.y;- int jump3 = blockDim.z;+ int x = blockIdx.x * blockDim.x + threadIdx.x;+ int y = blockIdx.y * blockDim.y + threadIdx.y;+ int z = threadIdx.z;+ int jump = blockDim.x * gridDim.x;+ int jump2 = blockDim.y * gridDim.y;+ int jump3 = blockDim.z;- for (; x < batchCount; x += jump) {- for (y=blockIdx.y * blockDim.y + threadIdx.y; y < m; y += jump2) {- for (z=threadIdx.z; z < n; z += jump3) {- matmul_single(m, n, k, y, z, Aarray[x], lda, Barray[x], ldb, Carray[x], ldc);- }+ for (; x < batchCount; x += jump) {+ for (y=blockIdx.y * blockDim.y + threadIdx.y; y < m; y += jump2) {+ for (z=threadIdx.z; z < n; z += jump3) {+ matmul_single(m, n, k, y, z, Aarray[x], lda, Barray[x], ldb, Carray[x], ldc);+ }+ }}- }}cublasStatus_t tinyblasGemmBatchedEx(cudaStream_t stream,@@ -183,35 +197,35 @@ cublasStatus_t tinyblasGemmBatchedEx(cudaStream_t stream,int batchCount,cublasComputeType_t computeType,cublasGemmAlgo_t algo) {- if (!check_args(transa, transb, alpha, Atype, Btype, beta, Ctype,- computeType)) {- return CUBLAS_STATUS_NOT_SUPPORTED;- }+ if (!check_args(transa, transb, alpha, Atype, Btype, beta, Ctype,+ computeType)) {+ return CUBLAS_STATUS_NOT_SUPPORTED;+ }- // https://developer.nvidia.com/blog/cuda-pro-tip-write-flexible-kernels-grid-stride-loops/- int numSMs, devId;- cudaGetDevice(&devId);- cudaDeviceGetAttribute(&numSMs, cudaDevAttrMultiProcessorCount, devId);- dim3 maxblocks(numSMs, 16, 1);- dim3 maxthreads(4, 4, 64);+ // https://developer.nvidia.com/blog/cuda-pro-tip-write-flexible-kernels-grid-stride-loops/+ int numSMs, devId;+ cudaGetDevice(&devId);+ cudaDeviceGetAttribute(&numSMs, cudaDevAttrMultiProcessorCount, devId);+ dim3 maxblocks(numSMs, 16, 1);+ dim3 maxthreads(4, 4, 64);- tinyblasGBE_entry<<<maxblocks, maxthreads, 0, stream>>>(- m, n, k, (const half **)Aarray, lda, (const half **)Barray, ldb,- (half **)Carray, ldc, batchCount);- return CUBLAS_STATUS_SUCCESS;+ tinyblasGBE_entry<<<maxblocks, maxthreads, 0, stream>>>(+ m, n, k, (const half **)Aarray, lda, (const half **)Barray, ldb,+ (half **)Carray, ldc, batchCount);+ return CUBLAS_STATUS_SUCCESS;}// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmstridedbatchedex#define STRIDE0(A, i, stride) ((A) + (i) * (stride))-#define STRIDE(A, type, i, stride) \- ((type) == CUDA_R_16F \- ? (void *)STRIDE0((half *)(A), (i), (stride)) \- : (void *)STRIDE0((float *)(A), (i), (stride)))-#define STRIDE_CONST(A, type, i, stride) \- ((type) == CUDA_R_16F \- ? (const void *)STRIDE0((const half *)(A), (i), (stride)) \- : (const void *)STRIDE0((const float *)(A), (i), (stride)))+#define STRIDE(A, type, i, stride) \+ ((type) == CUDA_R_16F \+ ? (void *)STRIDE0((half *)(A), (i), (stride)) \+ : (void *)STRIDE0((float *)(A), (i), (stride)))+#define STRIDE_CONST(A, type, i, stride) \+ ((type) == CUDA_R_16F \+ ? (const void *)STRIDE0((const half *)(A), (i), (stride)) \+ : (const void *)STRIDE0((const float *)(A), (i), (stride)))static __global__ void tinyblasGSBE_entry(int m, int n, int k,const half *A,@@ -224,21 +238,21 @@ static __global__ void tinyblasGSBE_entry(int m, int n, int k,int ldc,long long int strideC,int batchCount) {- int x = blockIdx.x * blockDim.x + threadIdx.x;- int y = blockIdx.y * blockDim.y + threadIdx.y;- int z = threadIdx.z;- int jump = blockDim.x * gridDim.x;- int jump2 = blockDim.y * gridDim.y;- int jump3 = blockDim.z;+ int x = blockIdx.x * blockDim.x + threadIdx.x;+ int y = blockIdx.y * blockDim.y + threadIdx.y;+ int z = threadIdx.z;+ int jump = blockDim.x * gridDim.x;+ int jump2 = blockDim.y * gridDim.y;+ int jump3 = blockDim.z;- for (; x < batchCount; x += jump) {- for (y=blockIdx.y * blockDim.y + threadIdx.y; y < m; y += jump2) {- for (z=threadIdx.z; z < n; z += jump3) {- matmul_single(m, n, k, y, z, A + x * strideA, lda, B + x * strideB, ldb,- C + x * strideC, ldc);- }+ for (; x < batchCount; x += jump) {+ for (y=blockIdx.y * blockDim.y + threadIdx.y; y < m; y += jump2) {+ for (z=threadIdx.z; z < n; z += jump3) {+ matmul_single(m, n, k, y, z, A + x * strideA, lda, B + x * strideB,+ ldb, C + x * strideC, ldc);+ }+ }}- }}cublasStatus_t tinyblasGemmStridedBatchedEx(cudaStream_t stream,@@ -262,22 +276,22 @@ cublasStatus_t tinyblasGemmStridedBatchedEx(cudaStream_t stream,int batchCount,cublasComputeType_t computeType,cublasGemmAlgo_t algo) {- if (!check_args(transa, transb, pAlpha, Atype, Btype, pBeta, Ctype,- computeType)) {- return CUBLAS_STATUS_NOT_SUPPORTED;- }+ if (!check_args(transa, transb, pAlpha, Atype, Btype, pBeta, Ctype,+ computeType)) {+ return CUBLAS_STATUS_NOT_SUPPORTED;+ }- // https://developer.nvidia.com/blog/cuda-pro-tip-write-flexible-kernels-grid-stride-loops/- int numSMs, devId;- cudaGetDevice(&devId);- cudaDeviceGetAttribute(&numSMs, cudaDevAttrMultiProcessorCount, devId);- dim3 maxblocks(numSMs, 16, 1);- dim3 maxthreads(4, 4, 64);+ // https://developer.nvidia.com/blog/cuda-pro-tip-write-flexible-kernels-grid-stride-loops/+ int numSMs, devId;+ cudaGetDevice(&devId);+ cudaDeviceGetAttribute(&numSMs, cudaDevAttrMultiProcessorCount, devId);+ dim3 maxblocks(numSMs, 16, 1);+ dim3 maxthreads(4, 4, 64);- // call the entry function- tinyblasGSBE_entry<<<maxblocks, maxthreads, 0, stream>>>(- m, n, k, (const half*)A, lda, strideA, (const half*)B, ldb, strideB,- (half *)C, ldc, strideC, batchCount);+ // call the entry function+ tinyblasGSBE_entry<<<maxblocks, maxthreads, 0, stream>>>(+ m, n, k, (const half*)A, lda, strideA, (const half*)B, ldb, strideB,+ (half *)C, ldc, strideC, batchCount);- return CUBLAS_STATUS_SUCCESS;+ return CUBLAS_STATUS_SUCCESS;}