Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Write more testing code
Justine Tunney1aee87f716db
diff --git a/build/cudacc b/build/cudaccnew file mode 100755index 00000000..3bdb2f55--- /dev/null+++ b/build/cudacc@@ -0,0 +1,54 @@+#!/bin/sh++find_nvcc() {+ CC=$(command -v nvcc 2>/dev/null) && return+ CC="$CUDA_PATH/bin/nvcc"+ [ -x "$CC" ] && return+ CC="/opt/cuda/bin/nvcc"+ [ -x "$CC" ] && return+ CC="/usr/local/cuda/bin/nvcc"+ [ -x "$CC" ] && return+ return 1+}++find_hipcc() {+ CC=$(command -v hipcc 2>/dev/null) && return+ CC="$HIP_PATH/bin/hipcc"+ [ -x "$CC" ] && return+ CC="/opt/rocm/bin/hipcc"+ [ -x "$CC" ] && return+ CC="/usr/local/rocm/bin/hipcc"+ [ -x "$CC" ] && return+ return 1+}++if find_hipcc; then+ VENDOR=AMD+ FLAGS=+elif find_nvcc; then+ VENDOR=NVIDIA+ FLAGS="--forward-unknown-to-host-compiler"+else+ echo 'error: need either hipcc (AMD) or nvcc (NVIDIA) on $PATH' >&2+ exit 1+fi++FIRST=1+for x; do+ if [ $FIRST -eq 1 ]; then+ set --+ FIRST=0+ fi+ if [ $VENDOR = AMD ]; then+ if [ x"$x" = x"-lcublas" ]; then+ set -- "$@" -lhipblas -lrocblas+ continue+ elif [ x"$x" = x"--use_fast_math" ]; then+ set -- "$@" -ffast-math -DHIP_FAST_MATH+ continue+ fi+ fi+ set -- "$@" "$x"+done++exec "$CC" $FLAGS "$@"diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cuindex ecf7d4ca..57781c1a 100644--- a/llama.cpp/ggml-cuda.cu+++ b/llama.cpp/ggml-cuda.cu@@ -53,7 +53,7 @@#define cublasSetStream tinyblasSetStream#define cublasSgemm tinyblasSgemm#define cublasStatus_t tinyblasStatus_t-#define cudaDataType_t tinyblasDatatype_t+#define cudaDataType_t tinyblasDataType_t#define cudaDeviceCanAccessPeer hipDeviceCanAccessPeer#define cudaDeviceDisablePeerAccess hipDeviceDisablePeerAccess#define cudaDeviceEnablePeerAccess hipDeviceEnablePeerAccess@@ -131,7 +131,7 @@#define cublasGemmBatchedEx tinyblasGemmBatchedEx#define cublasGemmStridedBatchedEx tinyblasGemmStridedBatchedEx#define cublasGetStatusString tinyblasGetStatusString-#define cudaDataType_t tinyblasDatatype_t+#define cudaDataType_t tinyblasDataType_t#define cublasSetMathMode(handle, mode) CUBLAS_STATUS_SUCCESS#elif defined(GGML_USE_HIPBLAS)diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex 7290d9f6..3e907432 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -44,17 +44,18 @@ o/$(MODE)/llamafile: \o/$(MODE)/llamafile/%.o: llamafile/%.cu llamafile/BUILD.mk@mkdir -p $(@D)- hipcc -O3 -march=native -ffast-math -c -o $@ $<--o/$(MODE)/llamafile/tester.o: llamafile/tester.cu llamafile/BUILD.mk- @mkdir -p $(@D)- hipcc -O3 -march=native -fopenmp -c -o $@ $<+ build/cudacc -g -O3 -march=native -ffast-math --use_fast_math -c -o $@ $<o/$(MODE)/llamafile/tinyblas_test: \o/$(MODE)/llamafile/tinyblas_test.o \o/$(MODE)/llamafile/tinyblas.o \o/$(MODE)/llamafile/tester.o- hipcc -fopenmp $(HIP_ARCH) -o $@ $^ -lhipblas -lrocblas+ build/cudacc -g -o $@ $^ -lcublas++o/$(MODE)/llamafile/cudaprops: \+ o/$(MODE)/llamafile/cudaprops.o \+ o/$(MODE)/llamafile/tester.o+ build/cudacc -g -o $@ $^ -lcublas.PHONY: o/$(MODE)/llamafile/checko/$(MODE)/llamafile/check: \diff --git a/llamafile/cuda.h b/llamafile/cuda.hnew file mode 100644index 00000000..babd08b7--- /dev/null+++ b/llamafile/cuda.h@@ -0,0 +1,72 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+#pragma once++#if defined(__NVCC__) || defined(__CUDA_ARCH__)+#include <cublas_v2.h>+#include <cuda_fp16.h>+#include <cuda_runtime.h>+#else+#include <hip/hip_fp16.h>+#include <hip/hip_runtime.h>+#include <hipblas/hipblas.h>+#include <rocblas/rocblas.h>+#define cudaFree hipFree+#define cublasMath_t int+#define cudaMalloc hipMalloc+#define cudaMemcpy hipMemcpy+#define cudaSuccess hipSuccess+#define cudaError_t hipError_t+#define cudaEvent_t hipEvent_t+#define cudaFreeHost hipHostFree+#define cudaStream_t hipStream_t+#define cublasSgemm hipblasSgemm+#define cudaSetDevice hipSetDevice+#define cublasCreate hipblasCreate+#define cudaDataType_t hipDataType+#define cublasDestroy hipblasDestroy+#define cudaDeviceProp hipDeviceProp_t+#define cublasHandle_t hipblasHandle_t+#define cublasStatus_t hipblasStatus_t+#define cudaEventCreate hipEventCreate+#define cudaEventRecord hipEventRecord+#define cudaGetDeviceCount hipGetDeviceCount+#define cudaGetDeviceProperties hipGetDeviceProperties+#define cudaGetLastError hipGetLastError+#define cudaDeviceReset hipDeviceReset+#define cudaEventDestroy hipEventDestroy+#define cublasSetStream hipblasSetStream+#define cudaStreamCreate hipStreamCreate+#define cudaMallocManaged hipMallocManaged+#define cudaStreamDestroy hipStreamDestroy+#define cublasComputeType_t hipblasComputeType_t+#define cudaEventSynchronize hipEventSynchronize+#define cudaEventElapsedTime hipEventElapsedTime+#define cudaDeviceSynchronize hipDeviceSynchronize+#define cudaStreamSynchronize hipStreamSynchronize+#define cublasGetStatusString hipblasStatusToString+#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost+#define cudaMemcpyHostToDevice hipMemcpyHostToDevice+#define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice+#define cublasSetMathMode(handle, mode) CUBLAS_STATUS_SUCCESS+#define cudaMallocHost(ptr, size) hipHostMalloc(ptr, size, hipHostMallocNonCoherent)+#define cudaGetErrorString hipGetErrorString+#define cudaGetDevice hipGetDevice+#define cudaDeviceReset hipDeviceReset+#define cublasGemmEx hipblasGemmEx_v2+#define CUDA_R_16F HIP_R_16F+#define CUDA_R_32F HIP_R_32F+#define CUBLAS_OP_N HIPBLAS_OP_N+#define CUBLAS_OP_T HIPBLAS_OP_T+#define CUBLAS_COMPUTE_16F HIPBLAS_COMPUTE_16F+#define CUBLAS_COMPUTE_32F HIPBLAS_COMPUTE_32F+#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_COMPUTE_32F_FAST_16F+#define CUBLAS_GEMM_DEFAULT HIPBLAS_GEMM_DEFAULT+#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS+#define CUBLAS_MATH_DISALLOW_REDUCED_PRECISION_REDUCTION 0+#define CUBLAS_TF32_TENSOR_OP_MATH 0+#define CUBLAS_PEDANTIC_MATH 0+#define CUBLAS_DEFAULT_MATH 0+#endif++#define WARPSIZE 32diff --git a/llamafile/cudaprops.cu b/llamafile/cudaprops.cunew file mode 100644index 00000000..3ec087c9--- /dev/null+++ b/llamafile/cudaprops.cu@@ -0,0 +1,80 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "cuda.h"+#include "tester.h"+#include <iostream>++#if defined(__NVCC__) || defined(__CUDA_ARCH__)+#define VENDOR "NVIDIA"+#else+#define VENDOR "AMD"+#endif++namespace {++void show_cuda_device(int idev) {+ cudaDeviceProp dp;+ CUDA_OR_DIE(cudaSetDevice(idev));+ CUDA_OR_DIE(cudaGetDeviceProperties(&dp, idev));++ fprintf(stderr,+ "\n" VENDOR " %s\n"+ "\t%.2f Ghz\n"+ "\t%g GB VRAM\n"+ "\t%d CPUs\n"+ "\tcompute_%d\n",+ dp.name, dp.clockRate * 1e-6, dp.totalGlobalMem / 1073741824., dp.multiProcessorCount,+ dp.major * 10 + dp.minor);++#define SHOW(s) std::cout << #s << " = " << dp.s << std::endl;++ SHOW(warpSize);+ SHOW(regsPerBlock);+ SHOW(regsPerMultiprocessor);+ SHOW(multiProcessorCount);+ SHOW(memoryBusWidth);+ SHOW(memoryClockRate);+ SHOW(sharedMemPerBlock);+ SHOW(sharedMemPerMultiprocessor);+ SHOW(totalConstMem);+ SHOW(maxThreadsPerBlock);+ SHOW(maxThreadsPerMultiProcessor);+ SHOW(maxThreadsDim[0]);+ SHOW(maxThreadsDim[1]);+ SHOW(maxThreadsDim[2]);+ SHOW(asyncEngineCount);+ SHOW(managedMemory);+ SHOW(concurrentManagedAccess);+ SHOW(directManagedMemAccessFromHost);+ SHOW(globalL1CacheSupported);+ SHOW(hostNativeAtomicSupported);+ // SHOW(hostRegisterReadOnlySupported);+ // SHOW(hostRegisterSupported);+ SHOW(integrated);+ SHOW(l2CacheSize);+}++} // namespace++int main() {+ int device_count;+ CUDA_OR_DIE(cudaGetDeviceCount(&device_count));+ for (int idev = 0; idev < device_count; ++idev) {+ show_cuda_device(idev);+ }+}diff --git a/llamafile/flt.h b/llamafile/flt.hnew file mode 100644index 00000000..b50df4ab--- /dev/null+++ b/llamafile/flt.h@@ -0,0 +1,75 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+#pragma once++// floating point utilities+//+// these functions let you fiddle with float bits. some of these+// functions e.g. flt::isnan(), won't be optimized away when the+// compiler is placed in -ffast-math mode.+//+// @see IEEE 754-2008++#if defined(__NVCC__) || defined(__CUDA_ARCH__)+#include <cuda_fp16.h>+#else+#include <hip/hip_fp16.h>+#endif++namespace flt {++inline int toint(float f) {+ union {+ float f;+ unsigned i;+ } u = {f};+ return u.i;+}++inline int toint(half f) {+ union {+ half f;+ unsigned short i;+ } u = {f};+ return u.i;+}++inline int signbit(float f) {+ return toint(f) & (-2147483647 - 1);+}++inline int signbit(half f) {+ return toint(f) & 0x8000;+}++inline int isnan(float f) {+ return (toint(f) & 0x7fffffff) > 0x7f800000;+}++inline int isnan(half f) {+ return (toint(f) & 0x7fff) > 0x7c00;+}++inline int isinf(float f) {+ return (toint(f) & 0x7fffffff) == 0x7f800000;+}++inline int isinf(half f) {+ return (toint(f) & 0x7fff) == 0x7c00;+}++inline int isnormal(float f) {+ int expo = toint(f) & 0x7f800000;+ return expo && expo != 0x7f800000;+}++inline int isnormal(half f) {+ int expo = toint(f) & 0x7c00;+ return expo && expo != 0x7c00;+}++template <typename T> inline int isdenormal(T f) {+ return f && !isnormal(f) && !isnan(f) && !isinf(f);+}++} // namespace fltdiff --git a/llamafile/gemm.h b/llamafile/gemm.hnew file mode 100644index 00000000..e9c3a660--- /dev/null+++ b/llamafile/gemm.h@@ -0,0 +1,70 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once++#include <algorithm>+#include <cassert>+#include <climits>+#include <cmath>++// multiplies matrix on cpu with column major ordering+//+// m×k * k×n → m×n+// k×m * k×n → m×n if aᵀ+// m×k * n×k → m×n if bᵀ+// k×m * n×k → m×n if aᵀ and bᵀ+//+template <typename T, typename TA, typename TB, typename TC>+void gemm(bool aᵀ, bool bᵀ, //+ int m, int n, int k, T α, //+ const TA *A, int lda, //+ const TB *B, int ldb, T β, //+ TC *C, int ldc) {+ assert(m >= 0 && n >= 0 && k >= 0);+ assert(lda >= std::max(1, aᵀ ? k : m));+ assert(ldb >= std::max(1, bᵀ ? n : k));+ assert(ldc >= std::max(1, m));+ assert(1ll * lda * (aᵀ ? m : k) <= INT_MAX);+ assert(1ll * ldb * (bᵀ ? k : n) <= INT_MAX);+ for (int i = 0; i < m; ++i)+ for (int j = 0; j < n; ++j) {+ T d = 0;+ for (int l = 0; l < k; ++l) {+ T a = aᵀ ? A[lda * i + l] : A[lda * l + i];+ T b = bᵀ ? B[ldb * l + j] : B[ldb * j + l];+ d = std::fma(a, b, d);+ }+ T c = C[ldc * j + i];+ C[ldc * j + i] = std::fma(α, d, β * c);+ }+}++// multiplies matrix on cpu with column row ordering+//+// k×m * n×k → n×m+// m×k * n×k → n×m if aᵀ+// k×m * k×n → n×m if bᵀ+// m×k * k×n → n×m if aᵀ and bᵀ+//+template <typename T, typename TA, typename TB, typename TC>+void gemmrm(bool aᵀ, bool bᵀ, //+ int m, int n, int k, T α, //+ const TA *A, int lda, //+ const TB *B, int ldb, T β, //+ TC *C, int ldc) {+ gemm(bᵀ, aᵀ, n, m, k, α, B, ldb, A, lda, β, C, ldc);+}diff --git a/llamafile/macros.h b/llamafile/macros.hnew file mode 100644index 00000000..8590ce47--- /dev/null+++ b/llamafile/macros.h@@ -0,0 +1,7 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+#pragma once++#define CEIL_DIV(M, N) (((M) + (N) - 1) / (N))+#define ROUNDUP(X, K) (((X) + (K) - 1) & -(K))+#define ARRAYLEN(A) ((sizeof(A) / sizeof(*(A))) / ((unsigned)!(sizeof(A) % sizeof(*(A)))))diff --git a/llamafile/naive.h b/llamafile/naive.hnew file mode 100644index 00000000..de45d328--- /dev/null+++ b/llamafile/naive.h@@ -0,0 +1,88 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+#pragma once++#include "macros.h"++//+// _ _ ___ _ _ ___+// | |_(_)_ _ _ _| _ ) | /_\ / __|+// | _| | ' \ || | _ \ |__ / _ \\__ \.+// \__|_|_||_\_, |___/____/_/ \_\___/+// |__/+//+// BASIC LINEAR ALGEBRA SUBPROGRAMS+//++// naive abstract generic matrix multiplication for gpu+//+// this matmul reference implementation is guaranteed to be bug-free and+// can be used for finding bugs in more advanced matmul implementations.++namespace naive {++__device__ __forceinline__ half madd(half x, half y, half z) {+ return (float)x * (float)y + (float)z;+}+__device__ __forceinline__ float madd(float x, float y, float z) {+ return __fmaf_rn(x, y, z);+}+__device__ __forceinline__ double madd(double x, double y, double z) {+ return __fma_rn(x, y, z);+}++template <typename T, typename TA, typename TB, typename TC>+__global__ void kernel(bool aᵀ, bool bᵀ, //+ int m, int n, int k, T α, //+ const TA *A, int lda, //+ const TB *B, int ldb, T β, //+ TC *C, int ldc) {+ const int i = blockIdx.x * blockDim.x + threadIdx.x;+ const int j = blockIdx.y * blockDim.y + threadIdx.y;+ if (i < m && j < n) {+ T d = 0;+ for (int l = 0; l < k; ++l) {+ T a = aᵀ ? A[lda * i + l] : A[lda * l + i];+ T b = bᵀ ? B[ldb * l + j] : B[ldb * j + l];+ d = madd(a, b, d);+ }+ T c = C[ldc * j + i];+ C[ldc * j + i] = madd(α, d, β * c);+ }+}++// multiplies matrix on gpu with column major ordering+//+// m×k * k×n → m×n+// k×m * k×n → m×n if aᵀ+// m×k * n×k → m×n if bᵀ+// k×m * n×k → m×n if aᵀ and bᵀ+//+template <typename T, typename TA, typename TB, typename TC>+cudaError_t gemm(cudaStream_t stream, //+ bool aᵀ, bool bᵀ, //+ int m, int n, int k, T α, //+ const TA *A, int lda, //+ const TB *B, int ldb, T β, //+ TC *C, int ldc) {+ dim3 blockDim(32, 32);+ dim3 gridDim(CEIL_DIV(m, 32), CEIL_DIV(n, 32));+ kernel<<<gridDim, blockDim, 0, stream>>>(aᵀ, bᵀ, m, n, k, α, A, lda, B, ldb, β, C, ldc);+ return cudaGetLastError();+}++} // namespace naivediff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cppindex 21a20d9e..80b21007 100644--- a/llamafile/sgemm.cpp+++ b/llamafile/sgemm.cpp@@ -17,6 +17,7 @@#include "llama.cpp/ggml-quants.h"#include "llamafile.h"+#include <assert.h>#include <cosmo.h>#include <immintrin.h>#include <stdio.h>@@ -31,11 +32,32 @@// BASIC LINEAR ALGEBRA SUBPROGRAMS//-#define ASSERT(x) \- if (!(x)) { \- fprintf(stderr, "%s:%d: assertion failed: %s\n", __FILE__, __LINE__, #x); \- __builtin_trap(); \- }+// matrix multiplication for x86 microprocessors+//+// this file implements optimized cpu kernels for llama.cpp that support+// column major ordering with the A matrix transposed. they work by+// computing multiple dot products at once, because it exploits+// instruction level parallelism, in addition to minimizing register+// loads since the adjacent dot products can share register loads.+//+// this technique usually slightly outperforms mkl and blis at single+// threaded performance, and usually goes 2x faster than mkl when openmp+// threading is used. however please note that this code is adapted to+// use the llama.cpp synchronization model, which isn't as advanced.+//+// ┌───────────┬──────────────────┬───────────────────┐+// │ software │ skinny matrices │ fat matrices │+// │ │ token generation │ prompt evaluation │+// │ ───────── │ ──────────────── │ ───────────────── │+// │ llama.cpp │ fast │ slow │+// │ MKL │ slow │ fast │+// │ tinyBLAS │ fast │ fast │+// └───────────┴──────────────────┴───────────────────┘+//+// these kernels do not have any of the latency overhead typically+// associated with the blas, which means they can provide the same+// performance as llama.cpp when performing the most common type of+// matmul used by text generation, which is matrix-vector product.#define END_KERNEL() }#define BEGIN_KERNEL(RM, RN) \@@ -100,17 +122,17 @@ template <typename T> class tinyBLAS {tinyBLAS(long k, const T *A, long lda, const T *B, long ldb, float *C, long ldc, long ith,long nth): k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith), nth(nth) {- ASSERT(A != nullptr);- ASSERT(B != nullptr);- ASSERT(C != nullptr);- ASSERT(k >= 0 && k % 8 == 0);- ASSERT(ith >= 0 && ith < nth);+ assert(A != nullptr);+ assert(B != nullptr);+ assert(C != nullptr);+ assert(k >= 0 && k % 8 == 0);+ assert(ith >= 0 && ith < nth);}void gemm(long m, long n) {- ASSERT(m >= 0);- ASSERT(n >= 0);- ASSERT(ldc >= m);+ assert(m >= 0);+ assert(n >= 0);+ assert(ldc >= m);mnpack(0, m, 0, n);}@@ -267,17 +289,17 @@ template <typename T, typename U, int avxvnni> class tinyBLASq {tinyBLASq(long k, const T *A, long lda, const U *B, long ldb, float *C, long ldc, long ith,long nth): k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith), nth(nth) {- ASSERT(A != nullptr);- ASSERT(B != nullptr);- ASSERT(C != nullptr);- ASSERT(k >= 0 && k % 32 == 0);- ASSERT(ith >= 0 && ith < nth);+ assert(A != nullptr);+ assert(B != nullptr);+ assert(C != nullptr);+ assert(k >= 0 && k % 32 == 0);+ assert(ith >= 0 && ith < nth);}void gemm(long m, long n) {- ASSERT(m >= 0);- ASSERT(n >= 0);- ASSERT(ldc >= m);+ assert(m >= 0);+ assert(n >= 0);+ assert(ldc >= m);mnpack(0, m, 0, n);}@@ -428,8 +450,8 @@ template <typename T, typename U, int avxvnni> class tinyBLASq {// computes m×k * n×k → n×mvoid llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda, const void *B,long ldb, float *C, long ldc, long ith, long nth) {- ASSERT(X86_HAVE(FMA));- ASSERT(X86_HAVE(AVX2));+ assert(X86_HAVE(FMA));+ assert(X86_HAVE(AVX2));switch (dtype) {#ifdef __x86_64__@@ -440,7 +462,7 @@ void llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda,}case GGML_TYPE_F16: {- ASSERT(X86_HAVE(F16C));+ assert(X86_HAVE(F16C));tinyBLAS<ggml_fp16_t> tb{k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth,};@@ -478,6 +500,6 @@ void llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda,#endifdefault:- ASSERT(!"unsupported dtype");+ assert(!"unsupported dtype");}}diff --git a/llamafile/tester.cu b/llamafile/tester.cuindex 5acd98c9..83955e89 100644--- a/llamafile/tester.cu+++ b/llamafile/tester.cu@@ -17,13 +17,12 @@#include "tester.h"-#include <algorithm>-#include <cmath>-#include <cstdio>-#include <cstdlib>+#include <atomic>#include <ctime>+#include <unistd.h>-const char *is_self_testing;+thread_local const char *is_self_testing;+const size_t kPageSize = std::max(sysconf(_SC_PAGESIZE), 4096l);long long micros(void) {struct timespec ts;@@ -46,197 +45,73 @@ float numba(void) { // (-1,1)return float01(rand32()) * 2 - 1;}-void fill(int m, int n, half *A, int lda) {- for (int i = 0; i < m; ++i)- for (int j = 0; j < n; ++j)- A[1ll * lda * j + i] = numba();-}--void fill(int m, int n, float *A, int lda) {- for (int i = 0; i < m; ++i)- for (int j = 0; j < n; ++j)- A[1ll * lda * j + i] = numba();-}--void broadcast(int m, int n, half *A, int lda, half x) {- for (int i = 0; i < m; ++i)- for (int j = 0; j < n; ++j)- A[1ll * lda * j + i] = x;-}--void broadcast(int m, int n, float *A, int lda, float x) {- for (int i = 0; i < m; ++i)- for (int j = 0; j < n; ++j)- A[1ll * lda * j + i] = x;-}--void dgemm(bool aᵀ, bool bᵀ, //- int m, int n, int k, float α, //- const float *A, int lda, //- const float *B, int ldb, float β, //- float *C, int ldc) {- ASSERT(m >= 0 && n >= 0 && k >= 0);- ASSERT(lda >= std::max(1, aᵀ ? k : m));- ASSERT(ldb >= std::max(1, bᵀ ? n : k));- ASSERT(ldc >= std::max(1, m));-#pragma omp parallel for collapse(2) if (m * n * k > 30000)- for (int i = 0; i < m; ++i)- for (int j = 0; j < n; ++j) {- double dot = 0;- for (int l = 0; l < k; ++l)- dot += (aᵀ ? A[1ll * lda * i + l] : A[1ll * lda * l + i]) *- (bᵀ ? B[1ll * ldb * l + j] : B[1ll * ldb * j + l]);- C[1ll * ldc * j + i] = α * dot + β * C[1ll * ldc * j + i];- }-}--void dgemm(bool aᵀ, bool bᵀ, //- int m, int n, int k, float α, //- const half *A, int lda, //- const half *B, int ldb, float β, //- float *C, int ldc) {- ASSERT(m >= 0 && n >= 0 && k >= 0);- ASSERT(lda >= std::max(1, aᵀ ? k : m));- ASSERT(ldb >= std::max(1, bᵀ ? n : k));- ASSERT(ldc >= std::max(1, m));-#pragma omp parallel for collapse(2) if (m * n * k > 30000)- for (int i = 0; i < m; ++i)- for (int j = 0; j < n; ++j) {- double dot = 0;- for (int l = 0; l < k; ++l)- dot += __half2float(aᵀ ? A[1ll * lda * i + l] : A[1ll * lda * l + i]) *- __half2float(bᵀ ? B[1ll * ldb * l + j] : B[1ll * ldb * j + l]);- C[1ll * ldc * j + i] = α * dot + β * C[1ll * ldc * j + i];- }-}--void dgemm(bool aᵀ, bool bᵀ, //- int m, int n, int k, float α, //- const half *A, int lda, //- const half *B, int ldb, float β, //- half *C, int ldc) {- ASSERT(m >= 0 && n >= 0 && k >= 0);- ASSERT(lda >= std::max(1, aᵀ ? k : m));- ASSERT(ldb >= std::max(1, bᵀ ? n : k));- ASSERT(ldc >= std::max(1, m));-#pragma omp parallel for collapse(2) if (m * n * k > 30000)- for (int i = 0; i < m; ++i)- for (int j = 0; j < n; ++j) {- float dot = 0;- for (int l = 0; l < k; ++l)- dot += __half2float(aᵀ ? A[1ll * lda * i + l] : A[1ll * lda * l + i]) *- __half2float(bᵀ ? B[1ll * ldb * l + j] : B[1ll * ldb * j + l]);- C[1ll * ldc * j + i] = α * dot + β * __half2float(C[1ll * ldc * j + i]);- }-}--template <typename T> double diff(int m, int n, const T *Wan, int lda, const T *Got, int ldb) {- double s = 0;- int got_nans = 0;- int wan_nans = 0;- if (!m || !n)- return 0;- for (int i = 0; i < m; ++i)- for (int j = 0; j < n; ++j)- if (IsNan(Wan[1ll * lda * j + i]))- ++wan_nans;- else if (IsNan(Got[1ll * ldb * j + i]))- ++got_nans;- else- s += std::fabs(static_cast<double>(Wan[1ll * lda * j + i]) -- static_cast<double>(Got[1ll * ldb * j + i]));- if (got_nans)- fprintf(stderr, "WARNING: got %d NaNs!\n", got_nans);- if (wan_nans)- fprintf(stderr, "WARNING: want array has %d NaNs!\n", wan_nans);- return s / (m * n);-}--template <typename T>-void show(FILE *f, int max, int m, int n, const T *A, int lda, const T *B, int ldb) {- flockfile(f);- fprintf(f, " ");- for (int i = 0; i < n && i < max; ++i)- fprintf(f, "%13d", i);- fprintf(f, "\n");- for (int i = 0; i < m; ++i) {- if (i == max) {- fprintf(f, "...\n");- break;- }- fprintf(f, "%5d ", i);- for (int j = 0; j < n; ++j) {- if (j == max) {- fprintf(f, " ...");- break;- }- char ba[32], bb[32];- snprintf(ba, 32, "%13.7f", static_cast<double>(A[1ll * lda * j + i]));- snprintf(bb, 32, "%13.7f", static_cast<double>(B[1ll * ldb * j + i]));- for (int k = 0; ba[k] && bb[k]; ++k) {- if (ba[k] != bb[k])- fputs_unlocked("\33[31m", f);- fputc_unlocked(ba[k], f);- if (ba[k] != bb[k])- fputs_unlocked("\33[0m", f);- }- }- fprintf(f, "\n");- }- funlockfile(f);-}--template <typename T>-void misfit(FILE *f, int max, int m, int n, const T *A, int lda, const T *B, int ldb,- const char *file, int line, double sad, double tol) {- fprintf(f, "%s:%d: sad %.17g exceeds %g (%s)\nwant\n", file, line, sad, tol,- is_self_testing ? is_self_testing : "n/a");- show(f, max, m, n, A, lda, B, ldb);- fprintf(f, "got\n");- show(f, max, m, n, B, ldb, A, lda);- fprintf(f, "\n");-}--template <typename T>-void checker(double tol, int m, int n, const T *A, int lda, const T *B, int ldb, const char *file,- int line) {- ASSERT(lda >= std::max(1, m));- ASSERT(ldb >= std::max(1, m));- double sad = diff(m, n, A, lda, B, ldb);- if (sad <= tol) {- if (!is_self_testing)- printf(" %g error\n", sad);- } else {- flockfile(stderr);- misfit(stderr, 16, m, n, A, lda, B, ldb, file, line, sad, tol);- const char *path = "/tmp/llamafile_tester.log";- FILE *f = fopen(path, "w");- if (f) {- misfit(f, 10000, m, n, A, lda, B, ldb, file, line, sad, tol);- printf("see also %s\n", path);+void *cudaMallocManagedOrDie(size_t n) {+ void *p;+ CUDA_OR_DIE(cudaMallocManaged(&p, n));+ return p;+}++void cudaFreeOrDie(void *p) {+ CUDA_OR_DIE(cudaFree(p));+}++[[noreturn]] void cuda_die(const char *stmt, const char *func, const char *file, int line,+ const char *msg) {+ int id = -1;+ (void)cudaGetDevice(&id);+ fprintf(stderr, "CUDA error: %s\n", msg);+ fprintf(stderr, " current device: %d, in function %s at %s:%d\n", id, func, file, line);+ fprintf(stderr, " %s\n", stmt);+ (void)cudaDeviceReset();+ _Exit(1);+}++void test_matmul(std::function<void(int m, int n, int k, int l, float α, float β)> f) {+ static const int kDims[] = {1, 2, 23, 77, 15, 2048, 512, 127, 129, 128, 16};+ static const float kAlphas[] = {1, .1};+ static const float kBetas[] = {0, .1};+ static const int kLeads[] = {0, 1};+ std::atomic_llong t = ATOMIC_VAR_INIT(0);+ for (int mi = 0; mi < ARRAYLEN(kDims); ++mi)+ for (int ni = 0; ni < ARRAYLEN(kDims); ++ni)+ for (int li = 0; li < ARRAYLEN(kLeads); ++li)+ for (int ki = 0; ki < ARRAYLEN(kDims); ++ki) {+ int m = kDims[mi];+ int n = kDims[ARRAYLEN(kDims) - 1 - ni];+ int k = kDims[ki];+ int l = kLeads[li];+ char name[256];+ sprintf(name, "testing %4d %4d %4d ld+%d", m, n, k, l);+ if (t++ % 7 == 0)+ fprintf(stderr, "%s\r", name);+ is_self_testing = name;+ f(m, n, k, l, 1, 0);+ is_self_testing = 0;+ }+ for (int ai = 0; ai < ARRAYLEN(kAlphas); ++ai)+ for (int bi = 0; bi < ARRAYLEN(kBetas); ++bi) {+ float α = kAlphas[ai];+ float β = kBetas[bi];+ int m = 128;+ int n = 128;+ int k = 128;+ char name[256];+ sprintf(name, "testing %4d %4d %4d α=%g β=%g", m, n, k, α, β);+ if (t++ % 7 == 0)+ fprintf(stderr, "%s\r", name);+ is_self_testing = name;+ f(m, n, k, 0, α, β);+ is_self_testing = 0;}- exit(1);- }-}--void check(double tol, int m, int n, const float *A, int lda, const float *B, int ldb,- const char *file, int line) {- checker(tol, m, n, A, lda, B, ldb, file, line);-}--void check(double tol, int m, int n, const half *A, int lda, const half *B, int ldb,- const char *file, int line) {- checker(tol, m, n, A, lda, B, ldb, file, line);-}--void run(const char *description) {- printf("\n%s\n", description);}-void bench(long long start, const char *description) {- printf("%8lld µs %s\n", (micros() - start + ITERATIONS - 1) / ITERATIONS, description);+static int cuda_tester_init() {+ CUDA_OR_DIE(cudaSetDevice(0));+#ifdef __HIP_PLATFORM_AMD__+ rocblas_initialize();+#endif+ CUDA_OR_DIE(cudaDeviceSynchronize());+ return 0;}-void passert(const char *file, int line, const char *expr) {- fprintf(stderr, "%s:%d: assertion failed: %s\n", file, line, expr);-}+const int cuda_tester_ = cuda_tester_init();diff --git a/llamafile/tester.h b/llamafile/tester.hindex 01a1c06e..8c816edb 100644--- a/llamafile/tester.h+++ b/llamafile/tester.h@@ -1,82 +1,314 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.#pragma once-#ifdef __NVCC__-#include <cuda_fp16.h>-#else-#include <hip/hip_fp16.h>-#endif+#include "cuda.h"+#include "flt.h"+#include "macros.h"+#include "naive.h"+#include "tinyblas.h"+#include <algorithm>+#include <cassert>+#include <cmath>+#include <cstdio>+#include <cstdlib>+#include <functional>++//+// _ _ ___ _ _ ___+// | |_(_)_ _ _ _| _ ) | /_\ / __|+// | _| | ' \ || | _ \ |__ / _ \\__ \.+// \__|_|_||_\_, |___/____/_/ \_\___/+// |__/+//+// BASIC LINEAR ALGEBRA SUBPROGRAMS+//++// c++ testing utilities for floating point math on cpu and gpu+//+// this file contains tools for testing matrix multiplication, measuring+// errors, and visualizing what went wrong in using ansi terminal codes.#define ITERATIONS 30+#define TOMBSTONE 1.666f-extern const char *is_self_testing;+extern const size_t kPageSize;+extern thread_local const char *is_self_testing;float float01(unsigned);float numba(void);int rand32(void);long long micros(void);-void bench(long long, const char *);-void broadcast(int, int, float *, int, float);-void broadcast(int, int, half *, int, half);-void check(double, int, int, const float *, int, const float *, int, const char *, int);-void check(double, int, int, const half *, int, const half *, int, const char *, int);-void fill(int, int, float *, int);-void fill(int, int, half *, int);-void passert(const char *, int, const char *);-void run(const char *);+void test_matmul(std::function<void(int, int, int, int, float, float)>);+void *cudaMallocManagedOrDie(size_t);+void cudaFreeOrDie(void *);-void dgemm(bool, bool, int, int, int, float, const half *, int, const half *, int, float, half *,- int);-void dgemm(bool, bool, int, int, int, float, const half *, int, const half *, int, float, float *,- int);-void dgemm(bool, bool, int, int, int, float, const float *, int, const float *, int, float, float *,- int);+inline int hamming(int x, int y) {+ return __builtin_popcount(x ^ y);+}-inline int toint(half f) {- union {- half f;- unsigned short i;- } u = {f};- return u.i;+template <typename T> void randomize(T *A, int n) {+ for (int i = 0; i < n; ++i)+ A[i] = numba();}-inline int toint(float f) {- union {- float f;- unsigned i;- } u = {f};- return u.i;+template <typename T> void randomize(int m, int n, T *A, int lda) {+ for (int j = 0; j < n; ++j)+ for (int i = 0; i < m; ++i)+ A[lda * j + i] = numba();}-inline int IsNan(half f) {- return (toint(f) & 0x7fff) > 0x7c00;+template <typename T, typename U> void broadcast(T *A, int n, U x) {+ for (int i = 0; i < n; ++i)+ A[i] = x;}-inline int IsNan(float f) {- return (toint(f) & 0x7fffffff) > 0x7f800000;+template <typename T, typename U> void broadcast(int m, int n, T *A, int lda, U x) {+ for (int j = 0; j < n; ++j)+ for (int i = 0; i < m; ++i)+ A[lda * j + i] = x;}-inline int IsNegativeZero(half f) {- return toint(f) == 0x8000;+template <typename T> struct cuda_memory {+ const size_t size;+ T *const p;+ explicit cuda_memory(int len)+#ifdef __SANITIZE_ADDRESS__+ : size(len * sizeof(T)),+#else+ : size(ROUNDUP(len * sizeof(T) + 64, kPageSize) - 64),+#endif+ p(static_cast<T *>(cudaMallocManagedOrDie(size))) {+ broadcast(p, size / sizeof(T), NAN);+ }+ ~cuda_memory() {+ cudaFreeOrDie(p);+ }+};++template <typename T>+void verify(double tol, int m, int n, const T *A, int lda, const T *B, int ldb) {+ T worst_x = 0;+ T worst_y = 0;+ int worst_i = 0;+ int worst_j = 0;+ long long nans = 0;+ long long infs = 0;+ long long worst = 0;+ long long flips = 0;+ long long diffsum = 0;+ long long denormals = 0;+ long long hamhist[32] = {0};+ for (int i = 0; i < m; i++)+ for (int j = 0; j < n; j++) {+ T x = A[lda * j + i];+ T y = B[ldb * j + i];+ long long xi = flt::toint(x);+ long long yi = flt::toint(y);+ long long d = std::abs(xi - yi);+ if (flt::isnormal(x) && flt::isnormal(y) && flt::signbit(x) == flt::signbit(y)) {+ diffsum += d;+ if (d > worst) {+ worst = d;+ worst_x = x;+ worst_y = y;+ worst_i = i;+ worst_j = j;+ }+ }+ nans -= flt::isnan(x);+ nans += flt::isnan(y);+ infs -= flt::isinf(x);+ infs += flt::isinf(y);+ flips += flt::signbit(x) != flt::signbit(y);+ denormals -= flt::isdenormal(x);+ denormals += flt::isdenormal(y);+ ++hamhist[hamming(xi, yi)];+ }+ double avg = static_cast<double>(diffsum) / (m * n);+ if (avg > tol || nans) {+ flockfile(stderr);+ fprintf(stderr, "error: %d×%d matrix difference exceeds tolerance\n", m, n);+ fprintf(stderr, " worst element at index (%d, %d)\n", worst_i, worst_j);+ fprintf(stderr, " difference %14e (%lld ulp)\n",+ std::fabs(static_cast<double>(worst_x) - static_cast<double>(worst_y)), worst);+ fprintf(stderr, " expected %14e (%x)\n", static_cast<double>(worst_x),+ flt::toint(worst_x));+ fprintf(stderr, " actual %14e (%x)\n", static_cast<double>(worst_y),+ flt::toint(worst_y));+ fprintf(stderr, " average %14g ulp\n", avg);+ fprintf(stderr, " denormals %14lld elements\n", denormals);+ fprintf(stderr, " flips %14lld elements\n", flips);+ fprintf(stderr, " nans %14lld elements\n", nans);+ fprintf(stderr, " infs %14lld elements\n", infs);+ for (int i = 0; i < 32; ++i)+ if (hamhist[i])+ fprintf(stderr, " hamhist%-2d %14lld elements\n", i, hamhist[i]);+ fflush(stderr);+ (void)cudaDeviceReset();+ _Exit(1);+ }}-inline int IsNegativeZero(float f) {- return toint(f) == -2147483647 - 1;+template <typename T>+void diff(int m, int n, int k, const T *Wan, int lda, const T *Got, int ldb, double *out_avg,+ double *out_sad, double *out_worst, int *out_infs, int *out_flips, int *out_zeroes,+ int *out_denormals, int *out_nans) {+ double sad = 0;+ long long ulp = 0;+ long long worst = 0;+ int infs = 0;+ int flips = 0;+ int zeroes = 0;+ int got_nans = 0;+ int wan_nans = 0;+ int denormals = 0;+ int considered = 0;+ if (m && n) {+ for (int i = 0; i < m; ++i)+ for (int j = 0; j < n; ++j) {+ T x = Wan[lda * j + i];+ T y = Got[ldb * j + i];+ if (flt::isnan(x)) {+ ++wan_nans;+ } else if (flt::isnan(y)) {+ ++got_nans;+ } else {+ zeroes -= !x;+ zeroes += !y;+ infs -= flt::isinf(x);+ infs += flt::isinf(y);+ denormals -= flt::isdenormal(x);+ denormals += flt::isdenormal(y);+ flips += flt::signbit(x) != flt::signbit(y);+ long long xi = flt::toint(x);+ long long yi = flt::toint(y);+ if (flt::isnormal(x) && flt::isnormal(y) &&+ flt::signbit(x) == flt::signbit(y)) {+ ++considered;+ ulp += std::abs(xi - yi);+ worst = std::max(worst, std::abs(xi - yi));+ sad += std::fabs(static_cast<double>(x) - static_cast<double>(y));+ }+ }+ }+ if (got_nans)+ fprintf(stderr, "WARNING: got %d NaNs!\n", got_nans);+ if (wan_nans)+ fprintf(stderr, "WARNING: want array has %d NaNs!\n", wan_nans);+ }+ int ops_per_element = k + 2;+ *out_avg = static_cast<double>(ulp) / considered / ops_per_element;+ *out_worst = static_cast<double>(worst) / ops_per_element;+ *out_nans = got_nans + wan_nans;+ *out_sad = sad / considered;+ *out_denormals = denormals;+ *out_zeroes = zeroes;+ *out_flips = flips;+ *out_infs = infs;}-#define ASSERT(x) \- do { \- if (!(x)) { \- passert(__FILE__, __LINE__, #x); \- __builtin_trap(); \- } \- } while (0)+template <typename T>+void show(FILE *f, int max, int m, int n, const T *A, int lda, const T *B, int ldb) {+ flockfile(f);+ fprintf(f, " ");+ for (int i = 0; i < n && i < max; ++i)+ fprintf(f, "%13d", i);+ fprintf(f, "\n");+ for (int i = 0; i < m; ++i) {+ if (i == max) {+ fprintf(f, "...\n");+ break;+ }+ fprintf(f, "%5d ", i);+ for (int j = 0; j < n; ++j) {+ if (j == max) {+ fprintf(f, " ...");+ break;+ }+ char ba[32], bb[32];+ snprintf(ba, 32, "%13.7f", static_cast<double>(A[lda * j + i]));+ snprintf(bb, 32, "%13.7f", static_cast<double>(B[ldb * j + i]));+ for (int k = 0; ba[k] && bb[k]; ++k) {+ if (ba[k] != bb[k])+ fputs_unlocked("\33[31m", f);+ fputc_unlocked(ba[k], f);+ if (ba[k] != bb[k])+ fputs_unlocked("\33[0m", f);+ }+ }+ fprintf(f, "\n");+ }+ funlockfile(f);+}++template <typename T>+void misfit(FILE *f, int max, int m, int n, const T *A, int lda, const T *B, int ldb,+ const char *file, int line, double avg, double sad, double worst, int infs, int flips,+ int zeroes, int denormals, int nans) {+ fprintf(f, "%s:%d: matrix errors exceed tolerance %s\n", file, line,+ is_self_testing ? is_self_testing : "");+ fprintf(f,+ " %g ulp (worst=%g sad=%g nans=%d infs=%d flips=%d zeroes=%d denormals=%d)\n",+ avg, worst, sad, nans, infs, flips, zeroes, denormals);+ fprintf(f, "want\n");+ show(f, max, m, n, A, lda, B, ldb);+ fprintf(f, "got\n");+ show(f, max, m, n, B, ldb, A, lda);+ fprintf(f, "\n");+}-#define CHECK(tol, m, n, A, lda, B, ldb) check(tol, m, n, A, lda, B, ldb, __FILE__, __LINE__)+template <typename T>+void check(double tol, //+ int m, int n, int k, //+ const T *A, int lda, //+ const T *B, int ldb, //+ const char *file, int line) {+ double avg, sad, worst;+ int nans, infs, flips, zeroes, denormals;+ assert(lda >= std::max(1, m));+ assert(ldb >= std::max(1, m));+ diff(m, n, k, A, lda, B, ldb, &avg, &sad, &worst, &infs, &flips, &zeroes, &denormals, &nans);+ if (!nans && avg <= tol) {+ if (!is_self_testing)+ printf(" %g ulp (worst=%g sad=%g nans=%d infs=%d flips=%d zeroes=%d "+ "denormals=%d)\n",+ avg, worst, sad, nans, infs, flips, zeroes, denormals);+ } else {+ flockfile(stderr);+ misfit(stderr, 16, m, n, A, lda, B, ldb, file, line, avg, sad, worst, infs, flips, zeroes,+ denormals, nans);+ const char *path = "/tmp/wompwomp.log";+ FILE *f = fopen(path, "w");+ if (f) {+ misfit(f, 10000, m, n, A, lda, B, ldb, file, line, avg, sad, worst, infs, flips, zeroes,+ denormals, nans);+ fclose(f);+ fprintf(stderr, "see also %s\n", path);+ }+ fflush(stderr);+ (void)cudaDeviceReset();+ _Exit(1);+ }+}++#define CHECK(tol, m, n, k, A, lda, B, ldb) check(tol, m, n, k, A, lda, B, ldb, __FILE__, __LINE__)-#define BENCH(x) \+#define BENCH_CPU(x) \do { \x; \__asm__ volatile("" ::: "memory"); \@@ -86,9 +318,197 @@ inline int IsNegativeZero(float f) {x; \__asm__ volatile("" ::: "memory"); \} \- bench(start, #x); \+ printf("%8lld µs %s\n", (micros() - start + ITERATIONS - 1) / ITERATIONS, #x); \+ } while (0)++#define CUDA_OR_DIE(x) \+ do { \+ cudaError_t err_ = (x); \+ if (err_ != cudaSuccess) \+ cuda_die(#x, __FUNCTION__, __FILE__, __LINE__, cudaGetErrorString(err_)); \+ } while (0)++#define CUBLAS_OR_DIE(x) \+ do { \+ cublasStatus_t status_ = (x); \+ if (status_ != CUBLAS_STATUS_SUCCESS) \+ cuda_die(#x, __FUNCTION__, __FILE__, __LINE__, cublasGetStatusString(status_)); \+ } while (0)++#define TINYBLAS_OR_DIE(x) \+ do { \+ tinyblasStatus_t status_ = (x); \+ if (status_ != TINYBLAS_STATUS_SUCCESS) \+ cuda_die(#x, __FUNCTION__, __FILE__, __LINE__, tinyblasGetStatusString(status_)); \+ } while (0)++#define BENCH_CUDA(x) \+ do { \+ if (is_self_testing) { \+ x; \+ } else { \+ cudaEvent_t start_, stop_; \+ CUDA_OR_DIE(cudaEventCreate(&stop_)); \+ CUDA_OR_DIE(cudaEventCreate(&start_)); \+ CUDA_OR_DIE(cudaEventRecord(start_, stream)); \+ for (int i_ = 0; i_ < ITERATIONS; ++i_) \+ x; \+ float msecTotal_ = 0; \+ CUDA_OR_DIE(cudaEventRecord(stop_, stream)); \+ CUDA_OR_DIE(cudaEventSynchronize(stop_)); \+ CUDA_OR_DIE(cudaEventElapsedTime(&msecTotal_, start_, stop_)); \+ printf("%8d µs %s(%d, %d, %d)\n", \+ static_cast<int>(std::ceil(msecTotal_ / ITERATIONS * 1000)), __FUNCTION__, m, \+ n, k); \+ CUDA_OR_DIE(cudaEventDestroy(start_)); \+ CUDA_OR_DIE(cudaEventDestroy(stop_)); \+ } \} while (0)#define RUN(x) \- run(#x); \+ printf(" \r\n%s\n", #x); \x++[[noreturn]] void cuda_die(const char *, const char *, const char *, int, const char *);++template <typename T> struct cuda_data_type;+template <> struct cuda_data_type<half> {+ static constexpr cudaDataType_t id = CUDA_R_16F;+};+template <> struct cuda_data_type<float> {+ static constexpr cudaDataType_t id = CUDA_R_32F;+};++template <typename T> struct cublas_compute_type;+template <> struct cublas_compute_type<half> {+ static constexpr cublasComputeType_t id = CUBLAS_COMPUTE_16F;+};+template <> struct cublas_compute_type<float> {+ static constexpr cublasComputeType_t id = CUBLAS_COMPUTE_32F;+};++template <typename T> struct tinyblas_data_type;+template <> struct tinyblas_data_type<half> {+ static constexpr tinyblasDataType_t id = TINYBLAS_R_16F;+};+template <> struct tinyblas_data_type<float> {+ static constexpr tinyblasDataType_t id = TINYBLAS_R_32F;+};++template <typename T> struct tinyblas_compute_type;+template <> struct tinyblas_compute_type<half> {+ static constexpr tinyblasComputeType_t id = TINYBLAS_COMPUTE_16F;+};+template <> struct tinyblas_compute_type<float> {+ static constexpr tinyblasComputeType_t id = TINYBLAS_COMPUTE_32F;+};++template <typename T, typename TA, typename TB, typename TC>+void cublas(bool aᵀ, bool bᵀ, //+ int m, int n, int k, T α, //+ const TA *A, int lda, //+ const TB *B, int ldb, T β, //+ TC *C, int ldc) {+ cudaStream_t stream;+ cublasHandle_t blas;+ assert(m >= 0 && n >= 0 && k >= 0);+ assert(lda >= std::max(1, aᵀ ? k : m));+ assert(ldb >= std::max(1, bᵀ ? n : k));+ assert(ldc >= std::max(1, m));+ CUDA_OR_DIE(cudaSetDevice(0));+ CUBLAS_OR_DIE(cublasCreate(&blas));+ CUDA_OR_DIE(cudaStreamCreate(&stream));+ CUBLAS_OR_DIE(cublasSetMathMode(blas, CUBLAS_DEFAULT_MATH));+ CUBLAS_OR_DIE(cublasSetStream(blas, stream));+ BENCH_CUDA(CUBLAS_OR_DIE(cublasGemmEx(+ blas, aᵀ ? CUBLAS_OP_T : CUBLAS_OP_N, bᵀ ? CUBLAS_OP_T : CUBLAS_OP_N, m, n, k, &α, A,+ cuda_data_type<TA>::id, lda, B, cuda_data_type<TB>::id, ldb, &β, C, cuda_data_type<TC>::id,+ ldc, cublas_compute_type<T>::id, CUBLAS_GEMM_DEFAULT)));+ CUDA_OR_DIE(cudaStreamSynchronize(stream));+ CUDA_OR_DIE(cudaStreamDestroy(stream));+ CUBLAS_OR_DIE(cublasDestroy(blas));+}++template <typename T, typename TA, typename TB, typename TC>+void cublasrm(bool aᵀ, bool bᵀ, //+ int m, int n, int k, T α, //+ const TA *A, int lda, //+ const TB *B, int ldb, T β, //+ TC *C, int ldc) {+ cublas(bᵀ, aᵀ, n, m, k, α, B, ldb, A, lda, β, C, ldc);+}++template <typename T, typename TA, typename TB, typename TC>+void tinyblas(bool aᵀ, bool bᵀ, //+ int m, int n, int k, T α, //+ const TA *A, int lda, //+ const TB *B, int ldb, T β, //+ TC *C, int ldc) {+ cudaStream_t stream;+ tinyblasHandle_t blas;+ assert(m >= 0 && n >= 0 && k >= 0);+ assert(lda >= std::max(1, aᵀ ? k : m));+ assert(ldb >= std::max(1, bᵀ ? n : k));+ assert(ldc >= std::max(1, m));+ CUDA_OR_DIE(cudaSetDevice(0));+ TINYBLAS_OR_DIE(tinyblasCreate(&blas));+ CUDA_OR_DIE(cudaStreamCreate(&stream));+ TINYBLAS_OR_DIE(tinyblasSetStream(blas, stream));+ BENCH_CUDA(TINYBLAS_OR_DIE(tinyblasGemmEx(+ blas, aᵀ ? TINYBLAS_OP_T : TINYBLAS_OP_N, bᵀ ? TINYBLAS_OP_T : TINYBLAS_OP_N, m, n, k, &α,+ A, tinyblas_data_type<TA>::id, lda, B, tinyblas_data_type<TB>::id, ldb, &β, C,+ tinyblas_data_type<TC>::id, ldc, tinyblas_compute_type<T>::id, TINYBLAS_GEMM_DEFAULT)));+ CUDA_OR_DIE(cudaStreamSynchronize(stream));+ CUDA_OR_DIE(cudaStreamDestroy(stream));+ TINYBLAS_OR_DIE(tinyblasDestroy(blas));+}++template <typename T, typename TA, typename TB, typename TC>+void tinyblasrm(bool aᵀ, bool bᵀ, //+ int m, int n, int k, T α, //+ const TA *A, int lda, //+ const TB *B, int ldb, T β, //+ TC *C, int ldc) {+ tinyblas(bᵀ, aᵀ, n, m, k, α, B, ldb, A, lda, β, C, ldc);+}++// multiplies matrix on gpu with column major ordering+//+// m×k * k×n → m×n+// k×m * k×n → m×n if aᵀ+// m×k * n×k → m×n if bᵀ+// k×m * n×k → m×n if aᵀ and bᵀ+//+template <typename T, typename TA, typename TB, typename TC>+void gemmref(bool aᵀ, bool bᵀ, //+ int m, int n, int k, T α, //+ const TA *A, int lda, //+ const TB *B, int ldb, T β, //+ TC *C, int ldc) {+ cudaStream_t stream;+ assert(m >= 0 && n >= 0 && k >= 0);+ assert(lda >= std::max(1, aᵀ ? k : m));+ assert(ldb >= std::max(1, bᵀ ? n : k));+ assert(ldc >= std::max(1, m));+ CUDA_OR_DIE(cudaSetDevice(0));+ CUDA_OR_DIE(cudaStreamCreate(&stream));+ BENCH_CUDA(CUDA_OR_DIE(naive::gemm(stream, aᵀ, bᵀ, m, n, k, α, A, lda, B, ldb, β, C, ldc)));+ CUDA_OR_DIE(cudaStreamSynchronize(stream));+ CUDA_OR_DIE(cudaStreamDestroy(stream));+}++// multiplies matrix on gpu with row major ordering+//+// k×m * n×k → n×m+// m×k * n×k → n×m if aᵀ+// k×m * k×n → n×m if bᵀ+// m×k * k×n → n×m if aᵀ and bᵀ+//+template <typename T, typename TA, typename TB, typename TC>+void gemmrefrm(bool aᵀ, bool bᵀ, //+ int m, int n, int k, T α, //+ const TA *A, int lda, //+ const TB *B, int ldb, T β, //+ TC *C, int ldc) {+ gemmref(bᵀ, aᵀ, n, m, k, α, B, ldb, A, lda, β, C, ldc);+}diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cuindex 970291ab..0a3c6f36 100644--- a/llamafile/tinyblas.cu+++ b/llamafile/tinyblas.cu@@ -21,14 +21,16 @@#include <cstdlib>#include <type_traits>-#ifdef __NVCC__+#if defined(__NVCC__) || defined(__CUDA_ARCH__)#include <cuda_fp16.h>#include <cuda_runtime.h>#define tinyblasStream_t cudaStream_t#else#include <hip/hip_fp16.h>#include <hip/hip_runtime.h>+#define cudaSuccess hipSuccess#define tinyblasStream_t hipStream_t+#define cudaGetLastError hipGetLastError#endif//@@ -41,7 +43,8 @@// BASIC LINEAR ALGEBRA SUBPROGRAMS//-#define SHARELEN (BK * BM + BK * BN)+#define THREAD_COUNT ((BM * BN) / (TM * TN))+#define KERNEL __launch_bounds__(THREAD_COUNT)#define CEIL_DIV(M, N) (((M) + (N) - 1) / (N))struct tinyblasContext {@@ -73,99 +76,6 @@ tinyblasStatus_t tinyblasGetStream(tinyblasHandle_t handle, void **out_stream) {return TINYBLAS_STATUS_SUCCESS;}-template <int BM, int BN, int BK, int TM, int TN, typename WORD, typename SRC, typename DST>-static __device__ void matmul_block2d(tinyblasOperation_t transa, tinyblasOperation_t transb, int m,- int n, int k, int x, int y, WORD alpha, const SRC *A, int lda,- const SRC *B, int ldb, WORD beta, DST *C, int ldc) {- static_assert((BM % TM == 0) && (BN % TN == 0), "can't divide work for threads");- static_assert((BM * BN) <= (BM * BK) + (BK * BN),- "didn't allocate enough shared mem for threads");- const int ii0 = threadIdx.x / (BN / TN); // {0, ..., (BM/TM) - 1}- const int ii1 = threadIdx.x % (BN / TN); // {0, ..., (BN/TN) - 1}- extern __shared__ char svals[]; // shared across all threads in a block- SRC *As /*[BK][BM]*/ = (SRC *)svals;- SRC *Bs /*[BK][BN]*/ = (SRC *)svals + BM * BK;-- WORD Cs[TM][TN] = {0};- SRC At[TM], Bt[TN];- int i, h, j, l, blob;-- for (blob = 0; blob < k; blob += BK) {- for (i = threadIdx.x; i < BK; i += blockDim.x)- for (j = 0; j < BM; ++j)- As[BM * i + j] = 0;- for (i = threadIdx.x; i < BK && blob + i < k; i += blockDim.x)- for (j = 0; j < BM && x + j < m; ++j)- As[BM * i + j] = transa ? A[1ll * lda * (x + j) + (blob + i)]- : A[1ll * lda * (blob + i) + (x + j)];- for (i = threadIdx.x; i < BK; i += blockDim.x)- for (j = 0; j < BN; ++j)- Bs[BN * i + j] = 0;- for (i = threadIdx.x; i < BK && blob + i < k; i += blockDim.x)- for (j = 0; j < BN && y + j < n; ++j)- Bs[BN * i + j] = transb ? B[1ll * ldb * (blob + i) + (y + j)]- : B[1ll * ldb * (y + j) + (blob + i)];- __syncthreads();-- for (l = 0; l < BK; ++l) {- for (j = 0; j < TM; ++j)- At[j] = As[BM * l + TM * ii0 + j];- for (h = 0; h < TN; ++h)- Bt[h] = Bs[BN * l + TN * ii1 + h];- for (j = 0; j < TM; ++j) {- WORD a = At[j];- for (h = 0; h < TN; ++h) {- WORD b = Bt[h];- Cs[j][h] += a * b;- }- }- }- __syncthreads();- }-- x += ii0 * TM;- y += ii1 * TN;- for (l = 0; l < TN && y + l < n; ++l)- for (j = 0; j < TM && x + j < m; ++j)- if (beta) {- WORD c = C[1ll * ldc * (y + l) + (x + j)];- C[1ll * ldc * (y + l) + (x + j)] = alpha * Cs[j][l] + beta * c;- } else {- C[1ll * ldc * (y + l) + (x + j)] = alpha * Cs[j][l];- }-}--template <int BM, int BN, int BK, int TM, int TN>-static __global__ void tinyblasS_entry(tinyblasOperation_t transa, tinyblasOperation_t transb,- int m, int n, int k, float alpha, const float *A, int lda,- const float *B, int ldb, float beta, float *C, int ldc) {-- int x = blockIdx.x * BM;- const int jump1 = gridDim.x * BM;-- int y = blockIdx.y * BN;- const int jump2 = gridDim.y * BN;-- // each block handles a sub-matrix of C, of size BM * BN- // each thread handles a sub-matrix of size TM * TN- for (x = blockIdx.x * BM; x < m; x += jump1)- for (y = blockIdx.y * BN; y < n; y += jump2)- matmul_block2d<BM, BN, BK, TM, TN, float>(transa, transb, m, n, k, x, y, alpha, A, lda,- B, ldb, beta, C, ldc);-}--template <int BM, int BN, int BK, int TM, int TN>-static void tinyblasS_wrapper(tinyblasHandle_t handle, tinyblasOperation_t transa,- tinyblasOperation_t transb, int m, int n, int k, const float alpha,- const float *A, int lda, const float *B, int ldb, const float beta,- float *C, int ldc) {- dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 1);- int sharedmem = sizeof(float) * SHARELEN;- int maxthreads = (BM * BN) / (TM * TN);- tinyblasS_entry<BM, BN, BK, TM, TN><<<maxblocks, maxthreads, sharedmem, handle->stream>>>(- transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc);-}-tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t handle, //tinyblasOperation_t transa, //tinyblasOperation_t transb, //@@ -175,175 +85,204 @@ tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t handle, //const float *B, int ldb, //const float *beta, //float *C, int ldc) {- tinyblasS_wrapper<32, 32, 16, 2, 8>(handle, transa, transb, m, n, k, *alpha, A, lda, B, ldb,- *beta, C, ldc);- return TINYBLAS_STATUS_SUCCESS;+ return tinyblasGemmEx(handle, transa, transb, m, n, k, alpha, A, TINYBLAS_R_32F, lda, B,+ TINYBLAS_R_32F, ldb, beta, C, TINYBLAS_R_32F, ldc, TINYBLAS_COMPUTE_32F,+ TINYBLAS_GEMM_DEFAULT);}-static bool is_valid(tinyblasOperation_t transa, //- tinyblasOperation_t transb, //- int m, int n, int k, //- int lda, int ldb, int ldc) {- return m >= 0 && //- n >= 0 && //- k >= 0 && //- lda >= std::max(1, transa ? k : m) && //- ldb >= std::max(1, transb ? n : k) && //- ldc >= std::max(1, m);+const char *tinyblasGetStatusString(tinyblasStatus_t err) {+ switch (err) {+ case TINYBLAS_STATUS_SUCCESS:+ return "Success";+ case TINYBLAS_STATUS_ALLOC_FAILED:+ return "Alloc failed";+ case TINYBLAS_STATUS_INVALID_VALUE:+ return "Invalid value";+ case TINYBLAS_STATUS_NOT_SUPPORTED:+ return "Not supported";+ case TINYBLAS_STATUS_EXECUTION_FAILED:+ return "Execution failed";+ case TINYBLAS_STATUS_DIMENSION_OVERFLOW:+ return "Dimension overflow";+ default:+ return "Unknown error";+ }}-static bool is_supported(tinyblasDatatype_t Atype, tinyblasDatatype_t Btype) {- return Atype == TINYBLAS_R_16F && //- Btype == TINYBLAS_R_16F;+__device__ __forceinline__ float madd(float x, float y, float z) {+ return __fmaf_rn(x, y, z);}-// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmex-template <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST, typename T>-static __global__ void tinyblasGE_entry(tinyblasOperation_t transa, tinyblasOperation_t transb,- int m, int n, int k, T alpha, const SRC *A, int lda,- const SRC *B, int ldb, T beta, DST *C, int ldc) {+template <int BM, int BN, int TM, int TN, typename WORD, typename SRC, typename DST>+static __device__ void matmul_block2d(tinyblasOperation_t transa, tinyblasOperation_t transb, int m,+ int n, int k, WORD alpha, const SRC *A, int lda, const SRC *B,+ int ldb, WORD beta, DST *C, int ldc) {++ constexpr int BK = THREAD_COUNT;+ static_assert(BM % TM == 0, "can't divide work for threads");+ static_assert(BN % TN == 0, "can't divide work for threads");+ static_assert(BM > 0 && BN > 0 && BK > 0 && TM > 0 && TN > 0,+ "one of the constexpr configuration values was non-positive");+ static_assert((BK * BM * sizeof(SRC)) + (BK * BN * sizeof(SRC)) <= 65536,+ "you're almost almost certainly using too much shared memory");++ const int th = threadIdx.x;+ const int ii = blockIdx.x * BM;+ const int jj = blockIdx.y * BN;+ const int ti = th / (BN / TN) * TM;+ const int tj = th % (BN / TN) * TN;++ __shared__ SRC As[BK * BM];+ __shared__ SRC Bs[BK * BN];++ WORD At[TM];+ WORD Bt[TN];+ WORD Cs[TM * TN] = {0};++ for (int ll = 0; ll < k; ll += BK) {++ for (int i = 0; i < BM; ++i)+ As[BM * th + i] = 0;+ for (int i = 0; i < BM && ll + th < k && ii + i < m; ++i)+ As[BM * th + i] = A[transa ? lda * (ii + i) + (ll + th) : lda * (ll + th) + (ii + i)];++ for (int j = 0; j < BN; ++j)+ Bs[BN * th + j] = 0;+ for (int j = 0; j < BN && ll + th < k && jj + j < n; ++j)+ Bs[BN * th + j] = B[transb ? ldb * (ll + th) + (jj + j) : ldb * (jj + j) + (ll + th)];- int x = blockIdx.x * BM;- const int jump1 = gridDim.x * BM;+ __syncthreads();- int y = blockIdx.y * BN;- const int jump2 = gridDim.y * BN;+ for (int l = 0; l < BK; ++l) {+ for (int j = 0; j < TM; ++j)+ At[j] = As[BM * l + ti + j];+ for (int h = 0; h < TN; ++h)+ Bt[h] = Bs[BN * l + tj + h];+ for (int j = 0; j < TM; ++j) {+ WORD a = At[j];+ for (int h = 0; h < TN; ++h) {+ WORD b = Bt[h];+ Cs[TN * j + h] = madd(a, b, Cs[TN * j + h]);+ }+ }+ }- // each block handles a sub-matrix of C, of size BM * BN- for (x = blockIdx.x * BM; x < m; x += jump1)- for (y = blockIdx.y * BN; y < n; y += jump2)- matmul_block2d<BM, BN, BK, TM, TN, float>(transa, transb, m, n, k, x, y, alpha, A, lda,- B, ldb, beta, C, ldc);+ __syncthreads();+ }++ if (alpha != (WORD)1)+ for (int i = 0; i < TM * TN; ++i)+ Cs[i] *= alpha;++ for (int j = 0; j < TN && jj + tj + j < n; ++j)+ for (int i = 0; i < TM && ii + ti + i < m; ++i)+ if (beta) {+ WORD c = C[ldc * (jj + tj + j) + (ii + ti + i)];+ C[ldc * (jj + tj + j) + (ii + ti + i)] = madd(c, beta, Cs[TN * i + j]);+ } else {+ C[ldc * (jj + tj + j) + (ii + ti + i)] = Cs[TN * i + j];+ }}-template <int BM, int BN, int BK, int TM, int TN, typename SRC>-static tinyblasStatus_t-tinyblasGE_wrapper(tinyblasHandle_t handle, tinyblasOperation_t transa, tinyblasOperation_t transb,- int m, int n, int k, const void *pAlpha, const SRC *A, int lda, const SRC *B,- int ldb, const void *pBeta, void *C, tinyblasDatatype_t Ctype, int ldc,- tinyblasComputeType_t computeType) {- dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 1);- int sharedmem = sizeof(SRC) * SHARELEN;- int maxthreads = (BM * BN) / (TM * TN);- switch (Ctype) {- case TINYBLAS_R_16F:- switch (computeType) {- case TINYBLAS_COMPUTE_16F:- tinyblasGE_entry<BM, BN, BK, TM, TN>- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(- transa, transb, m, n, k, *(const half *)pAlpha, A, lda, B, ldb,- *(const half *)pBeta, (half *)C, ldc);- return TINYBLAS_STATUS_SUCCESS;- case TINYBLAS_COMPUTE_32F:- tinyblasGE_entry<BM, BN, BK, TM, TN>- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(- transa, transb, m, n, k, *(const float *)pAlpha, A, lda, B, ldb,- *(const float *)pBeta, (half *)C, ldc);- return TINYBLAS_STATUS_SUCCESS;- default:- return TINYBLAS_STATUS_INVALID_VALUE;- }- case TINYBLAS_R_32F:- switch (computeType) {- case TINYBLAS_COMPUTE_16F:- return TINYBLAS_STATUS_NOT_SUPPORTED;- case TINYBLAS_COMPUTE_32F:- tinyblasGE_entry<BM, BN, BK, TM, TN>- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(- transa, transb, m, n, k, *(const float *)pAlpha, A, lda, B, ldb,- *(const float *)pBeta, (float *)C, ldc);- return TINYBLAS_STATUS_SUCCESS;- default:- return TINYBLAS_STATUS_INVALID_VALUE;- }- default:- return TINYBLAS_STATUS_INVALID_VALUE;- }+template <int BM, int BN, int TM, int TN, typename WORD, typename SRC, typename DST>+static __global__ void KERNEL tinyblasGE_entry(tinyblasOperation_t transa,+ tinyblasOperation_t transb, int m, int n, int k,+ WORD alpha, const SRC *A, int lda, const SRC *B,+ int ldb, WORD beta, DST *C, int ldc) {+ matmul_block2d<BM, BN, TM, TN>(transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc);+}++template <typename WORD, typename SRC, typename DST>+static tinyblasStatus_t tinyblasGE_launch(tinyblasHandle_t handle, tinyblasOperation_t transa,+ tinyblasOperation_t transb, int m, int n, int k,+ WORD alpha, const SRC *A, int lda, const SRC *B, int ldb,+ WORD beta, DST *C, int ldc) {+ constexpr int BM = 48;+ constexpr int BN = 32;+ constexpr int TM = 3;+ constexpr int TN = 8;+ dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN));+ tinyblasGE_entry<BM, BN, TM, TN><<<maxblocks, THREAD_COUNT, 0, handle->stream>>>(+ transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc);+ if (cudaGetLastError() != cudaSuccess)+ return TINYBLAS_STATUS_EXECUTION_FAILED;+ return TINYBLAS_STATUS_SUCCESS;}tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t handle, //tinyblasOperation_t transa, //tinyblasOperation_t transb, //int m, int n, int k, //- const void *pAlpha, //- const void *A, tinyblasDatatype_t Atype, int lda, //- const void *B, tinyblasDatatype_t Btype, int ldb, //- const void *pBeta, //- void *C, tinyblasDatatype_t Ctype, int ldc, //+ const void *alpha, //+ const void *A, tinyblasDataType_t Atype, int lda, //+ const void *B, tinyblasDataType_t Btype, int ldb, //+ const void *beta, //+ void *C, tinyblasDataType_t Ctype, int ldc, //tinyblasComputeType_t computeType, //tinyblasGemmAlgo_t algo) {- if (!is_valid(transa, transb, m, n, k, lda, ldb, ldc))++ if (m < 0 || n < 0 || k < 0)return TINYBLAS_STATUS_INVALID_VALUE;- if (!is_supported(Atype, Btype))+ if (lda < std::max(1, transa ? k : m))+ return TINYBLAS_STATUS_INVALID_VALUE;+ if (ldb < std::max(1, transb ? n : k))+ return TINYBLAS_STATUS_INVALID_VALUE;+ if (ldc < std::max(1, m))+ return TINYBLAS_STATUS_INVALID_VALUE;+ if (1ll * lda * ((transa ? k : m) - 1) + ((transa ? m : k) - 1) > INT_MAX)+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;+ if (1ll * ldb * ((transb ? n : k) - 1) + ((transb ? k : n) - 1) > INT_MAX)+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;+ if (1ll * ldc * (n - 1) + (m - 1) > INT_MAX)+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;+ if (Atype != Btype)return TINYBLAS_STATUS_NOT_SUPPORTED;- return tinyblasGE_wrapper<48, 32, 64, 3, 8>(handle, transa, transb, m, n, k, pAlpha,- (const half *)A, lda, (const half *)B, ldb, pBeta,- C, Ctype, ldc, computeType);-}-// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmbatchedex-template <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST, typename T>-static __global__ void tinyblasGBE_entry(tinyblasOperation_t transa, tinyblasOperation_t transb,- int m, int n, int k, T alpha, const SRC *const Aarray[],- int lda, const SRC *const Barray[], int ldb, T beta,- DST *const Carray[], int ldc, int batchCount) {-- int x = blockIdx.x * BM;- const int jump1 = gridDim.x * BM;-- int y = blockIdx.y * BN;- const int jump2 = gridDim.y * BN;-- int z = blockIdx.z;- const int jump3 = gridDim.z;-- // each block handles a sub-matrix of C, of size BM * BN- for (z = blockIdx.z; z < batchCount; z += jump3)- for (x = blockIdx.x * BM; x < m; x += jump1)- for (y = blockIdx.y * BN; y < n; y += jump2)- matmul_block2d<BM, BN, BK, TM, TN, float>(transa, transb, m, n, k, x, y, alpha,- Aarray[z], lda, Barray[z], ldb, beta,- Carray[z], ldc);-}--template <int BM, int BN, int BK, int TM, int TN, typename SRC>-static tinyblasStatus_t tinyblasGBE_wrapper(tinyblasHandle_t handle, tinyblasOperation_t transa,- tinyblasOperation_t transb, int m, int n, int k,- const void *pAlpha, const SRC *const Aarray[], int lda,- const SRC *const Barray[], int ldb, const void *pBeta,- void *const Carray[], tinyblasDatatype_t Ctype, int ldc,- int batchCount, tinyblasComputeType_t computeType) {- dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 32);- int sharedmem = sizeof(SRC) * SHARELEN;- int maxthreads = (BM * BN) / (TM * TN);- switch (Ctype) {+ switch (Atype) {case TINYBLAS_R_16F:- switch (computeType) {- case TINYBLAS_COMPUTE_16F:- tinyblasGBE_entry<BM, BN, BK, TM, TN>- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(- transa, transb, m, n, k, *(const half *)pAlpha, Aarray, lda, Barray, ldb,- *(const half *)pBeta, (half *const *)Carray, ldc, batchCount);- return TINYBLAS_STATUS_SUCCESS;- case TINYBLAS_COMPUTE_32F:- tinyblasGBE_entry<BM, BN, BK, TM, TN>- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(- transa, transb, m, n, k, *(const float *)pAlpha, Aarray, lda, Barray, ldb,- *(const float *)pBeta, (half *const *)Carray, ldc, batchCount);- return TINYBLAS_STATUS_SUCCESS;+ switch (Ctype) {+ case TINYBLAS_R_16F:+ switch (computeType) {+ case TINYBLAS_COMPUTE_16F:+ return tinyblasGE_launch(+ handle, transa, transb, m, n, k, (float)*(const half *)alpha, (const half *)A,+ lda, (const half *)B, ldb, (float)*(const half *)beta, (half *)C, ldc);+ case TINYBLAS_COMPUTE_32F:+ return tinyblasGE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,+ (const half *)A, lda, (const half *)B, ldb,+ *(const float *)beta, (half *)C, ldc);+ default:+ return TINYBLAS_STATUS_INVALID_VALUE;+ }+ case TINYBLAS_R_32F:+ switch (computeType) {+ case TINYBLAS_COMPUTE_16F:+ return TINYBLAS_STATUS_NOT_SUPPORTED;+ case TINYBLAS_COMPUTE_32F:+ return tinyblasGE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,+ (const half *)A, lda, (const half *)B, ldb,+ *(const float *)beta, (float *)C, ldc);+ default:+ return TINYBLAS_STATUS_INVALID_VALUE;+ }default:return TINYBLAS_STATUS_INVALID_VALUE;}case TINYBLAS_R_32F:- switch (computeType) {- case TINYBLAS_COMPUTE_16F:+ switch (Ctype) {+ case TINYBLAS_R_16F:return TINYBLAS_STATUS_NOT_SUPPORTED;- case TINYBLAS_COMPUTE_32F:- tinyblasGBE_entry<BM, BN, BK, TM, TN>- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(- transa, transb, m, n, k, *(const float *)pAlpha, Aarray, lda, Barray, ldb,- *(const float *)pBeta, (float *const *)Carray, ldc, batchCount);- return TINYBLAS_STATUS_SUCCESS;+ case TINYBLAS_R_32F:+ switch (computeType) {+ case TINYBLAS_COMPUTE_16F:+ return TINYBLAS_STATUS_NOT_SUPPORTED;+ case TINYBLAS_COMPUTE_32F:+ return tinyblasGE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,+ (const float *)A, lda, (const float *)B, ldb,+ *(const float *)beta, (float *)C, ldc);+ default:+ return TINYBLAS_STATUS_INVALID_VALUE;+ }default:return TINYBLAS_STATUS_INVALID_VALUE;}@@ -352,96 +291,117 @@ static tinyblasStatus_t tinyblasGBE_wrapper(tinyblasHandle_t handle, tinyblasOpe}}+template <int BM, int BN, int TM, int TN, typename WORD, typename SRC, typename DST>+static __global__ void KERNEL tinyblasGBE_entry(tinyblasOperation_t transa,+ tinyblasOperation_t transb, int m, int n, int k,+ WORD alpha, const SRC *const Aarray[], int lda,+ const SRC *const Barray[], int ldb, WORD beta,+ DST *const Carray[], int ldc, int batchCount) {+ for (int z = blockIdx.z; z < batchCount; z += gridDim.z)+ matmul_block2d<BM, BN, TM, TN>(transa, transb, m, n, k, alpha, Aarray[z], lda, Barray[z],+ ldb, beta, Carray[z], ldc);+}++template <typename WORD, typename SRC, typename DST>+static tinyblasStatus_t tinyblasGBE_launch(tinyblasHandle_t handle, tinyblasOperation_t transa,+ tinyblasOperation_t transb, int m, int n, int k,+ WORD alpha, const SRC *const *Aarray, int lda,+ const SRC *const *Barray, int ldb, WORD beta,+ DST *const *Carray, int ldc, int batchCount) {+ constexpr int BC = 32;+ constexpr int BM = 32;+ constexpr int BN = 32;+ constexpr int TM = 2;+ constexpr int TN = 8;+ dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), BC);+ tinyblasGBE_entry<BM, BN, TM, TN><<<maxblocks, THREAD_COUNT, 0, handle->stream>>>(+ transa, transb, m, n, k, alpha, Aarray, lda, Barray, ldb, beta, Carray, ldc, batchCount);+ if (cudaGetLastError() != cudaSuccess)+ return TINYBLAS_STATUS_EXECUTION_FAILED;+ return TINYBLAS_STATUS_SUCCESS;+}+tinyblasStatus_t tinyblasGemmBatchedEx(tinyblasHandle_t handle, //tinyblasOperation_t transa, //tinyblasOperation_t transb, //int m, int n, int k, //- const void *pAlpha, //- const void *const Aarray[], tinyblasDatatype_t Atype,+ const void *alpha, //+ const void *const Aarray[], tinyblasDataType_t Atype,int lda, //- const void *const Barray[], tinyblasDatatype_t Btype,+ const void *const Barray[], tinyblasDataType_t Btype,int ldb, //- const void *pBeta, //- void *const Carray[], tinyblasDatatype_t Ctype, int ldc, //+ const void *beta, //+ void *const Carray[], tinyblasDataType_t Ctype, int ldc, //int batchCount, //tinyblasComputeType_t computeType, //tinyblasGemmAlgo_t algo) {- if (!is_valid(transa, transb, m, n, k, lda, ldb, ldc))++ if (m < 0 || n < 0 || k < 0)+ return TINYBLAS_STATUS_INVALID_VALUE;+ if (lda < std::max(1, transa ? k : m))+ return TINYBLAS_STATUS_INVALID_VALUE;+ if (ldb < std::max(1, transb ? n : k))+ return TINYBLAS_STATUS_INVALID_VALUE;+ if (ldc < std::max(1, m))return TINYBLAS_STATUS_INVALID_VALUE;- if (!is_supported(Atype, Btype))+ if (1ll * lda * ((transa ? k : m) - 1) + ((transa ? m : k) - 1) > INT_MAX)+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;+ if (1ll * ldb * ((transb ? n : k) - 1) + ((transb ? k : n) - 1) > INT_MAX)+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;+ if (1ll * ldc * (n - 1) + (m - 1) > INT_MAX)+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;+ if (Atype != Btype)return TINYBLAS_STATUS_NOT_SUPPORTED;- return tinyblasGBE_wrapper<48, 32, 64, 3, 8>(handle, transa, transb, //- m, n, k, pAlpha, //- (const half *const *)Aarray, lda, //- (const half *const *)Barray, ldb, //- pBeta, Carray, Ctype, ldc, //- batchCount, computeType);-}--// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmstridedbatchedex-template <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST, typename T>-static __global__ void-tinyblasGSBE_entry(tinyblasOperation_t transa, tinyblasOperation_t transb, int m, int n, int k,- T alpha, const SRC *A, int lda, long long strideA, const SRC *B, int ldb,- long long strideB, T beta, DST *C, int ldc, long long strideC, int batchCount) {-- int x = blockIdx.x * BM;- const int jump1 = gridDim.x * BM;-- int y = blockIdx.y * BN;- const int jump2 = gridDim.y * BN;-- int z = blockIdx.z;- const int jump3 = gridDim.z;-- // each block handles a sub-matrix of C, of size BM * BN- for (z = blockIdx.z; z < batchCount; z += jump3)- for (x = blockIdx.x * BM; x < m; x += jump1)- for (y = blockIdx.y * BN; y < n; y += jump2)- matmul_block2d<BM, BN, BK, TM, TN, float>(transa, transb, m, n, k, x, y, alpha,- A + z * strideA, lda, B + z * strideB,- ldb, beta, C + z * strideC, ldc);-}-template <int BM, int BN, int BK, int TM, int TN, typename SRC>-static tinyblasStatus_t tinyblasGSBE_wrapper(tinyblasHandle_t handle, tinyblasOperation_t transa,- tinyblasOperation_t transb, int m, int n, int k,- const void *pAlpha, const SRC *A, int lda,- long long strideA, const SRC *B, int ldb,- long long strideB, const void *pBeta, void *C,- tinyblasDatatype_t Ctype, int ldc, long long strideC,- int batchCount, tinyblasComputeType_t computeType) {- dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 32);- int sharedmem = sizeof(SRC) * SHARELEN;- int maxthreads = (BM * BN) / (TM * TN);- switch (Ctype) {+ switch (Atype) {case TINYBLAS_R_16F:- switch (computeType) {- case TINYBLAS_COMPUTE_16F:- tinyblasGSBE_entry<BM, BN, BK, TM, TN>- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(- transa, transb, m, n, k, *(const half *)pAlpha, A, lda, strideA, B, ldb,- strideB, *(const half *)pBeta, (half *)C, ldc, strideC, batchCount);- return TINYBLAS_STATUS_SUCCESS;- case TINYBLAS_COMPUTE_32F:- tinyblasGSBE_entry<BM, BN, BK, TM, TN>- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(- transa, transb, m, n, k, *(const float *)pAlpha, A, lda, strideA, B, ldb,- strideB, *(const float *)pBeta, (half *)C, ldc, strideC, batchCount);- return TINYBLAS_STATUS_SUCCESS;+ switch (Ctype) {+ case TINYBLAS_R_16F:+ switch (computeType) {+ case TINYBLAS_COMPUTE_16F:+ return tinyblasGBE_launch(+ handle, transa, transb, m, n, k, (float)*(const half *)alpha,+ (const half *const *)Aarray, lda, (const half *const *)Barray, ldb,+ (float)*(const half *)beta, (half *const *)Carray, ldc, batchCount);+ case TINYBLAS_COMPUTE_32F:+ return tinyblasGBE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,+ (const half *const *)Aarray, lda,+ (const half *const *)Barray, ldb, *(const float *)beta,+ (half *const *)Carray, ldc, batchCount);+ default:+ return TINYBLAS_STATUS_INVALID_VALUE;+ }+ case TINYBLAS_R_32F:+ switch (computeType) {+ case TINYBLAS_COMPUTE_16F:+ return TINYBLAS_STATUS_NOT_SUPPORTED;+ case TINYBLAS_COMPUTE_32F:+ return tinyblasGBE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,+ (const half *const *)Aarray, lda,+ (const half *const *)Barray, ldb, *(const float *)beta,+ (float *const *)Carray, ldc, batchCount);+ default:+ return TINYBLAS_STATUS_INVALID_VALUE;+ }default:return TINYBLAS_STATUS_INVALID_VALUE;}case TINYBLAS_R_32F:- switch (computeType) {- case TINYBLAS_COMPUTE_16F:+ switch (Ctype) {+ case TINYBLAS_R_16F:return TINYBLAS_STATUS_NOT_SUPPORTED;- case TINYBLAS_COMPUTE_32F:- tinyblasGSBE_entry<BM, BN, BK, TM, TN>- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(- transa, transb, m, n, k, *(const float *)pAlpha, A, lda, strideA, B, ldb,- strideB, *(const float *)pBeta, (float *)C, ldc, strideC, batchCount);- return TINYBLAS_STATUS_SUCCESS;+ case TINYBLAS_R_32F:+ switch (computeType) {+ case TINYBLAS_COMPUTE_16F:+ return TINYBLAS_STATUS_NOT_SUPPORTED;+ case TINYBLAS_COMPUTE_32F:+ return tinyblasGBE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,+ (const float *const *)Aarray, lda,+ (const float *const *)Barray, ldb, *(const float *)beta,+ (float *const *)Carray, ldc, batchCount);+ default:+ return TINYBLAS_STATUS_INVALID_VALUE;+ }default:return TINYBLAS_STATUS_INVALID_VALUE;}@@ -450,41 +410,124 @@ static tinyblasStatus_t tinyblasGSBE_wrapper(tinyblasHandle_t handle, tinyblasOp}}+template <int BM, int BN, int TM, int TN, typename SRC, typename DST, typename WORD>+static __global__ void KERNEL tinyblasGSBE_entry(tinyblasOperation_t transa,+ tinyblasOperation_t transb, int m, int n, int k,+ WORD alpha, const SRC *A, int lda,+ long long strideA, const SRC *B, int ldb,+ long long strideB, WORD beta, DST *C, int ldc,+ long long strideC, int batchCount) {+ for (int z = blockIdx.z; z < batchCount; z += gridDim.z)+ matmul_block2d<BM, BN, TM, TN>(transa, transb, m, n, k, alpha, A + z * strideA, lda,+ B + z * strideB, ldb, beta, C + z * strideC, ldc);+}++template <typename WORD, typename SRC, typename DST>+static tinyblasStatus_t tinyblasGSBE_launch(tinyblasHandle_t handle, tinyblasOperation_t transa,+ tinyblasOperation_t transb, int m, int n, int k,+ WORD alpha, const SRC *A, int lda, long long strideA,+ const SRC *B, int ldb, long long strideB, WORD beta,+ DST *C, int ldc, long long strideC, int batchCount) {+ constexpr int BC = 32;+ constexpr int BM = 32;+ constexpr int BN = 32;+ constexpr int TM = 2;+ constexpr int TN = 8;+ dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), BC);+ tinyblasGSBE_entry<BM, BN, TM, TN><<<maxblocks, THREAD_COUNT, 0, handle->stream>>>(+ transa, transb, m, n, k, alpha, A, lda, strideA, B, ldb, strideB, beta, C, ldc, strideC,+ batchCount);+ if (cudaGetLastError() != cudaSuccess)+ return TINYBLAS_STATUS_EXECUTION_FAILED;+ return TINYBLAS_STATUS_SUCCESS;+}+tinyblasStatus_t tinyblasGemmStridedBatchedEx(tinyblasHandle_t handle, //tinyblasOperation_t transa, //tinyblasOperation_t transb, //int m, int n, int k, //- const void *pAlpha, //- const void *A, tinyblasDatatype_t Atype, int lda,+ const void *alpha, //+ const void *A, tinyblasDataType_t Atype, int lda,long long strideA, //- const void *B, tinyblasDatatype_t Btype, int ldb,+ const void *B, tinyblasDataType_t Btype, int ldb,long long strideB, //- const void *pBeta, //- void *C, tinyblasDatatype_t Ctype, int ldc,+ const void *beta, //+ void *C, tinyblasDataType_t Ctype, int ldc,long long strideC, //int batchCount, //tinyblasComputeType_t computeType, //tinyblasGemmAlgo_t algo) {- if (!is_valid(transa, transb, m, n, k, lda, ldb, ldc))++ if (m < 0 || n < 0 || k < 0)+ return TINYBLAS_STATUS_INVALID_VALUE;+ if (lda < std::max(1, transa ? k : m))+ return TINYBLAS_STATUS_INVALID_VALUE;+ if (ldb < std::max(1, transb ? n : k))return TINYBLAS_STATUS_INVALID_VALUE;- if (!is_supported(Atype, Btype))+ if (ldc < std::max(1, m))+ return TINYBLAS_STATUS_INVALID_VALUE;+ if (1ll * lda * ((transa ? k : m) - 1) + ((transa ? m : k) - 1) > INT_MAX)+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;+ if (1ll * ldb * ((transb ? n : k) - 1) + ((transb ? k : n) - 1) > INT_MAX)+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;+ if (1ll * ldc * (n - 1) + (m - 1) > INT_MAX)+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;+ if (Atype != Btype)return TINYBLAS_STATUS_NOT_SUPPORTED;- return tinyblasGSBE_wrapper<32, 4, 64, 1, 2>(- handle, transa, transb, m, n, k, pAlpha, (const half *)A, lda, strideA, (const half *)B,- ldb, strideB, pBeta, C, Ctype, ldc, strideC, batchCount, computeType);-}-const char *tinyblasGetStatusString(tinyblasStatus_t err) {- switch (err) {- case TINYBLAS_STATUS_SUCCESS:- return "TINYBLAS_STATUS_SUCCESS";- case TINYBLAS_STATUS_ALLOC_FAILED:- return "TINYBLAS_STATUS_ALLOC_FAILED";- case TINYBLAS_STATUS_INVALID_VALUE:- return "TINYBLAS_STATUS_INVALID_VALUE";- case TINYBLAS_STATUS_NOT_SUPPORTED:- return "TINYBLAS_STATUS_NOT_SUPPORTED";+ switch (Atype) {+ case TINYBLAS_R_16F:+ switch (Ctype) {+ case TINYBLAS_R_16F:+ switch (computeType) {+ case TINYBLAS_COMPUTE_16F:+ return tinyblasGSBE_launch(+ handle, transa, transb, m, n, k, (float)*(const half *)alpha, (const half *)A,+ lda, strideA, (const half *)B, ldb, strideB, (float)*(const half *)beta,+ (half *)C, ldc, strideC, batchCount);+ case TINYBLAS_COMPUTE_32F:+ return tinyblasGSBE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,+ (const half *)A, lda, strideA, (const half *)B, ldb,+ strideB, *(const float *)beta, (half *)C, ldc, strideC,+ batchCount);+ default:+ return TINYBLAS_STATUS_INVALID_VALUE;+ }+ case TINYBLAS_R_32F:+ switch (computeType) {+ case TINYBLAS_COMPUTE_16F:+ return TINYBLAS_STATUS_NOT_SUPPORTED;+ case TINYBLAS_COMPUTE_32F:+ return tinyblasGSBE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,+ (const half *)A, lda, strideA, (const half *)B, ldb,+ strideB, *(const float *)beta, (float *)C, ldc, strideC,+ batchCount);+ default:+ return TINYBLAS_STATUS_INVALID_VALUE;+ }+ default:+ return TINYBLAS_STATUS_INVALID_VALUE;+ }+ case TINYBLAS_R_32F:+ switch (Ctype) {+ case TINYBLAS_R_16F:+ return TINYBLAS_STATUS_NOT_SUPPORTED;+ case TINYBLAS_R_32F:+ switch (computeType) {+ case TINYBLAS_COMPUTE_16F:+ return TINYBLAS_STATUS_NOT_SUPPORTED;+ case TINYBLAS_COMPUTE_32F:+ return tinyblasGSBE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,+ (const float *)A, lda, strideA, (const float *)B, ldb,+ strideB, *(const float *)beta, (float *)C, ldc, strideC,+ batchCount);+ default:+ return TINYBLAS_STATUS_INVALID_VALUE;+ }+ default:+ return TINYBLAS_STATUS_INVALID_VALUE;+ }default:- return "unknown tinyblas error";+ return TINYBLAS_STATUS_INVALID_VALUE;}}diff --git a/llamafile/tinyblas.h b/llamafile/tinyblas.hindex 79dfad12..55de17ae 100644--- a/llamafile/tinyblas.h+++ b/llamafile/tinyblas.h@@ -7,7 +7,7 @@ enum tinyblasOperation_t {TINYBLAS_OP_T,};-enum tinyblasDatatype_t {+enum tinyblasDataType_t {TINYBLAS_R_32F,TINYBLAS_R_16F,};@@ -26,6 +26,8 @@ enum tinyblasStatus_t {TINYBLAS_STATUS_ALLOC_FAILED,TINYBLAS_STATUS_INVALID_VALUE,TINYBLAS_STATUS_NOT_SUPPORTED,+ TINYBLAS_STATUS_EXECUTION_FAILED,+ TINYBLAS_STATUS_DIMENSION_OVERFLOW,};struct tinyblasContext;@@ -43,21 +45,21 @@ tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t, tinyblasOperation_t, tinyblasOpconst float *, float *, int);tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t, tinyblasOperation_t, tinyblasOperation_t, int,- int, int, const void *, const void *, tinyblasDatatype_t, int,- const void *, tinyblasDatatype_t, int, const void *, void *,- tinyblasDatatype_t, int, tinyblasComputeType_t, tinyblasGemmAlgo_t);+ int, int, const void *, const void *, tinyblasDataType_t, int,+ const void *, tinyblasDataType_t, int, const void *, void *,+ tinyblasDataType_t, int, tinyblasComputeType_t, tinyblasGemmAlgo_t);tinyblasStatus_t tinyblasGemmBatchedEx(tinyblasHandle_t, tinyblasOperation_t, tinyblasOperation_t,int, int, int, const void *, const void *const[],- tinyblasDatatype_t, int, const void *const[],- tinyblasDatatype_t, int, const void *, void *const[],- tinyblasDatatype_t, int, int, tinyblasComputeType_t,+ tinyblasDataType_t, int, const void *const[],+ tinyblasDataType_t, int, const void *, void *const[],+ tinyblasDataType_t, int, int, tinyblasComputeType_t,tinyblasGemmAlgo_t);tinyblasStatus_t tinyblasGemmStridedBatchedEx(tinyblasHandle_t, tinyblasOperation_t,tinyblasOperation_t, int, int, int, const void *,- const void *, tinyblasDatatype_t, int, long long,- const void *, tinyblasDatatype_t, int, long long,- const void *, void *, tinyblasDatatype_t, int,+ const void *, tinyblasDataType_t, int, long long,+ const void *, tinyblasDataType_t, int, long long,+ const void *, void *, tinyblasDataType_t, int,long long, int, tinyblasComputeType_t,tinyblasGemmAlgo_t);diff --git a/llamafile/tinyblas_test.cu b/llamafile/tinyblas_test.cuindex 17875805..3dbcac0d 100644--- a/llamafile/tinyblas_test.cu+++ b/llamafile/tinyblas_test.cu@@ -15,246 +15,100 @@// See the License for the specific language governing permissions and// limitations under the License.-#include "tinyblas.h"--#include <atomic>-#include <cstdio>-#include <cstdlib>-#include <ctime>-#include "tester.h"-#define TOMBSTONE 1.666-#define PRECISION_F16 0.2-#define PRECISION_F32 9e-6--#ifdef __NVCC__-#include <cublas_v2.h>-#include <cuda_runtime.h>-#else-#define HIPBLAS_V2-#include <hip/hip_runtime.h>-#include <hipblas/hipblas.h>-#define cudaFree hipFree-#define cudaSuccess hipSuccess-#define cudaError_t hipError_t-#define cudaEvent_t hipEvent_t-#define cudaStream_t hipStream_t-#define cublasSgemm hipblasSgemm-#define cublasCreate hipblasCreate-#define cublasDestroy hipblasDestroy-#define cublasHandle_t hipblasHandle_t-#define cublasStatus_t hipblasStatus_t-#define cudaEventCreate hipEventCreate-#define cudaEventRecord hipEventRecord-#define cudaEventDestroy hipEventDestroy-#define cublasSetStream hipblasSetStream-#define cudaStreamCreate hipStreamCreate-#define cudaMallocManaged hipMallocManaged-#define cudaStreamDestroy hipStreamDestroy-#define cudaEventSynchronize hipEventSynchronize-#define cudaEventElapsedTime hipEventElapsedTime-#define cudaStreamSynchronize hipStreamSynchronize-#define cublasGetStatusString hipblasStatusToString-#define cudaGetErrorString hipGetErrorString-#define cudaGetDevice hipGetDevice-#define cudaDeviceReset hipDeviceReset-#define cublasGemmEx hipblasGemmEx-#define CUBLAS_OP_N HIPBLAS_OP_N-#define CUBLAS_OP_T HIPBLAS_OP_T-#define CUDA_R_16F HIPBLAS_R_16F-#define CUDA_R_32F HIPBLAS_R_32F-#define CUBLAS_COMPUTE_16F HIPBLAS_COMPUTE_16F-#define CUBLAS_COMPUTE_32F HIPBLAS_COMPUTE_32F-#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_COMPUTE_32F_FAST_16F-#define CUBLAS_GEMM_DEFAULT HIPBLAS_GEMM_DEFAULT-#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS-#endif--#define ARRAYLEN(A) ((sizeof(A) / sizeof(*(A))) / ((unsigned)!(sizeof(A) % sizeof(*(A)))))--#define CUDA_OR_DIE(x) \- do { \- cudaError_t err = (x); \- if (err != cudaSuccess) \- die(#x, __FUNCTION__, __FILE__, __LINE__, cudaGetErrorString(err)); \- } while (0)--#define CUBLAS_OR_DIE(x) \- do { \- cublasStatus_t status = (x); \- if (status != CUBLAS_STATUS_SUCCESS) \- die(#x, __FUNCTION__, __FILE__, __LINE__, cublasGetStatusString(status)); \- } while (0)--#define TINYBLAS_OR_DIE(x) \- do { \- tinyblasStatus_t status = (x); \- if (status != TINYBLAS_STATUS_SUCCESS) \- die(#x, __FUNCTION__, __FILE__, __LINE__, tinyblasGetStatusString(status)); \- } while (0)--static const int kDims[] = {1, 2, 23, 77, 15, 127, 128, 129};-static const float kAlphas[] = {1, .5};-static const float kBetas[] = {0, .1};--[[noreturn]]-void die(const char *stmt, const char *func, const char *file, int line, const char *msg) {- int id = -1;- (void)cudaGetDevice(&id);- fprintf(stderr, "CUDA error: %s\n", msg);- fprintf(stderr, " current device: %d, in function %s at %s:%d\n", id, func, file, line);- fprintf(stderr, " %s\n", stmt);- (void)cudaDeviceReset();- exit(1);-}--void checkFloat01() {- for (int i = 0; i < 10000; ++i) {- float x = float01(rand32());- ASSERT(x >= -1);- ASSERT(x <= +1);+void checkTinyblasWorksHHHH() {+ double TOLERANCE = .5;++ {+ half α = 1;+ half β = 0;+ int m = 577;+ int n = 577;+ int k = 64;+ int lda = k;+ int ldb = k;+ int ldc = m;+ cuda_memory<half> A{lda * m};+ cuda_memory<half> B{ldb * n};+ cuda_memory<half> C{ldc * n};+ cuda_memory<half> G{ldc * n};+ randomize(k, m, A.p, lda);+ randomize(k, n, B.p, ldb);+ cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ broadcast(m, n, C.p, ldc, TOMBSTONE);+ broadcast(m, n, G.p, ldc, TOMBSTONE);+ gemmref<float>(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);}-}--void checkCublasWorksSSS() {-- cublasHandle_t hand;- CUBLAS_OR_DIE(cublasCreate(&hand));-- cudaStream_t stream;- CUDA_OR_DIE(cudaStreamCreate(&stream));- CUBLAS_OR_DIE(cublasSetStream(hand, stream));- float *A, *B, *C, *G;- long long maxdim = 4096;- CUDA_OR_DIE(cudaMallocManaged(&A, maxdim * maxdim * sizeof(float)));- CUDA_OR_DIE(cudaMallocManaged(&B, maxdim * maxdim * sizeof(float)));- CUDA_OR_DIE(cudaMallocManaged(&C, maxdim * maxdim * sizeof(float)));- CUDA_OR_DIE(cudaMallocManaged(&G, maxdim * maxdim * sizeof(float)));-- std::atomic_llong t = ATOMIC_VAR_INIT(0);- for (int ai = 0; ai < ARRAYLEN(kAlphas); ++ai)- for (int bi = 0; bi < ARRAYLEN(kBetas); ++bi)- for (int mi = 0; mi < ARRAYLEN(kDims); ++mi)- for (int ni = 0; ni < ARRAYLEN(kDims); ++ni)- for (int ki = 0; ki < ARRAYLEN(kDims); ++ki) {- float α = kAlphas[ai];- float β = kBetas[bi];- int m = kDims[mi];- int n = kDims[ARRAYLEN(kDims) - 1 - ni];- int k = kDims[ki];- int lda = m;- int ldb = k;- int ldc = m;- char name[192];- snprintf(name, sizeof(name), "testing %4d %4d %4d α=%g β=%g", m, n, k, α,- β);- if (t++ % 7 == 0)- fprintf(stderr, "%s\r", name);- is_self_testing = name;- fill(m, k, A, lda);- fill(k, n, B, ldb);- broadcast(m, n, G, n, TOMBSTONE);- broadcast(m, n, C, n, TOMBSTONE);- dgemm(false, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);- CUBLAS_OR_DIE(cublasSgemm(hand, CUBLAS_OP_N, CUBLAS_OP_N, m, n, k, &α, A,- lda, B, ldb, &β, C, ldc));- CUDA_OR_DIE(cudaStreamSynchronize(stream));- CHECK(PRECISION_F32, m, n, G, ldc, C, ldc);- is_self_testing = 0;- }+ {+ half α = 1;+ half β = 0;+ int m = 4096;+ int n = 1024;+ int k = 2048;+ int lda = k;+ int ldb = k;+ int ldc = m;+ cuda_memory<half> A{lda * m};+ cuda_memory<half> B{ldb * n};+ cuda_memory<half> C{ldc * n};+ cuda_memory<half> G{ldc * n};+ randomize(k, m, A.p, lda);+ randomize(k, n, B.p, ldb);+ cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ broadcast(m, n, C.p, ldc, TOMBSTONE);+ broadcast(m, n, G.p, ldc, TOMBSTONE);+ gemmref<float>(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);+ }- float α = 1;- float β = 0;- int m = 4096;- int n = 1024;- int k = 577;- int lda = k;- int ldb = k;- int ldc = m;- fill(k, m, A, lda);- fill(k, n, B, ldb);- fill(m, n, C, ldc);- fill(m, n, G, ldc);- broadcast(m, n, G, n, TOMBSTONE);- broadcast(m, n, C, n, TOMBSTONE);- dgemm(true, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);- CUBLAS_OR_DIE(- cublasSgemm(hand, CUBLAS_OP_T, CUBLAS_OP_N, m, n, k, &α, A, lda, B, ldb, &β, C, ldc));- CUDA_OR_DIE(cudaStreamSynchronize(stream));- CHECK(PRECISION_F32, m, n, G, ldc, C, ldc);- cudaEvent_t start, stop;- CUDA_OR_DIE(cudaEventCreate(&stop));- CUDA_OR_DIE(cudaEventCreate(&start));- CUDA_OR_DIE(cudaEventRecord(start, NULL));- for (int i = 0; i < ITERATIONS; ++i)- CUBLAS_OR_DIE(- cublasSgemm(hand, CUBLAS_OP_T, CUBLAS_OP_N, m, n, k, &α, A, lda, B, ldb, &β, C, ldc));- float msecTotal = 0;- CUDA_OR_DIE(cudaEventRecord(stop, NULL));- CUDA_OR_DIE(cudaEventSynchronize(stop));- CUDA_OR_DIE(cudaEventElapsedTime(&msecTotal, start, stop));- printf("%8lld µs cublasSgemm(%d, %d, %d)\n",- (long long)std::ceil(msecTotal / ITERATIONS * 1000), m, n, k);- CUDA_OR_DIE(cudaEventDestroy(start));- CUDA_OR_DIE(cudaEventDestroy(stop));+ {+ half α = 1;+ half β = 0;+ int m = 4096;+ int n = 1024;+ int k = 577;+ int lda = k;+ int ldb = k;+ int ldc = m;+ cuda_memory<half> A{lda * m};+ cuda_memory<half> B{ldb * n};+ cuda_memory<half> C{ldc * n};+ cuda_memory<half> G{ldc * n};+ randomize(k, m, A.p, lda);+ randomize(k, n, B.p, ldb);+ cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ broadcast(m, n, C.p, ldc, TOMBSTONE);+ broadcast(m, n, G.p, ldc, TOMBSTONE);+ gemmref<float>(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);+ }- CUDA_OR_DIE(cudaFree(G));- CUDA_OR_DIE(cudaFree(C));- CUDA_OR_DIE(cudaFree(B));- CUDA_OR_DIE(cudaFree(A));- CUDA_OR_DIE(cudaStreamDestroy(stream));- CUBLAS_OR_DIE(cublasDestroy(hand));+ test_matmul([&](int m, int n, int k, int l, half α, half β) {+ int lda = m + l;+ int ldb = k + l;+ int ldc = m + l;+ cuda_memory<half> A{lda * k};+ cuda_memory<half> B{ldb * n};+ cuda_memory<half> C{ldc * n};+ cuda_memory<half> G{ldc * n};+ randomize(m, k, A.p, lda);+ randomize(k, n, B.p, ldb);+ broadcast(m, n, G.p, ldc, TOMBSTONE);+ broadcast(m, n, C.p, ldc, TOMBSTONE);+ gemmref<float>(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ tinyblas(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);+ });}-void checkTinyblasWorksSSS() {-- tinyblasHandle_t hand;- TINYBLAS_OR_DIE(tinyblasCreate(&hand));-- cudaStream_t stream;- CUDA_OR_DIE(cudaStreamCreate(&stream));- TINYBLAS_OR_DIE(tinyblasSetStream(hand, stream));-- float *A, *B, *C, *G;- long long maxdim = 4096;- CUDA_OR_DIE(cudaMallocManaged(&A, maxdim * maxdim * sizeof(float)));- CUDA_OR_DIE(cudaMallocManaged(&B, maxdim * maxdim * sizeof(float)));- CUDA_OR_DIE(cudaMallocManaged(&C, maxdim * maxdim * sizeof(float)));- CUDA_OR_DIE(cudaMallocManaged(&G, maxdim * maxdim * sizeof(float)));-- std::atomic_llong t = ATOMIC_VAR_INIT(0);- for (int ai = 0; ai < ARRAYLEN(kAlphas); ++ai)- for (int bi = 0; bi < ARRAYLEN(kBetas); ++bi)- for (int mi = 0; mi < ARRAYLEN(kDims); ++mi)- for (int ni = 0; ni < ARRAYLEN(kDims); ++ni)- for (int ki = 0; ki < ARRAYLEN(kDims); ++ki) {- float α = kAlphas[ai];- float β = kBetas[bi];- int m = kDims[mi];- int n = kDims[ARRAYLEN(kDims) - 1 - ni];- int k = kDims[ki];- int lda = m;- int ldb = k;- int ldc = m;- char name[192];- snprintf(name, sizeof(name), "testing %4d %4d %4d α=%g β=%g", m, n, k, α,- β);- if (t++ % 7 == 0)- fprintf(stderr, "%s\r", name);- is_self_testing = name;- fill(m, k, A, lda);- fill(k, n, B, ldb);- broadcast(m, n, G, n, TOMBSTONE);- broadcast(m, n, C, n, TOMBSTONE);- dgemm(false, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);- TINYBLAS_OR_DIE(tinyblasSgemm(hand, TINYBLAS_OP_N, TINYBLAS_OP_N, m, n, k,- &α, A, lda, B, ldb, &β, C, ldc));- CUDA_OR_DIE(cudaStreamSynchronize(stream));- CHECK(PRECISION_F32, m, n, G, ldc, C, ldc);- is_self_testing = 0;- }+void checkTinyblasWorksHHHS() {+ double TOLERANCE = .5;float α = 1;float β = 0;@@ -264,435 +118,124 @@ void checkTinyblasWorksSSS() {int lda = k;int ldb = k;int ldc = m;- fill(k, m, A, lda);- fill(k, n, B, ldb);- fill(m, n, C, ldc);- fill(m, n, G, ldc);- broadcast(m, n, G, n, TOMBSTONE);- broadcast(m, n, C, n, TOMBSTONE);- dgemm(true, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);- TINYBLAS_OR_DIE(- tinyblasSgemm(hand, TINYBLAS_OP_T, TINYBLAS_OP_N, m, n, k, &α, A, lda, B, ldb, &β, C, ldc));- CUDA_OR_DIE(cudaStreamSynchronize(stream));- CHECK(PRECISION_F32, m, n, G, ldc, C, ldc);- cudaEvent_t start, stop;- CUDA_OR_DIE(cudaEventCreate(&stop));- CUDA_OR_DIE(cudaEventCreate(&start));- CUDA_OR_DIE(cudaEventRecord(start, NULL));- for (int i = 0; i < ITERATIONS; ++i)- TINYBLAS_OR_DIE(tinyblasSgemm(hand, TINYBLAS_OP_T, TINYBLAS_OP_N, m, n, k, &α, A, lda, B,- ldb, &β, C, ldc));- float msecTotal = 0;- CUDA_OR_DIE(cudaEventRecord(stop, NULL));- CUDA_OR_DIE(cudaEventSynchronize(stop));- CUDA_OR_DIE(cudaEventElapsedTime(&msecTotal, start, stop));- printf("%8lld µs tinyblasSgemm(%d, %d, %d)\n",- (long long)std::ceil(msecTotal / ITERATIONS * 1000), m, n, k);- CUDA_OR_DIE(cudaEventDestroy(start));- CUDA_OR_DIE(cudaEventDestroy(stop));-- CUDA_OR_DIE(cudaFree(G));- CUDA_OR_DIE(cudaFree(C));- CUDA_OR_DIE(cudaFree(B));- CUDA_OR_DIE(cudaFree(A));- CUDA_OR_DIE(cudaStreamDestroy(stream));- TINYBLAS_OR_DIE(tinyblasDestroy(hand));+ cuda_memory<half> A{lda * m};+ cuda_memory<half> B{ldb * n};+ cuda_memory<half> C{ldc * n};+ cuda_memory<half> G{ldc * n};+ randomize(k, m, A.p, lda);+ randomize(k, n, B.p, ldb);+ cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ broadcast(m, n, C.p, ldc, TOMBSTONE);+ broadcast(m, n, G.p, ldc, TOMBSTONE);+ gemmref(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);++ test_matmul([&](int m, int n, int k, int l, float α, float β) {+ int lda = m + l;+ int ldb = k + l;+ int ldc = m + l;+ cuda_memory<half> A{lda * k};+ cuda_memory<half> B{ldb * n};+ cuda_memory<half> C{ldc * n};+ cuda_memory<half> G{ldc * n};+ randomize(m, k, A.p, lda);+ randomize(k, n, B.p, ldb);+ broadcast(m, n, G.p, ldc, TOMBSTONE);+ broadcast(m, n, C.p, ldc, TOMBSTONE);+ gemmref(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ tinyblas(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);+ });}-void checkCublasWorksHHS() {-- cublasHandle_t hand;- CUBLAS_OR_DIE(cublasCreate(&hand));-- cudaStream_t stream;- CUDA_OR_DIE(cudaStreamCreate(&stream));- CUBLAS_OR_DIE(cublasSetStream(hand, stream));-- half *A, *B;- float *C, *G;- long long maxdim = 4096;- CUDA_OR_DIE(cudaMallocManaged(&A, maxdim * maxdim * sizeof(half)));- CUDA_OR_DIE(cudaMallocManaged(&B, maxdim * maxdim * sizeof(half)));- CUDA_OR_DIE(cudaMallocManaged(&C, maxdim * maxdim * sizeof(float)));- CUDA_OR_DIE(cudaMallocManaged(&G, maxdim * maxdim * sizeof(float)));-- std::atomic_llong t = ATOMIC_VAR_INIT(0);- for (int ai = 0; ai < ARRAYLEN(kAlphas); ++ai)- for (int bi = 0; bi < ARRAYLEN(kBetas); ++bi)- for (int mi = 0; mi < ARRAYLEN(kDims); ++mi)- for (int ni = 0; ni < ARRAYLEN(kDims); ++ni)- for (int ki = 0; ki < ARRAYLEN(kDims); ++ki) {- float α = kAlphas[ai];- float β = kBetas[bi];- int m = kDims[mi];- int n = kDims[ARRAYLEN(kDims) - 1 - ni];- int k = kDims[ki];- int lda = m;- int ldb = k;- int ldc = m;-- char name[192];- snprintf(name, sizeof(name), "testing %4d %4d %4d α=%g β=%g", m, n, k, α,- β);- if (t++ % 7 == 0)- fprintf(stderr, "%s\r", name);- is_self_testing = name;-- fill(m, k, A, lda);- fill(k, n, B, ldb);- broadcast(m, n, G, n, TOMBSTONE);- broadcast(m, n, C, n, TOMBSTONE);- dgemm(false, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);-- CUBLAS_OR_DIE(cublasGemmEx(hand, CUBLAS_OP_N, CUBLAS_OP_N, m, n, k, &α, A,- CUDA_R_16F, lda, B, CUDA_R_16F, ldb, &β, C,- CUDA_R_32F, ldc, CUBLAS_COMPUTE_32F,- CUBLAS_GEMM_DEFAULT));- CUDA_OR_DIE(cudaStreamSynchronize(stream));-- CHECK(PRECISION_F32, m, n, G, ldc, C, ldc);- is_self_testing = 0;- }+void checkTinyblasWorksHHSS() {+ double TOLERANCE = .5;float α = 1;float β = 0;int m = 4096;int n = 1024;- int k = 577;+ int k = 1536;int lda = k;int ldb = k;int ldc = m;- fill(k, m, A, lda);- fill(k, n, B, ldb);- fill(m, n, C, ldc);- fill(m, n, G, ldc);- broadcast(m, n, G, n, TOMBSTONE);- broadcast(m, n, C, n, TOMBSTONE);- dgemm(true, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);- CUBLAS_OR_DIE(cublasGemmEx(hand, CUBLAS_OP_T, CUBLAS_OP_N, m, n, k, &α, A, CUDA_R_16F, lda, B,- CUDA_R_16F, ldb, &β, C, CUDA_R_32F, ldc, CUBLAS_COMPUTE_32F,- CUBLAS_GEMM_DEFAULT));- CUDA_OR_DIE(cudaStreamSynchronize(stream));- CHECK(PRECISION_F32, m, n, G, ldc, C, ldc);- cudaEvent_t start, stop;- CUDA_OR_DIE(cudaEventCreate(&stop));- CUDA_OR_DIE(cudaEventCreate(&start));- CUDA_OR_DIE(cudaEventRecord(start, NULL));- for (int i = 0; i < ITERATIONS; ++i)- CUBLAS_OR_DIE(cublasGemmEx(hand, CUBLAS_OP_T, CUBLAS_OP_N, m, n, k, &α, A, CUDA_R_16F, lda,- B, CUDA_R_16F, ldb, &β, C, CUDA_R_32F, ldc, CUBLAS_COMPUTE_32F,- CUBLAS_GEMM_DEFAULT));- float msecTotal = 0;- CUDA_OR_DIE(cudaEventRecord(stop, NULL));- CUDA_OR_DIE(cudaEventSynchronize(stop));- CUDA_OR_DIE(cudaEventElapsedTime(&msecTotal, start, stop));- printf("%8lld µs cublasGemmExHHS(%d, %d, %d)\n",- (long long)std::ceil(msecTotal / ITERATIONS * 1000), m, n, k);- CUDA_OR_DIE(cudaEventDestroy(start));- CUDA_OR_DIE(cudaEventDestroy(stop));-- CUDA_OR_DIE(cudaFree(G));- CUDA_OR_DIE(cudaFree(C));- CUDA_OR_DIE(cudaFree(B));- CUDA_OR_DIE(cudaFree(A));- CUDA_OR_DIE(cudaStreamDestroy(stream));- CUBLAS_OR_DIE(cublasDestroy(hand));+ cuda_memory<half> A{lda * m};+ cuda_memory<half> B{ldb * n};+ cuda_memory<float> C{ldc * n};+ cuda_memory<float> G{ldc * n};+ randomize(k, m, A.p, lda);+ randomize(k, n, B.p, ldb);+ cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ broadcast(m, n, G.p, ldc, TOMBSTONE);+ broadcast(m, n, C.p, ldc, TOMBSTONE);+ gemmref(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);++ test_matmul([&](int m, int n, int k, int l, float α, float β) {+ int lda = k + l;+ int ldb = k + l;+ int ldc = m + l;+ cuda_memory<half> A{lda * m};+ cuda_memory<half> B{ldb * n};+ cuda_memory<float> C{ldc * n};+ cuda_memory<float> G{ldc * n};+ randomize(k, m, A.p, lda);+ randomize(k, n, B.p, ldb);+ broadcast(m, n, G.p, ldc, TOMBSTONE);+ broadcast(m, n, C.p, ldc, TOMBSTONE);+ gemmref(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);+ });}-void checkTinyblasWorksHHS() {-- tinyblasHandle_t hand;- TINYBLAS_OR_DIE(tinyblasCreate(&hand));-- cudaStream_t stream;- CUDA_OR_DIE(cudaStreamCreate(&stream));- TINYBLAS_OR_DIE(tinyblasSetStream(hand, stream));-- half *A, *B;- float *C, *G;- long long maxdim = 4096;- CUDA_OR_DIE(cudaMallocManaged(&A, maxdim * maxdim * sizeof(half)));- CUDA_OR_DIE(cudaMallocManaged(&B, maxdim * maxdim * sizeof(half)));- CUDA_OR_DIE(cudaMallocManaged(&C, maxdim * maxdim * sizeof(float)));- CUDA_OR_DIE(cudaMallocManaged(&G, maxdim * maxdim * sizeof(float)));-- std::atomic_llong t = ATOMIC_VAR_INIT(0);- for (int ai = 0; ai < ARRAYLEN(kAlphas); ++ai)- for (int bi = 0; bi < ARRAYLEN(kBetas); ++bi)- for (int mi = 0; mi < ARRAYLEN(kDims); ++mi)- for (int ni = 0; ni < ARRAYLEN(kDims); ++ni)- for (int ki = 0; ki < ARRAYLEN(kDims); ++ki) {- float α = kAlphas[ai];- float β = kBetas[bi];- int m = kDims[mi];- int n = kDims[ARRAYLEN(kDims) - 1 - ni];- int k = kDims[ki];- int lda = k;- int ldb = k;- int ldc = m;- char name[192];- snprintf(name, sizeof(name), "testing %4d %4d %4d α=%g β=%g", m, n, k, α,- β);- if (t++ % 7 == 0)- fprintf(stderr, "%s\r", name);- is_self_testing = name;- fill(m, k, A, lda);- fill(k, n, B, ldb);- broadcast(m, n, G, n, TOMBSTONE);- broadcast(m, n, C, n, TOMBSTONE);- dgemm(true, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);- TINYBLAS_OR_DIE(tinyblasGemmEx(- hand, TINYBLAS_OP_T, TINYBLAS_OP_N, m, n, k, &α, A, TINYBLAS_R_16F, lda,- B, TINYBLAS_R_16F, ldb, &β, C, TINYBLAS_R_32F, ldc,- TINYBLAS_COMPUTE_32F, TINYBLAS_GEMM_DEFAULT));- CUDA_OR_DIE(cudaStreamSynchronize(stream));- CHECK(PRECISION_F32, m, n, G, ldc, C, ldc);- is_self_testing = 0;- }+void checkTinyblasWorksSSSS() {+ double TOLERANCE = .5;float α = 1;float β = 0;int m = 4096;int n = 1024;- int k = 577;+ int k = 1020;int lda = k;int ldb = k;int ldc = m;- fill(k, m, A, lda);- fill(k, n, B, ldb);- fill(m, n, C, ldc);- fill(m, n, G, ldc);- broadcast(m, n, G, n, TOMBSTONE);- broadcast(m, n, C, n, TOMBSTONE);- dgemm(true, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);- TINYBLAS_OR_DIE(tinyblasGemmEx(- hand, TINYBLAS_OP_T, TINYBLAS_OP_N, m, n, k, &α, A, TINYBLAS_R_16F, lda, B, TINYBLAS_R_16F,- ldb, &β, C, TINYBLAS_R_32F, ldc, TINYBLAS_COMPUTE_32F, TINYBLAS_GEMM_DEFAULT));- CUDA_OR_DIE(cudaStreamSynchronize(stream));- CHECK(PRECISION_F32, m, n, G, ldc, C, ldc);- cudaEvent_t start, stop;- CUDA_OR_DIE(cudaEventCreate(&stop));- CUDA_OR_DIE(cudaEventCreate(&start));- CUDA_OR_DIE(cudaEventRecord(start, NULL));- for (int i = 0; i < ITERATIONS; ++i)- TINYBLAS_OR_DIE(tinyblasGemmEx(hand, TINYBLAS_OP_T, TINYBLAS_OP_N, m, n, k, &α, A,- TINYBLAS_R_16F, lda, B, TINYBLAS_R_16F, ldb, &β, C,- TINYBLAS_R_32F, ldc, TINYBLAS_COMPUTE_32F,- TINYBLAS_GEMM_DEFAULT));- float msecTotal = 0;- CUDA_OR_DIE(cudaEventRecord(stop, NULL));- CUDA_OR_DIE(cudaEventSynchronize(stop));- CUDA_OR_DIE(cudaEventElapsedTime(&msecTotal, start, stop));- printf("%8lld µs tinyblasGemmExHHS(%d, %d, %d)\n",- (long long)std::ceil(msecTotal / ITERATIONS * 1000), m, n, k);- CUDA_OR_DIE(cudaEventDestroy(start));- CUDA_OR_DIE(cudaEventDestroy(stop));-- CUDA_OR_DIE(cudaFree(G));- CUDA_OR_DIE(cudaFree(C));- CUDA_OR_DIE(cudaFree(B));- CUDA_OR_DIE(cudaFree(A));- CUDA_OR_DIE(cudaStreamDestroy(stream));- TINYBLAS_OR_DIE(tinyblasDestroy(hand));-}--void checkCublasWorksHHH() {-- cublasHandle_t hand;- CUBLAS_OR_DIE(cublasCreate(&hand));-- cudaStream_t stream;- CUDA_OR_DIE(cudaStreamCreate(&stream));- CUBLAS_OR_DIE(cublasSetStream(hand, stream));-- half *A, *B, *C, *G;- long long maxdim = 32000;- CUDA_OR_DIE(cudaMallocManaged(&A, maxdim * maxdim * sizeof(half)));- CUDA_OR_DIE(cudaMallocManaged(&B, maxdim * maxdim * sizeof(half)));- CUDA_OR_DIE(cudaMallocManaged(&C, maxdim * maxdim * sizeof(half)));- CUDA_OR_DIE(cudaMallocManaged(&G, maxdim * maxdim * sizeof(half)));-- std::atomic_llong t = ATOMIC_VAR_INIT(0);- for (int ai = 0; ai < ARRAYLEN(kAlphas); ++ai)- for (int bi = 0; bi < ARRAYLEN(kBetas); ++bi)- for (int mi = 0; mi < ARRAYLEN(kDims); ++mi)- for (int ni = 0; ni < ARRAYLEN(kDims); ++ni)- for (int ki = 0; ki < ARRAYLEN(kDims); ++ki) {- half α = kAlphas[ai];- half β = kBetas[bi];- int m = kDims[mi];- int n = kDims[ARRAYLEN(kDims) - 1 - ni];- int k = kDims[ki];- int lda = m;- int ldb = k;- int ldc = m;- char name[192];- snprintf(name, sizeof(name), "testing %4d %4d %4d α=%g β=%g", m, n, k,- __half2float(α), __half2float(β));- if (t++ % 7 == 0)- fprintf(stderr, "%s\r", name);- is_self_testing = name;- fill(m, k, A, lda);- fill(k, n, B, ldb);- broadcast(m, n, G, n, TOMBSTONE);- broadcast(m, n, C, n, TOMBSTONE);- dgemm(false, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);- CUBLAS_OR_DIE(cublasGemmEx(hand, CUBLAS_OP_N, CUBLAS_OP_N, m, n, k, &α, A,- CUDA_R_16F, lda, B, CUDA_R_16F, ldb, &β, C,- CUDA_R_16F, ldc, CUBLAS_COMPUTE_16F,- CUBLAS_GEMM_DEFAULT));- CUDA_OR_DIE(cudaStreamSynchronize(stream));- CHECK(PRECISION_F16, m, n, G, ldc, C, ldc);- is_self_testing = 0;- }-- half α = 1;- half β = 0;- int m = 4096;- int n = 1024;- int k = 577;- int lda = k;- int ldb = k;- int ldc = m;- fill(k, m, A, lda);- fill(k, n, B, ldb);- fill(m, n, C, ldc);- fill(m, n, G, ldc);- broadcast(m, n, G, n, TOMBSTONE);- broadcast(m, n, C, n, TOMBSTONE);- dgemm(true, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);- CUBLAS_OR_DIE(cublasGemmEx(hand, CUBLAS_OP_T, CUBLAS_OP_N, m, n, k, &α, A, CUDA_R_16F, lda, B,- CUDA_R_16F, ldb, &β, C, CUDA_R_16F, ldc, CUBLAS_COMPUTE_16F,- CUBLAS_GEMM_DEFAULT));- CUDA_OR_DIE(cudaStreamSynchronize(stream));- CHECK(PRECISION_F16, m, n, G, ldc, C, ldc);- cudaEvent_t start, stop;- CUDA_OR_DIE(cudaEventCreate(&stop));- CUDA_OR_DIE(cudaEventCreate(&start));- CUDA_OR_DIE(cudaEventRecord(start, NULL));- for (int i = 0; i < ITERATIONS; ++i)- CUBLAS_OR_DIE(cublasGemmEx(hand, CUBLAS_OP_T, CUBLAS_OP_N, m, n, k, &α, A, CUDA_R_16F, lda,- B, CUDA_R_16F, ldb, &β, C, CUDA_R_16F, ldc, CUBLAS_COMPUTE_16F,- CUBLAS_GEMM_DEFAULT));- float msecTotal = 0;- CUDA_OR_DIE(cudaEventRecord(stop, NULL));- CUDA_OR_DIE(cudaEventSynchronize(stop));- CUDA_OR_DIE(cudaEventElapsedTime(&msecTotal, start, stop));- printf("%8lld µs cublasGemmExHHH(%d, %d, %d)\n",- (long long)std::ceil(msecTotal / ITERATIONS * 1000), m, n, k);- CUDA_OR_DIE(cudaEventDestroy(start));- CUDA_OR_DIE(cudaEventDestroy(stop));-- CUDA_OR_DIE(cudaFree(G));- CUDA_OR_DIE(cudaFree(C));- CUDA_OR_DIE(cudaFree(B));- CUDA_OR_DIE(cudaFree(A));- CUDA_OR_DIE(cudaStreamDestroy(stream));- CUBLAS_OR_DIE(cublasDestroy(hand));-}--void checkTinyblasWorksHHH() {-- tinyblasHandle_t hand;- TINYBLAS_OR_DIE(tinyblasCreate(&hand));-- cudaStream_t stream;- CUDA_OR_DIE(cudaStreamCreate(&stream));- TINYBLAS_OR_DIE(tinyblasSetStream(hand, stream));-- half *A, *B, *C, *G;- long long maxdim = 32000;- CUDA_OR_DIE(cudaMallocManaged(&A, maxdim * maxdim * sizeof(half)));- CUDA_OR_DIE(cudaMallocManaged(&B, maxdim * maxdim * sizeof(half)));- CUDA_OR_DIE(cudaMallocManaged(&C, maxdim * maxdim * sizeof(half)));- CUDA_OR_DIE(cudaMallocManaged(&G, maxdim * maxdim * sizeof(half)));-- std::atomic_llong t = ATOMIC_VAR_INIT(0);- for (int ai = 0; ai < ARRAYLEN(kAlphas); ++ai)- for (int bi = 0; bi < ARRAYLEN(kBetas); ++bi)- for (int mi = 0; mi < ARRAYLEN(kDims); ++mi)- for (int ni = 0; ni < ARRAYLEN(kDims); ++ni)- for (int ki = 0; ki < ARRAYLEN(kDims); ++ki) {- half α = kAlphas[ai];- half β = kBetas[bi];- int m = kDims[mi];- int n = kDims[ARRAYLEN(kDims) - 1 - ni];- int k = kDims[ki];- int lda = m;- int ldb = k;- int ldc = m;- char name[192];- snprintf(name, sizeof(name), "testing %4d %4d %4d α=%g β=%g", m, n, k,- __half2float(α), __half2float(β));- if (t++ % 7 == 0)- fprintf(stderr, "%s\r", name);- is_self_testing = name;- fill(m, k, A, lda);- fill(k, n, B, ldb);- broadcast(m, n, G, n, TOMBSTONE);- broadcast(m, n, C, n, TOMBSTONE);- dgemm(false, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);- TINYBLAS_OR_DIE(tinyblasGemmEx(- hand, TINYBLAS_OP_N, TINYBLAS_OP_N, m, n, k, &α, A, TINYBLAS_R_16F, lda,- B, TINYBLAS_R_16F, ldb, &β, C, TINYBLAS_R_16F, ldc,- TINYBLAS_COMPUTE_16F, TINYBLAS_GEMM_DEFAULT));- CUDA_OR_DIE(cudaStreamSynchronize(stream));- CHECK(PRECISION_F16, m, n, G, ldc, C, ldc);- is_self_testing = 0;- }-- half α = 1;- half β = 0;- int m = 4096;- int n = 1024;- int k = 577;- int lda = k;- int ldb = k;- int ldc = m;- fill(k, m, A, lda);- fill(k, n, B, ldb);- fill(m, n, C, ldc);- fill(m, n, G, ldc);- broadcast(m, n, G, n, TOMBSTONE);- broadcast(m, n, C, n, TOMBSTONE);- dgemm(true, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);- TINYBLAS_OR_DIE(tinyblasGemmEx(- hand, TINYBLAS_OP_T, TINYBLAS_OP_N, m, n, k, &α, A, TINYBLAS_R_16F, lda, B, TINYBLAS_R_16F,- ldb, &β, C, TINYBLAS_R_16F, ldc, TINYBLAS_COMPUTE_16F, TINYBLAS_GEMM_DEFAULT));- CUDA_OR_DIE(cudaStreamSynchronize(stream));- CHECK(PRECISION_F16, m, n, G, ldc, C, ldc);- cudaEvent_t start, stop;- CUDA_OR_DIE(cudaEventCreate(&stop));- CUDA_OR_DIE(cudaEventCreate(&start));- CUDA_OR_DIE(cudaEventRecord(start, NULL));- for (int i = 0; i < ITERATIONS; ++i)- TINYBLAS_OR_DIE(tinyblasGemmEx(hand, TINYBLAS_OP_T, TINYBLAS_OP_N, m, n, k, &α, A,- TINYBLAS_R_16F, lda, B, TINYBLAS_R_16F, ldb, &β, C,- TINYBLAS_R_16F, ldc, TINYBLAS_COMPUTE_16F,- TINYBLAS_GEMM_DEFAULT));- float msecTotal = 0;- CUDA_OR_DIE(cudaEventRecord(stop, NULL));- CUDA_OR_DIE(cudaEventSynchronize(stop));- CUDA_OR_DIE(cudaEventElapsedTime(&msecTotal, start, stop));- printf("%8lld µs tinyblasGemmExHHH(%d, %d, %d)\n",- (long long)std::ceil(msecTotal / ITERATIONS * 1000), m, n, k);- CUDA_OR_DIE(cudaEventDestroy(start));- CUDA_OR_DIE(cudaEventDestroy(stop));-- CUDA_OR_DIE(cudaFree(G));- CUDA_OR_DIE(cudaFree(C));- CUDA_OR_DIE(cudaFree(B));- CUDA_OR_DIE(cudaFree(A));- CUDA_OR_DIE(cudaStreamDestroy(stream));- TINYBLAS_OR_DIE(tinyblasDestroy(hand));+ cuda_memory<float> A{lda * m};+ cuda_memory<float> B{ldb * n};+ cuda_memory<float> C{ldc * n};+ cuda_memory<float> G{ldc * n};+ randomize(k, m, A.p, lda);+ randomize(k, n, B.p, ldb);+ cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ broadcast(m, n, C.p, ldc, TOMBSTONE);+ broadcast(m, n, G.p, ldc, TOMBSTONE);+ gemmref(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);++ test_matmul([&](int m, int n, int k, int l, float α, float β) {+ int lda = m + l;+ int ldb = k + l;+ int ldc = m + l;+ cuda_memory<float> A{lda * k};+ cuda_memory<float> B{ldb * n};+ cuda_memory<float> C{ldc * n};+ cuda_memory<float> G{ldc * n};+ randomize(m, k, A.p, lda);+ randomize(k, n, B.p, ldb);+ broadcast(m, n, G.p, ldc, TOMBSTONE);+ broadcast(m, n, C.p, ldc, TOMBSTONE);+ gemmref(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ tinyblas(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);+ });}int main(int argc, char *argv[]) {- RUN(checkFloat01());- RUN(checkCublasWorksSSS());- RUN(checkTinyblasWorksSSS());- RUN(checkCublasWorksHHS());- RUN(checkTinyblasWorksHHS());- RUN(checkCublasWorksHHH());- RUN(checkTinyblasWorksHHH());+ RUN(checkTinyblasWorksHHHH());+ RUN(checkTinyblasWorksSSSS());+ RUN(checkTinyblasWorksHHSS());+ RUN(checkTinyblasWorksHHHS());}