Skip to content

Distribute and run LLMs with a single file.

Write more testing code

Justine Tunney1aee87f716db
diff --git a/build/cudacc b/build/cudacc
new file mode 100755
index 00000000..3bdb2f55
--- /dev/null
+++ b/build/cudacc
@@ -0,0 +1,54 @@
+#!/bin/sh
+
+find_nvcc() {
+ CC=$(command -v nvcc 2>/dev/null) && return
+ CC="$CUDA_PATH/bin/nvcc"
+ [ -x "$CC" ] && return
+ CC="/opt/cuda/bin/nvcc"
+ [ -x "$CC" ] && return
+ CC="/usr/local/cuda/bin/nvcc"
+ [ -x "$CC" ] && return
+ return 1
+}
+
+find_hipcc() {
+ CC=$(command -v hipcc 2>/dev/null) && return
+ CC="$HIP_PATH/bin/hipcc"
+ [ -x "$CC" ] && return
+ CC="/opt/rocm/bin/hipcc"
+ [ -x "$CC" ] && return
+ CC="/usr/local/rocm/bin/hipcc"
+ [ -x "$CC" ] && return
+ return 1
+}
+
+if find_hipcc; then
+ VENDOR=AMD
+ FLAGS=
+elif find_nvcc; then
+ VENDOR=NVIDIA
+ FLAGS="--forward-unknown-to-host-compiler"
+else
+ echo 'error: need either hipcc (AMD) or nvcc (NVIDIA) on $PATH' >&2
+ exit 1
+fi
+
+FIRST=1
+for x; do
+ if [ $FIRST -eq 1 ]; then
+ set --
+ FIRST=0
+ fi
+ if [ $VENDOR = AMD ]; then
+ if [ x"$x" = x"-lcublas" ]; then
+ set -- "$@" -lhipblas -lrocblas
+ continue
+ elif [ x"$x" = x"--use_fast_math" ]; then
+ set -- "$@" -ffast-math -DHIP_FAST_MATH
+ continue
+ fi
+ fi
+ set -- "$@" "$x"
+done
+
+exec "$CC" $FLAGS "$@"
diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cu
index ecf7d4ca..57781c1a 100644
--- a/llama.cpp/ggml-cuda.cu
+++ b/llama.cpp/ggml-cuda.cu
@@ -53,7 +53,7 @@
#define cublasSetStream tinyblasSetStream
#define cublasSgemm tinyblasSgemm
#define cublasStatus_t tinyblasStatus_t
-#define cudaDataType_t tinyblasDatatype_t
+#define cudaDataType_t tinyblasDataType_t
#define cudaDeviceCanAccessPeer hipDeviceCanAccessPeer
#define cudaDeviceDisablePeerAccess hipDeviceDisablePeerAccess
#define cudaDeviceEnablePeerAccess hipDeviceEnablePeerAccess
@@ -131,7 +131,7 @@
#define cublasGemmBatchedEx tinyblasGemmBatchedEx
#define cublasGemmStridedBatchedEx tinyblasGemmStridedBatchedEx
#define cublasGetStatusString tinyblasGetStatusString
-#define cudaDataType_t tinyblasDatatype_t
+#define cudaDataType_t tinyblasDataType_t
#define cublasSetMathMode(handle, mode) CUBLAS_STATUS_SUCCESS
#elif defined(GGML_USE_HIPBLAS)
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index 7290d9f6..3e907432 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -44,17 +44,18 @@ o/$(MODE)/llamafile: \
o/$(MODE)/llamafile/%.o: llamafile/%.cu llamafile/BUILD.mk
@mkdir -p $(@D)
- hipcc -O3 -march=native -ffast-math -c -o $@ $<
-
-o/$(MODE)/llamafile/tester.o: llamafile/tester.cu llamafile/BUILD.mk
- @mkdir -p $(@D)
- hipcc -O3 -march=native -fopenmp -c -o $@ $<
+ build/cudacc -g -O3 -march=native -ffast-math --use_fast_math -c -o $@ $<
o/$(MODE)/llamafile/tinyblas_test: \
o/$(MODE)/llamafile/tinyblas_test.o \
o/$(MODE)/llamafile/tinyblas.o \
o/$(MODE)/llamafile/tester.o
- hipcc -fopenmp $(HIP_ARCH) -o $@ $^ -lhipblas -lrocblas
+ build/cudacc -g -o $@ $^ -lcublas
+
+o/$(MODE)/llamafile/cudaprops: \
+ o/$(MODE)/llamafile/cudaprops.o \
+ o/$(MODE)/llamafile/tester.o
+ build/cudacc -g -o $@ $^ -lcublas
.PHONY: o/$(MODE)/llamafile/check
o/$(MODE)/llamafile/check: \
diff --git a/llamafile/cuda.h b/llamafile/cuda.h
new file mode 100644
index 00000000..babd08b7
--- /dev/null
+++ b/llamafile/cuda.h
@@ -0,0 +1,72 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#pragma once
+
+#if defined(__NVCC__) || defined(__CUDA_ARCH__)
+#include <cublas_v2.h>
+#include <cuda_fp16.h>
+#include <cuda_runtime.h>
+#else
+#include <hip/hip_fp16.h>
+#include <hip/hip_runtime.h>
+#include <hipblas/hipblas.h>
+#include <rocblas/rocblas.h>
+#define cudaFree hipFree
+#define cublasMath_t int
+#define cudaMalloc hipMalloc
+#define cudaMemcpy hipMemcpy
+#define cudaSuccess hipSuccess
+#define cudaError_t hipError_t
+#define cudaEvent_t hipEvent_t
+#define cudaFreeHost hipHostFree
+#define cudaStream_t hipStream_t
+#define cublasSgemm hipblasSgemm
+#define cudaSetDevice hipSetDevice
+#define cublasCreate hipblasCreate
+#define cudaDataType_t hipDataType
+#define cublasDestroy hipblasDestroy
+#define cudaDeviceProp hipDeviceProp_t
+#define cublasHandle_t hipblasHandle_t
+#define cublasStatus_t hipblasStatus_t
+#define cudaEventCreate hipEventCreate
+#define cudaEventRecord hipEventRecord
+#define cudaGetDeviceCount hipGetDeviceCount
+#define cudaGetDeviceProperties hipGetDeviceProperties
+#define cudaGetLastError hipGetLastError
+#define cudaDeviceReset hipDeviceReset
+#define cudaEventDestroy hipEventDestroy
+#define cublasSetStream hipblasSetStream
+#define cudaStreamCreate hipStreamCreate
+#define cudaMallocManaged hipMallocManaged
+#define cudaStreamDestroy hipStreamDestroy
+#define cublasComputeType_t hipblasComputeType_t
+#define cudaEventSynchronize hipEventSynchronize
+#define cudaEventElapsedTime hipEventElapsedTime
+#define cudaDeviceSynchronize hipDeviceSynchronize
+#define cudaStreamSynchronize hipStreamSynchronize
+#define cublasGetStatusString hipblasStatusToString
+#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost
+#define cudaMemcpyHostToDevice hipMemcpyHostToDevice
+#define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice
+#define cublasSetMathMode(handle, mode) CUBLAS_STATUS_SUCCESS
+#define cudaMallocHost(ptr, size) hipHostMalloc(ptr, size, hipHostMallocNonCoherent)
+#define cudaGetErrorString hipGetErrorString
+#define cudaGetDevice hipGetDevice
+#define cudaDeviceReset hipDeviceReset
+#define cublasGemmEx hipblasGemmEx_v2
+#define CUDA_R_16F HIP_R_16F
+#define CUDA_R_32F HIP_R_32F
+#define CUBLAS_OP_N HIPBLAS_OP_N
+#define CUBLAS_OP_T HIPBLAS_OP_T
+#define CUBLAS_COMPUTE_16F HIPBLAS_COMPUTE_16F
+#define CUBLAS_COMPUTE_32F HIPBLAS_COMPUTE_32F
+#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_COMPUTE_32F_FAST_16F
+#define CUBLAS_GEMM_DEFAULT HIPBLAS_GEMM_DEFAULT
+#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS
+#define CUBLAS_MATH_DISALLOW_REDUCED_PRECISION_REDUCTION 0
+#define CUBLAS_TF32_TENSOR_OP_MATH 0
+#define CUBLAS_PEDANTIC_MATH 0
+#define CUBLAS_DEFAULT_MATH 0
+#endif
+
+#define WARPSIZE 32
diff --git a/llamafile/cudaprops.cu b/llamafile/cudaprops.cu
new file mode 100644
index 00000000..3ec087c9
--- /dev/null
+++ b/llamafile/cudaprops.cu
@@ -0,0 +1,80 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cuda.h"
+#include "tester.h"
+#include <iostream>
+
+#if defined(__NVCC__) || defined(__CUDA_ARCH__)
+#define VENDOR "NVIDIA"
+#else
+#define VENDOR "AMD"
+#endif
+
+namespace {
+
+void show_cuda_device(int idev) {
+ cudaDeviceProp dp;
+ CUDA_OR_DIE(cudaSetDevice(idev));
+ CUDA_OR_DIE(cudaGetDeviceProperties(&dp, idev));
+
+ fprintf(stderr,
+ "\n" VENDOR " %s\n"
+ "\t%.2f Ghz\n"
+ "\t%g GB VRAM\n"
+ "\t%d CPUs\n"
+ "\tcompute_%d\n",
+ dp.name, dp.clockRate * 1e-6, dp.totalGlobalMem / 1073741824., dp.multiProcessorCount,
+ dp.major * 10 + dp.minor);
+
+#define SHOW(s) std::cout << #s << " = " << dp.s << std::endl;
+
+ SHOW(warpSize);
+ SHOW(regsPerBlock);
+ SHOW(regsPerMultiprocessor);
+ SHOW(multiProcessorCount);
+ SHOW(memoryBusWidth);
+ SHOW(memoryClockRate);
+ SHOW(sharedMemPerBlock);
+ SHOW(sharedMemPerMultiprocessor);
+ SHOW(totalConstMem);
+ SHOW(maxThreadsPerBlock);
+ SHOW(maxThreadsPerMultiProcessor);
+ SHOW(maxThreadsDim[0]);
+ SHOW(maxThreadsDim[1]);
+ SHOW(maxThreadsDim[2]);
+ SHOW(asyncEngineCount);
+ SHOW(managedMemory);
+ SHOW(concurrentManagedAccess);
+ SHOW(directManagedMemAccessFromHost);
+ SHOW(globalL1CacheSupported);
+ SHOW(hostNativeAtomicSupported);
+ // SHOW(hostRegisterReadOnlySupported);
+ // SHOW(hostRegisterSupported);
+ SHOW(integrated);
+ SHOW(l2CacheSize);
+}
+
+} // namespace
+
+int main() {
+ int device_count;
+ CUDA_OR_DIE(cudaGetDeviceCount(&device_count));
+ for (int idev = 0; idev < device_count; ++idev) {
+ show_cuda_device(idev);
+ }
+}
diff --git a/llamafile/flt.h b/llamafile/flt.h
new file mode 100644
index 00000000..b50df4ab
--- /dev/null
+++ b/llamafile/flt.h
@@ -0,0 +1,75 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#pragma once
+
+// floating point utilities
+//
+// these functions let you fiddle with float bits. some of these
+// functions e.g. flt::isnan(), won't be optimized away when the
+// compiler is placed in -ffast-math mode.
+//
+// @see IEEE 754-2008
+
+#if defined(__NVCC__) || defined(__CUDA_ARCH__)
+#include <cuda_fp16.h>
+#else
+#include <hip/hip_fp16.h>
+#endif
+
+namespace flt {
+
+inline int toint(float f) {
+ union {
+ float f;
+ unsigned i;
+ } u = {f};
+ return u.i;
+}
+
+inline int toint(half f) {
+ union {
+ half f;
+ unsigned short i;
+ } u = {f};
+ return u.i;
+}
+
+inline int signbit(float f) {
+ return toint(f) & (-2147483647 - 1);
+}
+
+inline int signbit(half f) {
+ return toint(f) & 0x8000;
+}
+
+inline int isnan(float f) {
+ return (toint(f) & 0x7fffffff) > 0x7f800000;
+}
+
+inline int isnan(half f) {
+ return (toint(f) & 0x7fff) > 0x7c00;
+}
+
+inline int isinf(float f) {
+ return (toint(f) & 0x7fffffff) == 0x7f800000;
+}
+
+inline int isinf(half f) {
+ return (toint(f) & 0x7fff) == 0x7c00;
+}
+
+inline int isnormal(float f) {
+ int expo = toint(f) & 0x7f800000;
+ return expo && expo != 0x7f800000;
+}
+
+inline int isnormal(half f) {
+ int expo = toint(f) & 0x7c00;
+ return expo && expo != 0x7c00;
+}
+
+template <typename T> inline int isdenormal(T f) {
+ return f && !isnormal(f) && !isnan(f) && !isinf(f);
+}
+
+} // namespace flt
diff --git a/llamafile/gemm.h b/llamafile/gemm.h
new file mode 100644
index 00000000..e9c3a660
--- /dev/null
+++ b/llamafile/gemm.h
@@ -0,0 +1,70 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include <algorithm>
+#include <cassert>
+#include <climits>
+#include <cmath>
+
+// multiplies matrix on cpu with column major ordering
+//
+// m×k * k×n → m×n
+// k×m * k×n → m×n if aᵀ
+// m×k * n×k → m×n if bᵀ
+// k×m * n×k → m×n if aᵀ and bᵀ
+//
+template <typename T, typename TA, typename TB, typename TC>
+void gemm(bool aᵀ, bool bᵀ, //
+ int m, int n, int k, T α, //
+ const TA *A, int lda, //
+ const TB *B, int ldb, T β, //
+ TC *C, int ldc) {
+ assert(m >= 0 && n >= 0 && k >= 0);
+ assert(lda >= std::max(1, aᵀ ? k : m));
+ assert(ldb >= std::max(1, bᵀ ? n : k));
+ assert(ldc >= std::max(1, m));
+ assert(1ll * lda * (aᵀ ? m : k) <= INT_MAX);
+ assert(1ll * ldb * (bᵀ ? k : n) <= INT_MAX);
+ for (int i = 0; i < m; ++i)
+ for (int j = 0; j < n; ++j) {
+ T d = 0;
+ for (int l = 0; l < k; ++l) {
+ T a = aᵀ ? A[lda * i + l] : A[lda * l + i];
+ T b = bᵀ ? B[ldb * l + j] : B[ldb * j + l];
+ d = std::fma(a, b, d);
+ }
+ T c = C[ldc * j + i];
+ C[ldc * j + i] = std::fma(α, d, β * c);
+ }
+}
+
+// multiplies matrix on cpu with column row ordering
+//
+// k×m * n×k → n×m
+// m×k * n×k → n×m if aᵀ
+// k×m * k×n → n×m if bᵀ
+// m×k * k×n → n×m if aᵀ and bᵀ
+//
+template <typename T, typename TA, typename TB, typename TC>
+void gemmrm(bool aᵀ, bool bᵀ, //
+ int m, int n, int k, T α, //
+ const TA *A, int lda, //
+ const TB *B, int ldb, T β, //
+ TC *C, int ldc) {
+ gemm(bᵀ, aᵀ, n, m, k, α, B, ldb, A, lda, β, C, ldc);
+}
diff --git a/llamafile/macros.h b/llamafile/macros.h
new file mode 100644
index 00000000..8590ce47
--- /dev/null
+++ b/llamafile/macros.h
@@ -0,0 +1,7 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#pragma once
+
+#define CEIL_DIV(M, N) (((M) + (N) - 1) / (N))
+#define ROUNDUP(X, K) (((X) + (K) - 1) & -(K))
+#define ARRAYLEN(A) ((sizeof(A) / sizeof(*(A))) / ((unsigned)!(sizeof(A) % sizeof(*(A)))))
diff --git a/llamafile/naive.h b/llamafile/naive.h
new file mode 100644
index 00000000..de45d328
--- /dev/null
+++ b/llamafile/naive.h
@@ -0,0 +1,88 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#pragma once
+
+#include "macros.h"
+
+//
+// _ _ ___ _ _ ___
+// | |_(_)_ _ _ _| _ ) | /_\ / __|
+// | _| | ' \ || | _ \ |__ / _ \\__ \.
+// \__|_|_||_\_, |___/____/_/ \_\___/
+// |__/
+//
+// BASIC LINEAR ALGEBRA SUBPROGRAMS
+//
+
+// naive abstract generic matrix multiplication for gpu
+//
+// this matmul reference implementation is guaranteed to be bug-free and
+// can be used for finding bugs in more advanced matmul implementations.
+
+namespace naive {
+
+__device__ __forceinline__ half madd(half x, half y, half z) {
+ return (float)x * (float)y + (float)z;
+}
+__device__ __forceinline__ float madd(float x, float y, float z) {
+ return __fmaf_rn(x, y, z);
+}
+__device__ __forceinline__ double madd(double x, double y, double z) {
+ return __fma_rn(x, y, z);
+}
+
+template <typename T, typename TA, typename TB, typename TC>
+__global__ void kernel(bool aᵀ, bool bᵀ, //
+ int m, int n, int k, T α, //
+ const TA *A, int lda, //
+ const TB *B, int ldb, T β, //
+ TC *C, int ldc) {
+ const int i = blockIdx.x * blockDim.x + threadIdx.x;
+ const int j = blockIdx.y * blockDim.y + threadIdx.y;
+ if (i < m && j < n) {
+ T d = 0;
+ for (int l = 0; l < k; ++l) {
+ T a = aᵀ ? A[lda * i + l] : A[lda * l + i];
+ T b = bᵀ ? B[ldb * l + j] : B[ldb * j + l];
+ d = madd(a, b, d);
+ }
+ T c = C[ldc * j + i];
+ C[ldc * j + i] = madd(α, d, β * c);
+ }
+}
+
+// multiplies matrix on gpu with column major ordering
+//
+// m×k * k×n → m×n
+// k×m * k×n → m×n if aᵀ
+// m×k * n×k → m×n if bᵀ
+// k×m * n×k → m×n if aᵀ and bᵀ
+//
+template <typename T, typename TA, typename TB, typename TC>
+cudaError_t gemm(cudaStream_t stream, //
+ bool aᵀ, bool bᵀ, //
+ int m, int n, int k, T α, //
+ const TA *A, int lda, //
+ const TB *B, int ldb, T β, //
+ TC *C, int ldc) {
+ dim3 blockDim(32, 32);
+ dim3 gridDim(CEIL_DIV(m, 32), CEIL_DIV(n, 32));
+ kernel<<<gridDim, blockDim, 0, stream>>>(aᵀ, bᵀ, m, n, k, α, A, lda, B, ldb, β, C, ldc);
+ return cudaGetLastError();
+}
+
+} // namespace naive
diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cpp
index 21a20d9e..80b21007 100644
--- a/llamafile/sgemm.cpp
+++ b/llamafile/sgemm.cpp
@@ -17,6 +17,7 @@
#include "llama.cpp/ggml-quants.h"
#include "llamafile.h"
+#include <assert.h>
#include <cosmo.h>
#include <immintrin.h>
#include <stdio.h>
@@ -31,11 +32,32 @@
// BASIC LINEAR ALGEBRA SUBPROGRAMS
//
-#define ASSERT(x) \
- if (!(x)) { \
- fprintf(stderr, "%s:%d: assertion failed: %s\n", __FILE__, __LINE__, #x); \
- __builtin_trap(); \
- }
+// matrix multiplication for x86 microprocessors
+//
+// this file implements optimized cpu kernels for llama.cpp that support
+// column major ordering with the A matrix transposed. they work by
+// computing multiple dot products at once, because it exploits
+// instruction level parallelism, in addition to minimizing register
+// loads since the adjacent dot products can share register loads.
+//
+// this technique usually slightly outperforms mkl and blis at single
+// threaded performance, and usually goes 2x faster than mkl when openmp
+// threading is used. however please note that this code is adapted to
+// use the llama.cpp synchronization model, which isn't as advanced.
+//
+// ┌───────────┬──────────────────┬───────────────────┐
+// │ software │ skinny matrices │ fat matrices │
+// │ │ token generation │ prompt evaluation │
+// │ ───────── │ ──────────────── │ ───────────────── │
+// │ llama.cpp │ fast │ slow │
+// │ MKL │ slow │ fast │
+// │ tinyBLAS │ fast │ fast │
+// └───────────┴──────────────────┴───────────────────┘
+//
+// these kernels do not have any of the latency overhead typically
+// associated with the blas, which means they can provide the same
+// performance as llama.cpp when performing the most common type of
+// matmul used by text generation, which is matrix-vector product.
#define END_KERNEL() }
#define BEGIN_KERNEL(RM, RN) \
@@ -100,17 +122,17 @@ template <typename T> class tinyBLAS {
tinyBLAS(long k, const T *A, long lda, const T *B, long ldb, float *C, long ldc, long ith,
long nth)
: k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith), nth(nth) {
- ASSERT(A != nullptr);
- ASSERT(B != nullptr);
- ASSERT(C != nullptr);
- ASSERT(k >= 0 && k % 8 == 0);
- ASSERT(ith >= 0 && ith < nth);
+ assert(A != nullptr);
+ assert(B != nullptr);
+ assert(C != nullptr);
+ assert(k >= 0 && k % 8 == 0);
+ assert(ith >= 0 && ith < nth);
}
void gemm(long m, long n) {
- ASSERT(m >= 0);
- ASSERT(n >= 0);
- ASSERT(ldc >= m);
+ assert(m >= 0);
+ assert(n >= 0);
+ assert(ldc >= m);
mnpack(0, m, 0, n);
}
@@ -267,17 +289,17 @@ template <typename T, typename U, int avxvnni> class tinyBLASq {
tinyBLASq(long k, const T *A, long lda, const U *B, long ldb, float *C, long ldc, long ith,
long nth)
: k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith), nth(nth) {
- ASSERT(A != nullptr);
- ASSERT(B != nullptr);
- ASSERT(C != nullptr);
- ASSERT(k >= 0 && k % 32 == 0);
- ASSERT(ith >= 0 && ith < nth);
+ assert(A != nullptr);
+ assert(B != nullptr);
+ assert(C != nullptr);
+ assert(k >= 0 && k % 32 == 0);
+ assert(ith >= 0 && ith < nth);
}
void gemm(long m, long n) {
- ASSERT(m >= 0);
- ASSERT(n >= 0);
- ASSERT(ldc >= m);
+ assert(m >= 0);
+ assert(n >= 0);
+ assert(ldc >= m);
mnpack(0, m, 0, n);
}
@@ -428,8 +450,8 @@ template <typename T, typename U, int avxvnni> class tinyBLASq {
// computes m×k * n×k → n×m
void llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda, const void *B,
long ldb, float *C, long ldc, long ith, long nth) {
- ASSERT(X86_HAVE(FMA));
- ASSERT(X86_HAVE(AVX2));
+ assert(X86_HAVE(FMA));
+ assert(X86_HAVE(AVX2));
switch (dtype) {
#ifdef __x86_64__
@@ -440,7 +462,7 @@ void llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda,
}
case GGML_TYPE_F16: {
- ASSERT(X86_HAVE(F16C));
+ assert(X86_HAVE(F16C));
tinyBLAS<ggml_fp16_t> tb{
k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth,
};
@@ -478,6 +500,6 @@ void llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda,
#endif
default:
- ASSERT(!"unsupported dtype");
+ assert(!"unsupported dtype");
}
}
diff --git a/llamafile/tester.cu b/llamafile/tester.cu
index 5acd98c9..83955e89 100644
--- a/llamafile/tester.cu
+++ b/llamafile/tester.cu
@@ -17,13 +17,12 @@
#include "tester.h"
-#include <algorithm>
-#include <cmath>
-#include <cstdio>
-#include <cstdlib>
+#include <atomic>
#include <ctime>
+#include <unistd.h>
-const char *is_self_testing;
+thread_local const char *is_self_testing;
+const size_t kPageSize = std::max(sysconf(_SC_PAGESIZE), 4096l);
long long micros(void) {
struct timespec ts;
@@ -46,197 +45,73 @@ float numba(void) { // (-1,1)
return float01(rand32()) * 2 - 1;
}
-void fill(int m, int n, half *A, int lda) {
- for (int i = 0; i < m; ++i)
- for (int j = 0; j < n; ++j)
- A[1ll * lda * j + i] = numba();
-}
-
-void fill(int m, int n, float *A, int lda) {
- for (int i = 0; i < m; ++i)
- for (int j = 0; j < n; ++j)
- A[1ll * lda * j + i] = numba();
-}
-
-void broadcast(int m, int n, half *A, int lda, half x) {
- for (int i = 0; i < m; ++i)
- for (int j = 0; j < n; ++j)
- A[1ll * lda * j + i] = x;
-}
-
-void broadcast(int m, int n, float *A, int lda, float x) {
- for (int i = 0; i < m; ++i)
- for (int j = 0; j < n; ++j)
- A[1ll * lda * j + i] = x;
-}
-
-void dgemm(bool aᵀ, bool bᵀ, //
- int m, int n, int k, float α, //
- const float *A, int lda, //
- const float *B, int ldb, float β, //
- float *C, int ldc) {
- ASSERT(m >= 0 && n >= 0 && k >= 0);
- ASSERT(lda >= std::max(1, aᵀ ? k : m));
- ASSERT(ldb >= std::max(1, bᵀ ? n : k));
- ASSERT(ldc >= std::max(1, m));
-#pragma omp parallel for collapse(2) if (m * n * k > 30000)
- for (int i = 0; i < m; ++i)
- for (int j = 0; j < n; ++j) {
- double dot = 0;
- for (int l = 0; l < k; ++l)
- dot += (aᵀ ? A[1ll * lda * i + l] : A[1ll * lda * l + i]) *
- (bᵀ ? B[1ll * ldb * l + j] : B[1ll * ldb * j + l]);
- C[1ll * ldc * j + i] = α * dot + β * C[1ll * ldc * j + i];
- }
-}
-
-void dgemm(bool aᵀ, bool bᵀ, //
- int m, int n, int k, float α, //
- const half *A, int lda, //
- const half *B, int ldb, float β, //
- float *C, int ldc) {
- ASSERT(m >= 0 && n >= 0 && k >= 0);
- ASSERT(lda >= std::max(1, aᵀ ? k : m));
- ASSERT(ldb >= std::max(1, bᵀ ? n : k));
- ASSERT(ldc >= std::max(1, m));
-#pragma omp parallel for collapse(2) if (m * n * k > 30000)
- for (int i = 0; i < m; ++i)
- for (int j = 0; j < n; ++j) {
- double dot = 0;
- for (int l = 0; l < k; ++l)
- dot += __half2float(aᵀ ? A[1ll * lda * i + l] : A[1ll * lda * l + i]) *
- __half2float(bᵀ ? B[1ll * ldb * l + j] : B[1ll * ldb * j + l]);
- C[1ll * ldc * j + i] = α * dot + β * C[1ll * ldc * j + i];
- }
-}
-
-void dgemm(bool aᵀ, bool bᵀ, //
- int m, int n, int k, float α, //
- const half *A, int lda, //
- const half *B, int ldb, float β, //
- half *C, int ldc) {
- ASSERT(m >= 0 && n >= 0 && k >= 0);
- ASSERT(lda >= std::max(1, aᵀ ? k : m));
- ASSERT(ldb >= std::max(1, bᵀ ? n : k));
- ASSERT(ldc >= std::max(1, m));
-#pragma omp parallel for collapse(2) if (m * n * k > 30000)
- for (int i = 0; i < m; ++i)
- for (int j = 0; j < n; ++j) {
- float dot = 0;
- for (int l = 0; l < k; ++l)
- dot += __half2float(aᵀ ? A[1ll * lda * i + l] : A[1ll * lda * l + i]) *
- __half2float(bᵀ ? B[1ll * ldb * l + j] : B[1ll * ldb * j + l]);
- C[1ll * ldc * j + i] = α * dot + β * __half2float(C[1ll * ldc * j + i]);
- }
-}
-
-template <typename T> double diff(int m, int n, const T *Wan, int lda, const T *Got, int ldb) {
- double s = 0;
- int got_nans = 0;
- int wan_nans = 0;
- if (!m || !n)
- return 0;
- for (int i = 0; i < m; ++i)
- for (int j = 0; j < n; ++j)
- if (IsNan(Wan[1ll * lda * j + i]))
- ++wan_nans;
- else if (IsNan(Got[1ll * ldb * j + i]))
- ++got_nans;
- else
- s += std::fabs(static_cast<double>(Wan[1ll * lda * j + i]) -
- static_cast<double>(Got[1ll * ldb * j + i]));
- if (got_nans)
- fprintf(stderr, "WARNING: got %d NaNs!\n", got_nans);
- if (wan_nans)
- fprintf(stderr, "WARNING: want array has %d NaNs!\n", wan_nans);
- return s / (m * n);
-}
-
-template <typename T>
-void show(FILE *f, int max, int m, int n, const T *A, int lda, const T *B, int ldb) {
- flockfile(f);
- fprintf(f, " ");
- for (int i = 0; i < n && i < max; ++i)
- fprintf(f, "%13d", i);
- fprintf(f, "\n");
- for (int i = 0; i < m; ++i) {
- if (i == max) {
- fprintf(f, "...\n");
- break;
- }
- fprintf(f, "%5d ", i);
- for (int j = 0; j < n; ++j) {
- if (j == max) {
- fprintf(f, " ...");
- break;
- }
- char ba[32], bb[32];
- snprintf(ba, 32, "%13.7f", static_cast<double>(A[1ll * lda * j + i]));
- snprintf(bb, 32, "%13.7f", static_cast<double>(B[1ll * ldb * j + i]));
- for (int k = 0; ba[k] && bb[k]; ++k) {
- if (ba[k] != bb[k])
- fputs_unlocked("\33[31m", f);
- fputc_unlocked(ba[k], f);
- if (ba[k] != bb[k])
- fputs_unlocked("\33[0m", f);
- }
- }
- fprintf(f, "\n");
- }
- funlockfile(f);
-}
-
-template <typename T>
-void misfit(FILE *f, int max, int m, int n, const T *A, int lda, const T *B, int ldb,
- const char *file, int line, double sad, double tol) {
- fprintf(f, "%s:%d: sad %.17g exceeds %g (%s)\nwant\n", file, line, sad, tol,
- is_self_testing ? is_self_testing : "n/a");
- show(f, max, m, n, A, lda, B, ldb);
- fprintf(f, "got\n");
- show(f, max, m, n, B, ldb, A, lda);
- fprintf(f, "\n");
-}
-
-template <typename T>
-void checker(double tol, int m, int n, const T *A, int lda, const T *B, int ldb, const char *file,
- int line) {
- ASSERT(lda >= std::max(1, m));
- ASSERT(ldb >= std::max(1, m));
- double sad = diff(m, n, A, lda, B, ldb);
- if (sad <= tol) {
- if (!is_self_testing)
- printf(" %g error\n", sad);
- } else {
- flockfile(stderr);
- misfit(stderr, 16, m, n, A, lda, B, ldb, file, line, sad, tol);
- const char *path = "/tmp/llamafile_tester.log";
- FILE *f = fopen(path, "w");
- if (f) {
- misfit(f, 10000, m, n, A, lda, B, ldb, file, line, sad, tol);
- printf("see also %s\n", path);
+void *cudaMallocManagedOrDie(size_t n) {
+ void *p;
+ CUDA_OR_DIE(cudaMallocManaged(&p, n));
+ return p;
+}
+
+void cudaFreeOrDie(void *p) {
+ CUDA_OR_DIE(cudaFree(p));
+}
+
+[[noreturn]] void cuda_die(const char *stmt, const char *func, const char *file, int line,
+ const char *msg) {
+ int id = -1;
+ (void)cudaGetDevice(&id);
+ fprintf(stderr, "CUDA error: %s\n", msg);
+ fprintf(stderr, " current device: %d, in function %s at %s:%d\n", id, func, file, line);
+ fprintf(stderr, " %s\n", stmt);
+ (void)cudaDeviceReset();
+ _Exit(1);
+}
+
+void test_matmul(std::function<void(int m, int n, int k, int l, float α, float β)> f) {
+ static const int kDims[] = {1, 2, 23, 77, 15, 2048, 512, 127, 129, 128, 16};
+ static const float kAlphas[] = {1, .1};
+ static const float kBetas[] = {0, .1};
+ static const int kLeads[] = {0, 1};
+ std::atomic_llong t = ATOMIC_VAR_INIT(0);
+ for (int mi = 0; mi < ARRAYLEN(kDims); ++mi)
+ for (int ni = 0; ni < ARRAYLEN(kDims); ++ni)
+ for (int li = 0; li < ARRAYLEN(kLeads); ++li)
+ for (int ki = 0; ki < ARRAYLEN(kDims); ++ki) {
+ int m = kDims[mi];
+ int n = kDims[ARRAYLEN(kDims) - 1 - ni];
+ int k = kDims[ki];
+ int l = kLeads[li];
+ char name[256];
+ sprintf(name, "testing %4d %4d %4d ld+%d", m, n, k, l);
+ if (t++ % 7 == 0)
+ fprintf(stderr, "%s\r", name);
+ is_self_testing = name;
+ f(m, n, k, l, 1, 0);
+ is_self_testing = 0;
+ }
+ for (int ai = 0; ai < ARRAYLEN(kAlphas); ++ai)
+ for (int bi = 0; bi < ARRAYLEN(kBetas); ++bi) {
+ float α = kAlphas[ai];
+ float β = kBetas[bi];
+ int m = 128;
+ int n = 128;
+ int k = 128;
+ char name[256];
+ sprintf(name, "testing %4d %4d %4d α=%g β=%g", m, n, k, α, β);
+ if (t++ % 7 == 0)
+ fprintf(stderr, "%s\r", name);
+ is_self_testing = name;
+ f(m, n, k, 0, α, β);
+ is_self_testing = 0;
}
- exit(1);
- }
-}
-
-void check(double tol, int m, int n, const float *A, int lda, const float *B, int ldb,
- const char *file, int line) {
- checker(tol, m, n, A, lda, B, ldb, file, line);
-}
-
-void check(double tol, int m, int n, const half *A, int lda, const half *B, int ldb,
- const char *file, int line) {
- checker(tol, m, n, A, lda, B, ldb, file, line);
-}
-
-void run(const char *description) {
- printf("\n%s\n", description);
}
-void bench(long long start, const char *description) {
- printf("%8lld µs %s\n", (micros() - start + ITERATIONS - 1) / ITERATIONS, description);
+static int cuda_tester_init() {
+ CUDA_OR_DIE(cudaSetDevice(0));
+#ifdef __HIP_PLATFORM_AMD__
+ rocblas_initialize();
+#endif
+ CUDA_OR_DIE(cudaDeviceSynchronize());
+ return 0;
}
-void passert(const char *file, int line, const char *expr) {
- fprintf(stderr, "%s:%d: assertion failed: %s\n", file, line, expr);
-}
+const int cuda_tester_ = cuda_tester_init();
diff --git a/llamafile/tester.h b/llamafile/tester.h
index 01a1c06e..8c816edb 100644
--- a/llamafile/tester.h
+++ b/llamafile/tester.h
@@ -1,82 +1,314 @@
// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
#pragma once
-#ifdef __NVCC__
-#include <cuda_fp16.h>
-#else
-#include <hip/hip_fp16.h>
-#endif
+#include "cuda.h"
+#include "flt.h"
+#include "macros.h"
+#include "naive.h"
+#include "tinyblas.h"
+#include <algorithm>
+#include <cassert>
+#include <cmath>
+#include <cstdio>
+#include <cstdlib>
+#include <functional>
+
+//
+// _ _ ___ _ _ ___
+// | |_(_)_ _ _ _| _ ) | /_\ / __|
+// | _| | ' \ || | _ \ |__ / _ \\__ \.
+// \__|_|_||_\_, |___/____/_/ \_\___/
+// |__/
+//
+// BASIC LINEAR ALGEBRA SUBPROGRAMS
+//
+
+// c++ testing utilities for floating point math on cpu and gpu
+//
+// this file contains tools for testing matrix multiplication, measuring
+// errors, and visualizing what went wrong in using ansi terminal codes.
#define ITERATIONS 30
+#define TOMBSTONE 1.666f
-extern const char *is_self_testing;
+extern const size_t kPageSize;
+extern thread_local const char *is_self_testing;
float float01(unsigned);
float numba(void);
int rand32(void);
long long micros(void);
-void bench(long long, const char *);
-void broadcast(int, int, float *, int, float);
-void broadcast(int, int, half *, int, half);
-void check(double, int, int, const float *, int, const float *, int, const char *, int);
-void check(double, int, int, const half *, int, const half *, int, const char *, int);
-void fill(int, int, float *, int);
-void fill(int, int, half *, int);
-void passert(const char *, int, const char *);
-void run(const char *);
+void test_matmul(std::function<void(int, int, int, int, float, float)>);
+void *cudaMallocManagedOrDie(size_t);
+void cudaFreeOrDie(void *);
-void dgemm(bool, bool, int, int, int, float, const half *, int, const half *, int, float, half *,
- int);
-void dgemm(bool, bool, int, int, int, float, const half *, int, const half *, int, float, float *,
- int);
-void dgemm(bool, bool, int, int, int, float, const float *, int, const float *, int, float, float *,
- int);
+inline int hamming(int x, int y) {
+ return __builtin_popcount(x ^ y);
+}
-inline int toint(half f) {
- union {
- half f;
- unsigned short i;
- } u = {f};
- return u.i;
+template <typename T> void randomize(T *A, int n) {
+ for (int i = 0; i < n; ++i)
+ A[i] = numba();
}
-inline int toint(float f) {
- union {
- float f;
- unsigned i;
- } u = {f};
- return u.i;
+template <typename T> void randomize(int m, int n, T *A, int lda) {
+ for (int j = 0; j < n; ++j)
+ for (int i = 0; i < m; ++i)
+ A[lda * j + i] = numba();
}
-inline int IsNan(half f) {
- return (toint(f) & 0x7fff) > 0x7c00;
+template <typename T, typename U> void broadcast(T *A, int n, U x) {
+ for (int i = 0; i < n; ++i)
+ A[i] = x;
}
-inline int IsNan(float f) {
- return (toint(f) & 0x7fffffff) > 0x7f800000;
+template <typename T, typename U> void broadcast(int m, int n, T *A, int lda, U x) {
+ for (int j = 0; j < n; ++j)
+ for (int i = 0; i < m; ++i)
+ A[lda * j + i] = x;
}
-inline int IsNegativeZero(half f) {
- return toint(f) == 0x8000;
+template <typename T> struct cuda_memory {
+ const size_t size;
+ T *const p;
+ explicit cuda_memory(int len)
+#ifdef __SANITIZE_ADDRESS__
+ : size(len * sizeof(T)),
+#else
+ : size(ROUNDUP(len * sizeof(T) + 64, kPageSize) - 64),
+#endif
+ p(static_cast<T *>(cudaMallocManagedOrDie(size))) {
+ broadcast(p, size / sizeof(T), NAN);
+ }
+ ~cuda_memory() {
+ cudaFreeOrDie(p);
+ }
+};
+
+template <typename T>
+void verify(double tol, int m, int n, const T *A, int lda, const T *B, int ldb) {
+ T worst_x = 0;
+ T worst_y = 0;
+ int worst_i = 0;
+ int worst_j = 0;
+ long long nans = 0;
+ long long infs = 0;
+ long long worst = 0;
+ long long flips = 0;
+ long long diffsum = 0;
+ long long denormals = 0;
+ long long hamhist[32] = {0};
+ for (int i = 0; i < m; i++)
+ for (int j = 0; j < n; j++) {
+ T x = A[lda * j + i];
+ T y = B[ldb * j + i];
+ long long xi = flt::toint(x);
+ long long yi = flt::toint(y);
+ long long d = std::abs(xi - yi);
+ if (flt::isnormal(x) && flt::isnormal(y) && flt::signbit(x) == flt::signbit(y)) {
+ diffsum += d;
+ if (d > worst) {
+ worst = d;
+ worst_x = x;
+ worst_y = y;
+ worst_i = i;
+ worst_j = j;
+ }
+ }
+ nans -= flt::isnan(x);
+ nans += flt::isnan(y);
+ infs -= flt::isinf(x);
+ infs += flt::isinf(y);
+ flips += flt::signbit(x) != flt::signbit(y);
+ denormals -= flt::isdenormal(x);
+ denormals += flt::isdenormal(y);
+ ++hamhist[hamming(xi, yi)];
+ }
+ double avg = static_cast<double>(diffsum) / (m * n);
+ if (avg > tol || nans) {
+ flockfile(stderr);
+ fprintf(stderr, "error: %d×%d matrix difference exceeds tolerance\n", m, n);
+ fprintf(stderr, " worst element at index (%d, %d)\n", worst_i, worst_j);
+ fprintf(stderr, " difference %14e (%lld ulp)\n",
+ std::fabs(static_cast<double>(worst_x) - static_cast<double>(worst_y)), worst);
+ fprintf(stderr, " expected %14e (%x)\n", static_cast<double>(worst_x),
+ flt::toint(worst_x));
+ fprintf(stderr, " actual %14e (%x)\n", static_cast<double>(worst_y),
+ flt::toint(worst_y));
+ fprintf(stderr, " average %14g ulp\n", avg);
+ fprintf(stderr, " denormals %14lld elements\n", denormals);
+ fprintf(stderr, " flips %14lld elements\n", flips);
+ fprintf(stderr, " nans %14lld elements\n", nans);
+ fprintf(stderr, " infs %14lld elements\n", infs);
+ for (int i = 0; i < 32; ++i)
+ if (hamhist[i])
+ fprintf(stderr, " hamhist%-2d %14lld elements\n", i, hamhist[i]);
+ fflush(stderr);
+ (void)cudaDeviceReset();
+ _Exit(1);
+ }
}
-inline int IsNegativeZero(float f) {
- return toint(f) == -2147483647 - 1;
+template <typename T>
+void diff(int m, int n, int k, const T *Wan, int lda, const T *Got, int ldb, double *out_avg,
+ double *out_sad, double *out_worst, int *out_infs, int *out_flips, int *out_zeroes,
+ int *out_denormals, int *out_nans) {
+ double sad = 0;
+ long long ulp = 0;
+ long long worst = 0;
+ int infs = 0;
+ int flips = 0;
+ int zeroes = 0;
+ int got_nans = 0;
+ int wan_nans = 0;
+ int denormals = 0;
+ int considered = 0;
+ if (m && n) {
+ for (int i = 0; i < m; ++i)
+ for (int j = 0; j < n; ++j) {
+ T x = Wan[lda * j + i];
+ T y = Got[ldb * j + i];
+ if (flt::isnan(x)) {
+ ++wan_nans;
+ } else if (flt::isnan(y)) {
+ ++got_nans;
+ } else {
+ zeroes -= !x;
+ zeroes += !y;
+ infs -= flt::isinf(x);
+ infs += flt::isinf(y);
+ denormals -= flt::isdenormal(x);
+ denormals += flt::isdenormal(y);
+ flips += flt::signbit(x) != flt::signbit(y);
+ long long xi = flt::toint(x);
+ long long yi = flt::toint(y);
+ if (flt::isnormal(x) && flt::isnormal(y) &&
+ flt::signbit(x) == flt::signbit(y)) {
+ ++considered;
+ ulp += std::abs(xi - yi);
+ worst = std::max(worst, std::abs(xi - yi));
+ sad += std::fabs(static_cast<double>(x) - static_cast<double>(y));
+ }
+ }
+ }
+ if (got_nans)
+ fprintf(stderr, "WARNING: got %d NaNs!\n", got_nans);
+ if (wan_nans)
+ fprintf(stderr, "WARNING: want array has %d NaNs!\n", wan_nans);
+ }
+ int ops_per_element = k + 2;
+ *out_avg = static_cast<double>(ulp) / considered / ops_per_element;
+ *out_worst = static_cast<double>(worst) / ops_per_element;
+ *out_nans = got_nans + wan_nans;
+ *out_sad = sad / considered;
+ *out_denormals = denormals;
+ *out_zeroes = zeroes;
+ *out_flips = flips;
+ *out_infs = infs;
}
-#define ASSERT(x) \
- do { \
- if (!(x)) { \
- passert(__FILE__, __LINE__, #x); \
- __builtin_trap(); \
- } \
- } while (0)
+template <typename T>
+void show(FILE *f, int max, int m, int n, const T *A, int lda, const T *B, int ldb) {
+ flockfile(f);
+ fprintf(f, " ");
+ for (int i = 0; i < n && i < max; ++i)
+ fprintf(f, "%13d", i);
+ fprintf(f, "\n");
+ for (int i = 0; i < m; ++i) {
+ if (i == max) {
+ fprintf(f, "...\n");
+ break;
+ }
+ fprintf(f, "%5d ", i);
+ for (int j = 0; j < n; ++j) {
+ if (j == max) {
+ fprintf(f, " ...");
+ break;
+ }
+ char ba[32], bb[32];
+ snprintf(ba, 32, "%13.7f", static_cast<double>(A[lda * j + i]));
+ snprintf(bb, 32, "%13.7f", static_cast<double>(B[ldb * j + i]));
+ for (int k = 0; ba[k] && bb[k]; ++k) {
+ if (ba[k] != bb[k])
+ fputs_unlocked("\33[31m", f);
+ fputc_unlocked(ba[k], f);
+ if (ba[k] != bb[k])
+ fputs_unlocked("\33[0m", f);
+ }
+ }
+ fprintf(f, "\n");
+ }
+ funlockfile(f);
+}
+
+template <typename T>
+void misfit(FILE *f, int max, int m, int n, const T *A, int lda, const T *B, int ldb,
+ const char *file, int line, double avg, double sad, double worst, int infs, int flips,
+ int zeroes, int denormals, int nans) {
+ fprintf(f, "%s:%d: matrix errors exceed tolerance %s\n", file, line,
+ is_self_testing ? is_self_testing : "");
+ fprintf(f,
+ " %g ulp (worst=%g sad=%g nans=%d infs=%d flips=%d zeroes=%d denormals=%d)\n",
+ avg, worst, sad, nans, infs, flips, zeroes, denormals);
+ fprintf(f, "want\n");
+ show(f, max, m, n, A, lda, B, ldb);
+ fprintf(f, "got\n");
+ show(f, max, m, n, B, ldb, A, lda);
+ fprintf(f, "\n");
+}
-#define CHECK(tol, m, n, A, lda, B, ldb) check(tol, m, n, A, lda, B, ldb, __FILE__, __LINE__)
+template <typename T>
+void check(double tol, //
+ int m, int n, int k, //
+ const T *A, int lda, //
+ const T *B, int ldb, //
+ const char *file, int line) {
+ double avg, sad, worst;
+ int nans, infs, flips, zeroes, denormals;
+ assert(lda >= std::max(1, m));
+ assert(ldb >= std::max(1, m));
+ diff(m, n, k, A, lda, B, ldb, &avg, &sad, &worst, &infs, &flips, &zeroes, &denormals, &nans);
+ if (!nans && avg <= tol) {
+ if (!is_self_testing)
+ printf(" %g ulp (worst=%g sad=%g nans=%d infs=%d flips=%d zeroes=%d "
+ "denormals=%d)\n",
+ avg, worst, sad, nans, infs, flips, zeroes, denormals);
+ } else {
+ flockfile(stderr);
+ misfit(stderr, 16, m, n, A, lda, B, ldb, file, line, avg, sad, worst, infs, flips, zeroes,
+ denormals, nans);
+ const char *path = "/tmp/wompwomp.log";
+ FILE *f = fopen(path, "w");
+ if (f) {
+ misfit(f, 10000, m, n, A, lda, B, ldb, file, line, avg, sad, worst, infs, flips, zeroes,
+ denormals, nans);
+ fclose(f);
+ fprintf(stderr, "see also %s\n", path);
+ }
+ fflush(stderr);
+ (void)cudaDeviceReset();
+ _Exit(1);
+ }
+}
+
+#define CHECK(tol, m, n, k, A, lda, B, ldb) check(tol, m, n, k, A, lda, B, ldb, __FILE__, __LINE__)
-#define BENCH(x) \
+#define BENCH_CPU(x) \
do { \
x; \
__asm__ volatile("" ::: "memory"); \
@@ -86,9 +318,197 @@ inline int IsNegativeZero(float f) {
x; \
__asm__ volatile("" ::: "memory"); \
} \
- bench(start, #x); \
+ printf("%8lld µs %s\n", (micros() - start + ITERATIONS - 1) / ITERATIONS, #x); \
+ } while (0)
+
+#define CUDA_OR_DIE(x) \
+ do { \
+ cudaError_t err_ = (x); \
+ if (err_ != cudaSuccess) \
+ cuda_die(#x, __FUNCTION__, __FILE__, __LINE__, cudaGetErrorString(err_)); \
+ } while (0)
+
+#define CUBLAS_OR_DIE(x) \
+ do { \
+ cublasStatus_t status_ = (x); \
+ if (status_ != CUBLAS_STATUS_SUCCESS) \
+ cuda_die(#x, __FUNCTION__, __FILE__, __LINE__, cublasGetStatusString(status_)); \
+ } while (0)
+
+#define TINYBLAS_OR_DIE(x) \
+ do { \
+ tinyblasStatus_t status_ = (x); \
+ if (status_ != TINYBLAS_STATUS_SUCCESS) \
+ cuda_die(#x, __FUNCTION__, __FILE__, __LINE__, tinyblasGetStatusString(status_)); \
+ } while (0)
+
+#define BENCH_CUDA(x) \
+ do { \
+ if (is_self_testing) { \
+ x; \
+ } else { \
+ cudaEvent_t start_, stop_; \
+ CUDA_OR_DIE(cudaEventCreate(&stop_)); \
+ CUDA_OR_DIE(cudaEventCreate(&start_)); \
+ CUDA_OR_DIE(cudaEventRecord(start_, stream)); \
+ for (int i_ = 0; i_ < ITERATIONS; ++i_) \
+ x; \
+ float msecTotal_ = 0; \
+ CUDA_OR_DIE(cudaEventRecord(stop_, stream)); \
+ CUDA_OR_DIE(cudaEventSynchronize(stop_)); \
+ CUDA_OR_DIE(cudaEventElapsedTime(&msecTotal_, start_, stop_)); \
+ printf("%8d µs %s(%d, %d, %d)\n", \
+ static_cast<int>(std::ceil(msecTotal_ / ITERATIONS * 1000)), __FUNCTION__, m, \
+ n, k); \
+ CUDA_OR_DIE(cudaEventDestroy(start_)); \
+ CUDA_OR_DIE(cudaEventDestroy(stop_)); \
+ } \
} while (0)
#define RUN(x) \
- run(#x); \
+ printf(" \r\n%s\n", #x); \
x
+
+[[noreturn]] void cuda_die(const char *, const char *, const char *, int, const char *);
+
+template <typename T> struct cuda_data_type;
+template <> struct cuda_data_type<half> {
+ static constexpr cudaDataType_t id = CUDA_R_16F;
+};
+template <> struct cuda_data_type<float> {
+ static constexpr cudaDataType_t id = CUDA_R_32F;
+};
+
+template <typename T> struct cublas_compute_type;
+template <> struct cublas_compute_type<half> {
+ static constexpr cublasComputeType_t id = CUBLAS_COMPUTE_16F;
+};
+template <> struct cublas_compute_type<float> {
+ static constexpr cublasComputeType_t id = CUBLAS_COMPUTE_32F;
+};
+
+template <typename T> struct tinyblas_data_type;
+template <> struct tinyblas_data_type<half> {
+ static constexpr tinyblasDataType_t id = TINYBLAS_R_16F;
+};
+template <> struct tinyblas_data_type<float> {
+ static constexpr tinyblasDataType_t id = TINYBLAS_R_32F;
+};
+
+template <typename T> struct tinyblas_compute_type;
+template <> struct tinyblas_compute_type<half> {
+ static constexpr tinyblasComputeType_t id = TINYBLAS_COMPUTE_16F;
+};
+template <> struct tinyblas_compute_type<float> {
+ static constexpr tinyblasComputeType_t id = TINYBLAS_COMPUTE_32F;
+};
+
+template <typename T, typename TA, typename TB, typename TC>
+void cublas(bool aᵀ, bool bᵀ, //
+ int m, int n, int k, T α, //
+ const TA *A, int lda, //
+ const TB *B, int ldb, T β, //
+ TC *C, int ldc) {
+ cudaStream_t stream;
+ cublasHandle_t blas;
+ assert(m >= 0 && n >= 0 && k >= 0);
+ assert(lda >= std::max(1, aᵀ ? k : m));
+ assert(ldb >= std::max(1, bᵀ ? n : k));
+ assert(ldc >= std::max(1, m));
+ CUDA_OR_DIE(cudaSetDevice(0));
+ CUBLAS_OR_DIE(cublasCreate(&blas));
+ CUDA_OR_DIE(cudaStreamCreate(&stream));
+ CUBLAS_OR_DIE(cublasSetMathMode(blas, CUBLAS_DEFAULT_MATH));
+ CUBLAS_OR_DIE(cublasSetStream(blas, stream));
+ BENCH_CUDA(CUBLAS_OR_DIE(cublasGemmEx(
+ blas, aᵀ ? CUBLAS_OP_T : CUBLAS_OP_N, bᵀ ? CUBLAS_OP_T : CUBLAS_OP_N, m, n, k, &α, A,
+ cuda_data_type<TA>::id, lda, B, cuda_data_type<TB>::id, ldb, &β, C, cuda_data_type<TC>::id,
+ ldc, cublas_compute_type<T>::id, CUBLAS_GEMM_DEFAULT)));
+ CUDA_OR_DIE(cudaStreamSynchronize(stream));
+ CUDA_OR_DIE(cudaStreamDestroy(stream));
+ CUBLAS_OR_DIE(cublasDestroy(blas));
+}
+
+template <typename T, typename TA, typename TB, typename TC>
+void cublasrm(bool aᵀ, bool bᵀ, //
+ int m, int n, int k, T α, //
+ const TA *A, int lda, //
+ const TB *B, int ldb, T β, //
+ TC *C, int ldc) {
+ cublas(bᵀ, aᵀ, n, m, k, α, B, ldb, A, lda, β, C, ldc);
+}
+
+template <typename T, typename TA, typename TB, typename TC>
+void tinyblas(bool aᵀ, bool bᵀ, //
+ int m, int n, int k, T α, //
+ const TA *A, int lda, //
+ const TB *B, int ldb, T β, //
+ TC *C, int ldc) {
+ cudaStream_t stream;
+ tinyblasHandle_t blas;
+ assert(m >= 0 && n >= 0 && k >= 0);
+ assert(lda >= std::max(1, aᵀ ? k : m));
+ assert(ldb >= std::max(1, bᵀ ? n : k));
+ assert(ldc >= std::max(1, m));
+ CUDA_OR_DIE(cudaSetDevice(0));
+ TINYBLAS_OR_DIE(tinyblasCreate(&blas));
+ CUDA_OR_DIE(cudaStreamCreate(&stream));
+ TINYBLAS_OR_DIE(tinyblasSetStream(blas, stream));
+ BENCH_CUDA(TINYBLAS_OR_DIE(tinyblasGemmEx(
+ blas, aᵀ ? TINYBLAS_OP_T : TINYBLAS_OP_N, bᵀ ? TINYBLAS_OP_T : TINYBLAS_OP_N, m, n, k, &α,
+ A, tinyblas_data_type<TA>::id, lda, B, tinyblas_data_type<TB>::id, ldb, &β, C,
+ tinyblas_data_type<TC>::id, ldc, tinyblas_compute_type<T>::id, TINYBLAS_GEMM_DEFAULT)));
+ CUDA_OR_DIE(cudaStreamSynchronize(stream));
+ CUDA_OR_DIE(cudaStreamDestroy(stream));
+ TINYBLAS_OR_DIE(tinyblasDestroy(blas));
+}
+
+template <typename T, typename TA, typename TB, typename TC>
+void tinyblasrm(bool aᵀ, bool bᵀ, //
+ int m, int n, int k, T α, //
+ const TA *A, int lda, //
+ const TB *B, int ldb, T β, //
+ TC *C, int ldc) {
+ tinyblas(bᵀ, aᵀ, n, m, k, α, B, ldb, A, lda, β, C, ldc);
+}
+
+// multiplies matrix on gpu with column major ordering
+//
+// m×k * k×n → m×n
+// k×m * k×n → m×n if aᵀ
+// m×k * n×k → m×n if bᵀ
+// k×m * n×k → m×n if aᵀ and bᵀ
+//
+template <typename T, typename TA, typename TB, typename TC>
+void gemmref(bool aᵀ, bool bᵀ, //
+ int m, int n, int k, T α, //
+ const TA *A, int lda, //
+ const TB *B, int ldb, T β, //
+ TC *C, int ldc) {
+ cudaStream_t stream;
+ assert(m >= 0 && n >= 0 && k >= 0);
+ assert(lda >= std::max(1, aᵀ ? k : m));
+ assert(ldb >= std::max(1, bᵀ ? n : k));
+ assert(ldc >= std::max(1, m));
+ CUDA_OR_DIE(cudaSetDevice(0));
+ CUDA_OR_DIE(cudaStreamCreate(&stream));
+ BENCH_CUDA(CUDA_OR_DIE(naive::gemm(stream, aᵀ, bᵀ, m, n, k, α, A, lda, B, ldb, β, C, ldc)));
+ CUDA_OR_DIE(cudaStreamSynchronize(stream));
+ CUDA_OR_DIE(cudaStreamDestroy(stream));
+}
+
+// multiplies matrix on gpu with row major ordering
+//
+// k×m * n×k → n×m
+// m×k * n×k → n×m if aᵀ
+// k×m * k×n → n×m if bᵀ
+// m×k * k×n → n×m if aᵀ and bᵀ
+//
+template <typename T, typename TA, typename TB, typename TC>
+void gemmrefrm(bool aᵀ, bool bᵀ, //
+ int m, int n, int k, T α, //
+ const TA *A, int lda, //
+ const TB *B, int ldb, T β, //
+ TC *C, int ldc) {
+ gemmref(bᵀ, aᵀ, n, m, k, α, B, ldb, A, lda, β, C, ldc);
+}
diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cu
index 970291ab..0a3c6f36 100644
--- a/llamafile/tinyblas.cu
+++ b/llamafile/tinyblas.cu
@@ -21,14 +21,16 @@
#include <cstdlib>
#include <type_traits>
-#ifdef __NVCC__
+#if defined(__NVCC__) || defined(__CUDA_ARCH__)
#include <cuda_fp16.h>
#include <cuda_runtime.h>
#define tinyblasStream_t cudaStream_t
#else
#include <hip/hip_fp16.h>
#include <hip/hip_runtime.h>
+#define cudaSuccess hipSuccess
#define tinyblasStream_t hipStream_t
+#define cudaGetLastError hipGetLastError
#endif
//
@@ -41,7 +43,8 @@
// BASIC LINEAR ALGEBRA SUBPROGRAMS
//
-#define SHARELEN (BK * BM + BK * BN)
+#define THREAD_COUNT ((BM * BN) / (TM * TN))
+#define KERNEL __launch_bounds__(THREAD_COUNT)
#define CEIL_DIV(M, N) (((M) + (N) - 1) / (N))
struct tinyblasContext {
@@ -73,99 +76,6 @@ tinyblasStatus_t tinyblasGetStream(tinyblasHandle_t handle, void **out_stream) {
return TINYBLAS_STATUS_SUCCESS;
}
-template <int BM, int BN, int BK, int TM, int TN, typename WORD, typename SRC, typename DST>
-static __device__ void matmul_block2d(tinyblasOperation_t transa, tinyblasOperation_t transb, int m,
- int n, int k, int x, int y, WORD alpha, const SRC *A, int lda,
- const SRC *B, int ldb, WORD beta, DST *C, int ldc) {
- static_assert((BM % TM == 0) && (BN % TN == 0), "can't divide work for threads");
- static_assert((BM * BN) <= (BM * BK) + (BK * BN),
- "didn't allocate enough shared mem for threads");
- const int ii0 = threadIdx.x / (BN / TN); // {0, ..., (BM/TM) - 1}
- const int ii1 = threadIdx.x % (BN / TN); // {0, ..., (BN/TN) - 1}
- extern __shared__ char svals[]; // shared across all threads in a block
- SRC *As /*[BK][BM]*/ = (SRC *)svals;
- SRC *Bs /*[BK][BN]*/ = (SRC *)svals + BM * BK;
-
- WORD Cs[TM][TN] = {0};
- SRC At[TM], Bt[TN];
- int i, h, j, l, blob;
-
- for (blob = 0; blob < k; blob += BK) {
- for (i = threadIdx.x; i < BK; i += blockDim.x)
- for (j = 0; j < BM; ++j)
- As[BM * i + j] = 0;
- for (i = threadIdx.x; i < BK && blob + i < k; i += blockDim.x)
- for (j = 0; j < BM && x + j < m; ++j)
- As[BM * i + j] = transa ? A[1ll * lda * (x + j) + (blob + i)]
- : A[1ll * lda * (blob + i) + (x + j)];
- for (i = threadIdx.x; i < BK; i += blockDim.x)
- for (j = 0; j < BN; ++j)
- Bs[BN * i + j] = 0;
- for (i = threadIdx.x; i < BK && blob + i < k; i += blockDim.x)
- for (j = 0; j < BN && y + j < n; ++j)
- Bs[BN * i + j] = transb ? B[1ll * ldb * (blob + i) + (y + j)]
- : B[1ll * ldb * (y + j) + (blob + i)];
- __syncthreads();
-
- for (l = 0; l < BK; ++l) {
- for (j = 0; j < TM; ++j)
- At[j] = As[BM * l + TM * ii0 + j];
- for (h = 0; h < TN; ++h)
- Bt[h] = Bs[BN * l + TN * ii1 + h];
- for (j = 0; j < TM; ++j) {
- WORD a = At[j];
- for (h = 0; h < TN; ++h) {
- WORD b = Bt[h];
- Cs[j][h] += a * b;
- }
- }
- }
- __syncthreads();
- }
-
- x += ii0 * TM;
- y += ii1 * TN;
- for (l = 0; l < TN && y + l < n; ++l)
- for (j = 0; j < TM && x + j < m; ++j)
- if (beta) {
- WORD c = C[1ll * ldc * (y + l) + (x + j)];
- C[1ll * ldc * (y + l) + (x + j)] = alpha * Cs[j][l] + beta * c;
- } else {
- C[1ll * ldc * (y + l) + (x + j)] = alpha * Cs[j][l];
- }
-}
-
-template <int BM, int BN, int BK, int TM, int TN>
-static __global__ void tinyblasS_entry(tinyblasOperation_t transa, tinyblasOperation_t transb,
- int m, int n, int k, float alpha, const float *A, int lda,
- const float *B, int ldb, float beta, float *C, int ldc) {
-
- int x = blockIdx.x * BM;
- const int jump1 = gridDim.x * BM;
-
- int y = blockIdx.y * BN;
- const int jump2 = gridDim.y * BN;
-
- // each block handles a sub-matrix of C, of size BM * BN
- // each thread handles a sub-matrix of size TM * TN
- for (x = blockIdx.x * BM; x < m; x += jump1)
- for (y = blockIdx.y * BN; y < n; y += jump2)
- matmul_block2d<BM, BN, BK, TM, TN, float>(transa, transb, m, n, k, x, y, alpha, A, lda,
- B, ldb, beta, C, ldc);
-}
-
-template <int BM, int BN, int BK, int TM, int TN>
-static void tinyblasS_wrapper(tinyblasHandle_t handle, tinyblasOperation_t transa,
- tinyblasOperation_t transb, int m, int n, int k, const float alpha,
- const float *A, int lda, const float *B, int ldb, const float beta,
- float *C, int ldc) {
- dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 1);
- int sharedmem = sizeof(float) * SHARELEN;
- int maxthreads = (BM * BN) / (TM * TN);
- tinyblasS_entry<BM, BN, BK, TM, TN><<<maxblocks, maxthreads, sharedmem, handle->stream>>>(
- transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc);
-}
-
tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t handle, //
tinyblasOperation_t transa, //
tinyblasOperation_t transb, //
@@ -175,175 +85,204 @@ tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t handle, //
const float *B, int ldb, //
const float *beta, //
float *C, int ldc) {
- tinyblasS_wrapper<32, 32, 16, 2, 8>(handle, transa, transb, m, n, k, *alpha, A, lda, B, ldb,
- *beta, C, ldc);
- return TINYBLAS_STATUS_SUCCESS;
+ return tinyblasGemmEx(handle, transa, transb, m, n, k, alpha, A, TINYBLAS_R_32F, lda, B,
+ TINYBLAS_R_32F, ldb, beta, C, TINYBLAS_R_32F, ldc, TINYBLAS_COMPUTE_32F,
+ TINYBLAS_GEMM_DEFAULT);
}
-static bool is_valid(tinyblasOperation_t transa, //
- tinyblasOperation_t transb, //
- int m, int n, int k, //
- int lda, int ldb, int ldc) {
- return m >= 0 && //
- n >= 0 && //
- k >= 0 && //
- lda >= std::max(1, transa ? k : m) && //
- ldb >= std::max(1, transb ? n : k) && //
- ldc >= std::max(1, m);
+const char *tinyblasGetStatusString(tinyblasStatus_t err) {
+ switch (err) {
+ case TINYBLAS_STATUS_SUCCESS:
+ return "Success";
+ case TINYBLAS_STATUS_ALLOC_FAILED:
+ return "Alloc failed";
+ case TINYBLAS_STATUS_INVALID_VALUE:
+ return "Invalid value";
+ case TINYBLAS_STATUS_NOT_SUPPORTED:
+ return "Not supported";
+ case TINYBLAS_STATUS_EXECUTION_FAILED:
+ return "Execution failed";
+ case TINYBLAS_STATUS_DIMENSION_OVERFLOW:
+ return "Dimension overflow";
+ default:
+ return "Unknown error";
+ }
}
-static bool is_supported(tinyblasDatatype_t Atype, tinyblasDatatype_t Btype) {
- return Atype == TINYBLAS_R_16F && //
- Btype == TINYBLAS_R_16F;
+__device__ __forceinline__ float madd(float x, float y, float z) {
+ return __fmaf_rn(x, y, z);
}
-// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmex
-template <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST, typename T>
-static __global__ void tinyblasGE_entry(tinyblasOperation_t transa, tinyblasOperation_t transb,
- int m, int n, int k, T alpha, const SRC *A, int lda,
- const SRC *B, int ldb, T beta, DST *C, int ldc) {
+template <int BM, int BN, int TM, int TN, typename WORD, typename SRC, typename DST>
+static __device__ void matmul_block2d(tinyblasOperation_t transa, tinyblasOperation_t transb, int m,
+ int n, int k, WORD alpha, const SRC *A, int lda, const SRC *B,
+ int ldb, WORD beta, DST *C, int ldc) {
+
+ constexpr int BK = THREAD_COUNT;
+ static_assert(BM % TM == 0, "can't divide work for threads");
+ static_assert(BN % TN == 0, "can't divide work for threads");
+ static_assert(BM > 0 && BN > 0 && BK > 0 && TM > 0 && TN > 0,
+ "one of the constexpr configuration values was non-positive");
+ static_assert((BK * BM * sizeof(SRC)) + (BK * BN * sizeof(SRC)) <= 65536,
+ "you're almost almost certainly using too much shared memory");
+
+ const int th = threadIdx.x;
+ const int ii = blockIdx.x * BM;
+ const int jj = blockIdx.y * BN;
+ const int ti = th / (BN / TN) * TM;
+ const int tj = th % (BN / TN) * TN;
+
+ __shared__ SRC As[BK * BM];
+ __shared__ SRC Bs[BK * BN];
+
+ WORD At[TM];
+ WORD Bt[TN];
+ WORD Cs[TM * TN] = {0};
+
+ for (int ll = 0; ll < k; ll += BK) {
+
+ for (int i = 0; i < BM; ++i)
+ As[BM * th + i] = 0;
+ for (int i = 0; i < BM && ll + th < k && ii + i < m; ++i)
+ As[BM * th + i] = A[transa ? lda * (ii + i) + (ll + th) : lda * (ll + th) + (ii + i)];
+
+ for (int j = 0; j < BN; ++j)
+ Bs[BN * th + j] = 0;
+ for (int j = 0; j < BN && ll + th < k && jj + j < n; ++j)
+ Bs[BN * th + j] = B[transb ? ldb * (ll + th) + (jj + j) : ldb * (jj + j) + (ll + th)];
- int x = blockIdx.x * BM;
- const int jump1 = gridDim.x * BM;
+ __syncthreads();
- int y = blockIdx.y * BN;
- const int jump2 = gridDim.y * BN;
+ for (int l = 0; l < BK; ++l) {
+ for (int j = 0; j < TM; ++j)
+ At[j] = As[BM * l + ti + j];
+ for (int h = 0; h < TN; ++h)
+ Bt[h] = Bs[BN * l + tj + h];
+ for (int j = 0; j < TM; ++j) {
+ WORD a = At[j];
+ for (int h = 0; h < TN; ++h) {
+ WORD b = Bt[h];
+ Cs[TN * j + h] = madd(a, b, Cs[TN * j + h]);
+ }
+ }
+ }
- // each block handles a sub-matrix of C, of size BM * BN
- for (x = blockIdx.x * BM; x < m; x += jump1)
- for (y = blockIdx.y * BN; y < n; y += jump2)
- matmul_block2d<BM, BN, BK, TM, TN, float>(transa, transb, m, n, k, x, y, alpha, A, lda,
- B, ldb, beta, C, ldc);
+ __syncthreads();
+ }
+
+ if (alpha != (WORD)1)
+ for (int i = 0; i < TM * TN; ++i)
+ Cs[i] *= alpha;
+
+ for (int j = 0; j < TN && jj + tj + j < n; ++j)
+ for (int i = 0; i < TM && ii + ti + i < m; ++i)
+ if (beta) {
+ WORD c = C[ldc * (jj + tj + j) + (ii + ti + i)];
+ C[ldc * (jj + tj + j) + (ii + ti + i)] = madd(c, beta, Cs[TN * i + j]);
+ } else {
+ C[ldc * (jj + tj + j) + (ii + ti + i)] = Cs[TN * i + j];
+ }
}
-template <int BM, int BN, int BK, int TM, int TN, typename SRC>
-static tinyblasStatus_t
-tinyblasGE_wrapper(tinyblasHandle_t handle, tinyblasOperation_t transa, tinyblasOperation_t transb,
- int m, int n, int k, const void *pAlpha, const SRC *A, int lda, const SRC *B,
- int ldb, const void *pBeta, void *C, tinyblasDatatype_t Ctype, int ldc,
- tinyblasComputeType_t computeType) {
- dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 1);
- int sharedmem = sizeof(SRC) * SHARELEN;
- int maxthreads = (BM * BN) / (TM * TN);
- switch (Ctype) {
- case TINYBLAS_R_16F:
- switch (computeType) {
- case TINYBLAS_COMPUTE_16F:
- tinyblasGE_entry<BM, BN, BK, TM, TN>
- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(
- transa, transb, m, n, k, *(const half *)pAlpha, A, lda, B, ldb,
- *(const half *)pBeta, (half *)C, ldc);
- return TINYBLAS_STATUS_SUCCESS;
- case TINYBLAS_COMPUTE_32F:
- tinyblasGE_entry<BM, BN, BK, TM, TN>
- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(
- transa, transb, m, n, k, *(const float *)pAlpha, A, lda, B, ldb,
- *(const float *)pBeta, (half *)C, ldc);
- return TINYBLAS_STATUS_SUCCESS;
- default:
- return TINYBLAS_STATUS_INVALID_VALUE;
- }
- case TINYBLAS_R_32F:
- switch (computeType) {
- case TINYBLAS_COMPUTE_16F:
- return TINYBLAS_STATUS_NOT_SUPPORTED;
- case TINYBLAS_COMPUTE_32F:
- tinyblasGE_entry<BM, BN, BK, TM, TN>
- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(
- transa, transb, m, n, k, *(const float *)pAlpha, A, lda, B, ldb,
- *(const float *)pBeta, (float *)C, ldc);
- return TINYBLAS_STATUS_SUCCESS;
- default:
- return TINYBLAS_STATUS_INVALID_VALUE;
- }
- default:
- return TINYBLAS_STATUS_INVALID_VALUE;
- }
+template <int BM, int BN, int TM, int TN, typename WORD, typename SRC, typename DST>
+static __global__ void KERNEL tinyblasGE_entry(tinyblasOperation_t transa,
+ tinyblasOperation_t transb, int m, int n, int k,
+ WORD alpha, const SRC *A, int lda, const SRC *B,
+ int ldb, WORD beta, DST *C, int ldc) {
+ matmul_block2d<BM, BN, TM, TN>(transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc);
+}
+
+template <typename WORD, typename SRC, typename DST>
+static tinyblasStatus_t tinyblasGE_launch(tinyblasHandle_t handle, tinyblasOperation_t transa,
+ tinyblasOperation_t transb, int m, int n, int k,
+ WORD alpha, const SRC *A, int lda, const SRC *B, int ldb,
+ WORD beta, DST *C, int ldc) {
+ constexpr int BM = 48;
+ constexpr int BN = 32;
+ constexpr int TM = 3;
+ constexpr int TN = 8;
+ dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN));
+ tinyblasGE_entry<BM, BN, TM, TN><<<maxblocks, THREAD_COUNT, 0, handle->stream>>>(
+ transa, transb, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc);
+ if (cudaGetLastError() != cudaSuccess)
+ return TINYBLAS_STATUS_EXECUTION_FAILED;
+ return TINYBLAS_STATUS_SUCCESS;
}
tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t handle, //
tinyblasOperation_t transa, //
tinyblasOperation_t transb, //
int m, int n, int k, //
- const void *pAlpha, //
- const void *A, tinyblasDatatype_t Atype, int lda, //
- const void *B, tinyblasDatatype_t Btype, int ldb, //
- const void *pBeta, //
- void *C, tinyblasDatatype_t Ctype, int ldc, //
+ const void *alpha, //
+ const void *A, tinyblasDataType_t Atype, int lda, //
+ const void *B, tinyblasDataType_t Btype, int ldb, //
+ const void *beta, //
+ void *C, tinyblasDataType_t Ctype, int ldc, //
tinyblasComputeType_t computeType, //
tinyblasGemmAlgo_t algo) {
- if (!is_valid(transa, transb, m, n, k, lda, ldb, ldc))
+
+ if (m < 0 || n < 0 || k < 0)
return TINYBLAS_STATUS_INVALID_VALUE;
- if (!is_supported(Atype, Btype))
+ if (lda < std::max(1, transa ? k : m))
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ if (ldb < std::max(1, transb ? n : k))
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ if (ldc < std::max(1, m))
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ if (1ll * lda * ((transa ? k : m) - 1) + ((transa ? m : k) - 1) > INT_MAX)
+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;
+ if (1ll * ldb * ((transb ? n : k) - 1) + ((transb ? k : n) - 1) > INT_MAX)
+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;
+ if (1ll * ldc * (n - 1) + (m - 1) > INT_MAX)
+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;
+ if (Atype != Btype)
return TINYBLAS_STATUS_NOT_SUPPORTED;
- return tinyblasGE_wrapper<48, 32, 64, 3, 8>(handle, transa, transb, m, n, k, pAlpha,
- (const half *)A, lda, (const half *)B, ldb, pBeta,
- C, Ctype, ldc, computeType);
-}
-// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmbatchedex
-template <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST, typename T>
-static __global__ void tinyblasGBE_entry(tinyblasOperation_t transa, tinyblasOperation_t transb,
- int m, int n, int k, T alpha, const SRC *const Aarray[],
- int lda, const SRC *const Barray[], int ldb, T beta,
- DST *const Carray[], int ldc, int batchCount) {
-
- int x = blockIdx.x * BM;
- const int jump1 = gridDim.x * BM;
-
- int y = blockIdx.y * BN;
- const int jump2 = gridDim.y * BN;
-
- int z = blockIdx.z;
- const int jump3 = gridDim.z;
-
- // each block handles a sub-matrix of C, of size BM * BN
- for (z = blockIdx.z; z < batchCount; z += jump3)
- for (x = blockIdx.x * BM; x < m; x += jump1)
- for (y = blockIdx.y * BN; y < n; y += jump2)
- matmul_block2d<BM, BN, BK, TM, TN, float>(transa, transb, m, n, k, x, y, alpha,
- Aarray[z], lda, Barray[z], ldb, beta,
- Carray[z], ldc);
-}
-
-template <int BM, int BN, int BK, int TM, int TN, typename SRC>
-static tinyblasStatus_t tinyblasGBE_wrapper(tinyblasHandle_t handle, tinyblasOperation_t transa,
- tinyblasOperation_t transb, int m, int n, int k,
- const void *pAlpha, const SRC *const Aarray[], int lda,
- const SRC *const Barray[], int ldb, const void *pBeta,
- void *const Carray[], tinyblasDatatype_t Ctype, int ldc,
- int batchCount, tinyblasComputeType_t computeType) {
- dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 32);
- int sharedmem = sizeof(SRC) * SHARELEN;
- int maxthreads = (BM * BN) / (TM * TN);
- switch (Ctype) {
+ switch (Atype) {
case TINYBLAS_R_16F:
- switch (computeType) {
- case TINYBLAS_COMPUTE_16F:
- tinyblasGBE_entry<BM, BN, BK, TM, TN>
- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(
- transa, transb, m, n, k, *(const half *)pAlpha, Aarray, lda, Barray, ldb,
- *(const half *)pBeta, (half *const *)Carray, ldc, batchCount);
- return TINYBLAS_STATUS_SUCCESS;
- case TINYBLAS_COMPUTE_32F:
- tinyblasGBE_entry<BM, BN, BK, TM, TN>
- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(
- transa, transb, m, n, k, *(const float *)pAlpha, Aarray, lda, Barray, ldb,
- *(const float *)pBeta, (half *const *)Carray, ldc, batchCount);
- return TINYBLAS_STATUS_SUCCESS;
+ switch (Ctype) {
+ case TINYBLAS_R_16F:
+ switch (computeType) {
+ case TINYBLAS_COMPUTE_16F:
+ return tinyblasGE_launch(
+ handle, transa, transb, m, n, k, (float)*(const half *)alpha, (const half *)A,
+ lda, (const half *)B, ldb, (float)*(const half *)beta, (half *)C, ldc);
+ case TINYBLAS_COMPUTE_32F:
+ return tinyblasGE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,
+ (const half *)A, lda, (const half *)B, ldb,
+ *(const float *)beta, (half *)C, ldc);
+ default:
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ }
+ case TINYBLAS_R_32F:
+ switch (computeType) {
+ case TINYBLAS_COMPUTE_16F:
+ return TINYBLAS_STATUS_NOT_SUPPORTED;
+ case TINYBLAS_COMPUTE_32F:
+ return tinyblasGE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,
+ (const half *)A, lda, (const half *)B, ldb,
+ *(const float *)beta, (float *)C, ldc);
+ default:
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ }
default:
return TINYBLAS_STATUS_INVALID_VALUE;
}
case TINYBLAS_R_32F:
- switch (computeType) {
- case TINYBLAS_COMPUTE_16F:
+ switch (Ctype) {
+ case TINYBLAS_R_16F:
return TINYBLAS_STATUS_NOT_SUPPORTED;
- case TINYBLAS_COMPUTE_32F:
- tinyblasGBE_entry<BM, BN, BK, TM, TN>
- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(
- transa, transb, m, n, k, *(const float *)pAlpha, Aarray, lda, Barray, ldb,
- *(const float *)pBeta, (float *const *)Carray, ldc, batchCount);
- return TINYBLAS_STATUS_SUCCESS;
+ case TINYBLAS_R_32F:
+ switch (computeType) {
+ case TINYBLAS_COMPUTE_16F:
+ return TINYBLAS_STATUS_NOT_SUPPORTED;
+ case TINYBLAS_COMPUTE_32F:
+ return tinyblasGE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,
+ (const float *)A, lda, (const float *)B, ldb,
+ *(const float *)beta, (float *)C, ldc);
+ default:
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ }
default:
return TINYBLAS_STATUS_INVALID_VALUE;
}
@@ -352,96 +291,117 @@ static tinyblasStatus_t tinyblasGBE_wrapper(tinyblasHandle_t handle, tinyblasOpe
}
}
+template <int BM, int BN, int TM, int TN, typename WORD, typename SRC, typename DST>
+static __global__ void KERNEL tinyblasGBE_entry(tinyblasOperation_t transa,
+ tinyblasOperation_t transb, int m, int n, int k,
+ WORD alpha, const SRC *const Aarray[], int lda,
+ const SRC *const Barray[], int ldb, WORD beta,
+ DST *const Carray[], int ldc, int batchCount) {
+ for (int z = blockIdx.z; z < batchCount; z += gridDim.z)
+ matmul_block2d<BM, BN, TM, TN>(transa, transb, m, n, k, alpha, Aarray[z], lda, Barray[z],
+ ldb, beta, Carray[z], ldc);
+}
+
+template <typename WORD, typename SRC, typename DST>
+static tinyblasStatus_t tinyblasGBE_launch(tinyblasHandle_t handle, tinyblasOperation_t transa,
+ tinyblasOperation_t transb, int m, int n, int k,
+ WORD alpha, const SRC *const *Aarray, int lda,
+ const SRC *const *Barray, int ldb, WORD beta,
+ DST *const *Carray, int ldc, int batchCount) {
+ constexpr int BC = 32;
+ constexpr int BM = 32;
+ constexpr int BN = 32;
+ constexpr int TM = 2;
+ constexpr int TN = 8;
+ dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), BC);
+ tinyblasGBE_entry<BM, BN, TM, TN><<<maxblocks, THREAD_COUNT, 0, handle->stream>>>(
+ transa, transb, m, n, k, alpha, Aarray, lda, Barray, ldb, beta, Carray, ldc, batchCount);
+ if (cudaGetLastError() != cudaSuccess)
+ return TINYBLAS_STATUS_EXECUTION_FAILED;
+ return TINYBLAS_STATUS_SUCCESS;
+}
+
tinyblasStatus_t tinyblasGemmBatchedEx(tinyblasHandle_t handle, //
tinyblasOperation_t transa, //
tinyblasOperation_t transb, //
int m, int n, int k, //
- const void *pAlpha, //
- const void *const Aarray[], tinyblasDatatype_t Atype,
+ const void *alpha, //
+ const void *const Aarray[], tinyblasDataType_t Atype,
int lda, //
- const void *const Barray[], tinyblasDatatype_t Btype,
+ const void *const Barray[], tinyblasDataType_t Btype,
int ldb, //
- const void *pBeta, //
- void *const Carray[], tinyblasDatatype_t Ctype, int ldc, //
+ const void *beta, //
+ void *const Carray[], tinyblasDataType_t Ctype, int ldc, //
int batchCount, //
tinyblasComputeType_t computeType, //
tinyblasGemmAlgo_t algo) {
- if (!is_valid(transa, transb, m, n, k, lda, ldb, ldc))
+
+ if (m < 0 || n < 0 || k < 0)
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ if (lda < std::max(1, transa ? k : m))
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ if (ldb < std::max(1, transb ? n : k))
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ if (ldc < std::max(1, m))
return TINYBLAS_STATUS_INVALID_VALUE;
- if (!is_supported(Atype, Btype))
+ if (1ll * lda * ((transa ? k : m) - 1) + ((transa ? m : k) - 1) > INT_MAX)
+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;
+ if (1ll * ldb * ((transb ? n : k) - 1) + ((transb ? k : n) - 1) > INT_MAX)
+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;
+ if (1ll * ldc * (n - 1) + (m - 1) > INT_MAX)
+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;
+ if (Atype != Btype)
return TINYBLAS_STATUS_NOT_SUPPORTED;
- return tinyblasGBE_wrapper<48, 32, 64, 3, 8>(handle, transa, transb, //
- m, n, k, pAlpha, //
- (const half *const *)Aarray, lda, //
- (const half *const *)Barray, ldb, //
- pBeta, Carray, Ctype, ldc, //
- batchCount, computeType);
-}
-
-// https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmstridedbatchedex
-template <int BM, int BN, int BK, int TM, int TN, typename SRC, typename DST, typename T>
-static __global__ void
-tinyblasGSBE_entry(tinyblasOperation_t transa, tinyblasOperation_t transb, int m, int n, int k,
- T alpha, const SRC *A, int lda, long long strideA, const SRC *B, int ldb,
- long long strideB, T beta, DST *C, int ldc, long long strideC, int batchCount) {
-
- int x = blockIdx.x * BM;
- const int jump1 = gridDim.x * BM;
-
- int y = blockIdx.y * BN;
- const int jump2 = gridDim.y * BN;
-
- int z = blockIdx.z;
- const int jump3 = gridDim.z;
-
- // each block handles a sub-matrix of C, of size BM * BN
- for (z = blockIdx.z; z < batchCount; z += jump3)
- for (x = blockIdx.x * BM; x < m; x += jump1)
- for (y = blockIdx.y * BN; y < n; y += jump2)
- matmul_block2d<BM, BN, BK, TM, TN, float>(transa, transb, m, n, k, x, y, alpha,
- A + z * strideA, lda, B + z * strideB,
- ldb, beta, C + z * strideC, ldc);
-}
-template <int BM, int BN, int BK, int TM, int TN, typename SRC>
-static tinyblasStatus_t tinyblasGSBE_wrapper(tinyblasHandle_t handle, tinyblasOperation_t transa,
- tinyblasOperation_t transb, int m, int n, int k,
- const void *pAlpha, const SRC *A, int lda,
- long long strideA, const SRC *B, int ldb,
- long long strideB, const void *pBeta, void *C,
- tinyblasDatatype_t Ctype, int ldc, long long strideC,
- int batchCount, tinyblasComputeType_t computeType) {
- dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), 32);
- int sharedmem = sizeof(SRC) * SHARELEN;
- int maxthreads = (BM * BN) / (TM * TN);
- switch (Ctype) {
+ switch (Atype) {
case TINYBLAS_R_16F:
- switch (computeType) {
- case TINYBLAS_COMPUTE_16F:
- tinyblasGSBE_entry<BM, BN, BK, TM, TN>
- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(
- transa, transb, m, n, k, *(const half *)pAlpha, A, lda, strideA, B, ldb,
- strideB, *(const half *)pBeta, (half *)C, ldc, strideC, batchCount);
- return TINYBLAS_STATUS_SUCCESS;
- case TINYBLAS_COMPUTE_32F:
- tinyblasGSBE_entry<BM, BN, BK, TM, TN>
- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(
- transa, transb, m, n, k, *(const float *)pAlpha, A, lda, strideA, B, ldb,
- strideB, *(const float *)pBeta, (half *)C, ldc, strideC, batchCount);
- return TINYBLAS_STATUS_SUCCESS;
+ switch (Ctype) {
+ case TINYBLAS_R_16F:
+ switch (computeType) {
+ case TINYBLAS_COMPUTE_16F:
+ return tinyblasGBE_launch(
+ handle, transa, transb, m, n, k, (float)*(const half *)alpha,
+ (const half *const *)Aarray, lda, (const half *const *)Barray, ldb,
+ (float)*(const half *)beta, (half *const *)Carray, ldc, batchCount);
+ case TINYBLAS_COMPUTE_32F:
+ return tinyblasGBE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,
+ (const half *const *)Aarray, lda,
+ (const half *const *)Barray, ldb, *(const float *)beta,
+ (half *const *)Carray, ldc, batchCount);
+ default:
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ }
+ case TINYBLAS_R_32F:
+ switch (computeType) {
+ case TINYBLAS_COMPUTE_16F:
+ return TINYBLAS_STATUS_NOT_SUPPORTED;
+ case TINYBLAS_COMPUTE_32F:
+ return tinyblasGBE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,
+ (const half *const *)Aarray, lda,
+ (const half *const *)Barray, ldb, *(const float *)beta,
+ (float *const *)Carray, ldc, batchCount);
+ default:
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ }
default:
return TINYBLAS_STATUS_INVALID_VALUE;
}
case TINYBLAS_R_32F:
- switch (computeType) {
- case TINYBLAS_COMPUTE_16F:
+ switch (Ctype) {
+ case TINYBLAS_R_16F:
return TINYBLAS_STATUS_NOT_SUPPORTED;
- case TINYBLAS_COMPUTE_32F:
- tinyblasGSBE_entry<BM, BN, BK, TM, TN>
- <<<maxblocks, maxthreads, sharedmem, handle->stream>>>(
- transa, transb, m, n, k, *(const float *)pAlpha, A, lda, strideA, B, ldb,
- strideB, *(const float *)pBeta, (float *)C, ldc, strideC, batchCount);
- return TINYBLAS_STATUS_SUCCESS;
+ case TINYBLAS_R_32F:
+ switch (computeType) {
+ case TINYBLAS_COMPUTE_16F:
+ return TINYBLAS_STATUS_NOT_SUPPORTED;
+ case TINYBLAS_COMPUTE_32F:
+ return tinyblasGBE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,
+ (const float *const *)Aarray, lda,
+ (const float *const *)Barray, ldb, *(const float *)beta,
+ (float *const *)Carray, ldc, batchCount);
+ default:
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ }
default:
return TINYBLAS_STATUS_INVALID_VALUE;
}
@@ -450,41 +410,124 @@ static tinyblasStatus_t tinyblasGSBE_wrapper(tinyblasHandle_t handle, tinyblasOp
}
}
+template <int BM, int BN, int TM, int TN, typename SRC, typename DST, typename WORD>
+static __global__ void KERNEL tinyblasGSBE_entry(tinyblasOperation_t transa,
+ tinyblasOperation_t transb, int m, int n, int k,
+ WORD alpha, const SRC *A, int lda,
+ long long strideA, const SRC *B, int ldb,
+ long long strideB, WORD beta, DST *C, int ldc,
+ long long strideC, int batchCount) {
+ for (int z = blockIdx.z; z < batchCount; z += gridDim.z)
+ matmul_block2d<BM, BN, TM, TN>(transa, transb, m, n, k, alpha, A + z * strideA, lda,
+ B + z * strideB, ldb, beta, C + z * strideC, ldc);
+}
+
+template <typename WORD, typename SRC, typename DST>
+static tinyblasStatus_t tinyblasGSBE_launch(tinyblasHandle_t handle, tinyblasOperation_t transa,
+ tinyblasOperation_t transb, int m, int n, int k,
+ WORD alpha, const SRC *A, int lda, long long strideA,
+ const SRC *B, int ldb, long long strideB, WORD beta,
+ DST *C, int ldc, long long strideC, int batchCount) {
+ constexpr int BC = 32;
+ constexpr int BM = 32;
+ constexpr int BN = 32;
+ constexpr int TM = 2;
+ constexpr int TN = 8;
+ dim3 maxblocks(CEIL_DIV(m, BM), CEIL_DIV(n, BN), BC);
+ tinyblasGSBE_entry<BM, BN, TM, TN><<<maxblocks, THREAD_COUNT, 0, handle->stream>>>(
+ transa, transb, m, n, k, alpha, A, lda, strideA, B, ldb, strideB, beta, C, ldc, strideC,
+ batchCount);
+ if (cudaGetLastError() != cudaSuccess)
+ return TINYBLAS_STATUS_EXECUTION_FAILED;
+ return TINYBLAS_STATUS_SUCCESS;
+}
+
tinyblasStatus_t tinyblasGemmStridedBatchedEx(tinyblasHandle_t handle, //
tinyblasOperation_t transa, //
tinyblasOperation_t transb, //
int m, int n, int k, //
- const void *pAlpha, //
- const void *A, tinyblasDatatype_t Atype, int lda,
+ const void *alpha, //
+ const void *A, tinyblasDataType_t Atype, int lda,
long long strideA, //
- const void *B, tinyblasDatatype_t Btype, int ldb,
+ const void *B, tinyblasDataType_t Btype, int ldb,
long long strideB, //
- const void *pBeta, //
- void *C, tinyblasDatatype_t Ctype, int ldc,
+ const void *beta, //
+ void *C, tinyblasDataType_t Ctype, int ldc,
long long strideC, //
int batchCount, //
tinyblasComputeType_t computeType, //
tinyblasGemmAlgo_t algo) {
- if (!is_valid(transa, transb, m, n, k, lda, ldb, ldc))
+
+ if (m < 0 || n < 0 || k < 0)
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ if (lda < std::max(1, transa ? k : m))
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ if (ldb < std::max(1, transb ? n : k))
return TINYBLAS_STATUS_INVALID_VALUE;
- if (!is_supported(Atype, Btype))
+ if (ldc < std::max(1, m))
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ if (1ll * lda * ((transa ? k : m) - 1) + ((transa ? m : k) - 1) > INT_MAX)
+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;
+ if (1ll * ldb * ((transb ? n : k) - 1) + ((transb ? k : n) - 1) > INT_MAX)
+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;
+ if (1ll * ldc * (n - 1) + (m - 1) > INT_MAX)
+ return TINYBLAS_STATUS_DIMENSION_OVERFLOW;
+ if (Atype != Btype)
return TINYBLAS_STATUS_NOT_SUPPORTED;
- return tinyblasGSBE_wrapper<32, 4, 64, 1, 2>(
- handle, transa, transb, m, n, k, pAlpha, (const half *)A, lda, strideA, (const half *)B,
- ldb, strideB, pBeta, C, Ctype, ldc, strideC, batchCount, computeType);
-}
-const char *tinyblasGetStatusString(tinyblasStatus_t err) {
- switch (err) {
- case TINYBLAS_STATUS_SUCCESS:
- return "TINYBLAS_STATUS_SUCCESS";
- case TINYBLAS_STATUS_ALLOC_FAILED:
- return "TINYBLAS_STATUS_ALLOC_FAILED";
- case TINYBLAS_STATUS_INVALID_VALUE:
- return "TINYBLAS_STATUS_INVALID_VALUE";
- case TINYBLAS_STATUS_NOT_SUPPORTED:
- return "TINYBLAS_STATUS_NOT_SUPPORTED";
+ switch (Atype) {
+ case TINYBLAS_R_16F:
+ switch (Ctype) {
+ case TINYBLAS_R_16F:
+ switch (computeType) {
+ case TINYBLAS_COMPUTE_16F:
+ return tinyblasGSBE_launch(
+ handle, transa, transb, m, n, k, (float)*(const half *)alpha, (const half *)A,
+ lda, strideA, (const half *)B, ldb, strideB, (float)*(const half *)beta,
+ (half *)C, ldc, strideC, batchCount);
+ case TINYBLAS_COMPUTE_32F:
+ return tinyblasGSBE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,
+ (const half *)A, lda, strideA, (const half *)B, ldb,
+ strideB, *(const float *)beta, (half *)C, ldc, strideC,
+ batchCount);
+ default:
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ }
+ case TINYBLAS_R_32F:
+ switch (computeType) {
+ case TINYBLAS_COMPUTE_16F:
+ return TINYBLAS_STATUS_NOT_SUPPORTED;
+ case TINYBLAS_COMPUTE_32F:
+ return tinyblasGSBE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,
+ (const half *)A, lda, strideA, (const half *)B, ldb,
+ strideB, *(const float *)beta, (float *)C, ldc, strideC,
+ batchCount);
+ default:
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ }
+ default:
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ }
+ case TINYBLAS_R_32F:
+ switch (Ctype) {
+ case TINYBLAS_R_16F:
+ return TINYBLAS_STATUS_NOT_SUPPORTED;
+ case TINYBLAS_R_32F:
+ switch (computeType) {
+ case TINYBLAS_COMPUTE_16F:
+ return TINYBLAS_STATUS_NOT_SUPPORTED;
+ case TINYBLAS_COMPUTE_32F:
+ return tinyblasGSBE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,
+ (const float *)A, lda, strideA, (const float *)B, ldb,
+ strideB, *(const float *)beta, (float *)C, ldc, strideC,
+ batchCount);
+ default:
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ }
+ default:
+ return TINYBLAS_STATUS_INVALID_VALUE;
+ }
default:
- return "unknown tinyblas error";
+ return TINYBLAS_STATUS_INVALID_VALUE;
}
}
diff --git a/llamafile/tinyblas.h b/llamafile/tinyblas.h
index 79dfad12..55de17ae 100644
--- a/llamafile/tinyblas.h
+++ b/llamafile/tinyblas.h
@@ -7,7 +7,7 @@ enum tinyblasOperation_t {
TINYBLAS_OP_T,
};
-enum tinyblasDatatype_t {
+enum tinyblasDataType_t {
TINYBLAS_R_32F,
TINYBLAS_R_16F,
};
@@ -26,6 +26,8 @@ enum tinyblasStatus_t {
TINYBLAS_STATUS_ALLOC_FAILED,
TINYBLAS_STATUS_INVALID_VALUE,
TINYBLAS_STATUS_NOT_SUPPORTED,
+ TINYBLAS_STATUS_EXECUTION_FAILED,
+ TINYBLAS_STATUS_DIMENSION_OVERFLOW,
};
struct tinyblasContext;
@@ -43,21 +45,21 @@ tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t, tinyblasOperation_t, tinyblasOp
const float *, float *, int);
tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t, tinyblasOperation_t, tinyblasOperation_t, int,
- int, int, const void *, const void *, tinyblasDatatype_t, int,
- const void *, tinyblasDatatype_t, int, const void *, void *,
- tinyblasDatatype_t, int, tinyblasComputeType_t, tinyblasGemmAlgo_t);
+ int, int, const void *, const void *, tinyblasDataType_t, int,
+ const void *, tinyblasDataType_t, int, const void *, void *,
+ tinyblasDataType_t, int, tinyblasComputeType_t, tinyblasGemmAlgo_t);
tinyblasStatus_t tinyblasGemmBatchedEx(tinyblasHandle_t, tinyblasOperation_t, tinyblasOperation_t,
int, int, int, const void *, const void *const[],
- tinyblasDatatype_t, int, const void *const[],
- tinyblasDatatype_t, int, const void *, void *const[],
- tinyblasDatatype_t, int, int, tinyblasComputeType_t,
+ tinyblasDataType_t, int, const void *const[],
+ tinyblasDataType_t, int, const void *, void *const[],
+ tinyblasDataType_t, int, int, tinyblasComputeType_t,
tinyblasGemmAlgo_t);
tinyblasStatus_t tinyblasGemmStridedBatchedEx(tinyblasHandle_t, tinyblasOperation_t,
tinyblasOperation_t, int, int, int, const void *,
- const void *, tinyblasDatatype_t, int, long long,
- const void *, tinyblasDatatype_t, int, long long,
- const void *, void *, tinyblasDatatype_t, int,
+ const void *, tinyblasDataType_t, int, long long,
+ const void *, tinyblasDataType_t, int, long long,
+ const void *, void *, tinyblasDataType_t, int,
long long, int, tinyblasComputeType_t,
tinyblasGemmAlgo_t);
diff --git a/llamafile/tinyblas_test.cu b/llamafile/tinyblas_test.cu
index 17875805..3dbcac0d 100644
--- a/llamafile/tinyblas_test.cu
+++ b/llamafile/tinyblas_test.cu
@@ -15,246 +15,100 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-#include "tinyblas.h"
-
-#include <atomic>
-#include <cstdio>
-#include <cstdlib>
-#include <ctime>
-
#include "tester.h"
-#define TOMBSTONE 1.666
-#define PRECISION_F16 0.2
-#define PRECISION_F32 9e-6
-
-#ifdef __NVCC__
-#include <cublas_v2.h>
-#include <cuda_runtime.h>
-#else
-#define HIPBLAS_V2
-#include <hip/hip_runtime.h>
-#include <hipblas/hipblas.h>
-#define cudaFree hipFree
-#define cudaSuccess hipSuccess
-#define cudaError_t hipError_t
-#define cudaEvent_t hipEvent_t
-#define cudaStream_t hipStream_t
-#define cublasSgemm hipblasSgemm
-#define cublasCreate hipblasCreate
-#define cublasDestroy hipblasDestroy
-#define cublasHandle_t hipblasHandle_t
-#define cublasStatus_t hipblasStatus_t
-#define cudaEventCreate hipEventCreate
-#define cudaEventRecord hipEventRecord
-#define cudaEventDestroy hipEventDestroy
-#define cublasSetStream hipblasSetStream
-#define cudaStreamCreate hipStreamCreate
-#define cudaMallocManaged hipMallocManaged
-#define cudaStreamDestroy hipStreamDestroy
-#define cudaEventSynchronize hipEventSynchronize
-#define cudaEventElapsedTime hipEventElapsedTime
-#define cudaStreamSynchronize hipStreamSynchronize
-#define cublasGetStatusString hipblasStatusToString
-#define cudaGetErrorString hipGetErrorString
-#define cudaGetDevice hipGetDevice
-#define cudaDeviceReset hipDeviceReset
-#define cublasGemmEx hipblasGemmEx
-#define CUBLAS_OP_N HIPBLAS_OP_N
-#define CUBLAS_OP_T HIPBLAS_OP_T
-#define CUDA_R_16F HIPBLAS_R_16F
-#define CUDA_R_32F HIPBLAS_R_32F
-#define CUBLAS_COMPUTE_16F HIPBLAS_COMPUTE_16F
-#define CUBLAS_COMPUTE_32F HIPBLAS_COMPUTE_32F
-#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_COMPUTE_32F_FAST_16F
-#define CUBLAS_GEMM_DEFAULT HIPBLAS_GEMM_DEFAULT
-#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS
-#endif
-
-#define ARRAYLEN(A) ((sizeof(A) / sizeof(*(A))) / ((unsigned)!(sizeof(A) % sizeof(*(A)))))
-
-#define CUDA_OR_DIE(x) \
- do { \
- cudaError_t err = (x); \
- if (err != cudaSuccess) \
- die(#x, __FUNCTION__, __FILE__, __LINE__, cudaGetErrorString(err)); \
- } while (0)
-
-#define CUBLAS_OR_DIE(x) \
- do { \
- cublasStatus_t status = (x); \
- if (status != CUBLAS_STATUS_SUCCESS) \
- die(#x, __FUNCTION__, __FILE__, __LINE__, cublasGetStatusString(status)); \
- } while (0)
-
-#define TINYBLAS_OR_DIE(x) \
- do { \
- tinyblasStatus_t status = (x); \
- if (status != TINYBLAS_STATUS_SUCCESS) \
- die(#x, __FUNCTION__, __FILE__, __LINE__, tinyblasGetStatusString(status)); \
- } while (0)
-
-static const int kDims[] = {1, 2, 23, 77, 15, 127, 128, 129};
-static const float kAlphas[] = {1, .5};
-static const float kBetas[] = {0, .1};
-
-[[noreturn]]
-void die(const char *stmt, const char *func, const char *file, int line, const char *msg) {
- int id = -1;
- (void)cudaGetDevice(&id);
- fprintf(stderr, "CUDA error: %s\n", msg);
- fprintf(stderr, " current device: %d, in function %s at %s:%d\n", id, func, file, line);
- fprintf(stderr, " %s\n", stmt);
- (void)cudaDeviceReset();
- exit(1);
-}
-
-void checkFloat01() {
- for (int i = 0; i < 10000; ++i) {
- float x = float01(rand32());
- ASSERT(x >= -1);
- ASSERT(x <= +1);
+void checkTinyblasWorksHHHH() {
+ double TOLERANCE = .5;
+
+ {
+ half α = 1;
+ half β = 0;
+ int m = 577;
+ int n = 577;
+ int k = 64;
+ int lda = k;
+ int ldb = k;
+ int ldc = m;
+ cuda_memory<half> A{lda * m};
+ cuda_memory<half> B{ldb * n};
+ cuda_memory<half> C{ldc * n};
+ cuda_memory<half> G{ldc * n};
+ randomize(k, m, A.p, lda);
+ randomize(k, n, B.p, ldb);
+ cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ broadcast(m, n, C.p, ldc, TOMBSTONE);
+ broadcast(m, n, G.p, ldc, TOMBSTONE);
+ gemmref<float>(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
}
-}
-
-void checkCublasWorksSSS() {
-
- cublasHandle_t hand;
- CUBLAS_OR_DIE(cublasCreate(&hand));
-
- cudaStream_t stream;
- CUDA_OR_DIE(cudaStreamCreate(&stream));
- CUBLAS_OR_DIE(cublasSetStream(hand, stream));
- float *A, *B, *C, *G;
- long long maxdim = 4096;
- CUDA_OR_DIE(cudaMallocManaged(&A, maxdim * maxdim * sizeof(float)));
- CUDA_OR_DIE(cudaMallocManaged(&B, maxdim * maxdim * sizeof(float)));
- CUDA_OR_DIE(cudaMallocManaged(&C, maxdim * maxdim * sizeof(float)));
- CUDA_OR_DIE(cudaMallocManaged(&G, maxdim * maxdim * sizeof(float)));
-
- std::atomic_llong t = ATOMIC_VAR_INIT(0);
- for (int ai = 0; ai < ARRAYLEN(kAlphas); ++ai)
- for (int bi = 0; bi < ARRAYLEN(kBetas); ++bi)
- for (int mi = 0; mi < ARRAYLEN(kDims); ++mi)
- for (int ni = 0; ni < ARRAYLEN(kDims); ++ni)
- for (int ki = 0; ki < ARRAYLEN(kDims); ++ki) {
- float α = kAlphas[ai];
- float β = kBetas[bi];
- int m = kDims[mi];
- int n = kDims[ARRAYLEN(kDims) - 1 - ni];
- int k = kDims[ki];
- int lda = m;
- int ldb = k;
- int ldc = m;
- char name[192];
- snprintf(name, sizeof(name), "testing %4d %4d %4d α=%g β=%g", m, n, k, α,
- β);
- if (t++ % 7 == 0)
- fprintf(stderr, "%s\r", name);
- is_self_testing = name;
- fill(m, k, A, lda);
- fill(k, n, B, ldb);
- broadcast(m, n, G, n, TOMBSTONE);
- broadcast(m, n, C, n, TOMBSTONE);
- dgemm(false, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);
- CUBLAS_OR_DIE(cublasSgemm(hand, CUBLAS_OP_N, CUBLAS_OP_N, m, n, k, &α, A,
- lda, B, ldb, &β, C, ldc));
- CUDA_OR_DIE(cudaStreamSynchronize(stream));
- CHECK(PRECISION_F32, m, n, G, ldc, C, ldc);
- is_self_testing = 0;
- }
+ {
+ half α = 1;
+ half β = 0;
+ int m = 4096;
+ int n = 1024;
+ int k = 2048;
+ int lda = k;
+ int ldb = k;
+ int ldc = m;
+ cuda_memory<half> A{lda * m};
+ cuda_memory<half> B{ldb * n};
+ cuda_memory<half> C{ldc * n};
+ cuda_memory<half> G{ldc * n};
+ randomize(k, m, A.p, lda);
+ randomize(k, n, B.p, ldb);
+ cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ broadcast(m, n, C.p, ldc, TOMBSTONE);
+ broadcast(m, n, G.p, ldc, TOMBSTONE);
+ gemmref<float>(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
+ }
- float α = 1;
- float β = 0;
- int m = 4096;
- int n = 1024;
- int k = 577;
- int lda = k;
- int ldb = k;
- int ldc = m;
- fill(k, m, A, lda);
- fill(k, n, B, ldb);
- fill(m, n, C, ldc);
- fill(m, n, G, ldc);
- broadcast(m, n, G, n, TOMBSTONE);
- broadcast(m, n, C, n, TOMBSTONE);
- dgemm(true, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);
- CUBLAS_OR_DIE(
- cublasSgemm(hand, CUBLAS_OP_T, CUBLAS_OP_N, m, n, k, &α, A, lda, B, ldb, &β, C, ldc));
- CUDA_OR_DIE(cudaStreamSynchronize(stream));
- CHECK(PRECISION_F32, m, n, G, ldc, C, ldc);
- cudaEvent_t start, stop;
- CUDA_OR_DIE(cudaEventCreate(&stop));
- CUDA_OR_DIE(cudaEventCreate(&start));
- CUDA_OR_DIE(cudaEventRecord(start, NULL));
- for (int i = 0; i < ITERATIONS; ++i)
- CUBLAS_OR_DIE(
- cublasSgemm(hand, CUBLAS_OP_T, CUBLAS_OP_N, m, n, k, &α, A, lda, B, ldb, &β, C, ldc));
- float msecTotal = 0;
- CUDA_OR_DIE(cudaEventRecord(stop, NULL));
- CUDA_OR_DIE(cudaEventSynchronize(stop));
- CUDA_OR_DIE(cudaEventElapsedTime(&msecTotal, start, stop));
- printf("%8lld µs cublasSgemm(%d, %d, %d)\n",
- (long long)std::ceil(msecTotal / ITERATIONS * 1000), m, n, k);
- CUDA_OR_DIE(cudaEventDestroy(start));
- CUDA_OR_DIE(cudaEventDestroy(stop));
+ {
+ half α = 1;
+ half β = 0;
+ int m = 4096;
+ int n = 1024;
+ int k = 577;
+ int lda = k;
+ int ldb = k;
+ int ldc = m;
+ cuda_memory<half> A{lda * m};
+ cuda_memory<half> B{ldb * n};
+ cuda_memory<half> C{ldc * n};
+ cuda_memory<half> G{ldc * n};
+ randomize(k, m, A.p, lda);
+ randomize(k, n, B.p, ldb);
+ cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ broadcast(m, n, C.p, ldc, TOMBSTONE);
+ broadcast(m, n, G.p, ldc, TOMBSTONE);
+ gemmref<float>(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
+ }
- CUDA_OR_DIE(cudaFree(G));
- CUDA_OR_DIE(cudaFree(C));
- CUDA_OR_DIE(cudaFree(B));
- CUDA_OR_DIE(cudaFree(A));
- CUDA_OR_DIE(cudaStreamDestroy(stream));
- CUBLAS_OR_DIE(cublasDestroy(hand));
+ test_matmul([&](int m, int n, int k, int l, half α, half β) {
+ int lda = m + l;
+ int ldb = k + l;
+ int ldc = m + l;
+ cuda_memory<half> A{lda * k};
+ cuda_memory<half> B{ldb * n};
+ cuda_memory<half> C{ldc * n};
+ cuda_memory<half> G{ldc * n};
+ randomize(m, k, A.p, lda);
+ randomize(k, n, B.p, ldb);
+ broadcast(m, n, G.p, ldc, TOMBSTONE);
+ broadcast(m, n, C.p, ldc, TOMBSTONE);
+ gemmref<float>(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ tinyblas(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
+ });
}
-void checkTinyblasWorksSSS() {
-
- tinyblasHandle_t hand;
- TINYBLAS_OR_DIE(tinyblasCreate(&hand));
-
- cudaStream_t stream;
- CUDA_OR_DIE(cudaStreamCreate(&stream));
- TINYBLAS_OR_DIE(tinyblasSetStream(hand, stream));
-
- float *A, *B, *C, *G;
- long long maxdim = 4096;
- CUDA_OR_DIE(cudaMallocManaged(&A, maxdim * maxdim * sizeof(float)));
- CUDA_OR_DIE(cudaMallocManaged(&B, maxdim * maxdim * sizeof(float)));
- CUDA_OR_DIE(cudaMallocManaged(&C, maxdim * maxdim * sizeof(float)));
- CUDA_OR_DIE(cudaMallocManaged(&G, maxdim * maxdim * sizeof(float)));
-
- std::atomic_llong t = ATOMIC_VAR_INIT(0);
- for (int ai = 0; ai < ARRAYLEN(kAlphas); ++ai)
- for (int bi = 0; bi < ARRAYLEN(kBetas); ++bi)
- for (int mi = 0; mi < ARRAYLEN(kDims); ++mi)
- for (int ni = 0; ni < ARRAYLEN(kDims); ++ni)
- for (int ki = 0; ki < ARRAYLEN(kDims); ++ki) {
- float α = kAlphas[ai];
- float β = kBetas[bi];
- int m = kDims[mi];
- int n = kDims[ARRAYLEN(kDims) - 1 - ni];
- int k = kDims[ki];
- int lda = m;
- int ldb = k;
- int ldc = m;
- char name[192];
- snprintf(name, sizeof(name), "testing %4d %4d %4d α=%g β=%g", m, n, k, α,
- β);
- if (t++ % 7 == 0)
- fprintf(stderr, "%s\r", name);
- is_self_testing = name;
- fill(m, k, A, lda);
- fill(k, n, B, ldb);
- broadcast(m, n, G, n, TOMBSTONE);
- broadcast(m, n, C, n, TOMBSTONE);
- dgemm(false, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);
- TINYBLAS_OR_DIE(tinyblasSgemm(hand, TINYBLAS_OP_N, TINYBLAS_OP_N, m, n, k,
- &α, A, lda, B, ldb, &β, C, ldc));
- CUDA_OR_DIE(cudaStreamSynchronize(stream));
- CHECK(PRECISION_F32, m, n, G, ldc, C, ldc);
- is_self_testing = 0;
- }
+void checkTinyblasWorksHHHS() {
+ double TOLERANCE = .5;
float α = 1;
float β = 0;
@@ -264,435 +118,124 @@ void checkTinyblasWorksSSS() {
int lda = k;
int ldb = k;
int ldc = m;
- fill(k, m, A, lda);
- fill(k, n, B, ldb);
- fill(m, n, C, ldc);
- fill(m, n, G, ldc);
- broadcast(m, n, G, n, TOMBSTONE);
- broadcast(m, n, C, n, TOMBSTONE);
- dgemm(true, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);
- TINYBLAS_OR_DIE(
- tinyblasSgemm(hand, TINYBLAS_OP_T, TINYBLAS_OP_N, m, n, k, &α, A, lda, B, ldb, &β, C, ldc));
- CUDA_OR_DIE(cudaStreamSynchronize(stream));
- CHECK(PRECISION_F32, m, n, G, ldc, C, ldc);
- cudaEvent_t start, stop;
- CUDA_OR_DIE(cudaEventCreate(&stop));
- CUDA_OR_DIE(cudaEventCreate(&start));
- CUDA_OR_DIE(cudaEventRecord(start, NULL));
- for (int i = 0; i < ITERATIONS; ++i)
- TINYBLAS_OR_DIE(tinyblasSgemm(hand, TINYBLAS_OP_T, TINYBLAS_OP_N, m, n, k, &α, A, lda, B,
- ldb, &β, C, ldc));
- float msecTotal = 0;
- CUDA_OR_DIE(cudaEventRecord(stop, NULL));
- CUDA_OR_DIE(cudaEventSynchronize(stop));
- CUDA_OR_DIE(cudaEventElapsedTime(&msecTotal, start, stop));
- printf("%8lld µs tinyblasSgemm(%d, %d, %d)\n",
- (long long)std::ceil(msecTotal / ITERATIONS * 1000), m, n, k);
- CUDA_OR_DIE(cudaEventDestroy(start));
- CUDA_OR_DIE(cudaEventDestroy(stop));
-
- CUDA_OR_DIE(cudaFree(G));
- CUDA_OR_DIE(cudaFree(C));
- CUDA_OR_DIE(cudaFree(B));
- CUDA_OR_DIE(cudaFree(A));
- CUDA_OR_DIE(cudaStreamDestroy(stream));
- TINYBLAS_OR_DIE(tinyblasDestroy(hand));
+ cuda_memory<half> A{lda * m};
+ cuda_memory<half> B{ldb * n};
+ cuda_memory<half> C{ldc * n};
+ cuda_memory<half> G{ldc * n};
+ randomize(k, m, A.p, lda);
+ randomize(k, n, B.p, ldb);
+ cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ broadcast(m, n, C.p, ldc, TOMBSTONE);
+ broadcast(m, n, G.p, ldc, TOMBSTONE);
+ gemmref(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
+
+ test_matmul([&](int m, int n, int k, int l, float α, float β) {
+ int lda = m + l;
+ int ldb = k + l;
+ int ldc = m + l;
+ cuda_memory<half> A{lda * k};
+ cuda_memory<half> B{ldb * n};
+ cuda_memory<half> C{ldc * n};
+ cuda_memory<half> G{ldc * n};
+ randomize(m, k, A.p, lda);
+ randomize(k, n, B.p, ldb);
+ broadcast(m, n, G.p, ldc, TOMBSTONE);
+ broadcast(m, n, C.p, ldc, TOMBSTONE);
+ gemmref(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ tinyblas(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
+ });
}
-void checkCublasWorksHHS() {
-
- cublasHandle_t hand;
- CUBLAS_OR_DIE(cublasCreate(&hand));
-
- cudaStream_t stream;
- CUDA_OR_DIE(cudaStreamCreate(&stream));
- CUBLAS_OR_DIE(cublasSetStream(hand, stream));
-
- half *A, *B;
- float *C, *G;
- long long maxdim = 4096;
- CUDA_OR_DIE(cudaMallocManaged(&A, maxdim * maxdim * sizeof(half)));
- CUDA_OR_DIE(cudaMallocManaged(&B, maxdim * maxdim * sizeof(half)));
- CUDA_OR_DIE(cudaMallocManaged(&C, maxdim * maxdim * sizeof(float)));
- CUDA_OR_DIE(cudaMallocManaged(&G, maxdim * maxdim * sizeof(float)));
-
- std::atomic_llong t = ATOMIC_VAR_INIT(0);
- for (int ai = 0; ai < ARRAYLEN(kAlphas); ++ai)
- for (int bi = 0; bi < ARRAYLEN(kBetas); ++bi)
- for (int mi = 0; mi < ARRAYLEN(kDims); ++mi)
- for (int ni = 0; ni < ARRAYLEN(kDims); ++ni)
- for (int ki = 0; ki < ARRAYLEN(kDims); ++ki) {
- float α = kAlphas[ai];
- float β = kBetas[bi];
- int m = kDims[mi];
- int n = kDims[ARRAYLEN(kDims) - 1 - ni];
- int k = kDims[ki];
- int lda = m;
- int ldb = k;
- int ldc = m;
-
- char name[192];
- snprintf(name, sizeof(name), "testing %4d %4d %4d α=%g β=%g", m, n, k, α,
- β);
- if (t++ % 7 == 0)
- fprintf(stderr, "%s\r", name);
- is_self_testing = name;
-
- fill(m, k, A, lda);
- fill(k, n, B, ldb);
- broadcast(m, n, G, n, TOMBSTONE);
- broadcast(m, n, C, n, TOMBSTONE);
- dgemm(false, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);
-
- CUBLAS_OR_DIE(cublasGemmEx(hand, CUBLAS_OP_N, CUBLAS_OP_N, m, n, k, &α, A,
- CUDA_R_16F, lda, B, CUDA_R_16F, ldb, &β, C,
- CUDA_R_32F, ldc, CUBLAS_COMPUTE_32F,
- CUBLAS_GEMM_DEFAULT));
- CUDA_OR_DIE(cudaStreamSynchronize(stream));
-
- CHECK(PRECISION_F32, m, n, G, ldc, C, ldc);
- is_self_testing = 0;
- }
+void checkTinyblasWorksHHSS() {
+ double TOLERANCE = .5;
float α = 1;
float β = 0;
int m = 4096;
int n = 1024;
- int k = 577;
+ int k = 1536;
int lda = k;
int ldb = k;
int ldc = m;
- fill(k, m, A, lda);
- fill(k, n, B, ldb);
- fill(m, n, C, ldc);
- fill(m, n, G, ldc);
- broadcast(m, n, G, n, TOMBSTONE);
- broadcast(m, n, C, n, TOMBSTONE);
- dgemm(true, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);
- CUBLAS_OR_DIE(cublasGemmEx(hand, CUBLAS_OP_T, CUBLAS_OP_N, m, n, k, &α, A, CUDA_R_16F, lda, B,
- CUDA_R_16F, ldb, &β, C, CUDA_R_32F, ldc, CUBLAS_COMPUTE_32F,
- CUBLAS_GEMM_DEFAULT));
- CUDA_OR_DIE(cudaStreamSynchronize(stream));
- CHECK(PRECISION_F32, m, n, G, ldc, C, ldc);
- cudaEvent_t start, stop;
- CUDA_OR_DIE(cudaEventCreate(&stop));
- CUDA_OR_DIE(cudaEventCreate(&start));
- CUDA_OR_DIE(cudaEventRecord(start, NULL));
- for (int i = 0; i < ITERATIONS; ++i)
- CUBLAS_OR_DIE(cublasGemmEx(hand, CUBLAS_OP_T, CUBLAS_OP_N, m, n, k, &α, A, CUDA_R_16F, lda,
- B, CUDA_R_16F, ldb, &β, C, CUDA_R_32F, ldc, CUBLAS_COMPUTE_32F,
- CUBLAS_GEMM_DEFAULT));
- float msecTotal = 0;
- CUDA_OR_DIE(cudaEventRecord(stop, NULL));
- CUDA_OR_DIE(cudaEventSynchronize(stop));
- CUDA_OR_DIE(cudaEventElapsedTime(&msecTotal, start, stop));
- printf("%8lld µs cublasGemmExHHS(%d, %d, %d)\n",
- (long long)std::ceil(msecTotal / ITERATIONS * 1000), m, n, k);
- CUDA_OR_DIE(cudaEventDestroy(start));
- CUDA_OR_DIE(cudaEventDestroy(stop));
-
- CUDA_OR_DIE(cudaFree(G));
- CUDA_OR_DIE(cudaFree(C));
- CUDA_OR_DIE(cudaFree(B));
- CUDA_OR_DIE(cudaFree(A));
- CUDA_OR_DIE(cudaStreamDestroy(stream));
- CUBLAS_OR_DIE(cublasDestroy(hand));
+ cuda_memory<half> A{lda * m};
+ cuda_memory<half> B{ldb * n};
+ cuda_memory<float> C{ldc * n};
+ cuda_memory<float> G{ldc * n};
+ randomize(k, m, A.p, lda);
+ randomize(k, n, B.p, ldb);
+ cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ broadcast(m, n, G.p, ldc, TOMBSTONE);
+ broadcast(m, n, C.p, ldc, TOMBSTONE);
+ gemmref(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
+
+ test_matmul([&](int m, int n, int k, int l, float α, float β) {
+ int lda = k + l;
+ int ldb = k + l;
+ int ldc = m + l;
+ cuda_memory<half> A{lda * m};
+ cuda_memory<half> B{ldb * n};
+ cuda_memory<float> C{ldc * n};
+ cuda_memory<float> G{ldc * n};
+ randomize(k, m, A.p, lda);
+ randomize(k, n, B.p, ldb);
+ broadcast(m, n, G.p, ldc, TOMBSTONE);
+ broadcast(m, n, C.p, ldc, TOMBSTONE);
+ gemmref(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
+ });
}
-void checkTinyblasWorksHHS() {
-
- tinyblasHandle_t hand;
- TINYBLAS_OR_DIE(tinyblasCreate(&hand));
-
- cudaStream_t stream;
- CUDA_OR_DIE(cudaStreamCreate(&stream));
- TINYBLAS_OR_DIE(tinyblasSetStream(hand, stream));
-
- half *A, *B;
- float *C, *G;
- long long maxdim = 4096;
- CUDA_OR_DIE(cudaMallocManaged(&A, maxdim * maxdim * sizeof(half)));
- CUDA_OR_DIE(cudaMallocManaged(&B, maxdim * maxdim * sizeof(half)));
- CUDA_OR_DIE(cudaMallocManaged(&C, maxdim * maxdim * sizeof(float)));
- CUDA_OR_DIE(cudaMallocManaged(&G, maxdim * maxdim * sizeof(float)));
-
- std::atomic_llong t = ATOMIC_VAR_INIT(0);
- for (int ai = 0; ai < ARRAYLEN(kAlphas); ++ai)
- for (int bi = 0; bi < ARRAYLEN(kBetas); ++bi)
- for (int mi = 0; mi < ARRAYLEN(kDims); ++mi)
- for (int ni = 0; ni < ARRAYLEN(kDims); ++ni)
- for (int ki = 0; ki < ARRAYLEN(kDims); ++ki) {
- float α = kAlphas[ai];
- float β = kBetas[bi];
- int m = kDims[mi];
- int n = kDims[ARRAYLEN(kDims) - 1 - ni];
- int k = kDims[ki];
- int lda = k;
- int ldb = k;
- int ldc = m;
- char name[192];
- snprintf(name, sizeof(name), "testing %4d %4d %4d α=%g β=%g", m, n, k, α,
- β);
- if (t++ % 7 == 0)
- fprintf(stderr, "%s\r", name);
- is_self_testing = name;
- fill(m, k, A, lda);
- fill(k, n, B, ldb);
- broadcast(m, n, G, n, TOMBSTONE);
- broadcast(m, n, C, n, TOMBSTONE);
- dgemm(true, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);
- TINYBLAS_OR_DIE(tinyblasGemmEx(
- hand, TINYBLAS_OP_T, TINYBLAS_OP_N, m, n, k, &α, A, TINYBLAS_R_16F, lda,
- B, TINYBLAS_R_16F, ldb, &β, C, TINYBLAS_R_32F, ldc,
- TINYBLAS_COMPUTE_32F, TINYBLAS_GEMM_DEFAULT));
- CUDA_OR_DIE(cudaStreamSynchronize(stream));
- CHECK(PRECISION_F32, m, n, G, ldc, C, ldc);
- is_self_testing = 0;
- }
+void checkTinyblasWorksSSSS() {
+ double TOLERANCE = .5;
float α = 1;
float β = 0;
int m = 4096;
int n = 1024;
- int k = 577;
+ int k = 1020;
int lda = k;
int ldb = k;
int ldc = m;
- fill(k, m, A, lda);
- fill(k, n, B, ldb);
- fill(m, n, C, ldc);
- fill(m, n, G, ldc);
- broadcast(m, n, G, n, TOMBSTONE);
- broadcast(m, n, C, n, TOMBSTONE);
- dgemm(true, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);
- TINYBLAS_OR_DIE(tinyblasGemmEx(
- hand, TINYBLAS_OP_T, TINYBLAS_OP_N, m, n, k, &α, A, TINYBLAS_R_16F, lda, B, TINYBLAS_R_16F,
- ldb, &β, C, TINYBLAS_R_32F, ldc, TINYBLAS_COMPUTE_32F, TINYBLAS_GEMM_DEFAULT));
- CUDA_OR_DIE(cudaStreamSynchronize(stream));
- CHECK(PRECISION_F32, m, n, G, ldc, C, ldc);
- cudaEvent_t start, stop;
- CUDA_OR_DIE(cudaEventCreate(&stop));
- CUDA_OR_DIE(cudaEventCreate(&start));
- CUDA_OR_DIE(cudaEventRecord(start, NULL));
- for (int i = 0; i < ITERATIONS; ++i)
- TINYBLAS_OR_DIE(tinyblasGemmEx(hand, TINYBLAS_OP_T, TINYBLAS_OP_N, m, n, k, &α, A,
- TINYBLAS_R_16F, lda, B, TINYBLAS_R_16F, ldb, &β, C,
- TINYBLAS_R_32F, ldc, TINYBLAS_COMPUTE_32F,
- TINYBLAS_GEMM_DEFAULT));
- float msecTotal = 0;
- CUDA_OR_DIE(cudaEventRecord(stop, NULL));
- CUDA_OR_DIE(cudaEventSynchronize(stop));
- CUDA_OR_DIE(cudaEventElapsedTime(&msecTotal, start, stop));
- printf("%8lld µs tinyblasGemmExHHS(%d, %d, %d)\n",
- (long long)std::ceil(msecTotal / ITERATIONS * 1000), m, n, k);
- CUDA_OR_DIE(cudaEventDestroy(start));
- CUDA_OR_DIE(cudaEventDestroy(stop));
-
- CUDA_OR_DIE(cudaFree(G));
- CUDA_OR_DIE(cudaFree(C));
- CUDA_OR_DIE(cudaFree(B));
- CUDA_OR_DIE(cudaFree(A));
- CUDA_OR_DIE(cudaStreamDestroy(stream));
- TINYBLAS_OR_DIE(tinyblasDestroy(hand));
-}
-
-void checkCublasWorksHHH() {
-
- cublasHandle_t hand;
- CUBLAS_OR_DIE(cublasCreate(&hand));
-
- cudaStream_t stream;
- CUDA_OR_DIE(cudaStreamCreate(&stream));
- CUBLAS_OR_DIE(cublasSetStream(hand, stream));
-
- half *A, *B, *C, *G;
- long long maxdim = 32000;
- CUDA_OR_DIE(cudaMallocManaged(&A, maxdim * maxdim * sizeof(half)));
- CUDA_OR_DIE(cudaMallocManaged(&B, maxdim * maxdim * sizeof(half)));
- CUDA_OR_DIE(cudaMallocManaged(&C, maxdim * maxdim * sizeof(half)));
- CUDA_OR_DIE(cudaMallocManaged(&G, maxdim * maxdim * sizeof(half)));
-
- std::atomic_llong t = ATOMIC_VAR_INIT(0);
- for (int ai = 0; ai < ARRAYLEN(kAlphas); ++ai)
- for (int bi = 0; bi < ARRAYLEN(kBetas); ++bi)
- for (int mi = 0; mi < ARRAYLEN(kDims); ++mi)
- for (int ni = 0; ni < ARRAYLEN(kDims); ++ni)
- for (int ki = 0; ki < ARRAYLEN(kDims); ++ki) {
- half α = kAlphas[ai];
- half β = kBetas[bi];
- int m = kDims[mi];
- int n = kDims[ARRAYLEN(kDims) - 1 - ni];
- int k = kDims[ki];
- int lda = m;
- int ldb = k;
- int ldc = m;
- char name[192];
- snprintf(name, sizeof(name), "testing %4d %4d %4d α=%g β=%g", m, n, k,
- __half2float(α), __half2float(β));
- if (t++ % 7 == 0)
- fprintf(stderr, "%s\r", name);
- is_self_testing = name;
- fill(m, k, A, lda);
- fill(k, n, B, ldb);
- broadcast(m, n, G, n, TOMBSTONE);
- broadcast(m, n, C, n, TOMBSTONE);
- dgemm(false, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);
- CUBLAS_OR_DIE(cublasGemmEx(hand, CUBLAS_OP_N, CUBLAS_OP_N, m, n, k, &α, A,
- CUDA_R_16F, lda, B, CUDA_R_16F, ldb, &β, C,
- CUDA_R_16F, ldc, CUBLAS_COMPUTE_16F,
- CUBLAS_GEMM_DEFAULT));
- CUDA_OR_DIE(cudaStreamSynchronize(stream));
- CHECK(PRECISION_F16, m, n, G, ldc, C, ldc);
- is_self_testing = 0;
- }
-
- half α = 1;
- half β = 0;
- int m = 4096;
- int n = 1024;
- int k = 577;
- int lda = k;
- int ldb = k;
- int ldc = m;
- fill(k, m, A, lda);
- fill(k, n, B, ldb);
- fill(m, n, C, ldc);
- fill(m, n, G, ldc);
- broadcast(m, n, G, n, TOMBSTONE);
- broadcast(m, n, C, n, TOMBSTONE);
- dgemm(true, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);
- CUBLAS_OR_DIE(cublasGemmEx(hand, CUBLAS_OP_T, CUBLAS_OP_N, m, n, k, &α, A, CUDA_R_16F, lda, B,
- CUDA_R_16F, ldb, &β, C, CUDA_R_16F, ldc, CUBLAS_COMPUTE_16F,
- CUBLAS_GEMM_DEFAULT));
- CUDA_OR_DIE(cudaStreamSynchronize(stream));
- CHECK(PRECISION_F16, m, n, G, ldc, C, ldc);
- cudaEvent_t start, stop;
- CUDA_OR_DIE(cudaEventCreate(&stop));
- CUDA_OR_DIE(cudaEventCreate(&start));
- CUDA_OR_DIE(cudaEventRecord(start, NULL));
- for (int i = 0; i < ITERATIONS; ++i)
- CUBLAS_OR_DIE(cublasGemmEx(hand, CUBLAS_OP_T, CUBLAS_OP_N, m, n, k, &α, A, CUDA_R_16F, lda,
- B, CUDA_R_16F, ldb, &β, C, CUDA_R_16F, ldc, CUBLAS_COMPUTE_16F,
- CUBLAS_GEMM_DEFAULT));
- float msecTotal = 0;
- CUDA_OR_DIE(cudaEventRecord(stop, NULL));
- CUDA_OR_DIE(cudaEventSynchronize(stop));
- CUDA_OR_DIE(cudaEventElapsedTime(&msecTotal, start, stop));
- printf("%8lld µs cublasGemmExHHH(%d, %d, %d)\n",
- (long long)std::ceil(msecTotal / ITERATIONS * 1000), m, n, k);
- CUDA_OR_DIE(cudaEventDestroy(start));
- CUDA_OR_DIE(cudaEventDestroy(stop));
-
- CUDA_OR_DIE(cudaFree(G));
- CUDA_OR_DIE(cudaFree(C));
- CUDA_OR_DIE(cudaFree(B));
- CUDA_OR_DIE(cudaFree(A));
- CUDA_OR_DIE(cudaStreamDestroy(stream));
- CUBLAS_OR_DIE(cublasDestroy(hand));
-}
-
-void checkTinyblasWorksHHH() {
-
- tinyblasHandle_t hand;
- TINYBLAS_OR_DIE(tinyblasCreate(&hand));
-
- cudaStream_t stream;
- CUDA_OR_DIE(cudaStreamCreate(&stream));
- TINYBLAS_OR_DIE(tinyblasSetStream(hand, stream));
-
- half *A, *B, *C, *G;
- long long maxdim = 32000;
- CUDA_OR_DIE(cudaMallocManaged(&A, maxdim * maxdim * sizeof(half)));
- CUDA_OR_DIE(cudaMallocManaged(&B, maxdim * maxdim * sizeof(half)));
- CUDA_OR_DIE(cudaMallocManaged(&C, maxdim * maxdim * sizeof(half)));
- CUDA_OR_DIE(cudaMallocManaged(&G, maxdim * maxdim * sizeof(half)));
-
- std::atomic_llong t = ATOMIC_VAR_INIT(0);
- for (int ai = 0; ai < ARRAYLEN(kAlphas); ++ai)
- for (int bi = 0; bi < ARRAYLEN(kBetas); ++bi)
- for (int mi = 0; mi < ARRAYLEN(kDims); ++mi)
- for (int ni = 0; ni < ARRAYLEN(kDims); ++ni)
- for (int ki = 0; ki < ARRAYLEN(kDims); ++ki) {
- half α = kAlphas[ai];
- half β = kBetas[bi];
- int m = kDims[mi];
- int n = kDims[ARRAYLEN(kDims) - 1 - ni];
- int k = kDims[ki];
- int lda = m;
- int ldb = k;
- int ldc = m;
- char name[192];
- snprintf(name, sizeof(name), "testing %4d %4d %4d α=%g β=%g", m, n, k,
- __half2float(α), __half2float(β));
- if (t++ % 7 == 0)
- fprintf(stderr, "%s\r", name);
- is_self_testing = name;
- fill(m, k, A, lda);
- fill(k, n, B, ldb);
- broadcast(m, n, G, n, TOMBSTONE);
- broadcast(m, n, C, n, TOMBSTONE);
- dgemm(false, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);
- TINYBLAS_OR_DIE(tinyblasGemmEx(
- hand, TINYBLAS_OP_N, TINYBLAS_OP_N, m, n, k, &α, A, TINYBLAS_R_16F, lda,
- B, TINYBLAS_R_16F, ldb, &β, C, TINYBLAS_R_16F, ldc,
- TINYBLAS_COMPUTE_16F, TINYBLAS_GEMM_DEFAULT));
- CUDA_OR_DIE(cudaStreamSynchronize(stream));
- CHECK(PRECISION_F16, m, n, G, ldc, C, ldc);
- is_self_testing = 0;
- }
-
- half α = 1;
- half β = 0;
- int m = 4096;
- int n = 1024;
- int k = 577;
- int lda = k;
- int ldb = k;
- int ldc = m;
- fill(k, m, A, lda);
- fill(k, n, B, ldb);
- fill(m, n, C, ldc);
- fill(m, n, G, ldc);
- broadcast(m, n, G, n, TOMBSTONE);
- broadcast(m, n, C, n, TOMBSTONE);
- dgemm(true, false, m, n, k, α, A, lda, B, ldb, β, G, ldc);
- TINYBLAS_OR_DIE(tinyblasGemmEx(
- hand, TINYBLAS_OP_T, TINYBLAS_OP_N, m, n, k, &α, A, TINYBLAS_R_16F, lda, B, TINYBLAS_R_16F,
- ldb, &β, C, TINYBLAS_R_16F, ldc, TINYBLAS_COMPUTE_16F, TINYBLAS_GEMM_DEFAULT));
- CUDA_OR_DIE(cudaStreamSynchronize(stream));
- CHECK(PRECISION_F16, m, n, G, ldc, C, ldc);
- cudaEvent_t start, stop;
- CUDA_OR_DIE(cudaEventCreate(&stop));
- CUDA_OR_DIE(cudaEventCreate(&start));
- CUDA_OR_DIE(cudaEventRecord(start, NULL));
- for (int i = 0; i < ITERATIONS; ++i)
- TINYBLAS_OR_DIE(tinyblasGemmEx(hand, TINYBLAS_OP_T, TINYBLAS_OP_N, m, n, k, &α, A,
- TINYBLAS_R_16F, lda, B, TINYBLAS_R_16F, ldb, &β, C,
- TINYBLAS_R_16F, ldc, TINYBLAS_COMPUTE_16F,
- TINYBLAS_GEMM_DEFAULT));
- float msecTotal = 0;
- CUDA_OR_DIE(cudaEventRecord(stop, NULL));
- CUDA_OR_DIE(cudaEventSynchronize(stop));
- CUDA_OR_DIE(cudaEventElapsedTime(&msecTotal, start, stop));
- printf("%8lld µs tinyblasGemmExHHH(%d, %d, %d)\n",
- (long long)std::ceil(msecTotal / ITERATIONS * 1000), m, n, k);
- CUDA_OR_DIE(cudaEventDestroy(start));
- CUDA_OR_DIE(cudaEventDestroy(stop));
-
- CUDA_OR_DIE(cudaFree(G));
- CUDA_OR_DIE(cudaFree(C));
- CUDA_OR_DIE(cudaFree(B));
- CUDA_OR_DIE(cudaFree(A));
- CUDA_OR_DIE(cudaStreamDestroy(stream));
- TINYBLAS_OR_DIE(tinyblasDestroy(hand));
+ cuda_memory<float> A{lda * m};
+ cuda_memory<float> B{ldb * n};
+ cuda_memory<float> C{ldc * n};
+ cuda_memory<float> G{ldc * n};
+ randomize(k, m, A.p, lda);
+ randomize(k, n, B.p, ldb);
+ cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ broadcast(m, n, C.p, ldc, TOMBSTONE);
+ broadcast(m, n, G.p, ldc, TOMBSTONE);
+ gemmref(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
+
+ test_matmul([&](int m, int n, int k, int l, float α, float β) {
+ int lda = m + l;
+ int ldb = k + l;
+ int ldc = m + l;
+ cuda_memory<float> A{lda * k};
+ cuda_memory<float> B{ldb * n};
+ cuda_memory<float> C{ldc * n};
+ cuda_memory<float> G{ldc * n};
+ randomize(m, k, A.p, lda);
+ randomize(k, n, B.p, ldb);
+ broadcast(m, n, G.p, ldc, TOMBSTONE);
+ broadcast(m, n, C.p, ldc, TOMBSTONE);
+ gemmref(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ tinyblas(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
+ });
}
int main(int argc, char *argv[]) {
- RUN(checkFloat01());
- RUN(checkCublasWorksSSS());
- RUN(checkTinyblasWorksSSS());
- RUN(checkCublasWorksHHS());
- RUN(checkTinyblasWorksHHS());
- RUN(checkCublasWorksHHH());
- RUN(checkTinyblasWorksHHH());
+ RUN(checkTinyblasWorksHHHH());
+ RUN(checkTinyblasWorksSSSS());
+ RUN(checkTinyblasWorksHHSS());
+ RUN(checkTinyblasWorksHHHS());
}