Skip to content

Distribute and run LLMs with a single file.

Port tinyBLAS code to MSVC

Justine Tunneyf33895292ad3
diff --git a/llamafile/cuda.h b/llamafile/cuda.h
index babd08b7..b9066f82 100644
--- a/llamafile/cuda.h
+++ b/llamafile/cuda.h
@@ -2,7 +2,7 @@
// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
#pragma once
-#if defined(__NVCC__) || defined(__CUDA_ARCH__)
+#ifndef __HIP__
#include <cublas_v2.h>
#include <cuda_fp16.h>
#include <cuda_runtime.h>
diff --git a/llamafile/cudaprops.cu b/llamafile/cudaprops.cu
index 3ec087c9..1578b8b3 100644
--- a/llamafile/cudaprops.cu
+++ b/llamafile/cudaprops.cu
@@ -19,10 +19,10 @@
#include "tester.h"
#include <iostream>
-#if defined(__NVCC__) || defined(__CUDA_ARCH__)
-#define VENDOR "NVIDIA"
-#else
+#ifdef __HIP__
#define VENDOR "AMD"
+#else
+#define VENDOR "NVIDIA"
#endif
namespace {
diff --git a/llamafile/flt.h b/llamafile/flt.h
index b50df4ab..ea0cc607 100644
--- a/llamafile/flt.h
+++ b/llamafile/flt.h
@@ -10,7 +10,7 @@
//
// @see IEEE 754-2008
-#if defined(__NVCC__) || defined(__CUDA_ARCH__)
+#ifndef __HIP__
#include <cuda_fp16.h>
#else
#include <hip/hip_fp16.h>
diff --git a/llamafile/gemm.h b/llamafile/gemm.h
index e9c3a660..a26a3d32 100644
--- a/llamafile/gemm.h
+++ b/llamafile/gemm.h
@@ -29,27 +29,27 @@
// k×m * n×k → m×n if aᵀ and bᵀ
//
template <typename T, typename TA, typename TB, typename TC>
-void gemm(bool aᵀ, bool bᵀ, //
- int m, int n, int k, T α, //
+void gemm(bool aT, bool bT, //
+ int m, int n, int k, T alpha, //
const TA *A, int lda, //
- const TB *B, int ldb, T β, //
+ const TB *B, int ldb, T beta, //
TC *C, int ldc) {
assert(m >= 0 && n >= 0 && k >= 0);
- assert(lda >= std::max(1, aᵀ ? k : m));
- assert(ldb >= std::max(1, bᵀ ? n : k));
+ assert(lda >= std::max(1, aT ? k : m));
+ assert(ldb >= std::max(1, bT ? n : k));
assert(ldc >= std::max(1, m));
- assert(1ll * lda * (aᵀ ? m : k) <= INT_MAX);
- assert(1ll * ldb * (bᵀ ? k : n) <= INT_MAX);
+ assert(1ll * lda * (aT ? m : k) <= INT_MAX);
+ assert(1ll * ldb * (bT ? k : n) <= INT_MAX);
for (int i = 0; i < m; ++i)
for (int j = 0; j < n; ++j) {
T d = 0;
for (int l = 0; l < k; ++l) {
- T a = aᵀ ? A[lda * i + l] : A[lda * l + i];
- T b = bᵀ ? B[ldb * l + j] : B[ldb * j + l];
+ T a = aT ? A[lda * i + l] : A[lda * l + i];
+ T b = bT ? B[ldb * l + j] : B[ldb * j + l];
d = std::fma(a, b, d);
}
T c = C[ldc * j + i];
- C[ldc * j + i] = std::fma(α, d, β * c);
+ C[ldc * j + i] = std::fma(alpha, d, beta * c);
}
}
@@ -61,10 +61,10 @@ void gemm(bool aᵀ, bool bᵀ, //
// m×k * k×n → n×m if aᵀ and bᵀ
//
template <typename T, typename TA, typename TB, typename TC>
-void gemmrm(bool aᵀ, bool bᵀ, //
- int m, int n, int k, T α, //
+void gemmrm(bool aT, bool bT, //
+ int m, int n, int k, T alpha, //
const TA *A, int lda, //
- const TB *B, int ldb, T β, //
+ const TB *B, int ldb, T beta, //
TC *C, int ldc) {
- gemm(bᵀ, aᵀ, n, m, k, α, B, ldb, A, lda, β, C, ldc);
+ gemm(bT, aT, n, m, k, alpha, B, ldb, A, lda, beta, C, ldc);
}
diff --git a/llamafile/naive.h b/llamafile/naive.h
index de45d328..730f9f46 100644
--- a/llamafile/naive.h
+++ b/llamafile/naive.h
@@ -16,6 +16,7 @@
// limitations under the License.
#pragma once
+#include "cuda.h"
#include "macros.h"
//
@@ -34,37 +35,30 @@
// can be used for finding bugs in more advanced matmul implementations.
namespace naive {
-
-__device__ __forceinline__ half madd(half x, half y, half z) {
- return (float)x * (float)y + (float)z;
-}
-__device__ __forceinline__ float madd(float x, float y, float z) {
- return __fmaf_rn(x, y, z);
-}
-__device__ __forceinline__ double madd(double x, double y, double z) {
- return __fma_rn(x, y, z);
-}
+namespace {
template <typename T, typename TA, typename TB, typename TC>
-__global__ void kernel(bool aᵀ, bool bᵀ, //
- int m, int n, int k, T α, //
+__global__ void kernel(bool aT, bool bT, //
+ int m, int n, int k, T alpha, //
const TA *A, int lda, //
- const TB *B, int ldb, T β, //
+ const TB *B, int ldb, T beta, //
TC *C, int ldc) {
const int i = blockIdx.x * blockDim.x + threadIdx.x;
const int j = blockIdx.y * blockDim.y + threadIdx.y;
if (i < m && j < n) {
T d = 0;
for (int l = 0; l < k; ++l) {
- T a = aᵀ ? A[lda * i + l] : A[lda * l + i];
- T b = bᵀ ? B[ldb * l + j] : B[ldb * j + l];
- d = madd(a, b, d);
+ T a = aT ? A[lda * i + l] : A[lda * l + i];
+ T b = bT ? B[ldb * l + j] : B[ldb * j + l];
+ d += a * b;
}
T c = C[ldc * j + i];
- C[ldc * j + i] = madd(α, d, β * c);
+ C[ldc * j + i] = alpha * d + beta * c;
}
}
+} // namespace
+
// multiplies matrix on gpu with column major ordering
//
// m×k * k×n → m×n
@@ -74,14 +68,14 @@ __global__ void kernel(bool aᵀ, bool bᵀ, //
//
template <typename T, typename TA, typename TB, typename TC>
cudaError_t gemm(cudaStream_t stream, //
- bool aᵀ, bool bᵀ, //
- int m, int n, int k, T α, //
+ bool aT, bool bT, //
+ int m, int n, int k, T alpha, //
const TA *A, int lda, //
- const TB *B, int ldb, T β, //
+ const TB *B, int ldb, T beta, //
TC *C, int ldc) {
dim3 blockDim(32, 32);
dim3 gridDim(CEIL_DIV(m, 32), CEIL_DIV(n, 32));
- kernel<<<gridDim, blockDim, 0, stream>>>(aᵀ, bᵀ, m, n, k, α, A, lda, B, ldb, β, C, ldc);
+ kernel<<<gridDim, blockDim, 0, stream>>>(aT, bT, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc);
return cudaGetLastError();
}
diff --git a/llamafile/tester.cu b/llamafile/tester.cu
index 83955e89..f4e12698 100644
--- a/llamafile/tester.cu
+++ b/llamafile/tester.cu
@@ -17,18 +17,25 @@
#include "tester.h"
+#include <algorithm>
#include <atomic>
#include <ctime>
+
+#ifndef _WIN32
#include <unistd.h>
+#else
+#include "windows.h"
+#endif
+std::recursive_mutex g_log_lock;
thread_local const char *is_self_testing;
-const size_t kPageSize = std::max(sysconf(_SC_PAGESIZE), 4096l);
-long long micros(void) {
- struct timespec ts;
- clock_gettime(CLOCK_REALTIME, &ts);
- return ts.tv_sec * 1000000 + (ts.tv_nsec + 999) / 1000;
-}
+const int kPageSize =
+#ifndef _WIN32
+ std::max(sysconf(_SC_PAGESIZE), 4096l);
+#else
+ 4096;
+#endif
int rand32(void) {
static unsigned long long lcg = 1;
@@ -45,14 +52,51 @@ float numba(void) { // (-1,1)
return float01(rand32()) * 2 - 1;
}
+int hamming(int x, int y) {
+ return popcount(x ^ y);
+}
+
+int popcount(unsigned x) {
+ x = x - ((x >> 1) & 0x55555555);
+ x = ((x >> 2) & 0x33333333) + (x & 0x33333333);
+ x = (x + (x >> 4)) & 0x0F0F0F0F;
+ x = (x + (x >> 16));
+ return (x + (x >> 8)) & 0x0000003F;
+}
+
+void cudaFreeOrDie(void *p) {
+ CUDA_OR_DIE(cudaFree(p));
+}
+
void *cudaMallocManagedOrDie(size_t n) {
void *p;
CUDA_OR_DIE(cudaMallocManaged(&p, n));
return p;
}
-void cudaFreeOrDie(void *p) {
- CUDA_OR_DIE(cudaFree(p));
+#ifdef _WIN32
+static long long GetQueryPerformanceFrequency() {
+ LARGE_INTEGER t;
+ QueryPerformanceFrequency(&t);
+ return t.QuadPart;
+}
+static long long GetQueryPerformanceCounter() {
+ LARGE_INTEGER t;
+ QueryPerformanceCounter(&t);
+ return t.QuadPart;
+}
+#endif
+
+long long micros(void) {
+#ifndef _WIN32
+ struct timespec ts;
+ clock_gettime(CLOCK_REALTIME, &ts);
+ return ts.tv_sec * 1000000 + (ts.tv_nsec + 999) / 1000;
+#else
+ static long long timer_freq = GetQueryPerformanceFrequency();
+ static long long timer_start = GetQueryPerformanceCounter();
+ return ((GetQueryPerformanceCounter() - timer_start) * 1000000) / timer_freq;
+#endif
}
[[noreturn]] void cuda_die(const char *stmt, const char *func, const char *file, int line,
@@ -66,7 +110,7 @@ void cudaFreeOrDie(void *p) {
_Exit(1);
}
-void test_matmul(std::function<void(int m, int n, int k, int l, float α, float β)> f) {
+void test_matmul(std::function<void(int m, int n, int k, int l, float alpha, float beta)> f) {
static const int kDims[] = {1, 2, 23, 77, 15, 2048, 512, 127, 129, 128, 16};
static const float kAlphas[] = {1, .1};
static const float kBetas[] = {0, .1};
@@ -90,24 +134,24 @@ void test_matmul(std::function<void(int m, int n, int k, int l, float α, float
}
for (int ai = 0; ai < ARRAYLEN(kAlphas); ++ai)
for (int bi = 0; bi < ARRAYLEN(kBetas); ++bi) {
- float α = kAlphas[ai];
- float β = kBetas[bi];
+ float alpha = kAlphas[ai];
+ float beta = kBetas[bi];
int m = 128;
int n = 128;
int k = 128;
char name[256];
- sprintf(name, "testing %4d %4d %4d α=%g β=%g", m, n, k, α, β);
+ sprintf(name, "testing %4d %4d %4d alpha=%g beta=%g", m, n, k, alpha, beta);
if (t++ % 7 == 0)
fprintf(stderr, "%s\r", name);
is_self_testing = name;
- f(m, n, k, 0, α, β);
+ f(m, n, k, 0, alpha, beta);
is_self_testing = 0;
}
}
static int cuda_tester_init() {
CUDA_OR_DIE(cudaSetDevice(0));
-#ifdef __HIP_PLATFORM_AMD__
+#ifdef __HIP__
rocblas_initialize();
#endif
CUDA_OR_DIE(cudaDeviceSynchronize());
diff --git a/llamafile/tester.h b/llamafile/tester.h
index 8c816edb..1d672496 100644
--- a/llamafile/tester.h
+++ b/llamafile/tester.h
@@ -27,6 +27,7 @@
#include <cstdio>
#include <cstdlib>
#include <functional>
+#include <mutex>
//
// _ _ ___ _ _ ___
@@ -46,20 +47,19 @@
#define ITERATIONS 30
#define TOMBSTONE 1.666f
-extern const size_t kPageSize;
+extern const int kPageSize;
+extern std::recursive_mutex g_log_lock;
extern thread_local const char *is_self_testing;
float float01(unsigned);
float numba(void);
+int hamming(int, int);
+int popcount(unsigned);
int rand32(void);
long long micros(void);
-void test_matmul(std::function<void(int, int, int, int, float, float)>);
void *cudaMallocManagedOrDie(size_t);
void cudaFreeOrDie(void *);
-
-inline int hamming(int x, int y) {
- return __builtin_popcount(x ^ y);
-}
+void test_matmul(std::function<void(int, int, int, int, float, float)>);
template <typename T> void randomize(T *A, int n) {
for (int i = 0; i < n; ++i)
@@ -141,7 +141,7 @@ void verify(double tol, int m, int n, const T *A, int lda, const T *B, int ldb)
}
double avg = static_cast<double>(diffsum) / (m * n);
if (avg > tol || nans) {
- flockfile(stderr);
+ std::unique_lock<std::recursive_mutex> lock(g_log_lock);
fprintf(stderr, "error: %d×%d matrix difference exceeds tolerance\n", m, n);
fprintf(stderr, " worst element at index (%d, %d)\n", worst_i, worst_j);
fprintf(stderr, " difference %14e (%lld ulp)\n",
@@ -224,7 +224,7 @@ void diff(int m, int n, int k, const T *Wan, int lda, const T *Got, int ldb, dou
template <typename T>
void show(FILE *f, int max, int m, int n, const T *A, int lda, const T *B, int ldb) {
- flockfile(f);
+ std::unique_lock<std::recursive_mutex> lock(g_log_lock);
fprintf(f, " ");
for (int i = 0; i < n && i < max; ++i)
fprintf(f, "%13d", i);
@@ -245,15 +245,14 @@ void show(FILE *f, int max, int m, int n, const T *A, int lda, const T *B, int l
snprintf(bb, 32, "%13.7f", static_cast<double>(B[ldb * j + i]));
for (int k = 0; ba[k] && bb[k]; ++k) {
if (ba[k] != bb[k])
- fputs_unlocked("\33[31m", f);
- fputc_unlocked(ba[k], f);
+ fputs("\33[31m", f);
+ fputc(ba[k], f);
if (ba[k] != bb[k])
- fputs_unlocked("\33[0m", f);
+ fputs("\33[0m", f);
}
}
fprintf(f, "\n");
}
- funlockfile(f);
}
template <typename T>
@@ -289,7 +288,7 @@ void check(double tol, //
"denormals=%d)\n",
avg, worst, sad, nans, infs, flips, zeroes, denormals);
} else {
- flockfile(stderr);
+ std::unique_lock<std::recursive_mutex> lock(g_log_lock);
misfit(stderr, 16, m, n, A, lda, B, ldb, file, line, avg, sad, worst, infs, flips, zeroes,
denormals, nans);
const char *path = "/tmp/wompwomp.log";
@@ -318,7 +317,7 @@ void check(double tol, //
x; \
__asm__ volatile("" ::: "memory"); \
} \
- printf("%8lld µs %s\n", (micros() - start + ITERATIONS - 1) / ITERATIONS, #x); \
+ printf("%8lld us %s\n", (micros() - start + ITERATIONS - 1) / ITERATIONS, #x); \
} while (0)
#define CUDA_OR_DIE(x) \
@@ -344,9 +343,8 @@ void check(double tol, //
#define BENCH_CUDA(x) \
do { \
- if (is_self_testing) { \
- x; \
- } else { \
+ x; \
+ if (!is_self_testing) { \
cudaEvent_t start_, stop_; \
CUDA_OR_DIE(cudaEventCreate(&stop_)); \
CUDA_OR_DIE(cudaEventCreate(&start_)); \
@@ -357,7 +355,7 @@ void check(double tol, //
CUDA_OR_DIE(cudaEventRecord(stop_, stream)); \
CUDA_OR_DIE(cudaEventSynchronize(stop_)); \
CUDA_OR_DIE(cudaEventElapsedTime(&msecTotal_, start_, stop_)); \
- printf("%8d µs %s(%d, %d, %d)\n", \
+ printf("%8d us %s(%d, %d, %d)\n", \
static_cast<int>(std::ceil(msecTotal_ / ITERATIONS * 1000)), __FUNCTION__, m, \
n, k); \
CUDA_OR_DIE(cudaEventDestroy(start_)); \
@@ -404,16 +402,16 @@ template <> struct tinyblas_compute_type<float> {
};
template <typename T, typename TA, typename TB, typename TC>
-void cublas(bool aᵀ, bool bᵀ, //
- int m, int n, int k, T α, //
+void cublas(bool aT, bool bT, //
+ int m, int n, int k, T alpha, //
const TA *A, int lda, //
- const TB *B, int ldb, T β, //
+ const TB *B, int ldb, T beta, //
TC *C, int ldc) {
cudaStream_t stream;
cublasHandle_t blas;
assert(m >= 0 && n >= 0 && k >= 0);
- assert(lda >= std::max(1, aᵀ ? k : m));
- assert(ldb >= std::max(1, bᵀ ? n : k));
+ assert(lda >= std::max(1, aT ? k : m));
+ assert(ldb >= std::max(1, bT ? n : k));
assert(ldc >= std::max(1, m));
CUDA_OR_DIE(cudaSetDevice(0));
CUBLAS_OR_DIE(cublasCreate(&blas));
@@ -421,42 +419,42 @@ void cublas(bool aᵀ, bool bᵀ, //
CUBLAS_OR_DIE(cublasSetMathMode(blas, CUBLAS_DEFAULT_MATH));
CUBLAS_OR_DIE(cublasSetStream(blas, stream));
BENCH_CUDA(CUBLAS_OR_DIE(cublasGemmEx(
- blas, aᵀ ? CUBLAS_OP_T : CUBLAS_OP_N, bᵀ ? CUBLAS_OP_T : CUBLAS_OP_N, m, n, k, &α, A,
- cuda_data_type<TA>::id, lda, B, cuda_data_type<TB>::id, ldb, &β, C, cuda_data_type<TC>::id,
- ldc, cublas_compute_type<T>::id, CUBLAS_GEMM_DEFAULT)));
+ blas, aT ? CUBLAS_OP_T : CUBLAS_OP_N, bT ? CUBLAS_OP_T : CUBLAS_OP_N, m, n, k, &alpha, A,
+ cuda_data_type<TA>::id, lda, B, cuda_data_type<TB>::id, ldb, &beta, C,
+ cuda_data_type<TC>::id, ldc, cublas_compute_type<T>::id, CUBLAS_GEMM_DEFAULT)));
CUDA_OR_DIE(cudaStreamSynchronize(stream));
CUDA_OR_DIE(cudaStreamDestroy(stream));
CUBLAS_OR_DIE(cublasDestroy(blas));
}
template <typename T, typename TA, typename TB, typename TC>
-void cublasrm(bool aᵀ, bool bᵀ, //
- int m, int n, int k, T α, //
+void cublasrm(bool aT, bool bT, //
+ int m, int n, int k, T alpha, //
const TA *A, int lda, //
- const TB *B, int ldb, T β, //
+ const TB *B, int ldb, T beta, //
TC *C, int ldc) {
- cublas(bᵀ, aᵀ, n, m, k, α, B, ldb, A, lda, β, C, ldc);
+ cublas(bT, aT, n, m, k, alpha, B, ldb, A, lda, beta, C, ldc);
}
template <typename T, typename TA, typename TB, typename TC>
-void tinyblas(bool aᵀ, bool bᵀ, //
- int m, int n, int k, T α, //
+void tinyblas(bool aT, bool bT, //
+ int m, int n, int k, T alpha, //
const TA *A, int lda, //
- const TB *B, int ldb, T β, //
+ const TB *B, int ldb, T beta, //
TC *C, int ldc) {
cudaStream_t stream;
tinyblasHandle_t blas;
assert(m >= 0 && n >= 0 && k >= 0);
- assert(lda >= std::max(1, aᵀ ? k : m));
- assert(ldb >= std::max(1, bᵀ ? n : k));
+ assert(lda >= std::max(1, aT ? k : m));
+ assert(ldb >= std::max(1, bT ? n : k));
assert(ldc >= std::max(1, m));
CUDA_OR_DIE(cudaSetDevice(0));
TINYBLAS_OR_DIE(tinyblasCreate(&blas));
CUDA_OR_DIE(cudaStreamCreate(&stream));
TINYBLAS_OR_DIE(tinyblasSetStream(blas, stream));
BENCH_CUDA(TINYBLAS_OR_DIE(tinyblasGemmEx(
- blas, aᵀ ? TINYBLAS_OP_T : TINYBLAS_OP_N, bᵀ ? TINYBLAS_OP_T : TINYBLAS_OP_N, m, n, k, &α,
- A, tinyblas_data_type<TA>::id, lda, B, tinyblas_data_type<TB>::id, ldb, &β, C,
+ blas, aT ? TINYBLAS_OP_T : TINYBLAS_OP_N, bT ? TINYBLAS_OP_T : TINYBLAS_OP_N, m, n, k,
+ &alpha, A, tinyblas_data_type<TA>::id, lda, B, tinyblas_data_type<TB>::id, ldb, &beta, C,
tinyblas_data_type<TC>::id, ldc, tinyblas_compute_type<T>::id, TINYBLAS_GEMM_DEFAULT)));
CUDA_OR_DIE(cudaStreamSynchronize(stream));
CUDA_OR_DIE(cudaStreamDestroy(stream));
@@ -464,12 +462,12 @@ void tinyblas(bool aᵀ, bool bᵀ, //
}
template <typename T, typename TA, typename TB, typename TC>
-void tinyblasrm(bool aᵀ, bool bᵀ, //
- int m, int n, int k, T α, //
+void tinyblasrm(bool aT, bool bT, //
+ int m, int n, int k, T alpha, //
const TA *A, int lda, //
- const TB *B, int ldb, T β, //
+ const TB *B, int ldb, T beta, //
TC *C, int ldc) {
- tinyblas(bᵀ, aᵀ, n, m, k, α, B, ldb, A, lda, β, C, ldc);
+ tinyblas(bT, aT, n, m, k, alpha, B, ldb, A, lda, beta, C, ldc);
}
// multiplies matrix on gpu with column major ordering
@@ -480,19 +478,20 @@ void tinyblasrm(bool aᵀ, bool bᵀ, //
// k×m * n×k → m×n if aᵀ and bᵀ
//
template <typename T, typename TA, typename TB, typename TC>
-void gemmref(bool aᵀ, bool bᵀ, //
- int m, int n, int k, T α, //
+void gemmref(bool aT, bool bT, //
+ int m, int n, int k, T alpha, //
const TA *A, int lda, //
- const TB *B, int ldb, T β, //
+ const TB *B, int ldb, T beta, //
TC *C, int ldc) {
cudaStream_t stream;
assert(m >= 0 && n >= 0 && k >= 0);
- assert(lda >= std::max(1, aᵀ ? k : m));
- assert(ldb >= std::max(1, bᵀ ? n : k));
+ assert(lda >= std::max(1, aT ? k : m));
+ assert(ldb >= std::max(1, bT ? n : k));
assert(ldc >= std::max(1, m));
CUDA_OR_DIE(cudaSetDevice(0));
CUDA_OR_DIE(cudaStreamCreate(&stream));
- BENCH_CUDA(CUDA_OR_DIE(naive::gemm(stream, aᵀ, bᵀ, m, n, k, α, A, lda, B, ldb, β, C, ldc)));
+ BENCH_CUDA(
+ CUDA_OR_DIE(naive::gemm(stream, aT, bT, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc)));
CUDA_OR_DIE(cudaStreamSynchronize(stream));
CUDA_OR_DIE(cudaStreamDestroy(stream));
}
@@ -505,10 +504,10 @@ void gemmref(bool aᵀ, bool bᵀ, //
// m×k * k×n → n×m if aᵀ and bᵀ
//
template <typename T, typename TA, typename TB, typename TC>
-void gemmrefrm(bool aᵀ, bool bᵀ, //
- int m, int n, int k, T α, //
+void gemmrefrm(bool aT, bool bT, //
+ int m, int n, int k, T alpha, //
const TA *A, int lda, //
- const TB *B, int ldb, T β, //
+ const TB *B, int ldb, T beta, //
TC *C, int ldc) {
- gemmref(bᵀ, aᵀ, n, m, k, α, B, ldb, A, lda, β, C, ldc);
+ gemmref(bT, aT, n, m, k, alpha, B, ldb, A, lda, beta, C, ldc);
}
diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cu
index 0a3c6f36..f595461e 100644
--- a/llamafile/tinyblas.cu
+++ b/llamafile/tinyblas.cu
@@ -21,15 +21,14 @@
#include <cstdlib>
#include <type_traits>
-#if defined(__NVCC__) || defined(__CUDA_ARCH__)
+#ifndef __HIP__
#include <cuda_fp16.h>
#include <cuda_runtime.h>
-#define tinyblasStream_t cudaStream_t
#else
#include <hip/hip_fp16.h>
#include <hip/hip_runtime.h>
#define cudaSuccess hipSuccess
-#define tinyblasStream_t hipStream_t
+#define cudaStream_t hipStream_t
#define cudaGetLastError hipGetLastError
#endif
@@ -48,7 +47,7 @@
#define CEIL_DIV(M, N) (((M) + (N) - 1) / (N))
struct tinyblasContext {
- tinyblasStream_t stream;
+ cudaStream_t stream;
};
tinyblasStatus_t tinyblasCreate(tinyblasHandle_t *out_handle) {
@@ -67,7 +66,7 @@ tinyblasStatus_t tinyblasDestroy(tinyblasHandle_t handle) {
}
tinyblasStatus_t tinyblasSetStream(tinyblasHandle_t handle, void *stream) {
- handle->stream = (tinyblasStream_t)stream;
+ handle->stream = (cudaStream_t)stream;
return TINYBLAS_STATUS_SUCCESS;
}
@@ -109,10 +108,6 @@ const char *tinyblasGetStatusString(tinyblasStatus_t err) {
}
}
-__device__ __forceinline__ float madd(float x, float y, float z) {
- return __fmaf_rn(x, y, z);
-}
-
template <int BM, int BN, int TM, int TN, typename WORD, typename SRC, typename DST>
static __device__ void matmul_block2d(tinyblasOperation_t transa, tinyblasOperation_t transb, int m,
int n, int k, WORD alpha, const SRC *A, int lda, const SRC *B,
@@ -162,7 +157,7 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, tinyblasOperat
WORD a = At[j];
for (int h = 0; h < TN; ++h) {
WORD b = Bt[h];
- Cs[TN * j + h] = madd(a, b, Cs[TN * j + h]);
+ Cs[TN * j + h] += a * b;
}
}
}
@@ -178,7 +173,7 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, tinyblasOperat
for (int i = 0; i < TM && ii + ti + i < m; ++i)
if (beta) {
WORD c = C[ldc * (jj + tj + j) + (ii + ti + i)];
- C[ldc * (jj + tj + j) + (ii + ti + i)] = madd(c, beta, Cs[TN * i + j]);
+ C[ldc * (jj + tj + j) + (ii + ti + i)] = c * beta + Cs[TN * i + j];
} else {
C[ldc * (jj + tj + j) + (ii + ti + i)] = Cs[TN * i + j];
}
@@ -417,9 +412,9 @@ static __global__ void KERNEL tinyblasGSBE_entry(tinyblasOperation_t transa,
long long strideA, const SRC *B, int ldb,
long long strideB, WORD beta, DST *C, int ldc,
long long strideC, int batchCount) {
- for (int z = blockIdx.z; z < batchCount; z += gridDim.z)
- matmul_block2d<BM, BN, TM, TN>(transa, transb, m, n, k, alpha, A + z * strideA, lda,
- B + z * strideB, ldb, beta, C + z * strideC, ldc);
+ matmul_block2d<BM, BN, TM, TN>(transa, transb, m, n, k, alpha, A + blockIdx.z * strideA, lda,
+ B + blockIdx.z * strideB, ldb, beta, C + blockIdx.z * strideC,
+ ldc);
}
template <typename WORD, typename SRC, typename DST>
diff --git a/llamafile/tinyblas.h b/llamafile/tinyblas.h
index 55de17ae..38c02db0 100644
--- a/llamafile/tinyblas.h
+++ b/llamafile/tinyblas.h
@@ -2,33 +2,33 @@
// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
#pragma once
-enum tinyblasOperation_t {
+typedef enum tinyblasOperation {
TINYBLAS_OP_N,
TINYBLAS_OP_T,
-};
+} tinyblasOperation_t;
-enum tinyblasDataType_t {
+typedef enum tinyblasDataType {
TINYBLAS_R_32F,
TINYBLAS_R_16F,
-};
+} tinyblasDataType_t;
-enum tinyblasComputeType_t {
+typedef enum tinyblasComputeType {
TINYBLAS_COMPUTE_32F,
TINYBLAS_COMPUTE_16F,
-};
+} tinyblasComputeType_t;
-enum tinyblasGemmAlgo_t {
+typedef enum tinyblasGemmAlgo {
TINYBLAS_GEMM_DEFAULT,
-};
+} tinyblasGemmAlgo_t;
-enum tinyblasStatus_t {
+typedef enum tinyblasStatus {
TINYBLAS_STATUS_SUCCESS,
TINYBLAS_STATUS_ALLOC_FAILED,
TINYBLAS_STATUS_INVALID_VALUE,
TINYBLAS_STATUS_NOT_SUPPORTED,
TINYBLAS_STATUS_EXECUTION_FAILED,
TINYBLAS_STATUS_DIMENSION_OVERFLOW,
-};
+} tinyblasStatus_t;
struct tinyblasContext;
typedef struct tinyblasContext *tinyblasHandle_t;
diff --git a/llamafile/tinyblas_test.cu b/llamafile/tinyblas_test.cu
index 3dbcac0d..e49a1704 100644
--- a/llamafile/tinyblas_test.cu
+++ b/llamafile/tinyblas_test.cu
@@ -21,8 +21,8 @@ void checkTinyblasWorksHHHH() {
double TOLERANCE = .5;
{
- half α = 1;
- half β = 0;
+ half alpha = 1;
+ half beta = 0;
int m = 577;
int n = 577;
int k = 64;
@@ -35,17 +35,17 @@ void checkTinyblasWorksHHHH() {
cuda_memory<half> G{ldc * n};
randomize(k, m, A.p, lda);
randomize(k, n, B.p, ldb);
- cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ cublas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
broadcast(m, n, C.p, ldc, TOMBSTONE);
broadcast(m, n, G.p, ldc, TOMBSTONE);
- gemmref<float>(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
- tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ gemmref<float>(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
}
{
- half α = 1;
- half β = 0;
+ half alpha = 1;
+ half beta = 0;
int m = 4096;
int n = 1024;
int k = 2048;
@@ -58,17 +58,17 @@ void checkTinyblasWorksHHHH() {
cuda_memory<half> G{ldc * n};
randomize(k, m, A.p, lda);
randomize(k, n, B.p, ldb);
- cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ cublas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
broadcast(m, n, C.p, ldc, TOMBSTONE);
broadcast(m, n, G.p, ldc, TOMBSTONE);
- gemmref<float>(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
- tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ gemmref<float>(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
}
{
- half α = 1;
- half β = 0;
+ half alpha = 1;
+ half beta = 0;
int m = 4096;
int n = 1024;
int k = 577;
@@ -81,15 +81,15 @@ void checkTinyblasWorksHHHH() {
cuda_memory<half> G{ldc * n};
randomize(k, m, A.p, lda);
randomize(k, n, B.p, ldb);
- cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ cublas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
broadcast(m, n, C.p, ldc, TOMBSTONE);
broadcast(m, n, G.p, ldc, TOMBSTONE);
- gemmref<float>(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
- tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ gemmref<float>(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
}
- test_matmul([&](int m, int n, int k, int l, half α, half β) {
+ test_matmul([&](int m, int n, int k, int l, half alpha, half beta) {
int lda = m + l;
int ldb = k + l;
int ldc = m + l;
@@ -101,8 +101,8 @@ void checkTinyblasWorksHHHH() {
randomize(k, n, B.p, ldb);
broadcast(m, n, G.p, ldc, TOMBSTONE);
broadcast(m, n, C.p, ldc, TOMBSTONE);
- gemmref<float>(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
- tinyblas(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ gemmref<float>(false, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ tinyblas(false, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
});
}
@@ -110,8 +110,8 @@ void checkTinyblasWorksHHHH() {
void checkTinyblasWorksHHHS() {
double TOLERANCE = .5;
- float α = 1;
- float β = 0;
+ float alpha = 1;
+ float beta = 0;
int m = 4096;
int n = 1024;
int k = 577;
@@ -124,14 +124,14 @@ void checkTinyblasWorksHHHS() {
cuda_memory<half> G{ldc * n};
randomize(k, m, A.p, lda);
randomize(k, n, B.p, ldb);
- cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ cublas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
broadcast(m, n, C.p, ldc, TOMBSTONE);
broadcast(m, n, G.p, ldc, TOMBSTONE);
- gemmref(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
- tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ gemmref(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
- test_matmul([&](int m, int n, int k, int l, float α, float β) {
+ test_matmul([&](int m, int n, int k, int l, float alpha, float beta) {
int lda = m + l;
int ldb = k + l;
int ldc = m + l;
@@ -143,8 +143,8 @@ void checkTinyblasWorksHHHS() {
randomize(k, n, B.p, ldb);
broadcast(m, n, G.p, ldc, TOMBSTONE);
broadcast(m, n, C.p, ldc, TOMBSTONE);
- gemmref(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
- tinyblas(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ gemmref(false, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ tinyblas(false, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
});
}
@@ -152,8 +152,8 @@ void checkTinyblasWorksHHHS() {
void checkTinyblasWorksHHSS() {
double TOLERANCE = .5;
- float α = 1;
- float β = 0;
+ float alpha = 1;
+ float beta = 0;
int m = 4096;
int n = 1024;
int k = 1536;
@@ -166,14 +166,14 @@ void checkTinyblasWorksHHSS() {
cuda_memory<float> G{ldc * n};
randomize(k, m, A.p, lda);
randomize(k, n, B.p, ldb);
- cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ cublas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
broadcast(m, n, G.p, ldc, TOMBSTONE);
broadcast(m, n, C.p, ldc, TOMBSTONE);
- gemmref(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
- tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ gemmref(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
- test_matmul([&](int m, int n, int k, int l, float α, float β) {
+ test_matmul([&](int m, int n, int k, int l, float alpha, float beta) {
int lda = k + l;
int ldb = k + l;
int ldc = m + l;
@@ -185,8 +185,8 @@ void checkTinyblasWorksHHSS() {
randomize(k, n, B.p, ldb);
broadcast(m, n, G.p, ldc, TOMBSTONE);
broadcast(m, n, C.p, ldc, TOMBSTONE);
- gemmref(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
- tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ gemmref(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
});
}
@@ -194,8 +194,8 @@ void checkTinyblasWorksHHSS() {
void checkTinyblasWorksSSSS() {
double TOLERANCE = .5;
- float α = 1;
- float β = 0;
+ float alpha = 1;
+ float beta = 0;
int m = 4096;
int n = 1024;
int k = 1020;
@@ -208,14 +208,14 @@ void checkTinyblasWorksSSSS() {
cuda_memory<float> G{ldc * n};
randomize(k, m, A.p, lda);
randomize(k, n, B.p, ldb);
- cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
+ cublas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
broadcast(m, n, C.p, ldc, TOMBSTONE);
broadcast(m, n, G.p, ldc, TOMBSTONE);
- gemmref(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
- tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ gemmref(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
- test_matmul([&](int m, int n, int k, int l, float α, float β) {
+ test_matmul([&](int m, int n, int k, int l, float alpha, float beta) {
int lda = m + l;
int ldb = k + l;
int ldc = m + l;
@@ -227,13 +227,39 @@ void checkTinyblasWorksSSSS() {
randomize(k, n, B.p, ldb);
broadcast(m, n, G.p, ldc, TOMBSTONE);
broadcast(m, n, C.p, ldc, TOMBSTONE);
- gemmref(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);
- tinyblas(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);
+ gemmref(false, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ tinyblas(false, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
});
}
int main(int argc, char *argv[]) {
+
+ is_self_testing = "hi";
+
+ float alpha = 1;
+ float beta = 0;
+ int m = 48;
+ int n = 32;
+ int k = 64;
+ int lda = k;
+ int ldb = k;
+ int ldc = m;
+ cuda_memory<half> A{lda * m};
+ cuda_memory<half> B{ldb * n};
+ cuda_memory<half> C{ldc * n};
+ cuda_memory<half> G{ldc * n};
+ randomize(k, m, A.p, lda);
+ randomize(k, n, B.p, ldb);
+ broadcast(m, n, C.p, ldc, TOMBSTONE);
+ broadcast(m, n, G.p, ldc, TOMBSTONE);
+ gemmref(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
+ CHECK(0, m, n, k, G.p, ldc, C.p, ldc);
+
+ if (1)
+ return 0;
+
RUN(checkTinyblasWorksHHHH());
RUN(checkTinyblasWorksSSSS());
RUN(checkTinyblasWorksHHSS());