Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Port tinyBLAS code to MSVC
Justine Tunneyf33895292ad3
diff --git a/llamafile/cuda.h b/llamafile/cuda.hindex babd08b7..b9066f82 100644--- a/llamafile/cuda.h+++ b/llamafile/cuda.h@@ -2,7 +2,7 @@// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi#pragma once-#if defined(__NVCC__) || defined(__CUDA_ARCH__)+#ifndef __HIP__#include <cublas_v2.h>#include <cuda_fp16.h>#include <cuda_runtime.h>diff --git a/llamafile/cudaprops.cu b/llamafile/cudaprops.cuindex 3ec087c9..1578b8b3 100644--- a/llamafile/cudaprops.cu+++ b/llamafile/cudaprops.cu@@ -19,10 +19,10 @@#include "tester.h"#include <iostream>-#if defined(__NVCC__) || defined(__CUDA_ARCH__)-#define VENDOR "NVIDIA"-#else+#ifdef __HIP__#define VENDOR "AMD"+#else+#define VENDOR "NVIDIA"#endifnamespace {diff --git a/llamafile/flt.h b/llamafile/flt.hindex b50df4ab..ea0cc607 100644--- a/llamafile/flt.h+++ b/llamafile/flt.h@@ -10,7 +10,7 @@//// @see IEEE 754-2008-#if defined(__NVCC__) || defined(__CUDA_ARCH__)+#ifndef __HIP__#include <cuda_fp16.h>#else#include <hip/hip_fp16.h>diff --git a/llamafile/gemm.h b/llamafile/gemm.hindex e9c3a660..a26a3d32 100644--- a/llamafile/gemm.h+++ b/llamafile/gemm.h@@ -29,27 +29,27 @@// k×m * n×k → m×n if aᵀ and bᵀ//template <typename T, typename TA, typename TB, typename TC>-void gemm(bool aᵀ, bool bᵀ, //- int m, int n, int k, T α, //+void gemm(bool aT, bool bT, //+ int m, int n, int k, T alpha, //const TA *A, int lda, //- const TB *B, int ldb, T β, //+ const TB *B, int ldb, T beta, //TC *C, int ldc) {assert(m >= 0 && n >= 0 && k >= 0);- assert(lda >= std::max(1, aᵀ ? k : m));- assert(ldb >= std::max(1, bᵀ ? n : k));+ assert(lda >= std::max(1, aT ? k : m));+ assert(ldb >= std::max(1, bT ? n : k));assert(ldc >= std::max(1, m));- assert(1ll * lda * (aᵀ ? m : k) <= INT_MAX);- assert(1ll * ldb * (bᵀ ? k : n) <= INT_MAX);+ assert(1ll * lda * (aT ? m : k) <= INT_MAX);+ assert(1ll * ldb * (bT ? k : n) <= INT_MAX);for (int i = 0; i < m; ++i)for (int j = 0; j < n; ++j) {T d = 0;for (int l = 0; l < k; ++l) {- T a = aᵀ ? A[lda * i + l] : A[lda * l + i];- T b = bᵀ ? B[ldb * l + j] : B[ldb * j + l];+ T a = aT ? A[lda * i + l] : A[lda * l + i];+ T b = bT ? B[ldb * l + j] : B[ldb * j + l];d = std::fma(a, b, d);}T c = C[ldc * j + i];- C[ldc * j + i] = std::fma(α, d, β * c);+ C[ldc * j + i] = std::fma(alpha, d, beta * c);}}@@ -61,10 +61,10 @@ void gemm(bool aᵀ, bool bᵀ, //// m×k * k×n → n×m if aᵀ and bᵀ//template <typename T, typename TA, typename TB, typename TC>-void gemmrm(bool aᵀ, bool bᵀ, //- int m, int n, int k, T α, //+void gemmrm(bool aT, bool bT, //+ int m, int n, int k, T alpha, //const TA *A, int lda, //- const TB *B, int ldb, T β, //+ const TB *B, int ldb, T beta, //TC *C, int ldc) {- gemm(bᵀ, aᵀ, n, m, k, α, B, ldb, A, lda, β, C, ldc);+ gemm(bT, aT, n, m, k, alpha, B, ldb, A, lda, beta, C, ldc);}diff --git a/llamafile/naive.h b/llamafile/naive.hindex de45d328..730f9f46 100644--- a/llamafile/naive.h+++ b/llamafile/naive.h@@ -16,6 +16,7 @@// limitations under the License.#pragma once+#include "cuda.h"#include "macros.h"//@@ -34,37 +35,30 @@// can be used for finding bugs in more advanced matmul implementations.namespace naive {--__device__ __forceinline__ half madd(half x, half y, half z) {- return (float)x * (float)y + (float)z;-}-__device__ __forceinline__ float madd(float x, float y, float z) {- return __fmaf_rn(x, y, z);-}-__device__ __forceinline__ double madd(double x, double y, double z) {- return __fma_rn(x, y, z);-}+namespace {template <typename T, typename TA, typename TB, typename TC>-__global__ void kernel(bool aᵀ, bool bᵀ, //- int m, int n, int k, T α, //+__global__ void kernel(bool aT, bool bT, //+ int m, int n, int k, T alpha, //const TA *A, int lda, //- const TB *B, int ldb, T β, //+ const TB *B, int ldb, T beta, //TC *C, int ldc) {const int i = blockIdx.x * blockDim.x + threadIdx.x;const int j = blockIdx.y * blockDim.y + threadIdx.y;if (i < m && j < n) {T d = 0;for (int l = 0; l < k; ++l) {- T a = aᵀ ? A[lda * i + l] : A[lda * l + i];- T b = bᵀ ? B[ldb * l + j] : B[ldb * j + l];- d = madd(a, b, d);+ T a = aT ? A[lda * i + l] : A[lda * l + i];+ T b = bT ? B[ldb * l + j] : B[ldb * j + l];+ d += a * b;}T c = C[ldc * j + i];- C[ldc * j + i] = madd(α, d, β * c);+ C[ldc * j + i] = alpha * d + beta * c;}}+} // namespace+// multiplies matrix on gpu with column major ordering//// m×k * k×n → m×n@@ -74,14 +68,14 @@ __global__ void kernel(bool aᵀ, bool bᵀ, ////template <typename T, typename TA, typename TB, typename TC>cudaError_t gemm(cudaStream_t stream, //- bool aᵀ, bool bᵀ, //- int m, int n, int k, T α, //+ bool aT, bool bT, //+ int m, int n, int k, T alpha, //const TA *A, int lda, //- const TB *B, int ldb, T β, //+ const TB *B, int ldb, T beta, //TC *C, int ldc) {dim3 blockDim(32, 32);dim3 gridDim(CEIL_DIV(m, 32), CEIL_DIV(n, 32));- kernel<<<gridDim, blockDim, 0, stream>>>(aᵀ, bᵀ, m, n, k, α, A, lda, B, ldb, β, C, ldc);+ kernel<<<gridDim, blockDim, 0, stream>>>(aT, bT, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc);return cudaGetLastError();}diff --git a/llamafile/tester.cu b/llamafile/tester.cuindex 83955e89..f4e12698 100644--- a/llamafile/tester.cu+++ b/llamafile/tester.cu@@ -17,18 +17,25 @@#include "tester.h"+#include <algorithm>#include <atomic>#include <ctime>++#ifndef _WIN32#include <unistd.h>+#else+#include "windows.h"+#endif+std::recursive_mutex g_log_lock;thread_local const char *is_self_testing;-const size_t kPageSize = std::max(sysconf(_SC_PAGESIZE), 4096l);-long long micros(void) {- struct timespec ts;- clock_gettime(CLOCK_REALTIME, &ts);- return ts.tv_sec * 1000000 + (ts.tv_nsec + 999) / 1000;-}+const int kPageSize =+#ifndef _WIN32+ std::max(sysconf(_SC_PAGESIZE), 4096l);+#else+ 4096;+#endifint rand32(void) {static unsigned long long lcg = 1;@@ -45,14 +52,51 @@ float numba(void) { // (-1,1)return float01(rand32()) * 2 - 1;}+int hamming(int x, int y) {+ return popcount(x ^ y);+}++int popcount(unsigned x) {+ x = x - ((x >> 1) & 0x55555555);+ x = ((x >> 2) & 0x33333333) + (x & 0x33333333);+ x = (x + (x >> 4)) & 0x0F0F0F0F;+ x = (x + (x >> 16));+ return (x + (x >> 8)) & 0x0000003F;+}++void cudaFreeOrDie(void *p) {+ CUDA_OR_DIE(cudaFree(p));+}+void *cudaMallocManagedOrDie(size_t n) {void *p;CUDA_OR_DIE(cudaMallocManaged(&p, n));return p;}-void cudaFreeOrDie(void *p) {- CUDA_OR_DIE(cudaFree(p));+#ifdef _WIN32+static long long GetQueryPerformanceFrequency() {+ LARGE_INTEGER t;+ QueryPerformanceFrequency(&t);+ return t.QuadPart;+}+static long long GetQueryPerformanceCounter() {+ LARGE_INTEGER t;+ QueryPerformanceCounter(&t);+ return t.QuadPart;+}+#endif++long long micros(void) {+#ifndef _WIN32+ struct timespec ts;+ clock_gettime(CLOCK_REALTIME, &ts);+ return ts.tv_sec * 1000000 + (ts.tv_nsec + 999) / 1000;+#else+ static long long timer_freq = GetQueryPerformanceFrequency();+ static long long timer_start = GetQueryPerformanceCounter();+ return ((GetQueryPerformanceCounter() - timer_start) * 1000000) / timer_freq;+#endif}[[noreturn]] void cuda_die(const char *stmt, const char *func, const char *file, int line,@@ -66,7 +110,7 @@ void cudaFreeOrDie(void *p) {_Exit(1);}-void test_matmul(std::function<void(int m, int n, int k, int l, float α, float β)> f) {+void test_matmul(std::function<void(int m, int n, int k, int l, float alpha, float beta)> f) {static const int kDims[] = {1, 2, 23, 77, 15, 2048, 512, 127, 129, 128, 16};static const float kAlphas[] = {1, .1};static const float kBetas[] = {0, .1};@@ -90,24 +134,24 @@ void test_matmul(std::function<void(int m, int n, int k, int l, float α, float}for (int ai = 0; ai < ARRAYLEN(kAlphas); ++ai)for (int bi = 0; bi < ARRAYLEN(kBetas); ++bi) {- float α = kAlphas[ai];- float β = kBetas[bi];+ float alpha = kAlphas[ai];+ float beta = kBetas[bi];int m = 128;int n = 128;int k = 128;char name[256];- sprintf(name, "testing %4d %4d %4d α=%g β=%g", m, n, k, α, β);+ sprintf(name, "testing %4d %4d %4d alpha=%g beta=%g", m, n, k, alpha, beta);if (t++ % 7 == 0)fprintf(stderr, "%s\r", name);is_self_testing = name;- f(m, n, k, 0, α, β);+ f(m, n, k, 0, alpha, beta);is_self_testing = 0;}}static int cuda_tester_init() {CUDA_OR_DIE(cudaSetDevice(0));-#ifdef __HIP_PLATFORM_AMD__+#ifdef __HIP__rocblas_initialize();#endifCUDA_OR_DIE(cudaDeviceSynchronize());diff --git a/llamafile/tester.h b/llamafile/tester.hindex 8c816edb..1d672496 100644--- a/llamafile/tester.h+++ b/llamafile/tester.h@@ -27,6 +27,7 @@#include <cstdio>#include <cstdlib>#include <functional>+#include <mutex>//// _ _ ___ _ _ ___@@ -46,20 +47,19 @@#define ITERATIONS 30#define TOMBSTONE 1.666f-extern const size_t kPageSize;+extern const int kPageSize;+extern std::recursive_mutex g_log_lock;extern thread_local const char *is_self_testing;float float01(unsigned);float numba(void);+int hamming(int, int);+int popcount(unsigned);int rand32(void);long long micros(void);-void test_matmul(std::function<void(int, int, int, int, float, float)>);void *cudaMallocManagedOrDie(size_t);void cudaFreeOrDie(void *);--inline int hamming(int x, int y) {- return __builtin_popcount(x ^ y);-}+void test_matmul(std::function<void(int, int, int, int, float, float)>);template <typename T> void randomize(T *A, int n) {for (int i = 0; i < n; ++i)@@ -141,7 +141,7 @@ void verify(double tol, int m, int n, const T *A, int lda, const T *B, int ldb)}double avg = static_cast<double>(diffsum) / (m * n);if (avg > tol || nans) {- flockfile(stderr);+ std::unique_lock<std::recursive_mutex> lock(g_log_lock);fprintf(stderr, "error: %d×%d matrix difference exceeds tolerance\n", m, n);fprintf(stderr, " worst element at index (%d, %d)\n", worst_i, worst_j);fprintf(stderr, " difference %14e (%lld ulp)\n",@@ -224,7 +224,7 @@ void diff(int m, int n, int k, const T *Wan, int lda, const T *Got, int ldb, doutemplate <typename T>void show(FILE *f, int max, int m, int n, const T *A, int lda, const T *B, int ldb) {- flockfile(f);+ std::unique_lock<std::recursive_mutex> lock(g_log_lock);fprintf(f, " ");for (int i = 0; i < n && i < max; ++i)fprintf(f, "%13d", i);@@ -245,15 +245,14 @@ void show(FILE *f, int max, int m, int n, const T *A, int lda, const T *B, int lsnprintf(bb, 32, "%13.7f", static_cast<double>(B[ldb * j + i]));for (int k = 0; ba[k] && bb[k]; ++k) {if (ba[k] != bb[k])- fputs_unlocked("\33[31m", f);- fputc_unlocked(ba[k], f);+ fputs("\33[31m", f);+ fputc(ba[k], f);if (ba[k] != bb[k])- fputs_unlocked("\33[0m", f);+ fputs("\33[0m", f);}}fprintf(f, "\n");}- funlockfile(f);}template <typename T>@@ -289,7 +288,7 @@ void check(double tol, //"denormals=%d)\n",avg, worst, sad, nans, infs, flips, zeroes, denormals);} else {- flockfile(stderr);+ std::unique_lock<std::recursive_mutex> lock(g_log_lock);misfit(stderr, 16, m, n, A, lda, B, ldb, file, line, avg, sad, worst, infs, flips, zeroes,denormals, nans);const char *path = "/tmp/wompwomp.log";@@ -318,7 +317,7 @@ void check(double tol, //x; \__asm__ volatile("" ::: "memory"); \} \- printf("%8lld µs %s\n", (micros() - start + ITERATIONS - 1) / ITERATIONS, #x); \+ printf("%8lld us %s\n", (micros() - start + ITERATIONS - 1) / ITERATIONS, #x); \} while (0)#define CUDA_OR_DIE(x) \@@ -344,9 +343,8 @@ void check(double tol, //#define BENCH_CUDA(x) \do { \- if (is_self_testing) { \- x; \- } else { \+ x; \+ if (!is_self_testing) { \cudaEvent_t start_, stop_; \CUDA_OR_DIE(cudaEventCreate(&stop_)); \CUDA_OR_DIE(cudaEventCreate(&start_)); \@@ -357,7 +355,7 @@ void check(double tol, //CUDA_OR_DIE(cudaEventRecord(stop_, stream)); \CUDA_OR_DIE(cudaEventSynchronize(stop_)); \CUDA_OR_DIE(cudaEventElapsedTime(&msecTotal_, start_, stop_)); \- printf("%8d µs %s(%d, %d, %d)\n", \+ printf("%8d us %s(%d, %d, %d)\n", \static_cast<int>(std::ceil(msecTotal_ / ITERATIONS * 1000)), __FUNCTION__, m, \n, k); \CUDA_OR_DIE(cudaEventDestroy(start_)); \@@ -404,16 +402,16 @@ template <> struct tinyblas_compute_type<float> {};template <typename T, typename TA, typename TB, typename TC>-void cublas(bool aᵀ, bool bᵀ, //- int m, int n, int k, T α, //+void cublas(bool aT, bool bT, //+ int m, int n, int k, T alpha, //const TA *A, int lda, //- const TB *B, int ldb, T β, //+ const TB *B, int ldb, T beta, //TC *C, int ldc) {cudaStream_t stream;cublasHandle_t blas;assert(m >= 0 && n >= 0 && k >= 0);- assert(lda >= std::max(1, aᵀ ? k : m));- assert(ldb >= std::max(1, bᵀ ? n : k));+ assert(lda >= std::max(1, aT ? k : m));+ assert(ldb >= std::max(1, bT ? n : k));assert(ldc >= std::max(1, m));CUDA_OR_DIE(cudaSetDevice(0));CUBLAS_OR_DIE(cublasCreate(&blas));@@ -421,42 +419,42 @@ void cublas(bool aᵀ, bool bᵀ, //CUBLAS_OR_DIE(cublasSetMathMode(blas, CUBLAS_DEFAULT_MATH));CUBLAS_OR_DIE(cublasSetStream(blas, stream));BENCH_CUDA(CUBLAS_OR_DIE(cublasGemmEx(- blas, aᵀ ? CUBLAS_OP_T : CUBLAS_OP_N, bᵀ ? CUBLAS_OP_T : CUBLAS_OP_N, m, n, k, &α, A,- cuda_data_type<TA>::id, lda, B, cuda_data_type<TB>::id, ldb, &β, C, cuda_data_type<TC>::id,- ldc, cublas_compute_type<T>::id, CUBLAS_GEMM_DEFAULT)));+ blas, aT ? CUBLAS_OP_T : CUBLAS_OP_N, bT ? CUBLAS_OP_T : CUBLAS_OP_N, m, n, k, &alpha, A,+ cuda_data_type<TA>::id, lda, B, cuda_data_type<TB>::id, ldb, &beta, C,+ cuda_data_type<TC>::id, ldc, cublas_compute_type<T>::id, CUBLAS_GEMM_DEFAULT)));CUDA_OR_DIE(cudaStreamSynchronize(stream));CUDA_OR_DIE(cudaStreamDestroy(stream));CUBLAS_OR_DIE(cublasDestroy(blas));}template <typename T, typename TA, typename TB, typename TC>-void cublasrm(bool aᵀ, bool bᵀ, //- int m, int n, int k, T α, //+void cublasrm(bool aT, bool bT, //+ int m, int n, int k, T alpha, //const TA *A, int lda, //- const TB *B, int ldb, T β, //+ const TB *B, int ldb, T beta, //TC *C, int ldc) {- cublas(bᵀ, aᵀ, n, m, k, α, B, ldb, A, lda, β, C, ldc);+ cublas(bT, aT, n, m, k, alpha, B, ldb, A, lda, beta, C, ldc);}template <typename T, typename TA, typename TB, typename TC>-void tinyblas(bool aᵀ, bool bᵀ, //- int m, int n, int k, T α, //+void tinyblas(bool aT, bool bT, //+ int m, int n, int k, T alpha, //const TA *A, int lda, //- const TB *B, int ldb, T β, //+ const TB *B, int ldb, T beta, //TC *C, int ldc) {cudaStream_t stream;tinyblasHandle_t blas;assert(m >= 0 && n >= 0 && k >= 0);- assert(lda >= std::max(1, aᵀ ? k : m));- assert(ldb >= std::max(1, bᵀ ? n : k));+ assert(lda >= std::max(1, aT ? k : m));+ assert(ldb >= std::max(1, bT ? n : k));assert(ldc >= std::max(1, m));CUDA_OR_DIE(cudaSetDevice(0));TINYBLAS_OR_DIE(tinyblasCreate(&blas));CUDA_OR_DIE(cudaStreamCreate(&stream));TINYBLAS_OR_DIE(tinyblasSetStream(blas, stream));BENCH_CUDA(TINYBLAS_OR_DIE(tinyblasGemmEx(- blas, aᵀ ? TINYBLAS_OP_T : TINYBLAS_OP_N, bᵀ ? TINYBLAS_OP_T : TINYBLAS_OP_N, m, n, k, &α,- A, tinyblas_data_type<TA>::id, lda, B, tinyblas_data_type<TB>::id, ldb, &β, C,+ blas, aT ? TINYBLAS_OP_T : TINYBLAS_OP_N, bT ? TINYBLAS_OP_T : TINYBLAS_OP_N, m, n, k,+ &alpha, A, tinyblas_data_type<TA>::id, lda, B, tinyblas_data_type<TB>::id, ldb, &beta, C,tinyblas_data_type<TC>::id, ldc, tinyblas_compute_type<T>::id, TINYBLAS_GEMM_DEFAULT)));CUDA_OR_DIE(cudaStreamSynchronize(stream));CUDA_OR_DIE(cudaStreamDestroy(stream));@@ -464,12 +462,12 @@ void tinyblas(bool aᵀ, bool bᵀ, //}template <typename T, typename TA, typename TB, typename TC>-void tinyblasrm(bool aᵀ, bool bᵀ, //- int m, int n, int k, T α, //+void tinyblasrm(bool aT, bool bT, //+ int m, int n, int k, T alpha, //const TA *A, int lda, //- const TB *B, int ldb, T β, //+ const TB *B, int ldb, T beta, //TC *C, int ldc) {- tinyblas(bᵀ, aᵀ, n, m, k, α, B, ldb, A, lda, β, C, ldc);+ tinyblas(bT, aT, n, m, k, alpha, B, ldb, A, lda, beta, C, ldc);}// multiplies matrix on gpu with column major ordering@@ -480,19 +478,20 @@ void tinyblasrm(bool aᵀ, bool bᵀ, //// k×m * n×k → m×n if aᵀ and bᵀ//template <typename T, typename TA, typename TB, typename TC>-void gemmref(bool aᵀ, bool bᵀ, //- int m, int n, int k, T α, //+void gemmref(bool aT, bool bT, //+ int m, int n, int k, T alpha, //const TA *A, int lda, //- const TB *B, int ldb, T β, //+ const TB *B, int ldb, T beta, //TC *C, int ldc) {cudaStream_t stream;assert(m >= 0 && n >= 0 && k >= 0);- assert(lda >= std::max(1, aᵀ ? k : m));- assert(ldb >= std::max(1, bᵀ ? n : k));+ assert(lda >= std::max(1, aT ? k : m));+ assert(ldb >= std::max(1, bT ? n : k));assert(ldc >= std::max(1, m));CUDA_OR_DIE(cudaSetDevice(0));CUDA_OR_DIE(cudaStreamCreate(&stream));- BENCH_CUDA(CUDA_OR_DIE(naive::gemm(stream, aᵀ, bᵀ, m, n, k, α, A, lda, B, ldb, β, C, ldc)));+ BENCH_CUDA(+ CUDA_OR_DIE(naive::gemm(stream, aT, bT, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc)));CUDA_OR_DIE(cudaStreamSynchronize(stream));CUDA_OR_DIE(cudaStreamDestroy(stream));}@@ -505,10 +504,10 @@ void gemmref(bool aᵀ, bool bᵀ, //// m×k * k×n → n×m if aᵀ and bᵀ//template <typename T, typename TA, typename TB, typename TC>-void gemmrefrm(bool aᵀ, bool bᵀ, //- int m, int n, int k, T α, //+void gemmrefrm(bool aT, bool bT, //+ int m, int n, int k, T alpha, //const TA *A, int lda, //- const TB *B, int ldb, T β, //+ const TB *B, int ldb, T beta, //TC *C, int ldc) {- gemmref(bᵀ, aᵀ, n, m, k, α, B, ldb, A, lda, β, C, ldc);+ gemmref(bT, aT, n, m, k, alpha, B, ldb, A, lda, beta, C, ldc);}diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cuindex 0a3c6f36..f595461e 100644--- a/llamafile/tinyblas.cu+++ b/llamafile/tinyblas.cu@@ -21,15 +21,14 @@#include <cstdlib>#include <type_traits>-#if defined(__NVCC__) || defined(__CUDA_ARCH__)+#ifndef __HIP__#include <cuda_fp16.h>#include <cuda_runtime.h>-#define tinyblasStream_t cudaStream_t#else#include <hip/hip_fp16.h>#include <hip/hip_runtime.h>#define cudaSuccess hipSuccess-#define tinyblasStream_t hipStream_t+#define cudaStream_t hipStream_t#define cudaGetLastError hipGetLastError#endif@@ -48,7 +47,7 @@#define CEIL_DIV(M, N) (((M) + (N) - 1) / (N))struct tinyblasContext {- tinyblasStream_t stream;+ cudaStream_t stream;};tinyblasStatus_t tinyblasCreate(tinyblasHandle_t *out_handle) {@@ -67,7 +66,7 @@ tinyblasStatus_t tinyblasDestroy(tinyblasHandle_t handle) {}tinyblasStatus_t tinyblasSetStream(tinyblasHandle_t handle, void *stream) {- handle->stream = (tinyblasStream_t)stream;+ handle->stream = (cudaStream_t)stream;return TINYBLAS_STATUS_SUCCESS;}@@ -109,10 +108,6 @@ const char *tinyblasGetStatusString(tinyblasStatus_t err) {}}-__device__ __forceinline__ float madd(float x, float y, float z) {- return __fmaf_rn(x, y, z);-}-template <int BM, int BN, int TM, int TN, typename WORD, typename SRC, typename DST>static __device__ void matmul_block2d(tinyblasOperation_t transa, tinyblasOperation_t transb, int m,int n, int k, WORD alpha, const SRC *A, int lda, const SRC *B,@@ -162,7 +157,7 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, tinyblasOperatWORD a = At[j];for (int h = 0; h < TN; ++h) {WORD b = Bt[h];- Cs[TN * j + h] = madd(a, b, Cs[TN * j + h]);+ Cs[TN * j + h] += a * b;}}}@@ -178,7 +173,7 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, tinyblasOperatfor (int i = 0; i < TM && ii + ti + i < m; ++i)if (beta) {WORD c = C[ldc * (jj + tj + j) + (ii + ti + i)];- C[ldc * (jj + tj + j) + (ii + ti + i)] = madd(c, beta, Cs[TN * i + j]);+ C[ldc * (jj + tj + j) + (ii + ti + i)] = c * beta + Cs[TN * i + j];} else {C[ldc * (jj + tj + j) + (ii + ti + i)] = Cs[TN * i + j];}@@ -417,9 +412,9 @@ static __global__ void KERNEL tinyblasGSBE_entry(tinyblasOperation_t transa,long long strideA, const SRC *B, int ldb,long long strideB, WORD beta, DST *C, int ldc,long long strideC, int batchCount) {- for (int z = blockIdx.z; z < batchCount; z += gridDim.z)- matmul_block2d<BM, BN, TM, TN>(transa, transb, m, n, k, alpha, A + z * strideA, lda,- B + z * strideB, ldb, beta, C + z * strideC, ldc);+ matmul_block2d<BM, BN, TM, TN>(transa, transb, m, n, k, alpha, A + blockIdx.z * strideA, lda,+ B + blockIdx.z * strideB, ldb, beta, C + blockIdx.z * strideC,+ ldc);}template <typename WORD, typename SRC, typename DST>diff --git a/llamafile/tinyblas.h b/llamafile/tinyblas.hindex 55de17ae..38c02db0 100644--- a/llamafile/tinyblas.h+++ b/llamafile/tinyblas.h@@ -2,33 +2,33 @@// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi#pragma once-enum tinyblasOperation_t {+typedef enum tinyblasOperation {TINYBLAS_OP_N,TINYBLAS_OP_T,-};+} tinyblasOperation_t;-enum tinyblasDataType_t {+typedef enum tinyblasDataType {TINYBLAS_R_32F,TINYBLAS_R_16F,-};+} tinyblasDataType_t;-enum tinyblasComputeType_t {+typedef enum tinyblasComputeType {TINYBLAS_COMPUTE_32F,TINYBLAS_COMPUTE_16F,-};+} tinyblasComputeType_t;-enum tinyblasGemmAlgo_t {+typedef enum tinyblasGemmAlgo {TINYBLAS_GEMM_DEFAULT,-};+} tinyblasGemmAlgo_t;-enum tinyblasStatus_t {+typedef enum tinyblasStatus {TINYBLAS_STATUS_SUCCESS,TINYBLAS_STATUS_ALLOC_FAILED,TINYBLAS_STATUS_INVALID_VALUE,TINYBLAS_STATUS_NOT_SUPPORTED,TINYBLAS_STATUS_EXECUTION_FAILED,TINYBLAS_STATUS_DIMENSION_OVERFLOW,-};+} tinyblasStatus_t;struct tinyblasContext;typedef struct tinyblasContext *tinyblasHandle_t;diff --git a/llamafile/tinyblas_test.cu b/llamafile/tinyblas_test.cuindex 3dbcac0d..e49a1704 100644--- a/llamafile/tinyblas_test.cu+++ b/llamafile/tinyblas_test.cu@@ -21,8 +21,8 @@ void checkTinyblasWorksHHHH() {double TOLERANCE = .5;{- half α = 1;- half β = 0;+ half alpha = 1;+ half beta = 0;int m = 577;int n = 577;int k = 64;@@ -35,17 +35,17 @@ void checkTinyblasWorksHHHH() {cuda_memory<half> G{ldc * n};randomize(k, m, A.p, lda);randomize(k, n, B.p, ldb);- cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ cublas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);broadcast(m, n, C.p, ldc, TOMBSTONE);broadcast(m, n, G.p, ldc, TOMBSTONE);- gemmref<float>(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);- tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ gemmref<float>(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);}{- half α = 1;- half β = 0;+ half alpha = 1;+ half beta = 0;int m = 4096;int n = 1024;int k = 2048;@@ -58,17 +58,17 @@ void checkTinyblasWorksHHHH() {cuda_memory<half> G{ldc * n};randomize(k, m, A.p, lda);randomize(k, n, B.p, ldb);- cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ cublas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);broadcast(m, n, C.p, ldc, TOMBSTONE);broadcast(m, n, G.p, ldc, TOMBSTONE);- gemmref<float>(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);- tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ gemmref<float>(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);}{- half α = 1;- half β = 0;+ half alpha = 1;+ half beta = 0;int m = 4096;int n = 1024;int k = 577;@@ -81,15 +81,15 @@ void checkTinyblasWorksHHHH() {cuda_memory<half> G{ldc * n};randomize(k, m, A.p, lda);randomize(k, n, B.p, ldb);- cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ cublas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);broadcast(m, n, C.p, ldc, TOMBSTONE);broadcast(m, n, G.p, ldc, TOMBSTONE);- gemmref<float>(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);- tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ gemmref<float>(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);}- test_matmul([&](int m, int n, int k, int l, half α, half β) {+ test_matmul([&](int m, int n, int k, int l, half alpha, half beta) {int lda = m + l;int ldb = k + l;int ldc = m + l;@@ -101,8 +101,8 @@ void checkTinyblasWorksHHHH() {randomize(k, n, B.p, ldb);broadcast(m, n, G.p, ldc, TOMBSTONE);broadcast(m, n, C.p, ldc, TOMBSTONE);- gemmref<float>(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);- tinyblas(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ gemmref<float>(false, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ tinyblas(false, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);});}@@ -110,8 +110,8 @@ void checkTinyblasWorksHHHH() {void checkTinyblasWorksHHHS() {double TOLERANCE = .5;- float α = 1;- float β = 0;+ float alpha = 1;+ float beta = 0;int m = 4096;int n = 1024;int k = 577;@@ -124,14 +124,14 @@ void checkTinyblasWorksHHHS() {cuda_memory<half> G{ldc * n};randomize(k, m, A.p, lda);randomize(k, n, B.p, ldb);- cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ cublas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);broadcast(m, n, C.p, ldc, TOMBSTONE);broadcast(m, n, G.p, ldc, TOMBSTONE);- gemmref(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);- tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ gemmref(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);- test_matmul([&](int m, int n, int k, int l, float α, float β) {+ test_matmul([&](int m, int n, int k, int l, float alpha, float beta) {int lda = m + l;int ldb = k + l;int ldc = m + l;@@ -143,8 +143,8 @@ void checkTinyblasWorksHHHS() {randomize(k, n, B.p, ldb);broadcast(m, n, G.p, ldc, TOMBSTONE);broadcast(m, n, C.p, ldc, TOMBSTONE);- gemmref(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);- tinyblas(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ gemmref(false, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ tinyblas(false, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);});}@@ -152,8 +152,8 @@ void checkTinyblasWorksHHHS() {void checkTinyblasWorksHHSS() {double TOLERANCE = .5;- float α = 1;- float β = 0;+ float alpha = 1;+ float beta = 0;int m = 4096;int n = 1024;int k = 1536;@@ -166,14 +166,14 @@ void checkTinyblasWorksHHSS() {cuda_memory<float> G{ldc * n};randomize(k, m, A.p, lda);randomize(k, n, B.p, ldb);- cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ cublas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);broadcast(m, n, G.p, ldc, TOMBSTONE);broadcast(m, n, C.p, ldc, TOMBSTONE);- gemmref(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);- tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ gemmref(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);- test_matmul([&](int m, int n, int k, int l, float α, float β) {+ test_matmul([&](int m, int n, int k, int l, float alpha, float beta) {int lda = k + l;int ldb = k + l;int ldc = m + l;@@ -185,8 +185,8 @@ void checkTinyblasWorksHHSS() {randomize(k, n, B.p, ldb);broadcast(m, n, G.p, ldc, TOMBSTONE);broadcast(m, n, C.p, ldc, TOMBSTONE);- gemmref(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);- tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ gemmref(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);});}@@ -194,8 +194,8 @@ void checkTinyblasWorksHHSS() {void checkTinyblasWorksSSSS() {double TOLERANCE = .5;- float α = 1;- float β = 0;+ float alpha = 1;+ float beta = 0;int m = 4096;int n = 1024;int k = 1020;@@ -208,14 +208,14 @@ void checkTinyblasWorksSSSS() {cuda_memory<float> G{ldc * n};randomize(k, m, A.p, lda);randomize(k, n, B.p, ldb);- cublas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);+ cublas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);broadcast(m, n, C.p, ldc, TOMBSTONE);broadcast(m, n, G.p, ldc, TOMBSTONE);- gemmref(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);- tinyblas(true, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ gemmref(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);- test_matmul([&](int m, int n, int k, int l, float α, float β) {+ test_matmul([&](int m, int n, int k, int l, float alpha, float beta) {int lda = m + l;int ldb = k + l;int ldc = m + l;@@ -227,13 +227,39 @@ void checkTinyblasWorksSSSS() {randomize(k, n, B.p, ldb);broadcast(m, n, G.p, ldc, TOMBSTONE);broadcast(m, n, C.p, ldc, TOMBSTONE);- gemmref(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, G.p, ldc);- tinyblas(false, false, m, n, k, α, A.p, lda, B.p, ldb, β, C.p, ldc);+ gemmref(false, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ tinyblas(false, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);});}int main(int argc, char *argv[]) {++ is_self_testing = "hi";++ float alpha = 1;+ float beta = 0;+ int m = 48;+ int n = 32;+ int k = 64;+ int lda = k;+ int ldb = k;+ int ldc = m;+ cuda_memory<half> A{lda * m};+ cuda_memory<half> B{ldb * n};+ cuda_memory<half> C{ldc * n};+ cuda_memory<half> G{ldc * n};+ randomize(k, m, A.p, lda);+ randomize(k, n, B.p, ldb);+ broadcast(m, n, C.p, ldc, TOMBSTONE);+ broadcast(m, n, G.p, ldc, TOMBSTONE);+ gemmref(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ tinyblas(true, false, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);+ CHECK(0, m, n, k, G.p, ldc, C.p, ldc);++ if (1)+ return 0;+RUN(checkTinyblasWorksHHHH());RUN(checkTinyblasWorksSSSS());RUN(checkTinyblasWorksHHSS());