Skip to content

Distribute and run LLMs with a single file.

Reduce rounding errors by 10x

Justine Tunneycb817f5cd11c
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index abcd9c9e..2ecfd15f 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -19209,6 +19209,7 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cpl
pthread_attr_t attr;
pthread_attr_init(&attr);
+ pthread_attr_setstacksize(&attr, 128 * 1024);
pthread_attr_setguardsize(&attr, sysconf(_SC_PAGESIZE));
pthread_attr_setsigaltstacksize_np(&attr, sysconf(_SC_MINSIGSTKSZ) + 16384);
const int rc = ggml_thread_create((pthread_t *)&workers[j].thrd, &attr,
diff --git a/llamafile/ansiblas.h b/llamafile/ansiblas.h
index 644be158..388650ec 100644
--- a/llamafile/ansiblas.h
+++ b/llamafile/ansiblas.h
@@ -20,6 +20,8 @@
#include <cmath>
#include <unistd.h>
+int cpu_get_num_math();
+
namespace {
namespace ansiBLAS {
@@ -133,7 +135,7 @@ void sgemm(int m, int n, int k, //
const float *A, int lda, //
const float *B, int ldb, //
float *C, int ldc) {
- int nth = sysconf(_SC_NPROCESSORS_ONLN);
+ static int nth = cpu_get_num_math();
#pragma omp parallel for
for (int ith = 0; ith < nth; ++ith) {
ansiBLAS tb{k, A, lda, B, ldb, C, ldc, ith, nth};
diff --git a/llamafile/bench.h b/llamafile/bench.h
index 3e3765b0..75fb8188 100644
--- a/llamafile/bench.h
+++ b/llamafile/bench.h
@@ -16,5 +16,5 @@
x; \
__asm__ volatile("" ::: "memory"); \
} \
- printf("%9lld us %s\n", (micros() - start + ITERATIONS - 1) / ITERATIONS, #x); \
+ printf("%12lld us %s\n", (micros() - start + ITERATIONS - 1) / ITERATIONS, #x); \
} while (0)
diff --git a/llamafile/sgemm_matmul_test.cpp b/llamafile/sgemm_matmul_test.cpp
index 86fa22ec..66de305e 100644
--- a/llamafile/sgemm_matmul_test.cpp
+++ b/llamafile/sgemm_matmul_test.cpp
@@ -29,10 +29,12 @@
#define ITERATIONS 30
#define ALLOC(n) (float *)memalign(4096, sizeof(float) * (n))
+int cpu_get_num_math();
+
void llamafile_sgemm_openmp(long m, long n, long k, const void *A, long lda, const void *B,
long ldb, void *C, long ldc, int task, int Atype, int Btype, int Ctype,
int precision) {
- int nth = sysconf(_SC_NPROCESSORS_ONLN);
+ static int nth = cpu_get_num_math();
#pragma omp parallel for
for (int ith = 0; ith < nth; ++ith) {
bool res = llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,
@@ -44,7 +46,7 @@ void llamafile_sgemm_openmp(long m, long n, long k, const void *A, long lda, con
int test(void) {
int m = 256;
int n = 500;
- int k = 32768 * 2;
+ int k = 260000;
int lda = ROUNDUP(k, 16);
int ldb = ROUNDUP(k, 16);
int ldc = ROUNDUP(m, 16);
@@ -63,12 +65,15 @@ int test(void) {
BENCH(llamafile_sgemm_openmp(m, n, k, A, lda, B, ldb, C, ldc, GGML_TASK_TYPE_COMPUTE,
GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));
+ int flips = 0;
double err_sum = 0;
long long err_worst = 0;
for (int i = 0; i < m; ++i)
for (int j = 0; j < n; ++j) {
float g = G[ldc * j + i];
float c = C[ldc * j + i];
+ if (signbit(g) != signbit(c))
+ ++flips;
if (flt::isnan(g)) {
fprintf(stderr, "%s:%err: found nan in reference matrix: i=%err j=%err\n", __FILE__,
__LINE__, i, j);
@@ -90,8 +95,9 @@ int test(void) {
}
double err_avg = err_sum / (m * n);
- fprintf(stderr, "%9g ulp average\n", err_avg);
- fprintf(stderr, "%9lld ulp worst\n", err_worst);
+ fprintf(stderr, "%12g ulp average\n", err_avg);
+ fprintf(stderr, "%12lld ulp worst\n", err_worst);
+ fprintf(stderr, "%12d flips\n", flips);
// using one accumulator
// 87015 us gemm
@@ -125,15 +131,17 @@ int test(void) {
int main(int argc, char *argv[]) {
int rc;
- llamafile_trapping_enabled(+1);
+ // llamafile_trapping_enabled(+1);
+
+ printf("\n");
+ if ((rc = test()))
+ return rc;
- printf("\nFLAG_precise = false;\n");
- FLAG_precise = false;
+ printf("\n");
if ((rc = test()))
return rc;
- printf("\nFLAG_precise = true;\n");
- FLAG_precise = true;
+ printf("\n");
if ((rc = test()))
return rc;
}
diff --git a/llamafile/sgemm_sss_test.cpp b/llamafile/sgemm_sss_test.cpp
index 36edf88e..60b9ca6b 100644
--- a/llamafile/sgemm_sss_test.cpp
+++ b/llamafile/sgemm_sss_test.cpp
@@ -28,10 +28,12 @@
#define ITERATIONS 5
#define ALLOC(n) (float *)memalign(4096, sizeof(float) * (n))
+int cpu_get_num_math();
+
void llamafile_sgemm_openmp(long m, long n, long k, const void *A, long lda, const void *B,
long ldb, void *C, long ldc, int task, int Atype, int Btype, int Ctype,
int precision) {
- int nth = sysconf(_SC_NPROCESSORS_ONLN);
+ static int nth = cpu_get_num_math();
#pragma omp parallel for
for (int ith = 0; ith < nth; ++ith) {
bool res = llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,
@@ -44,7 +46,7 @@ int test(void) {
float tolerance = 2e-5;
int m = 510;
int n = 513;
- int k = 512 * 8;
+ int k = 260000;
int lda = ROUNDUP(k, 16);
int ldb = ROUNDUP(k, 16);
int ldc = ROUNDUP(m, 16);
@@ -60,8 +62,8 @@ int test(void) {
randomize(k, n, B, ldb);
BENCH(ansiBLAS::sgemm(m, n, k, A, lda, B, ldb, G, ldc));
- BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,
- GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));
+ BENCH(llamafile_sgemm_openmp(m, n, k, A, lda, B, ldb, C, ldc, GGML_TASK_TYPE_COMPUTE,
+ GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));
double err_sum = 0;
long long err_worst = 0;
@@ -104,15 +106,17 @@ int test(void) {
int main(int argc, char *argv[]) {
int rc;
- llamafile_trapping_enabled(+1);
+ // llamafile_trapping_enabled(+1);
+
+ printf("\n");
+ if ((rc = test()))
+ return rc;
- printf("\nFLAG_precise = false;\n");
- FLAG_precise = false;
+ printf("\n");
if ((rc = test()))
return rc;
- printf("\nFLAG_precise = true;\n");
- FLAG_precise = true;
+ printf("\n");
if ((rc = test()))
return rc;
}
diff --git a/llamafile/sgemm_vecdot_test.cpp b/llamafile/sgemm_vecdot_test.cpp
index 2905cb72..871c4af4 100644
--- a/llamafile/sgemm_vecdot_test.cpp
+++ b/llamafile/sgemm_vecdot_test.cpp
@@ -22,14 +22,29 @@
#include "macros.h"
#include "numba.h"
#include "sgemm.h"
+#include <assert.h>
#define ITERATIONS 30
#define ALLOC(n) (float *)memalign(4096, sizeof(float) * (n))
+int cpu_get_num_math();
+
+void llamafile_sgemm_openmp(long m, long n, long k, const void *A, long lda, const void *B,
+ long ldb, void *C, long ldc, int task, int Atype, int Btype, int Ctype,
+ int precision) {
+ static int nth = cpu_get_num_math();
+#pragma omp parallel for
+ for (int ith = 0; ith < nth; ++ith) {
+ bool res = llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,
+ Ctype, precision);
+ assert(res);
+ }
+}
+
int test(void) {
- int m = 1025;
+ int m = 1024;
int n = 1;
- int k = 32768;
+ int k = 260000;
int lda = ROUNDUP(k, 16);
int ldb = ROUNDUP(k, 16);
int ldc = ROUNDUP(m, 16);
@@ -45,8 +60,8 @@ int test(void) {
randomize(k, n, B, ldb);
BENCH(ansiBLAS::sgemm(m, n, k, A, lda, B, ldb, G, ldc));
- BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,
- GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));
+ BENCH(llamafile_sgemm_openmp(m, n, k, A, lda, B, ldb, C, ldc, GGML_TASK_TYPE_COMPUTE,
+ GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));
double err_sum = 0;
long long err_worst = 0;
@@ -75,8 +90,8 @@ int test(void) {
}
double err_avg = err_sum / (m * n);
- fprintf(stderr, "%9g ulp average\n", err_avg);
- fprintf(stderr, "%9lld ulp worst\n", err_worst);
+ fprintf(stderr, "%12g ulp average\n", err_avg);
+ fprintf(stderr, "%12lld ulp worst\n", err_worst);
// using one accumulator
// 40209 us gemm
@@ -110,13 +125,15 @@ int test(void) {
int main(int argc, char *argv[]) {
int rc;
- printf("\nFLAG_precise = false;\n");
- FLAG_precise = false;
+ printf("\n");
+ if ((rc = test()))
+ return rc;
+
+ printf("\n");
if ((rc = test()))
return rc;
- printf("\nFLAG_precise = true;\n");
- FLAG_precise = true;
+ printf("\n");
if ((rc = test()))
return rc;
}
diff --git a/llamafile/tinyblas_cpu.h b/llamafile/tinyblas_cpu.h
index fdffa0d1..288490ba 100644
--- a/llamafile/tinyblas_cpu.h
+++ b/llamafile/tinyblas_cpu.h
@@ -32,10 +32,8 @@
// have excellent performance[1] for matrices that fit in the CPU cache
// without imposing any overhead such as cache filling or malloc calls.
//
-// This implementation does not guarantee any upper bound with rounding
-// errors, which grow along with k. Our goal's to maximally exploit the
-// hardware for performance, and then use whatever resources remain for
-// improving numerical accuracy.
+// With the F32, F16, and BF16 data types, the accumulation of roundoff
+// errors will only grow logarithmically, thanks to the ruler function.
//
// [1] J. Tunney, ‘LLaMA Now Goes Faster on CPUs’, Mar. 2024. [Online].
// Available: https://justine.lol/matmul/. [Accessed: 29-Mar-2024].
@@ -51,6 +49,7 @@
#pragma GCC diagnostic ignored "-Wpedantic"
#pragma GCC diagnostic ignored "-Wignored-attributes"
+#define CHUNK 8
#define ROW_ALIGN 64
#define MATRIX_ALIGN 4096
#define MAX_ALIGN 4096
@@ -434,170 +433,93 @@ class tinyBLAS {
long mc, nc, mp, np;
#if VECTOR_REGISTERS == 32
- if (!FLAG_precise) {
- switch ((MIN(m - m0, 5) << 4) | MIN(n - n0, 5)) {
- case 0x55:
- mc = 5;
- nc = 5;
- gemm<5, 5, false>(m0, m, n0, n);
- break;
- case 0x54:
- case 0x53:
- case 0x52:
- case 0x45:
- case 0x44:
- case 0x43:
- case 0x42:
- case 0x35:
- case 0x34:
- case 0x33:
- case 0x32:
- case 0x25:
- case 0x24:
- case 0x23:
- case 0x22:
- mc = 2;
- nc = 2;
- gemm<2, 2, false>(m0, m, n0, n);
- break;
- case 0x51:
- case 0x41:
- case 0x31:
- case 0x21:
- mc = 2;
- nc = 1;
- gemm<2, 1, false>(m0, m, n0, n);
- break;
- case 0x15:
- case 0x14:
- case 0x13:
- case 0x12:
- mc = 1;
- nc = 2;
- gemm<1, 2, false>(m0, m, n0, n);
- break;
- case 0x11:
- mc = 1;
- nc = 1;
- gemm<1, 1, false>(m0, m, n0, n);
- break;
- default:
- return;
- }
- } else {
- switch ((MIN(m - m0, 4) << 4) | MIN(n - n0, 3)) {
- case 0x43:
- mc = 4;
- nc = 3;
- gemm<4, 3, true>(m0, m, n0, n);
- break;
- case 0x42:
- case 0x33:
- case 0x32:
- case 0x23:
- case 0x22:
- mc = 2;
- nc = 2;
- gemm<2, 2, true>(m0, m, n0, n);
- break;
- case 0x41:
- case 0x31:
- case 0x21:
- mc = 2;
- nc = 1;
- gemm<2, 1, true>(m0, m, n0, n);
- break;
- case 0x13:
- case 0x12:
- mc = 1;
- nc = 2;
- gemm<1, 2, true>(m0, m, n0, n);
- break;
- case 0x11:
- mc = 1;
- nc = 1;
- gemm<1, 1, true>(m0, m, n0, n);
- break;
- default:
- return;
- }
+ switch ((MIN(m - m0, 5) << 4) | MIN(n - n0, 5)) {
+ case 0x55:
+ mc = 5;
+ nc = 5;
+ gemm<5, 5>(m0, m, n0, n);
+ break;
+ case 0x54:
+ case 0x53:
+ case 0x52:
+ case 0x45:
+ case 0x44:
+ case 0x43:
+ case 0x42:
+ case 0x35:
+ case 0x34:
+ case 0x33:
+ case 0x32:
+ case 0x25:
+ case 0x24:
+ case 0x23:
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2>(m0, m, n0, n);
+ break;
+ case 0x51:
+ case 0x41:
+ case 0x31:
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1>(m0, m, n0, n);
+ break;
+ case 0x15:
+ case 0x14:
+ case 0x13:
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1>(m0, m, n0, n);
+ break;
+ default:
+ return;
}
#endif
#if VECTOR_REGISTERS == 16
- if (!FLAG_precise) {
- switch ((MIN(m - m0, 4) << 4) | MIN(n - n0, 3)) {
- case 0x43:
- mc = 4;
- nc = 3;
- gemm<4, 3, false>(m0, m, n0, n);
- break;
- case 0x42:
- case 0x33:
- case 0x32:
- case 0x23:
- case 0x22:
- mc = 2;
- nc = 2;
- gemm<2, 2, false>(m0, m, n0, n);
- break;
- case 0x41:
- case 0x31:
- case 0x21:
- mc = 2;
- nc = 1;
- gemm<2, 1, false>(m0, m, n0, n);
- break;
- case 0x13:
- case 0x12:
- mc = 1;
- nc = 2;
- gemm<1, 2, false>(m0, m, n0, n);
- break;
- case 0x11:
- mc = 1;
- nc = 1;
- gemm<1, 1, false>(m0, m, n0, n);
- break;
- default:
- return;
- }
- } else {
- switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 2)) {
- case 0x32:
- mc = 3;
- nc = 2;
- gemm<3, 2, true>(m0, m, n0, n);
- break;
- case 0x23:
- mc = 2;
- nc = 3;
- gemm<2, 3, true>(m0, m, n0, n);
- break;
- case 0x22:
- mc = 2;
- nc = 2;
- gemm<2, 2, true>(m0, m, n0, n);
- break;
- case 0x31:
- case 0x21:
- mc = 2;
- nc = 1;
- gemm<2, 1, true>(m0, m, n0, n);
- break;
- case 0x12:
- mc = 1;
- nc = 2;
- gemm<1, 2, true>(m0, m, n0, n);
- break;
- case 0x11:
- mc = 1;
- nc = 1;
- gemm<1, 1, true>(m0, m, n0, n);
- break;
- default:
- return;
- }
+ switch ((MIN(m - m0, 4) << 4) | MIN(n - n0, 3)) {
+ case 0x43:
+ mc = 4;
+ nc = 3;
+ gemm<4, 3>(m0, m, n0, n);
+ break;
+ case 0x42:
+ case 0x33:
+ case 0x32:
+ case 0x23:
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2>(m0, m, n0, n);
+ break;
+ case 0x41:
+ case 0x31:
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1>(m0, m, n0, n);
+ break;
+ case 0x13:
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1>(m0, m, n0, n);
+ break;
+ default:
+ return;
}
#endif
@@ -607,8 +529,9 @@ class tinyBLAS {
mnpack(m0, m, np, n);
}
- template <int RM, int RN, int PRECISE>
+ template <int RM, int RN>
NOINLINE void gemm(long m0, long m, long n0, long n) {
+ D stack[bsr(k / CHUNK + 1) + 1][RN][RM];
long ytiles = RM > 1 ? (m - m0) / RM : 1;
long xtiles = RN > 1 ? (n - n0) / RN : 1;
long tiles = xtiles * ytiles;
@@ -620,26 +543,61 @@ class tinyBLAS {
for (long job = start; job < end; ++job) {
long ii = m0 + job / xtiles * RM;
long jj = n0 + job % xtiles * RN;
- D Cv[RN][RM] = {};
- D Ce[RN][RM] = {};
- for (long l = 0; l < k; l += KN)
+
+ size_t chunk, sp = 0;
+ int i, j, rule, step = 2;
+ for (chunk = 0; chunk + KN * CHUNK * 4 <= k; chunk += KN * CHUNK * 4, step += 2, ++sp) {
+
+ D Cv[RN][RM] = {};
+ for (long l = 0; l < KN * CHUNK * 4; l += KN)
#pragma GCC unroll 100
- for (int j = 0; j < RN; ++j)
+ for (j = 0; j < RN; ++j)
#pragma GCC unroll 100
- for (int i = 0; i < RM; ++i)
- if (PRECISE)
- Cv[j][i] = madder(load<V>(INDEX(A, lda, ii + i, l)), //
- load<V>(INDEX(B, ldb, jj + j, l)), //
- Cv[j][i], &Ce[j][i]);
- else
- Cv[j][i] = madd(load<V>(INDEX(A, lda, ii + i, l)), //
- load<V>(INDEX(B, ldb, jj + j, l)), //
+ for (i = 0; i < RM; ++i)
+ Cv[j][i] = madd(load<V>(INDEX(A, lda, ii + i, chunk + l)), //
+ load<V>(INDEX(B, ldb, jj + j, chunk + l)), //
Cv[j][i]);
+
+ for (rule = bsr(step & -step); --rule;)
+ for (--sp, j = 0; j < RN; ++j)
+ for (i = 0; i < RM; ++i)
+ Cv[j][i] += stack[sp][j][i];
+
+ for (j = 0; j < RN; ++j)
+ for (i = 0; i < RM; ++i)
+ stack[sp][j][i] = Cv[j][i];
+ }
+
+ D Cv[RN][RM] = {};
+ for (; chunk + KN <= k; chunk += KN)
#pragma GCC unroll 100
- for (int j = 0; j < RN; ++j)
+ for (j = 0; j < RN; ++j)
#pragma GCC unroll 100
- for (int i = 0; i < RM; ++i)
- store(INDEX(C, ldc, jj + j, ii + i), hsum(Cv[j][i]));
+ for (i = 0; i < RM; ++i)
+ Cv[j][i] = madd(load<V>(INDEX(A, lda, ii + i, chunk)), //
+ load<V>(INDEX(B, ldb, jj + j, chunk)), //
+ Cv[j][i]);
+
+ while (sp--)
+ for (j = 0; j < RN; ++j)
+ for (i = 0; i < RM; ++i)
+ Cv[j][i] += stack[sp][j][i];
+
+ float Cf[RN][RM];
+ for (j = 0; j < RN; ++j)
+ for (i = 0; i < RM; ++i)
+ Cf[j][i] = hsum(Cv[j][i]);
+
+ for (; chunk < k; ++chunk)
+ for (j = 0; j < RN; ++j)
+ for (i = 0; i < RM; ++i)
+ Cf[j][i] = fmaf(load<float>(INDEX(A, lda, ii + i, chunk)), //
+ load<float>(INDEX(B, ldb, jj + j, chunk)), //
+ Cf[j][i]);
+
+ for (j = 0; j < RN; ++j)
+ for (i = 0; i < RM; ++i)
+ store(INDEX(C, ldc, jj + j, ii + i), Cf[j][i]);
}
}
diff --git a/llamafile/tinyblas_cpu_sgemm.inc b/llamafile/tinyblas_cpu_sgemm.inc
index 21915383..8fe864c2 100644
--- a/llamafile/tinyblas_cpu_sgemm.inc
+++ b/llamafile/tinyblas_cpu_sgemm.inc
@@ -55,22 +55,16 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const
if (Btype != GGML_TYPE_F32)
return NOT_SUPPORTED;
#if defined(__AVX512F__)
- if (k % 16)
- return NOT_SUPPORTED;
tinyBLAS<0, 16, __m512, __m512, float, float, TC> tb{
k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#elif defined(__AVX__) || defined(__AVX2__)
- if (k % 8)
- return NOT_SUPPORTED;
tinyBLAS<0, 8, __m256, __m256, float, float, TC> tb{
k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#elif defined(__ARM_NEON)
- if (k % 4)
- return NOT_SUPPORTED;
tinyBLAS<0, 4, float32x4_t, float32x4_t, float, float, TC> tb{
k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
@@ -82,7 +76,7 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const
case GGML_TYPE_BF16: {
#if defined(__AVX512BF16__)
- if (Btype == GGML_TYPE_F32 && n < 2 && !(k % 16)) {
+ if (Btype == GGML_TYPE_F32 && n < 2) {
tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, float, TC> tb{
k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
@@ -92,29 +86,23 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const
return WANT_QUANTIZATION;
if (Btype != GGML_TYPE_BF16)
return NOT_SUPPORTED;
- if (n > 1 && !FLAG_precise && !(k % 32)) {
+ if (n > 1) {
tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, TC> tb{
k, (const ggml_bf16_t *)A, lda, (const ggml_bf16_t *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
} else {
- if (k % 16)
- return NOT_SUPPORTED;
tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, ggml_bf16_t, TC> tb{
k, (const ggml_bf16_t *)A, lda, (const ggml_bf16_t *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
}
#elif defined(__AVX512F__)
- if (k % 16)
- return NOT_SUPPORTED;
tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, float, TC> tb{
k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
tb.matmul(m, n, task);
return true;
#elif defined(__AVX2__)
- if (k % 8)
- return NOT_SUPPORTED;
if (Btype != GGML_TYPE_F32)
return NOT_SUPPORTED;
tinyBLAS<0, 8, __m256, __m256, ggml_bf16_t, float, TC> tb{
@@ -122,8 +110,6 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const
tb.matmul(m, n, task);
return true;
#elif defined(__ARM_NEON) && !defined(_MSC_VER)
- if (k % 4)
- return NOT_SUPPORTED;
if (Btype != GGML_TYPE_F32)
return NOT_SUPPORTED;
tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_bf16_t, float, TC> tb{
@@ -137,8 +123,6 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const
case GGML_TYPE_F16: {
#if defined(__AVX512F__)
- if (k % 16)
- return NOT_SUPPORTED;
if (Btype == GGML_TYPE_F32 && n < 2) {
tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, float, TC> tb{
k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
@@ -155,8 +139,6 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const
return true;
#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)
if (X86_CHECK(F16C)) {
- if (k % 8)
- return NOT_SUPPORTED;
if (Btype == GGML_TYPE_F32 && n < 2) {
tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, TC> tb{
k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
@@ -175,12 +157,10 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const
return NOT_SUPPORTED;
}
#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
- if (n < 2 && !FLAG_precise)
+ if (n < 2)
// TODO(jart): Why is ggml_vec_dot_f16_unroll() so fast at matvec?
return NOT_SUPPORTED;
if (precision == GGML_PREC_F32) {
- if (k % 4)
- return NOT_SUPPORTED;
if (Btype != GGML_TYPE_F32)
return NOT_SUPPORTED;
tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, TC> tb{
@@ -188,8 +168,6 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const
tb.matmul(m, n, task);
return true;
} else {
- if (k % 8)
- return NOT_SUPPORTED;
if (Btype == GGML_TYPE_F32)
return WANT_QUANTIZATION;
if (Btype != GGML_TYPE_F16)
@@ -203,8 +181,6 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const
if (n < 2 && !FLAG_precise)
// TODO(jart): Why is ggml_vec_dot_f16_unroll() so fast at matvec?
return NOT_SUPPORTED;
- if (k % 4)
- return NOT_SUPPORTED;
if (Btype != GGML_TYPE_F32)
return NOT_SUPPORTED;
tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, TC> tb{