Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Reduce rounding errors by 10x
Justine Tunneycb817f5cd11c
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex abcd9c9e..2ecfd15f 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -19209,6 +19209,7 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplpthread_attr_t attr;pthread_attr_init(&attr);+ pthread_attr_setstacksize(&attr, 128 * 1024);pthread_attr_setguardsize(&attr, sysconf(_SC_PAGESIZE));pthread_attr_setsigaltstacksize_np(&attr, sysconf(_SC_MINSIGSTKSZ) + 16384);const int rc = ggml_thread_create((pthread_t *)&workers[j].thrd, &attr,diff --git a/llamafile/ansiblas.h b/llamafile/ansiblas.hindex 644be158..388650ec 100644--- a/llamafile/ansiblas.h+++ b/llamafile/ansiblas.h@@ -20,6 +20,8 @@#include <cmath>#include <unistd.h>+int cpu_get_num_math();+namespace {namespace ansiBLAS {@@ -133,7 +135,7 @@ void sgemm(int m, int n, int k, //const float *A, int lda, //const float *B, int ldb, //float *C, int ldc) {- int nth = sysconf(_SC_NPROCESSORS_ONLN);+ static int nth = cpu_get_num_math();#pragma omp parallel forfor (int ith = 0; ith < nth; ++ith) {ansiBLAS tb{k, A, lda, B, ldb, C, ldc, ith, nth};diff --git a/llamafile/bench.h b/llamafile/bench.hindex 3e3765b0..75fb8188 100644--- a/llamafile/bench.h+++ b/llamafile/bench.h@@ -16,5 +16,5 @@x; \__asm__ volatile("" ::: "memory"); \} \- printf("%9lld us %s\n", (micros() - start + ITERATIONS - 1) / ITERATIONS, #x); \+ printf("%12lld us %s\n", (micros() - start + ITERATIONS - 1) / ITERATIONS, #x); \} while (0)diff --git a/llamafile/sgemm_matmul_test.cpp b/llamafile/sgemm_matmul_test.cppindex 86fa22ec..66de305e 100644--- a/llamafile/sgemm_matmul_test.cpp+++ b/llamafile/sgemm_matmul_test.cpp@@ -29,10 +29,12 @@#define ITERATIONS 30#define ALLOC(n) (float *)memalign(4096, sizeof(float) * (n))+int cpu_get_num_math();+void llamafile_sgemm_openmp(long m, long n, long k, const void *A, long lda, const void *B,long ldb, void *C, long ldc, int task, int Atype, int Btype, int Ctype,int precision) {- int nth = sysconf(_SC_NPROCESSORS_ONLN);+ static int nth = cpu_get_num_math();#pragma omp parallel forfor (int ith = 0; ith < nth; ++ith) {bool res = llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,@@ -44,7 +46,7 @@ void llamafile_sgemm_openmp(long m, long n, long k, const void *A, long lda, conint test(void) {int m = 256;int n = 500;- int k = 32768 * 2;+ int k = 260000;int lda = ROUNDUP(k, 16);int ldb = ROUNDUP(k, 16);int ldc = ROUNDUP(m, 16);@@ -63,12 +65,15 @@ int test(void) {BENCH(llamafile_sgemm_openmp(m, n, k, A, lda, B, ldb, C, ldc, GGML_TASK_TYPE_COMPUTE,GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));+ int flips = 0;double err_sum = 0;long long err_worst = 0;for (int i = 0; i < m; ++i)for (int j = 0; j < n; ++j) {float g = G[ldc * j + i];float c = C[ldc * j + i];+ if (signbit(g) != signbit(c))+ ++flips;if (flt::isnan(g)) {fprintf(stderr, "%s:%err: found nan in reference matrix: i=%err j=%err\n", __FILE__,__LINE__, i, j);@@ -90,8 +95,9 @@ int test(void) {}double err_avg = err_sum / (m * n);- fprintf(stderr, "%9g ulp average\n", err_avg);- fprintf(stderr, "%9lld ulp worst\n", err_worst);+ fprintf(stderr, "%12g ulp average\n", err_avg);+ fprintf(stderr, "%12lld ulp worst\n", err_worst);+ fprintf(stderr, "%12d flips\n", flips);// using one accumulator// 87015 us gemm@@ -125,15 +131,17 @@ int test(void) {int main(int argc, char *argv[]) {int rc;- llamafile_trapping_enabled(+1);+ // llamafile_trapping_enabled(+1);++ printf("\n");+ if ((rc = test()))+ return rc;- printf("\nFLAG_precise = false;\n");- FLAG_precise = false;+ printf("\n");if ((rc = test()))return rc;- printf("\nFLAG_precise = true;\n");- FLAG_precise = true;+ printf("\n");if ((rc = test()))return rc;}diff --git a/llamafile/sgemm_sss_test.cpp b/llamafile/sgemm_sss_test.cppindex 36edf88e..60b9ca6b 100644--- a/llamafile/sgemm_sss_test.cpp+++ b/llamafile/sgemm_sss_test.cpp@@ -28,10 +28,12 @@#define ITERATIONS 5#define ALLOC(n) (float *)memalign(4096, sizeof(float) * (n))+int cpu_get_num_math();+void llamafile_sgemm_openmp(long m, long n, long k, const void *A, long lda, const void *B,long ldb, void *C, long ldc, int task, int Atype, int Btype, int Ctype,int precision) {- int nth = sysconf(_SC_NPROCESSORS_ONLN);+ static int nth = cpu_get_num_math();#pragma omp parallel forfor (int ith = 0; ith < nth; ++ith) {bool res = llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,@@ -44,7 +46,7 @@ int test(void) {float tolerance = 2e-5;int m = 510;int n = 513;- int k = 512 * 8;+ int k = 260000;int lda = ROUNDUP(k, 16);int ldb = ROUNDUP(k, 16);int ldc = ROUNDUP(m, 16);@@ -60,8 +62,8 @@ int test(void) {randomize(k, n, B, ldb);BENCH(ansiBLAS::sgemm(m, n, k, A, lda, B, ldb, G, ldc));- BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,- GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));+ BENCH(llamafile_sgemm_openmp(m, n, k, A, lda, B, ldb, C, ldc, GGML_TASK_TYPE_COMPUTE,+ GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));double err_sum = 0;long long err_worst = 0;@@ -104,15 +106,17 @@ int test(void) {int main(int argc, char *argv[]) {int rc;- llamafile_trapping_enabled(+1);+ // llamafile_trapping_enabled(+1);++ printf("\n");+ if ((rc = test()))+ return rc;- printf("\nFLAG_precise = false;\n");- FLAG_precise = false;+ printf("\n");if ((rc = test()))return rc;- printf("\nFLAG_precise = true;\n");- FLAG_precise = true;+ printf("\n");if ((rc = test()))return rc;}diff --git a/llamafile/sgemm_vecdot_test.cpp b/llamafile/sgemm_vecdot_test.cppindex 2905cb72..871c4af4 100644--- a/llamafile/sgemm_vecdot_test.cpp+++ b/llamafile/sgemm_vecdot_test.cpp@@ -22,14 +22,29 @@#include "macros.h"#include "numba.h"#include "sgemm.h"+#include <assert.h>#define ITERATIONS 30#define ALLOC(n) (float *)memalign(4096, sizeof(float) * (n))+int cpu_get_num_math();++void llamafile_sgemm_openmp(long m, long n, long k, const void *A, long lda, const void *B,+ long ldb, void *C, long ldc, int task, int Atype, int Btype, int Ctype,+ int precision) {+ static int nth = cpu_get_num_math();+#pragma omp parallel for+ for (int ith = 0; ith < nth; ++ith) {+ bool res = llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,+ Ctype, precision);+ assert(res);+ }+}+int test(void) {- int m = 1025;+ int m = 1024;int n = 1;- int k = 32768;+ int k = 260000;int lda = ROUNDUP(k, 16);int ldb = ROUNDUP(k, 16);int ldc = ROUNDUP(m, 16);@@ -45,8 +60,8 @@ int test(void) {randomize(k, n, B, ldb);BENCH(ansiBLAS::sgemm(m, n, k, A, lda, B, ldb, G, ldc));- BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,- GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));+ BENCH(llamafile_sgemm_openmp(m, n, k, A, lda, B, ldb, C, ldc, GGML_TASK_TYPE_COMPUTE,+ GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));double err_sum = 0;long long err_worst = 0;@@ -75,8 +90,8 @@ int test(void) {}double err_avg = err_sum / (m * n);- fprintf(stderr, "%9g ulp average\n", err_avg);- fprintf(stderr, "%9lld ulp worst\n", err_worst);+ fprintf(stderr, "%12g ulp average\n", err_avg);+ fprintf(stderr, "%12lld ulp worst\n", err_worst);// using one accumulator// 40209 us gemm@@ -110,13 +125,15 @@ int test(void) {int main(int argc, char *argv[]) {int rc;- printf("\nFLAG_precise = false;\n");- FLAG_precise = false;+ printf("\n");+ if ((rc = test()))+ return rc;++ printf("\n");if ((rc = test()))return rc;- printf("\nFLAG_precise = true;\n");- FLAG_precise = true;+ printf("\n");if ((rc = test()))return rc;}diff --git a/llamafile/tinyblas_cpu.h b/llamafile/tinyblas_cpu.hindex fdffa0d1..288490ba 100644--- a/llamafile/tinyblas_cpu.h+++ b/llamafile/tinyblas_cpu.h@@ -32,10 +32,8 @@// have excellent performance[1] for matrices that fit in the CPU cache// without imposing any overhead such as cache filling or malloc calls.//-// This implementation does not guarantee any upper bound with rounding-// errors, which grow along with k. Our goal's to maximally exploit the-// hardware for performance, and then use whatever resources remain for-// improving numerical accuracy.+// With the F32, F16, and BF16 data types, the accumulation of roundoff+// errors will only grow logarithmically, thanks to the ruler function.//// [1] J. Tunney, ‘LLaMA Now Goes Faster on CPUs’, Mar. 2024. [Online].// Available: https://justine.lol/matmul/. [Accessed: 29-Mar-2024].@@ -51,6 +49,7 @@#pragma GCC diagnostic ignored "-Wpedantic"#pragma GCC diagnostic ignored "-Wignored-attributes"+#define CHUNK 8#define ROW_ALIGN 64#define MATRIX_ALIGN 4096#define MAX_ALIGN 4096@@ -434,170 +433,93 @@ class tinyBLAS {long mc, nc, mp, np;#if VECTOR_REGISTERS == 32- if (!FLAG_precise) {- switch ((MIN(m - m0, 5) << 4) | MIN(n - n0, 5)) {- case 0x55:- mc = 5;- nc = 5;- gemm<5, 5, false>(m0, m, n0, n);- break;- case 0x54:- case 0x53:- case 0x52:- case 0x45:- case 0x44:- case 0x43:- case 0x42:- case 0x35:- case 0x34:- case 0x33:- case 0x32:- case 0x25:- case 0x24:- case 0x23:- case 0x22:- mc = 2;- nc = 2;- gemm<2, 2, false>(m0, m, n0, n);- break;- case 0x51:- case 0x41:- case 0x31:- case 0x21:- mc = 2;- nc = 1;- gemm<2, 1, false>(m0, m, n0, n);- break;- case 0x15:- case 0x14:- case 0x13:- case 0x12:- mc = 1;- nc = 2;- gemm<1, 2, false>(m0, m, n0, n);- break;- case 0x11:- mc = 1;- nc = 1;- gemm<1, 1, false>(m0, m, n0, n);- break;- default:- return;- }- } else {- switch ((MIN(m - m0, 4) << 4) | MIN(n - n0, 3)) {- case 0x43:- mc = 4;- nc = 3;- gemm<4, 3, true>(m0, m, n0, n);- break;- case 0x42:- case 0x33:- case 0x32:- case 0x23:- case 0x22:- mc = 2;- nc = 2;- gemm<2, 2, true>(m0, m, n0, n);- break;- case 0x41:- case 0x31:- case 0x21:- mc = 2;- nc = 1;- gemm<2, 1, true>(m0, m, n0, n);- break;- case 0x13:- case 0x12:- mc = 1;- nc = 2;- gemm<1, 2, true>(m0, m, n0, n);- break;- case 0x11:- mc = 1;- nc = 1;- gemm<1, 1, true>(m0, m, n0, n);- break;- default:- return;- }+ switch ((MIN(m - m0, 5) << 4) | MIN(n - n0, 5)) {+ case 0x55:+ mc = 5;+ nc = 5;+ gemm<5, 5>(m0, m, n0, n);+ break;+ case 0x54:+ case 0x53:+ case 0x52:+ case 0x45:+ case 0x44:+ case 0x43:+ case 0x42:+ case 0x35:+ case 0x34:+ case 0x33:+ case 0x32:+ case 0x25:+ case 0x24:+ case 0x23:+ case 0x22:+ mc = 2;+ nc = 2;+ gemm<2, 2>(m0, m, n0, n);+ break;+ case 0x51:+ case 0x41:+ case 0x31:+ case 0x21:+ mc = 2;+ nc = 1;+ gemm<2, 1>(m0, m, n0, n);+ break;+ case 0x15:+ case 0x14:+ case 0x13:+ case 0x12:+ mc = 1;+ nc = 2;+ gemm<1, 2>(m0, m, n0, n);+ break;+ case 0x11:+ mc = 1;+ nc = 1;+ gemm<1, 1>(m0, m, n0, n);+ break;+ default:+ return;}#endif#if VECTOR_REGISTERS == 16- if (!FLAG_precise) {- switch ((MIN(m - m0, 4) << 4) | MIN(n - n0, 3)) {- case 0x43:- mc = 4;- nc = 3;- gemm<4, 3, false>(m0, m, n0, n);- break;- case 0x42:- case 0x33:- case 0x32:- case 0x23:- case 0x22:- mc = 2;- nc = 2;- gemm<2, 2, false>(m0, m, n0, n);- break;- case 0x41:- case 0x31:- case 0x21:- mc = 2;- nc = 1;- gemm<2, 1, false>(m0, m, n0, n);- break;- case 0x13:- case 0x12:- mc = 1;- nc = 2;- gemm<1, 2, false>(m0, m, n0, n);- break;- case 0x11:- mc = 1;- nc = 1;- gemm<1, 1, false>(m0, m, n0, n);- break;- default:- return;- }- } else {- switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 2)) {- case 0x32:- mc = 3;- nc = 2;- gemm<3, 2, true>(m0, m, n0, n);- break;- case 0x23:- mc = 2;- nc = 3;- gemm<2, 3, true>(m0, m, n0, n);- break;- case 0x22:- mc = 2;- nc = 2;- gemm<2, 2, true>(m0, m, n0, n);- break;- case 0x31:- case 0x21:- mc = 2;- nc = 1;- gemm<2, 1, true>(m0, m, n0, n);- break;- case 0x12:- mc = 1;- nc = 2;- gemm<1, 2, true>(m0, m, n0, n);- break;- case 0x11:- mc = 1;- nc = 1;- gemm<1, 1, true>(m0, m, n0, n);- break;- default:- return;- }+ switch ((MIN(m - m0, 4) << 4) | MIN(n - n0, 3)) {+ case 0x43:+ mc = 4;+ nc = 3;+ gemm<4, 3>(m0, m, n0, n);+ break;+ case 0x42:+ case 0x33:+ case 0x32:+ case 0x23:+ case 0x22:+ mc = 2;+ nc = 2;+ gemm<2, 2>(m0, m, n0, n);+ break;+ case 0x41:+ case 0x31:+ case 0x21:+ mc = 2;+ nc = 1;+ gemm<2, 1>(m0, m, n0, n);+ break;+ case 0x13:+ case 0x12:+ mc = 1;+ nc = 2;+ gemm<1, 2>(m0, m, n0, n);+ break;+ case 0x11:+ mc = 1;+ nc = 1;+ gemm<1, 1>(m0, m, n0, n);+ break;+ default:+ return;}#endif@@ -607,8 +529,9 @@ class tinyBLAS {mnpack(m0, m, np, n);}- template <int RM, int RN, int PRECISE>+ template <int RM, int RN>NOINLINE void gemm(long m0, long m, long n0, long n) {+ D stack[bsr(k / CHUNK + 1) + 1][RN][RM];long ytiles = RM > 1 ? (m - m0) / RM : 1;long xtiles = RN > 1 ? (n - n0) / RN : 1;long tiles = xtiles * ytiles;@@ -620,26 +543,61 @@ class tinyBLAS {for (long job = start; job < end; ++job) {long ii = m0 + job / xtiles * RM;long jj = n0 + job % xtiles * RN;- D Cv[RN][RM] = {};- D Ce[RN][RM] = {};- for (long l = 0; l < k; l += KN)++ size_t chunk, sp = 0;+ int i, j, rule, step = 2;+ for (chunk = 0; chunk + KN * CHUNK * 4 <= k; chunk += KN * CHUNK * 4, step += 2, ++sp) {++ D Cv[RN][RM] = {};+ for (long l = 0; l < KN * CHUNK * 4; l += KN)#pragma GCC unroll 100- for (int j = 0; j < RN; ++j)+ for (j = 0; j < RN; ++j)#pragma GCC unroll 100- for (int i = 0; i < RM; ++i)- if (PRECISE)- Cv[j][i] = madder(load<V>(INDEX(A, lda, ii + i, l)), //- load<V>(INDEX(B, ldb, jj + j, l)), //- Cv[j][i], &Ce[j][i]);- else- Cv[j][i] = madd(load<V>(INDEX(A, lda, ii + i, l)), //- load<V>(INDEX(B, ldb, jj + j, l)), //+ for (i = 0; i < RM; ++i)+ Cv[j][i] = madd(load<V>(INDEX(A, lda, ii + i, chunk + l)), //+ load<V>(INDEX(B, ldb, jj + j, chunk + l)), //Cv[j][i]);++ for (rule = bsr(step & -step); --rule;)+ for (--sp, j = 0; j < RN; ++j)+ for (i = 0; i < RM; ++i)+ Cv[j][i] += stack[sp][j][i];++ for (j = 0; j < RN; ++j)+ for (i = 0; i < RM; ++i)+ stack[sp][j][i] = Cv[j][i];+ }++ D Cv[RN][RM] = {};+ for (; chunk + KN <= k; chunk += KN)#pragma GCC unroll 100- for (int j = 0; j < RN; ++j)+ for (j = 0; j < RN; ++j)#pragma GCC unroll 100- for (int i = 0; i < RM; ++i)- store(INDEX(C, ldc, jj + j, ii + i), hsum(Cv[j][i]));+ for (i = 0; i < RM; ++i)+ Cv[j][i] = madd(load<V>(INDEX(A, lda, ii + i, chunk)), //+ load<V>(INDEX(B, ldb, jj + j, chunk)), //+ Cv[j][i]);++ while (sp--)+ for (j = 0; j < RN; ++j)+ for (i = 0; i < RM; ++i)+ Cv[j][i] += stack[sp][j][i];++ float Cf[RN][RM];+ for (j = 0; j < RN; ++j)+ for (i = 0; i < RM; ++i)+ Cf[j][i] = hsum(Cv[j][i]);++ for (; chunk < k; ++chunk)+ for (j = 0; j < RN; ++j)+ for (i = 0; i < RM; ++i)+ Cf[j][i] = fmaf(load<float>(INDEX(A, lda, ii + i, chunk)), //+ load<float>(INDEX(B, ldb, jj + j, chunk)), //+ Cf[j][i]);++ for (j = 0; j < RN; ++j)+ for (i = 0; i < RM; ++i)+ store(INDEX(C, ldc, jj + j, ii + i), Cf[j][i]);}}diff --git a/llamafile/tinyblas_cpu_sgemm.inc b/llamafile/tinyblas_cpu_sgemm.incindex 21915383..8fe864c2 100644--- a/llamafile/tinyblas_cpu_sgemm.inc+++ b/llamafile/tinyblas_cpu_sgemm.inc@@ -55,22 +55,16 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, constif (Btype != GGML_TYPE_F32)return NOT_SUPPORTED;#if defined(__AVX512F__)- if (k % 16)- return NOT_SUPPORTED;tinyBLAS<0, 16, __m512, __m512, float, float, TC> tb{k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#elif defined(__AVX__) || defined(__AVX2__)- if (k % 8)- return NOT_SUPPORTED;tinyBLAS<0, 8, __m256, __m256, float, float, TC> tb{k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#elif defined(__ARM_NEON)- if (k % 4)- return NOT_SUPPORTED;tinyBLAS<0, 4, float32x4_t, float32x4_t, float, float, TC> tb{k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);@@ -82,7 +76,7 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, constcase GGML_TYPE_BF16: {#if defined(__AVX512BF16__)- if (Btype == GGML_TYPE_F32 && n < 2 && !(k % 16)) {+ if (Btype == GGML_TYPE_F32 && n < 2) {tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, float, TC> tb{k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);@@ -92,29 +86,23 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, constreturn WANT_QUANTIZATION;if (Btype != GGML_TYPE_BF16)return NOT_SUPPORTED;- if (n > 1 && !FLAG_precise && !(k % 32)) {+ if (n > 1) {tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, TC> tb{k, (const ggml_bf16_t *)A, lda, (const ggml_bf16_t *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;} else {- if (k % 16)- return NOT_SUPPORTED;tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, ggml_bf16_t, TC> tb{k, (const ggml_bf16_t *)A, lda, (const ggml_bf16_t *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;}#elif defined(__AVX512F__)- if (k % 16)- return NOT_SUPPORTED;tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, float, TC> tb{k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};tb.matmul(m, n, task);return true;#elif defined(__AVX2__)- if (k % 8)- return NOT_SUPPORTED;if (Btype != GGML_TYPE_F32)return NOT_SUPPORTED;tinyBLAS<0, 8, __m256, __m256, ggml_bf16_t, float, TC> tb{@@ -122,8 +110,6 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, consttb.matmul(m, n, task);return true;#elif defined(__ARM_NEON) && !defined(_MSC_VER)- if (k % 4)- return NOT_SUPPORTED;if (Btype != GGML_TYPE_F32)return NOT_SUPPORTED;tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_bf16_t, float, TC> tb{@@ -137,8 +123,6 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, constcase GGML_TYPE_F16: {#if defined(__AVX512F__)- if (k % 16)- return NOT_SUPPORTED;if (Btype == GGML_TYPE_F32 && n < 2) {tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, float, TC> tb{k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};@@ -155,8 +139,6 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, constreturn true;#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)if (X86_CHECK(F16C)) {- if (k % 8)- return NOT_SUPPORTED;if (Btype == GGML_TYPE_F32 && n < 2) {tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, TC> tb{k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};@@ -175,12 +157,10 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, constreturn NOT_SUPPORTED;}#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)- if (n < 2 && !FLAG_precise)+ if (n < 2)// TODO(jart): Why is ggml_vec_dot_f16_unroll() so fast at matvec?return NOT_SUPPORTED;if (precision == GGML_PREC_F32) {- if (k % 4)- return NOT_SUPPORTED;if (Btype != GGML_TYPE_F32)return NOT_SUPPORTED;tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, TC> tb{@@ -188,8 +168,6 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, consttb.matmul(m, n, task);return true;} else {- if (k % 8)- return NOT_SUPPORTED;if (Btype == GGML_TYPE_F32)return WANT_QUANTIZATION;if (Btype != GGML_TYPE_F16)@@ -203,8 +181,6 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, constif (n < 2 && !FLAG_precise)// TODO(jart): Why is ggml_vec_dot_f16_unroll() so fast at matvec?return NOT_SUPPORTED;- if (k % 4)- return NOT_SUPPORTED;if (Btype != GGML_TYPE_F32)return NOT_SUPPORTED;tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, TC> tb{