Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Make MoE prompt eval faster on CPUs
Justine Tunney4db03a114362
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 2b2769e1..9a5332fc 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -298,8 +298,56 @@ void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) {}}+#ifdef __x86_64__+#pragma GCC push_options+#pragma GCC target("avx512f")+static void ggml_bf16_to_fp32_row_avx512f(const ggml_bf16_t * x, float * y, int64_t n) {+ int64_t i = 0;+ for (; i + 16 <= n; i += 16) {+ _mm512_storeu_ps(y + i,+ _mm512_castsi512_ps(+ _mm512_slli_epi32(+ _mm512_cvtepu16_epi32(+ _mm256_loadu_si256(+ (const __m256i *)(x + i))),+ 16)));+ }+ for (; i < n; i++) {+ y[i] = GGML_BF16_TO_FP32(x[i]);+ }+}+#pragma GCC pop_options+#endif++#ifdef __x86_64__+#pragma GCC push_options+#pragma GCC target("avx2")+static void ggml_bf16_to_fp32_row_avx2(const ggml_bf16_t * x, float * y, int64_t n) {+ int64_t i = 0;+ for (; i + 8 <= n; i += 8) {+ _mm256_storeu_ps(y + i,+ _mm256_castsi256_ps(+ _mm256_slli_epi32(+ _mm256_cvtepu16_epi32(+ _mm_loadu_si128(+ (const __m128i *)(x + i))),+ 16)));+ }+ for (; i < n; i++) {+ y[i] = GGML_BF16_TO_FP32(x[i]);+ }+}+#pragma GCC pop_options+#endif+void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) {- for (int64_t i = 0; i < n; i++) {+#ifdef __x86_64__+ if (X86_HAVE(AVX512F))+ return ggml_bf16_to_fp32_row_avx512f(x, y, n);+ if (X86_HAVE(AVX2))+ return ggml_bf16_to_fp32_row_avx2(x, y, n);+#endif+ for (int i = 0; i < n; i++) {y[i] = GGML_BF16_TO_FP32(x[i]);}}@@ -11010,11 +11058,14 @@ static void ggml_compute_forward_mul_mat_id(const struct ggml_tensor * src1 = dst->src[1];const struct ggml_tensor * ids = dst->src[2];- GGML_TENSOR_BINARY_OP_LOCALS+ if (llamafile_mixmul(params, src0, src1, ids, dst))+ return;const int ith = params->ith;const int nth = params->nth;+ GGML_TENSOR_BINARY_OP_LOCALS+const enum ggml_type type = src0->type;const bool src1_cont = ggml_is_contiguous(src1);@@ -18558,6 +18609,7 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threacur = 0;const struct ggml_tensor * src0 = node->src[0];const struct ggml_tensor * src1 = node->src[1];+ const struct ggml_tensor * src2 = node->src[2];const enum ggml_type vec_dot_type = type_traits[src0->type].vec_dot_type;if (src1->type != vec_dot_type) {cur += ggml_row_size(vec_dot_type, ggml_nelements(src1));@@ -18566,6 +18618,8 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threacur += GGML_PAD(cur, sizeof(int64_t)); // aligncur += n_as * sizeof(int64_t); // matrix_row_countscur += n_as * src1->ne[2] * sizeof(int64_t); // matrix_rows+ size_t cur2 = llamafile_mixmul_needs(src0, src1, src2);+ cur = cur > cur2 ? cur : cur2;} break;case GGML_OP_OUT_PROD:{diff --git a/llamafile/.clang-format b/llamafile/.clang-formatindex 3b1dd74d..e2f7fba0 100644--- a/llamafile/.clang-format+++ b/llamafile/.clang-format@@ -7,4 +7,6 @@ Language: CppAllowShortFunctionsOnASingleLine: falseAlignTrailingComments: falseAlignEscapedNewlines: DontAlign+AlwaysBreakTemplateDeclarations: true+ConstructorInitializerAllOnOneLineOrOnePerLine: true---diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex 34923f7f..3cab491d 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -43,28 +43,12 @@ o/$(MODE)/llamafile: \################################################################################# microarchitectures-o/$(MODE)/llamafile/sgemm_sss_avx.o: private TARGET_ARCH += -Xx86_64-mavx-o/$(MODE)/llamafile/sgemm_sss_fma.o: private TARGET_ARCH += -Xx86_64-mfma-o/$(MODE)/llamafile/sgemm_bss_avx2.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma-o/$(MODE)/llamafile/sgemm_bss_avx512.o: private TARGET_ARCH += -Xx86_64-mavx512f-o/$(MODE)/llamafile/sgemm_bss_avx512bf16.o: private TARGET_ARCH += -Xx86_64-mavx512bf16 -Xx86_64-mavx512vl-o/$(MODE)/llamafile/sgemm_sss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f-o/$(MODE)/llamafile/sgemm_hss_f16c.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mf16c-o/$(MODE)/llamafile/sgemm_hss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f-o/$(MODE)/llamafile/sgemm_hhs_neon.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+fp16-o/$(MODE)/llamafile/sgemm_hsh_f16c.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mf16c-o/$(MODE)/llamafile/sgemm_hsh_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f-o/$(MODE)/llamafile/sgemm_hhh_neon.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+fp16-o/$(MODE)/llamafile/sgemm_q0q0s_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni-o/$(MODE)/llamafile/sgemm_q0q0s_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni-o/$(MODE)/llamafile/sgemm_q0q0s_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma-o/$(MODE)/llamafile/sgemm_q0q0s_dotprod.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+dotprod-o/$(MODE)/llamafile/sgemm_e0q0s_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni-o/$(MODE)/llamafile/sgemm_e0q0s_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni-o/$(MODE)/llamafile/sgemm_e0q0s_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma-o/$(MODE)/llamafile/sgemm_e1q1s_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni-o/$(MODE)/llamafile/sgemm_e1q1s_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni-o/$(MODE)/llamafile/sgemm_e1q1s_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma+o/$(MODE)/llamafile/tinyblas_cpu_amd_fma.o: private TARGET_ARCH += -Xx86_64-mfma+o/$(MODE)/llamafile/tinyblas_cpu_amd_avx2.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma+o/$(MODE)/llamafile/tinyblas_cpu_amd_avxvnni.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma -Xx86_64-mavxvnni+o/$(MODE)/llamafile/tinyblas_cpu_amd_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f+o/$(MODE)/llamafile/tinyblas_cpu_amd_zen4.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni -Xx86_64-mavx512bf16+o/$(MODE)/llamafile/tinyblas_cpu_arm82.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+dotprod+fp16################################################################################# testingdiff --git a/llamafile/bench.h b/llamafile/bench.hindex 461ff081..6e5a118c 100644--- a/llamafile/bench.h+++ b/llamafile/bench.h@@ -16,5 +16,5 @@x; \__asm__ volatile("" ::: "memory"); \} \- printf("%8lld us %s\n", (micros() - start + ITERATIONS - 1) / ITERATIONS, #x); \+ printf("%9lld us %s\n", (micros() - start + ITERATIONS - 1) / ITERATIONS, #x); \} while (0)diff --git a/llamafile/hsum.h b/llamafile/hsum.hdeleted file mode 100644index 347e32c1..00000000--- a/llamafile/hsum.h+++ /dev/null@@ -1,56 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-#pragma once--#ifdef __ARM_NEON-#include <arm_neon.h>-#endif-#ifdef __x86_64__-#include <immintrin.h>-#endif--inline float hsum(float x) {- return x;-}--#ifdef __ARM_NEON-inline float hsum(float32x4_t x) {- return vaddvq_f32(x);-}-#endif--#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC-inline float hsum(float16x8_t x) {- float32x4_t t = vcvt_f32_f16(vget_low_f16(x));- float32x4_t u = vcvt_f32_f16(vget_high_f16(x));- return vaddvq_f32(vaddq_f32(t, u));-}-#endif--#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)-inline float hsum(__m128 x) {-#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)- x = _mm_add_ps(x, _mm_movehl_ps(x, x));- x = _mm_add_ss(x, _mm_movehdup_ps(x));- return _mm_cvtss_f32(x);-#else- __m128 shuf = _mm_shuffle_ps(x, x, _MM_SHUFFLE(2, 3, 0, 1));- __m128 sums = _mm_add_ps(x, shuf);- shuf = _mm_movehl_ps(shuf, sums);- sums = _mm_add_ss(sums, shuf);- return _mm_cvtss_f32(sums);-#endif-}-#endif--#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)-inline float hsum(__m256 x) {- return hsum(_mm_add_ps(_mm256_extractf128_ps(x, 1), _mm256_castps256_ps128(x)));-}-#endif--#if defined(__AVX512F__)-inline float hsum(__m512 x) {- return _mm512_reduce_add_ps(x);-}-#endifdiff --git a/llamafile/kernel.h b/llamafile/kernel.hdeleted file mode 100644index 66d5ff4f..00000000--- a/llamafile/kernel.h+++ /dev/null@@ -1,32 +0,0 @@-#pragma once--#if defined(__ARM_NEON) || defined(__AVX512F__)-#define VECTOR_REGISTERS 32-#else-#define VECTOR_REGISTERS 16-#endif--#if defined(__AVX512F__)-#define VECTOR_WIDTH 64-#elif defined(__AVX__) || defined(__AVX2__)-#define VECTOR_WIDTH 32-#else-#define VECTOR_WIDTH 16-#endif--#define BEGIN_KERNEL(RM, RN) \- int ytiles = (m - m0) / RM; \- int xtiles = (n - n0) / RN; \- int tiles = ytiles * xtiles; \- int duty = (tiles + nth - 1) / nth; \- int start = duty * ith; \- int end = start + duty; \- if (end > tiles) \- end = tiles; \- for (int job = start; job < end; ++job) { \- int i = m0 + job / xtiles * RM; \- int j = n0 + job % xtiles * RN;--#define END_KERNEL() }--#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)diff --git a/llamafile/kernelcount.sh b/llamafile/kernelcount.shdeleted file mode 100755index 2bc4844e..00000000--- a/llamafile/kernelcount.sh+++ /dev/null@@ -1,11 +0,0 @@-#!/bin/sh-# prints number of llamafile_sgemm kernels--echo $(( $(aarch64-unknown-cosmo-nm \- -C o//llama.cpp/main/main.aarch64.elf |- grep GEMMER.*::gemm |- wc -l) +- $(x86_64-unknown-cosmo-nm \- -C o//llama.cpp/main/main.com.dbg |- grep GEMMER.*::gemm |- wc -l) ))diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex 71694858..c5b783e7 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -51,6 +51,13 @@ const char *llamafile_describe_gpu(void);bool llamafile_sgemm(int, int, int, const void *, int, const void *, int, void *, int, int, int,int, int, int, int);+struct ggml_tensor;+struct ggml_compute_params;+bool llamafile_mixmul(const struct ggml_compute_params *, const struct ggml_tensor *,+ const struct ggml_tensor *, const struct ggml_tensor *, struct ggml_tensor *);+size_t llamafile_mixmul_needs(const struct ggml_tensor *, const struct ggml_tensor *,+ const struct ggml_tensor *);+#ifdef __cplusplus}#endifdiff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cppindex 0a9b1a9a..d292bac7 100644--- a/llamafile/sgemm.cpp+++ b/llamafile/sgemm.cpp@@ -52,189 +52,73 @@ static const long hwcap = getauxval(AT_HWCAP);*/bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B, int ldb, void *C,int ldc, int ith, int nth, int task, int Atype, int Btype, int Ctype) {-- assert(m >= 0);- assert(n >= 0);- assert(k >= 0);- assert(lda >= k);- assert(ldb >= k);- assert(ldc >= m);- assert(nth > 0);- assert(ith < nth);-- switch (Atype) {-- case GGML_TYPE_F32:- if (Ctype != GGML_TYPE_F32)- return false;- if (Btype != GGML_TYPE_F32)- return false;#ifdef __x86_64__- if (!X86_HAVE(AVX))- return false;- if (X86_HAVE(AVX512F) && !(k % 16))- return llamafile_sgemm_sss_avx512f(m, n, k, (const float *)A, lda, (const float *)B,- ldb, (float *)C, ldc, ith, nth, task);- if (X86_HAVE(FMA) && !(k % 8))- return llamafile_sgemm_sss_fma(m, n, k, (const float *)A, lda, (const float *)B, ldb,- (float *)C, ldc, ith, nth, task);- if (!(k % 8))- return llamafile_sgemm_sss_avx(m, n, k, (const float *)A, lda, (const float *)B, ldb,- (float *)C, ldc, ith, nth, task);+ if (X86_HAVE(FMA)) {+ if (X86_HAVE(AVX2)) {+ if (X86_HAVE(AVX512F)) {+ if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && X86_HAVE(AVX512_BF16)) {+ return llamafile_sgemm_amd_zen4(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task,+ Atype, Btype, Ctype);+ } else {+ return llamafile_sgemm_amd_avx512f(m, n, k, A, lda, B, ldb, C, ldc, ith, nth,+ task, Atype, Btype, Ctype);+ }+ } else if (X86_HAVE(AVXVNNI)) {+ return llamafile_sgemm_amd_avxvnni(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task,+ Atype, Btype, Ctype);+ } else {+ return llamafile_sgemm_amd_avx2(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task,+ Atype, Btype, Ctype);+ }+ } else {+ return llamafile_sgemm_amd_fma(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype,+ Btype, Ctype);+ }+ } else {+ return llamafile_sgemm_amd_avx(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype,+ Btype, Ctype);+ }#elif defined(__aarch64__)- if (n > 1 && !(k % 4))- return llamafile_sgemm_sss_neon(m, n, k, (const float *)A, lda, (const float *)B, ldb,- (float *)C, ldc, ith, nth, task);+ if (hwcap & HWCAP_FPHP) {+ return llamafile_sgemm_arm82(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,+ Ctype);+ } else {+ return llamafile_sgemm_arm80(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,+ Ctype);+ }#endif- return false;+}- case GGML_TYPE_F16:- switch (Ctype) {- case GGML_TYPE_F32:-#ifdef __x86_64__- if (!X86_HAVE(AVX))- return false;- if (Btype != GGML_TYPE_F32)- return false;- if (X86_HAVE(AVX512F) && !(k % 16))- return llamafile_sgemm_hss_avx512f(m, n, k, (const unsigned short *)A, lda,- (const float *)B, ldb, (float *)C, ldc, ith, nth,- task);- if (X86_HAVE(FMA) && X86_HAVE(F16C) && !(k % 8))- return llamafile_sgemm_hss_f16c(m, n, k, (const unsigned short *)A, lda,- (const float *)B, ldb, (float *)C, ldc, ith, nth,- task);-#elif defined(__aarch64__)- if (n > 1 && !(k % 8) && (hwcap & HWCAP_FPHP) && Btype == GGML_TYPE_F16)- return llamafile_sgemm_hhs_neon(m, n, k, (const unsigned short *)A, lda,- (const unsigned short *)B, ldb, (float *)C, ldc,- ith, nth, task);- if (n > 1 && !(k % 4) && !(hwcap & HWCAP_FPHP) && Btype == GGML_TYPE_F32)- return llamafile_sgemm_hss_neon(m, n, k, (const unsigned short *)A, lda,- (const float *)B, ldb, (float *)C, ldc, ith, nth,- task);-#endif- return false;- case GGML_TYPE_F16:+/**+ * Performs "mixture of experts" tensor multiplication on CPU.+ */+bool llamafile_mixmul(const ggml_compute_params *params, const ggml_tensor *weights,+ const ggml_tensor *thought, const ggml_tensor *plan, ggml_tensor *result) {#ifdef __x86_64__- if (!X86_HAVE(AVX))- return false;- if (Btype != GGML_TYPE_F32)- return false;- if (X86_HAVE(AVX512F) && !(k % 16))- return llamafile_sgemm_hsh_avx512f(m, n, k, (const unsigned short *)A, lda,- (const float *)B, ldb, (llamafile_fp16 *)C, ldc,- ith, nth, task);- if (X86_HAVE(FMA) && X86_HAVE(F16C) && !(k % 8))- return llamafile_sgemm_hsh_f16c(m, n, k, (const unsigned short *)A, lda,- (const float *)B, ldb, (llamafile_fp16 *)C, ldc,- ith, nth, task);-#elif defined(__aarch64__)- if (n > 1 && !(k % 4) && !(hwcap & HWCAP_FPHP) && Btype == GGML_TYPE_F32)- return llamafile_sgemm_hsh_neon(m, n, k, (const unsigned short *)A, lda,- (const float *)B, ldb, (llamafile_fp16 *)C, ldc,- ith, nth, task);-#endif- return false;- default:- return false;+ if (X86_HAVE(FMA)) {+ if (X86_HAVE(AVX2)) {+ if (X86_HAVE(AVX512F)) {+ if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && X86_HAVE(AVX512_BF16)) {+ return llamafile_mixmul_amd_zen4(params, weights, thought, plan, result);+ } else {+ return llamafile_mixmul_amd_avx512f(params, weights, thought, plan, result);+ }+ } else if (X86_HAVE(AVXVNNI)) {+ return llamafile_mixmul_amd_avxvnni(params, weights, thought, plan, result);+ } else {+ return llamafile_mixmul_amd_avx2(params, weights, thought, plan, result);+ }+ } else {+ return llamafile_mixmul_amd_fma(params, weights, thought, plan, result);}-- case GGML_TYPE_BF16:- if (Ctype != GGML_TYPE_F32)- return false;-#ifdef __x86_64__- if (Btype != GGML_TYPE_F32)- return false;- if (X86_HAVE(AVX512_BF16) && !(k % 32))- return llamafile_sgemm_bss_avx512bf16(m, n, k, (const ggml_bf16_t *)A, lda,- (const float *)B, ldb, (float *)C, ldc, ith, nth,- task);- if (X86_HAVE(AVX512F) && !(k % 16))- return llamafile_sgemm_bss_avx512(m, n, k, (const ggml_bf16_t *)A, lda,- (const float *)B, ldb, (float *)C, ldc, ith, nth,- task);- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && !(k % 8))- return llamafile_sgemm_bss_avx2(m, n, k, (const ggml_bf16_t *)A, lda, (const float *)B,- ldb, (float *)C, ldc, ith, nth, task);-#endif- return false;-- case GGML_TYPE_Q8_0:- if (Ctype != GGML_TYPE_F32)- return false;- if (Btype != GGML_TYPE_Q8_0)- return false;-#ifdef __x86_64__- if (!X86_HAVE(AVX))- return false;- if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI))- return llamafile_sgemm_q0q0s_avx512vnni(m, n, k, (const block_q8_0 *)A, lda,- (const block_q8_0 *)B, ldb, (float *)C, ldc,- ith, nth, task);- if (X86_HAVE(FMA) && X86_HAVE(AVXVNNI))- return llamafile_sgemm_q0q0s_avxvnni(m, n, k, (const block_q8_0 *)A, lda,- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,- nth, task);- if (X86_HAVE(FMA) && X86_HAVE(AVX2))- return llamafile_sgemm_q0q0s_fma(m, n, k, (const block_q8_0 *)A, lda,- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,- task);+ } else {+ return llamafile_mixmul_amd_avx(params, weights, thought, plan, result);+ }#elif defined(__aarch64__)- if (hwcap & HWCAP_FPHP)- return llamafile_sgemm_q0q0s_dotprod(m, n, k, (const block_q8_0 *)A, lda,- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,- nth, task);-#endif- return false;-- case GGML_TYPE_Q4_0:- if (Ctype != GGML_TYPE_F32)- return false;- if (Btype != GGML_TYPE_Q8_0)- return false;-#ifdef __x86_64__- if (!X86_HAVE(AVX))- return false;- if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI))- return llamafile_sgemm_e0q0s_avx512vnni(m, n, k, (const block_q4_0 *)A, lda,- (const block_q8_0 *)B, ldb, (float *)C, ldc,- ith, nth, task);- if (X86_HAVE(FMA) && X86_HAVE(AVXVNNI))- return llamafile_sgemm_e0q0s_avxvnni(m, n, k, (const block_q4_0 *)A, lda,- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,- nth, task);- if (X86_HAVE(FMA) && X86_HAVE(AVX2))- return llamafile_sgemm_e0q0s_fma(m, n, k, (const block_q4_0 *)A, lda,- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,- task);-#endif- return false;-- case GGML_TYPE_Q4_1:- if (Ctype != GGML_TYPE_F32)- return false;- if (Btype != GGML_TYPE_Q8_1)- return false;-#ifdef __x86_64__- if (!X86_HAVE(AVX))- return false;- if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI))- return llamafile_sgemm_e1q1s_avx512vnni(m, n, k, (const block_q4_1 *)A, lda,- (const block_q8_1 *)B, ldb, (float *)C, ldc,- ith, nth, task);- if (X86_HAVE(FMA) && X86_HAVE(AVXVNNI))- return llamafile_sgemm_e1q1s_avxvnni(m, n, k, (const block_q4_1 *)A, lda,- (const block_q8_1 *)B, ldb, (float *)C, ldc, ith,- nth, task);- if (X86_HAVE(FMA) && X86_HAVE(AVX2))- return llamafile_sgemm_e1q1s_fma(m, n, k, (const block_q4_1 *)A, lda,- (const block_q8_1 *)B, ldb, (float *)C, ldc, ith, nth,- task);-#endif- return false;-- default:- return false;+ if (hwcap & HWCAP_FPHP) {+ return llamafile_mixmul_arm82(params, weights, thought, plan, result);+ } else {+ return llamafile_mixmul_arm80(params, weights, thought, plan, result);}+#endif}diff --git a/llamafile/sgemm.h b/llamafile/sgemm.hindex 398e2672..f384f890 100644--- a/llamafile/sgemm.h+++ b/llamafile/sgemm.h@@ -1,65 +1,52 @@#pragma once-#include "llama.cpp/ggml-quants.h"-#include "llama.cpp/ggml.h"-#include "llamafile/fp16.h"#ifdef __cplusplusextern "C" {#endif-bool llamafile_sgemm_sss_avx(int, int, int, const float *, int, const float *, int, float *, int,- int, int, int);-bool llamafile_sgemm_sss_fma(int, int, int, const float *, int, const float *, int, float *, int,- int, int, int);-bool llamafile_sgemm_sss_avx512f(int, int, int, const float *, int, const float *, int, float *,- int, int, int, int);-bool llamafile_sgemm_sss_neon(int, int, int, const float *, int, const float *, int, float *, int,- int, int, int);--bool llamafile_sgemm_hss_f16c(int, int, int, const unsigned short *, int, const float *, int,- float *, int, int, int, int);-bool llamafile_sgemm_hss_avx512f(int, int, int, const unsigned short *, int, const float *, int,- float *, int, int, int, int);-bool llamafile_sgemm_hss_neon(int, int, int, const unsigned short *, int, const float *, int,- float *, int, int, int, int);-bool llamafile_sgemm_hhs_neon(int, int, int, const unsigned short *, int, const unsigned short *,- int, float *, int, int, int, int);--bool llamafile_sgemm_hsh_f16c(int, int, int, const unsigned short *, int, const float *, int,- llamafile_fp16 *, int, int, int, int);-bool llamafile_sgemm_hsh_avx512f(int, int, int, const unsigned short *, int, const float *, int,- llamafile_fp16 *, int, int, int, int);-bool llamafile_sgemm_hsh_neon(int, int, int, const unsigned short *, int, const float *, int,- llamafile_fp16 *, int, int, int, int);--bool llamafile_sgemm_bss_avx2(int, int, int, const ggml_bf16_t *, int, const float *, int, float *,- int, int, int, int);-bool llamafile_sgemm_bss_avx512(int, int, int, const ggml_bf16_t *, int, const float *, int,- float *, int, int, int, int);-bool llamafile_sgemm_bss_avx512bf16(int, int, int, const ggml_bf16_t *, int, const float *, int,- float *, int, int, int, int);--bool llamafile_sgemm_q0q0s_avx512vnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *,- int, float *, int, int, int, int);-bool llamafile_sgemm_q0q0s_avxvnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,- float *, int, int, int, int);-bool llamafile_sgemm_q0q0s_fma(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,- float *, int, int, int, int);-bool llamafile_sgemm_q0q0s_dotprod(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,- float *, int, int, int, int);--bool llamafile_sgemm_e0q0s_avx512vnni(int, int, int, const block_q4_0 *, int, const block_q8_0 *,- int, float *, int, int, int, int);-bool llamafile_sgemm_e0q0s_avxvnni(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,- float *, int, int, int, int);-bool llamafile_sgemm_e0q0s_fma(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,- float *, int, int, int, int);--bool llamafile_sgemm_e1q1s_avx512vnni(int, int, int, const block_q4_1 *, int, const block_q8_1 *,- int, float *, int, int, int, int);-bool llamafile_sgemm_e1q1s_avxvnni(int, int, int, const block_q4_1 *, int, const block_q8_1 *, int,- float *, int, int, int, int);-bool llamafile_sgemm_e1q1s_fma(int, int, int, const block_q4_1 *, int, const block_q8_1 *, int,- float *, int, int, int, int);+struct ggml_tensor;+struct ggml_compute_params;++bool llamafile_sgemm_amd_avx(int, int, int, const void *, int, const void *, int, void *, int, int,+ int, int, int, int, int);+bool llamafile_sgemm_amd_fma(int, int, int, const void *, int, const void *, int, void *, int, int,+ int, int, int, int, int);+bool llamafile_sgemm_amd_avx2(int, int, int, const void *, int, const void *, int, void *, int, int,+ int, int, int, int, int);+bool llamafile_sgemm_amd_avxvnni(int, int, int, const void *, int, const void *, int, void *, int,+ int, int, int, int, int, int);+bool llamafile_sgemm_amd_avx512f(int, int, int, const void *, int, const void *, int, void *, int,+ int, int, int, int, int, int);+bool llamafile_sgemm_amd_zen4(int, int, int, const void *, int, const void *, int, void *, int, int,+ int, int, int, int, int);+bool llamafile_sgemm_arm80(int, int, int, const void *, int, const void *, int, void *, int, int,+ int, int, int, int, int);+bool llamafile_sgemm_arm82(int, int, int, const void *, int, const void *, int, void *, int, int,+ int, int, int, int, int);++bool llamafile_mixmul_amd_avx(const struct ggml_compute_params *, const struct ggml_tensor *,+ const struct ggml_tensor *, const struct ggml_tensor *,+ struct ggml_tensor *);+bool llamafile_mixmul_amd_fma(const struct ggml_compute_params *, const struct ggml_tensor *,+ const struct ggml_tensor *, const struct ggml_tensor *,+ struct ggml_tensor *);+bool llamafile_mixmul_amd_avx2(const struct ggml_compute_params *, const struct ggml_tensor *,+ const struct ggml_tensor *, const struct ggml_tensor *,+ struct ggml_tensor *);+bool llamafile_mixmul_amd_avxvnni(const struct ggml_compute_params *, const struct ggml_tensor *,+ const struct ggml_tensor *, const struct ggml_tensor *,+ struct ggml_tensor *);+bool llamafile_mixmul_amd_avx512f(const struct ggml_compute_params *, const struct ggml_tensor *,+ const struct ggml_tensor *, const struct ggml_tensor *,+ struct ggml_tensor *);+bool llamafile_mixmul_amd_zen4(const struct ggml_compute_params *, const struct ggml_tensor *,+ const struct ggml_tensor *, const struct ggml_tensor *,+ struct ggml_tensor *);+bool llamafile_mixmul_arm80(const struct ggml_compute_params *, const struct ggml_tensor *,+ const struct ggml_tensor *, const struct ggml_tensor *,+ struct ggml_tensor *);+bool llamafile_mixmul_arm82(const struct ggml_compute_params *, const struct ggml_tensor *,+ const struct ggml_tensor *, const struct ggml_tensor *,+ struct ggml_tensor *);#ifdef __cplusplus}diff --git a/llamafile/sgemm_bss_avx2.cpp b/llamafile/sgemm_bss_avx2.cppdeleted file mode 100644index 5630e34c..00000000--- a/llamafile/sgemm_bss_avx2.cpp+++ /dev/null@@ -1,51 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"-#include <immintrin.h>--#define KN 8--#define V __m256-#define D __m256-#define TA ggml_bf16_t-#define TB float-#define TC float--static inline __m256 load(const float *p) {- return _mm256_loadu_ps(p);-}--static inline __m256 load(const ggml_bf16_t *p) {- return _mm256_castsi256_ps(- _mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)p)), 16));-}--#include "sgemmer.inc"--bool llamafile_sgemm_bss_avx2(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_bss_avx512.cpp b/llamafile/sgemm_bss_avx512.cppdeleted file mode 100644index 96c563a9..00000000--- a/llamafile/sgemm_bss_avx512.cpp+++ /dev/null@@ -1,51 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"-#include <immintrin.h>--#define KN 16--#define V __m512-#define D __m512-#define TA ggml_bf16_t-#define TB float-#define TC float--static inline __m512 load(const float *p) {- return _mm512_loadu_ps(p);-}--static inline __m512 load(const ggml_bf16_t *p) {- return _mm512_castsi512_ps(- _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)p)), 16));-}--#include "sgemmer.inc"--bool llamafile_sgemm_bss_avx512(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_bss_avx512bf16.cpp b/llamafile/sgemm_bss_avx512bf16.cppdeleted file mode 100644index a130e3a8..00000000--- a/llamafile/sgemm_bss_avx512bf16.cpp+++ /dev/null@@ -1,59 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"-#include "varith.h"-#include <immintrin.h>--#define KN 32--#define D __m512-#define V __m512bh-#define TA ggml_bf16_t-#define TB float-#define TC float--static inline __m512bh load(const ggml_bf16_t *p) {- return (__m512bh)_mm512_loadu_ps((const float *)p);-}--static inline __m512bh load(const float *p) {- return _mm512_cvtne2ps_pbh(_mm512_loadu_ps(p + 16), _mm512_loadu_ps(p));-}--template <> inline __m512 madd(__m512bh x, __m512bh y, __m512 z) {- return _mm512_dpbf16_ps(z, x, y);-}--template <> inline __m512 madder(__m512bh x, __m512bh y, __m512 z, __m512 *e) {- return _mm512_dpbf16_ps(z, x, y);-}--#include "sgemmer.inc"--bool llamafile_sgemm_bss_avx512bf16(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_e0q0s_avx512vnni.cpp b/llamafile/sgemm_e0q0s_avx512vnni.cppdeleted file mode 100644index 5107717a..00000000--- a/llamafile/sgemm_e0q0s_avx512vnni.cpp+++ /dev/null@@ -1,37 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"--#define TA block_q4_0-#define TB block_q8_0-#define TC float--#include "sgemmer0.inc"--bool llamafile_sgemm_e0q0s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,- int ldb, TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_e0q0s_avxvnni.cpp b/llamafile/sgemm_e0q0s_avxvnni.cppdeleted file mode 100644index b5713bb7..00000000--- a/llamafile/sgemm_e0q0s_avxvnni.cpp+++ /dev/null@@ -1,37 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"--#define TA block_q4_0-#define TB block_q8_0-#define TC float--#include "sgemmer0.inc"--bool llamafile_sgemm_e0q0s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_e0q0s_fma.cpp b/llamafile/sgemm_e0q0s_fma.cppdeleted file mode 100644index 8b0ccee3..00000000--- a/llamafile/sgemm_e0q0s_fma.cpp+++ /dev/null@@ -1,37 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"--#define TA block_q4_0-#define TB block_q8_0-#define TC float--#include "sgemmer0.inc"--bool llamafile_sgemm_e0q0s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_e1q1s_avx512vnni.cpp b/llamafile/sgemm_e1q1s_avx512vnni.cppdeleted file mode 100644index 1aa2264c..00000000--- a/llamafile/sgemm_e1q1s_avx512vnni.cpp+++ /dev/null@@ -1,37 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"--#define TA block_q4_1-#define TB block_q8_1-#define TC float--#include "sgemmer1.inc"--bool llamafile_sgemm_e1q1s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,- int ldb, TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER1 tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_e1q1s_avxvnni.cpp b/llamafile/sgemm_e1q1s_avxvnni.cppdeleted file mode 100644index 49ced14d..00000000--- a/llamafile/sgemm_e1q1s_avxvnni.cpp+++ /dev/null@@ -1,37 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"--#define TA block_q4_1-#define TB block_q8_1-#define TC float--#include "sgemmer1.inc"--bool llamafile_sgemm_e1q1s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER1 tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_e1q1s_fma.cpp b/llamafile/sgemm_e1q1s_fma.cppdeleted file mode 100644index 178e23ac..00000000--- a/llamafile/sgemm_e1q1s_fma.cpp+++ /dev/null@@ -1,37 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"--#define TA block_q4_1-#define TB block_q8_1-#define TC float--#include "sgemmer1.inc"--bool llamafile_sgemm_e1q1s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER1 tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_hhs_neon.cpp b/llamafile/sgemm_hhs_neon.cppdeleted file mode 100644index 5507a045..00000000--- a/llamafile/sgemm_hhs_neon.cpp+++ /dev/null@@ -1,46 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __aarch64__--#include "sgemm.h"-#include <arm_neon.h>--#define KN 8--#define V float16x8_t-#define D float16x8_t-#define TA unsigned short-#define TB unsigned short-#define TC float--static inline V load(const unsigned short *p) {- return vld1q_f16((const __fp16 *)p);-}--#include "sgemmer.inc"--bool llamafile_sgemm_hhs_neon(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __aarch64__diff --git a/llamafile/sgemm_hsh_avx512f.cpp b/llamafile/sgemm_hsh_avx512f.cppdeleted file mode 100644index 463dcdf8..00000000--- a/llamafile/sgemm_hsh_avx512f.cpp+++ /dev/null@@ -1,50 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"-#include <immintrin.h>--#define KN 16--#define V __m512-#define D __m512-#define TA unsigned short-#define TB float-#define TC llamafile_fp16--static inline V load(const float *p) {- return _mm512_loadu_ps(p);-}--static inline V load(const unsigned short *p) {- return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)p));-}--#include "sgemmer.inc"--bool llamafile_sgemm_hsh_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_hsh_f16c.cpp b/llamafile/sgemm_hsh_f16c.cppdeleted file mode 100644index 7bb09b4e..00000000--- a/llamafile/sgemm_hsh_f16c.cpp+++ /dev/null@@ -1,50 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"-#include <immintrin.h>--#define KN 8--#define V __m256-#define D __m256-#define TA unsigned short-#define TB float-#define TC llamafile_fp16--static inline V load(const float *p) {- return _mm256_loadu_ps(p);-}--static inline V load(const unsigned short *p) {- return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));-}--#include "sgemmer.inc"--bool llamafile_sgemm_hsh_f16c(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_hsh_neon.cpp b/llamafile/sgemm_hsh_neon.cppdeleted file mode 100644index 35db6a4b..00000000--- a/llamafile/sgemm_hsh_neon.cpp+++ /dev/null@@ -1,50 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __aarch64__--#include "sgemm.h"-#include <arm_neon.h>--#define KN 4--#define V float32x4_t-#define D float32x4_t-#define TA unsigned short-#define TB float-#define TC llamafile_fp16--static inline V load(const float *p) {- return vld1q_f32(p);-}--static inline V load(const unsigned short *p) {- return vcvt_f32_f16(vld1_f16((const __fp16 *)p));-}--#include "sgemmer.inc"--bool llamafile_sgemm_hsh_neon(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __aarch64__diff --git a/llamafile/sgemm_hss_avx512f.cpp b/llamafile/sgemm_hss_avx512f.cppdeleted file mode 100644index f0a8bf33..00000000--- a/llamafile/sgemm_hss_avx512f.cpp+++ /dev/null@@ -1,50 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"-#include <immintrin.h>--#define KN 16--#define V __m512-#define D __m512-#define TA unsigned short-#define TB float-#define TC float--static inline V load(const float *p) {- return _mm512_loadu_ps(p);-}--static inline V load(const unsigned short *p) {- return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)p));-}--#include "sgemmer.inc"--bool llamafile_sgemm_hss_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_hss_f16c.cpp b/llamafile/sgemm_hss_f16c.cppdeleted file mode 100644index e9ba461b..00000000--- a/llamafile/sgemm_hss_f16c.cpp+++ /dev/null@@ -1,50 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"-#include <immintrin.h>--#define KN 8--#define V __m256-#define D __m256-#define TA unsigned short-#define TB float-#define TC float--static inline V load(const float *p) {- return _mm256_loadu_ps(p);-}--static inline V load(const unsigned short *p) {- return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));-}--#include "sgemmer.inc"--bool llamafile_sgemm_hss_f16c(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_hss_neon.cpp b/llamafile/sgemm_hss_neon.cppdeleted file mode 100644index e10e37e9..00000000--- a/llamafile/sgemm_hss_neon.cpp+++ /dev/null@@ -1,50 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __aarch64__--#include "sgemm.h"-#include <arm_neon.h>--#define KN 4--#define V float32x4_t-#define D float32x4_t-#define TA unsigned short-#define TB float-#define TC float--static inline V load(const float *p) {- return vld1q_f32(p);-}--static inline V load(const unsigned short *p) {- return vcvt_f32_f16(vld1_f16((const __fp16 *)p));-}--#include "sgemmer.inc"--bool llamafile_sgemm_hss_neon(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __aarch64__diff --git a/llamafile/sgemm_q0q0s_avx512vnni.cpp b/llamafile/sgemm_q0q0s_avx512vnni.cppdeleted file mode 100644index e5b3ed24..00000000--- a/llamafile/sgemm_q0q0s_avx512vnni.cpp+++ /dev/null@@ -1,37 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"--#define TA block_q8_0-#define TB block_q8_0-#define TC float--#include "sgemmer0.inc"--bool llamafile_sgemm_q0q0s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,- int ldb, TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_q0q0s_avxvnni.cpp b/llamafile/sgemm_q0q0s_avxvnni.cppdeleted file mode 100644index d53b2979..00000000--- a/llamafile/sgemm_q0q0s_avxvnni.cpp+++ /dev/null@@ -1,37 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"--#define TA block_q8_0-#define TB block_q8_0-#define TC float--#include "sgemmer0.inc"--bool llamafile_sgemm_q0q0s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_q0q0s_dotprod.cpp b/llamafile/sgemm_q0q0s_dotprod.cppdeleted file mode 100644index c737f66c..00000000--- a/llamafile/sgemm_q0q0s_dotprod.cpp+++ /dev/null@@ -1,182 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __aarch64__--#include <arm_neon.h>--#include "llama.cpp/ggml-impl.h"-#include "llama.cpp/ggml.h"--#include "hsum.h"-#include "kernel.h"-#include "sgemm.h"--namespace {--class GEMMERQ0ARM {- public:- GEMMERQ0ARM(int k, const block_q8_0 *A, int lda, const block_q8_0 *B, int ldb, float *C,- int ldc, int ith, int nth)- : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {- }-- void matmul(int m, int n) {- mnpack(0, m, 0, n);- }-- private:- dontinline void mnpack(int m0, int m, int n0, int n) {- if (m - m0 <= 0 || n - n0 <= 0)- return;- int mc, nc, mp, np;- if (m - m0 >= 3 && n - n0 >= 3) {- mc = 3;- nc = 3;- gemm3x3(m0, m, n0, n);- } else {- mc = 1;- nc = 1;- gemm1x1(m0, m, n0, n);- }- mp = m0 + (m - m0) / mc * mc;- np = n0 + (n - n0) / nc * nc;- mnpack(mp, m, n0, np);- mnpack(m0, m, np, n);- }-- dontinline void gemm3x3(int m0, int m, int n0, int n) {- BEGIN_KERNEL(3, 3)- int32x4_t zero = vdupq_n_s32(0);- float32x4_t c00 = vdupq_n_f32(0.f);- float32x4_t c01 = vdupq_n_f32(0.f);- float32x4_t c02 = vdupq_n_f32(0.f);- float32x4_t c10 = vdupq_n_f32(0.f);- float32x4_t c11 = vdupq_n_f32(0.f);- float32x4_t c12 = vdupq_n_f32(0.f);- float32x4_t c20 = vdupq_n_f32(0.f);- float32x4_t c21 = vdupq_n_f32(0.f);- float32x4_t c22 = vdupq_n_f32(0.f);- const block_q8_0 *Ap0 = A + lda * (i + 0);- const block_q8_0 *Ap1 = A + lda * (i + 1);- const block_q8_0 *Ap2 = A + lda * (i + 2);- const block_q8_0 *Bp0 = B + ldb * (j + 0);- const block_q8_0 *Bp1 = B + ldb * (j + 1);- const block_q8_0 *Bp2 = B + ldb * (j + 2);- for (int l = 0; l < k; ++l) {- c00 = vmlaq_n_f32(- c00,- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap0[l].qs), vld1q_s8(Bp0[l].qs)),- vld1q_s8(Ap0[l].qs + 16), vld1q_s8(Bp0[l].qs + 16))),- unhalf(Ap0[l].d) * unhalf(Bp0[l].d));- c01 = vmlaq_n_f32(- c01,- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap0[l].qs), vld1q_s8(Bp1[l].qs)),- vld1q_s8(Ap0[l].qs + 16), vld1q_s8(Bp1[l].qs + 16))),- unhalf(Ap0[l].d) * unhalf(Bp1[l].d));- c02 = vmlaq_n_f32(- c02,- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap0[l].qs), vld1q_s8(Bp2[l].qs)),- vld1q_s8(Ap0[l].qs + 16), vld1q_s8(Bp2[l].qs + 16))),- unhalf(Ap0[l].d) * unhalf(Bp2[l].d));- c10 = vmlaq_n_f32(- c10,- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap1[l].qs), vld1q_s8(Bp0[l].qs)),- vld1q_s8(Ap1[l].qs + 16), vld1q_s8(Bp0[l].qs + 16))),- unhalf(Ap1[l].d) * unhalf(Bp0[l].d));- c11 = vmlaq_n_f32(- c11,- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap1[l].qs), vld1q_s8(Bp1[l].qs)),- vld1q_s8(Ap1[l].qs + 16), vld1q_s8(Bp1[l].qs + 16))),- unhalf(Ap1[l].d) * unhalf(Bp1[l].d));- c12 = vmlaq_n_f32(- c12,- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap1[l].qs), vld1q_s8(Bp2[l].qs)),- vld1q_s8(Ap1[l].qs + 16), vld1q_s8(Bp2[l].qs + 16))),- unhalf(Ap1[l].d) * unhalf(Bp2[l].d));- c20 = vmlaq_n_f32(- c20,- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap2[l].qs), vld1q_s8(Bp0[l].qs)),- vld1q_s8(Ap2[l].qs + 16), vld1q_s8(Bp0[l].qs + 16))),- unhalf(Ap2[l].d) * unhalf(Bp0[l].d));- c21 = vmlaq_n_f32(- c21,- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap2[l].qs), vld1q_s8(Bp1[l].qs)),- vld1q_s8(Ap2[l].qs + 16), vld1q_s8(Bp1[l].qs + 16))),- unhalf(Ap2[l].d) * unhalf(Bp1[l].d));- c22 = vmlaq_n_f32(- c22,- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap2[l].qs), vld1q_s8(Bp2[l].qs)),- vld1q_s8(Ap2[l].qs + 16), vld1q_s8(Bp2[l].qs + 16))),- unhalf(Ap2[l].d) * unhalf(Bp2[l].d));- }- C[ldc * (j + 0) + (i + 0)] = hsum(c00);- C[ldc * (j + 0) + (i + 1)] = hsum(c10);- C[ldc * (j + 0) + (i + 2)] = hsum(c20);- C[ldc * (j + 1) + (i + 0)] = hsum(c01);- C[ldc * (j + 1) + (i + 1)] = hsum(c11);- C[ldc * (j + 1) + (i + 2)] = hsum(c21);- C[ldc * (j + 2) + (i + 0)] = hsum(c02);- C[ldc * (j + 2) + (i + 1)] = hsum(c12);- C[ldc * (j + 2) + (i + 2)] = hsum(c22);- END_KERNEL()- }-- dontinline void gemm1x1(int m0, int m, int n0, int n) {- BEGIN_KERNEL(1, 1)- float32x4_t acc = vdupq_n_f32(0.f);- const block_q8_0 *Ap = A + lda * i;- const block_q8_0 *Bp = B + ldb * j;- for (int l = 0; l < k; ++l) {- acc = vmlaq_n_f32(acc,- vcvtq_f32_s32(vdotq_s32(- vdotq_s32(vdupq_n_s32(0), vld1q_s8(Ap[l].qs), vld1q_s8(Bp[l].qs)),- vld1q_s8(Ap[l].qs + 16), vld1q_s8(Bp[l].qs + 16))),- unhalf(Ap[l].d) * unhalf(Bp[l].d));- }- C[ldc * j + i] = hsum(acc);- END_KERNEL()- }-- static inline float unhalf(unsigned short d) {- return GGML_FP16_TO_FP32(d);- }-- const block_q8_0 *const A;- const block_q8_0 *const B;- float *const C;- const int k;- const int lda;- const int ldb;- const int ldc;- const int ith;- const int nth;-};--} // namespace--bool llamafile_sgemm_q0q0s_dotprod(int m, int n, int k, const block_q8_0 *A, int lda,- const block_q8_0 *B, int ldb, float *C, int ldc, int ith,- int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- GEMMERQ0ARM tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __aarch64__diff --git a/llamafile/sgemm_q0q0s_fma.cpp b/llamafile/sgemm_q0q0s_fma.cppdeleted file mode 100644index 437ec389..00000000--- a/llamafile/sgemm_q0q0s_fma.cpp+++ /dev/null@@ -1,37 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"--#define TA block_q8_0-#define TB block_q8_0-#define TC float--#include "sgemmer0.inc"--bool llamafile_sgemm_q0q0s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_sss_avx.cpp b/llamafile/sgemm_sss_avx.cppdeleted file mode 100644index a49427c8..00000000--- a/llamafile/sgemm_sss_avx.cpp+++ /dev/null@@ -1,46 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"-#include <immintrin.h>--#define KN 8--#define V __m256-#define D __m256-#define TA float-#define TB float-#define TC float--static inline V load(const float *p) {- return _mm256_loadu_ps(p);-}--#include "sgemmer.inc"--bool llamafile_sgemm_sss_avx(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,- int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_sss_avx512f.cpp b/llamafile/sgemm_sss_avx512f.cppdeleted file mode 100644index a9dbc1a0..00000000--- a/llamafile/sgemm_sss_avx512f.cpp+++ /dev/null@@ -1,46 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"-#include <immintrin.h>--#define KN 16--#define V __m512-#define D __m512-#define TA float-#define TB float-#define TC float--static inline V load(const float *p) {- return _mm512_loadu_ps(p);-}--#include "sgemmer.inc"--bool llamafile_sgemm_sss_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_sss_fma.cpp b/llamafile/sgemm_sss_fma.cppdeleted file mode 100644index f12ad7d6..00000000--- a/llamafile/sgemm_sss_fma.cpp+++ /dev/null@@ -1,46 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __x86_64__--#include "sgemm.h"-#include <immintrin.h>--#define KN 8--#define V __m256-#define D __m256-#define TA float-#define TB float-#define TC float--static inline V load(const float *p) {- return _mm256_loadu_ps(p);-}--#include "sgemmer.inc"--bool llamafile_sgemm_sss_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,- int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __x86_64__diff --git a/llamafile/sgemm_sss_neon.cpp b/llamafile/sgemm_sss_neon.cppdeleted file mode 100644index 88644221..00000000--- a/llamafile/sgemm_sss_neon.cpp+++ /dev/null@@ -1,46 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#ifdef __aarch64__--#include "sgemm.h"-#include <arm_neon.h>--#define KN 4--#define V float32x4_t-#define D float32x4_t-#define TA float-#define TB float-#define TC float--static inline V load(const float *p) {- return vld1q_f32(p);-}--#include "sgemmer.inc"--bool llamafile_sgemm_sss_neon(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,- TC *C, int ldc, int ith, int nth, int task) {- if (task != GGML_TASK_TYPE_COMPUTE)- return true;- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};- tb.matmul(m, n);- return true;-}--#endif // __aarch64__diff --git a/llamafile/sgemm_sss_test.cpp b/llamafile/sgemm_sss_test.cppdeleted file mode 100644index 79c00839..00000000--- a/llamafile/sgemm_sss_test.cpp+++ /dev/null@@ -1,79 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#include "bench.h"-#include "float.h"-#include "gemm.h"-#include "llama.cpp/ggml.h"-#include "llamafile.h"-#include "numba.h"--#define ITERATIONS 5--int main(int argc, char *argv[]) {- float tolerance = 2e-5;- int m = 510;- int n = 513;- int k = 512;- int l = 3;- int lda = k + l;- int ldb = k + l;- int ldc = m + l;- float *A = new float[lda * m];- float *B = new float[ldb * n];- float *C = new float[ldc * n];- float *G = new float[ldc * n];- broadcast(A, lda * m, NAN);- broadcast(B, ldb * n, NAN);- broadcast(C, ldc * n, NAN);- broadcast(G, ldc * n, NAN);- randomize(k, m, A, lda);- randomize(k, n, B, ldb);-- BENCH(gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc));- BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,- GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32));-- for (int i = 0; i < m; ++i)- for (int j = 0; j < n; ++j) {- auto g = G[ldc * j + i];- auto c = C[ldc * j + i];- if (flt::isnan(g)) {- fprintf(stderr, "%s:%d: found nan in reference matrix: i=%d j=%d\n", __FILE__,- __LINE__, i, j);- return 3;- }- if (flt::isnan(c)) {- fprintf(stderr, "%s:%d: found nan in output matrix: i=%d j=%d\n", __FILE__,- __LINE__, i, j);- return 4;- }- auto d = g - c;- if (d < 0)- d = -d;- if (d > tolerance) {- fprintf(stderr, "%s:%d: difference exceeds tolerance: %g vs. %g (%g) i=%d j=%d\n",- __FILE__, __LINE__, g, c, d, i, j);- return 5;- }- }-- delete[] G;- delete[] C;- delete[] B;- delete[] A;-}diff --git a/llamafile/sgemm_vecdot_test.cpp b/llamafile/sgemm_vecdot_test.cppindex 79507d41..78c7587b 100644--- a/llamafile/sgemm_vecdot_test.cpp+++ b/llamafile/sgemm_vecdot_test.cpp@@ -74,24 +74,28 @@ int main(int argc, char *argv[]) {}double err_avg = err_sum / (m * n);- fprintf(stderr, "note: %g average ulp\n", err_avg);- fprintf(stderr, "note: %lld worst ulp\n", err_worst);+ fprintf(stderr, "%9g ulp average\n", err_avg);+ fprintf(stderr, "%9lld ulp worst\n", err_worst);// using one accumulator- // 52.7393 average ulp- // 9052 worst ulp+ // 40209 us gemm+ // 2851 us llamafile_sgemm+ // 42.0078 ulp average+ // 6731 ulp worst// using three accumulators// 22.291 average ulp// 1566 worst ulp// using kahan summation- // 2.14244 average ulp- // 134 worst ulp+ // 40190 us gemm+ // 3028 us llamafile_sgemm+ // 2.14244 ulp average+ // 134 ulp worst- if (err_avg > 30)+ if (err_avg > 2.15)return 5;- if (err_worst > 3000)+ if (err_worst > 134)return 6;delete[] G;diff --git a/llamafile/sgemmer.inc b/llamafile/sgemmer.incdeleted file mode 100644index 762cafaa..00000000--- a/llamafile/sgemmer.inc+++ /dev/null@@ -1,230 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#include <algorithm>--#include "llama.cpp/ggml.h"--#include "hsum.h"-#include "kernel.h"-#include "varith.h"--namespace {--class SGEMMER {- public:- SGEMMER(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)- : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {- }-- void matmul(int m, int n) {- mnpack(0, m, 0, n);- }-- private:- void mnpack(int m0, int m, int n0, int n) {- int mc, nc, mp, np;- switch ((std::min(m - m0, 5) << 4) | std::min(n - n0, 5)) {-#if VECTOR_REGISTERS == 32- case 0x55:- mc = 5;- nc = 5;- gemm<5, 5>(m0, m, n0, n);- break;- case 0x45:- mc = 4;- nc = 5;- gemm<4, 5>(m0, m, n0, n);- break;- case 0x54:- mc = 5;- nc = 4;- gemm<5, 4>(m0, m, n0, n);- break;- case 0x44:- mc = 4;- nc = 4;- gemm<4, 4>(m0, m, n0, n);- break;- case 0x53:- mc = 5;- nc = 3;- gemm<5, 3>(m0, m, n0, n);- break;- case 0x35:- mc = 3;- nc = 5;- gemm<3, 5>(m0, m, n0, n);- break;- case 0x43:- mc = 4;- nc = 3;- gemm<4, 3>(m0, m, n0, n);- break;-#else- case 0x55:- case 0x54:- case 0x53:- case 0x45:- case 0x44:- case 0x43:- mc = 4;- nc = 3;- gemm<4, 3>(m0, m, n0, n);- break;- case 0x35:-#endif- case 0x34:- mc = 3;- nc = 4;- gemm<3, 4>(m0, m, n0, n);- break;- case 0x52:- mc = 5;- nc = 2;- gemm<5, 2>(m0, m, n0, n);- break;- case 0x33:- mc = 3;- nc = 3;- gemm<3, 3>(m0, m, n0, n);- break;- case 0x25:- mc = 2;- nc = 5;- gemm<2, 5>(m0, m, n0, n);- break;- case 0x42:- mc = 4;- nc = 2;- gemm<4, 2>(m0, m, n0, n);- break;- case 0x24:- mc = 2;- nc = 4;- gemm<2, 4>(m0, m, n0, n);- break;- case 0x32:- mc = 3;- nc = 2;- gemm<3, 2>(m0, m, n0, n);- break;- case 0x23:- mc = 2;- nc = 3;- gemm<2, 3>(m0, m, n0, n);- break;- case 0x51:- mc = 5;- nc = 1;- gemm<5, 1>(m0, m, n0, n);- break;- case 0x41:- mc = 4;- nc = 1;- gemm<4, 1>(m0, m, n0, n);- break;- case 0x22:- mc = 2;- nc = 2;- gemm<2, 2>(m0, m, n0, n);- break;- case 0x15:- mc = 1;- nc = 5;- gemm<1, 5>(m0, m, n0, n);- break;- case 0x14:- mc = 1;- nc = 4;- gemm<1, 4>(m0, m, n0, n);- break;- case 0x31:- mc = 3;- nc = 1;- gemm<3, 1>(m0, m, n0, n);- break;- case 0x13:- mc = 1;- nc = 3;- gemm<1, 3>(m0, m, n0, n);- break;- case 0x21:- mc = 2;- nc = 1;- gemm<2, 1>(m0, m, n0, n);- break;- case 0x12:- mc = 1;- nc = 2;- gemm<1, 2>(m0, m, n0, n);- break;- case 0x11:- mc = 1;- nc = 1;- gemm<1, 1>(m0, m, n0, n);- break;- default:- return;- }- mp = m0 + (m - m0) / mc * mc;- np = n0 + (n - n0) / nc * nc;- mnpack(mp, m, n0, np);- mnpack(m0, m, np, n);- }-- template <int RM, int RN> dontinline void gemm(int m0, int m, int n0, int n) {- int ytiles = (m - m0) / RM;- int xtiles = (n - n0) / RN;- int tiles = xtiles * ytiles;- int duty = (tiles + nth - 1) / nth;- int start = duty * ith;- int end = start + duty;- if (end > tiles)- end = tiles;- for (int job = start; job < end; ++job) {- int ii = m0 + job / xtiles * RM;- int jj = n0 + job % xtiles * RN;- D Cv[RN][RM] = {0};- for (int l = 0; l < k; l += KN)- for (int j = 0; j < RN; ++j)- for (int i = 0; i < RM; ++i)- Cv[j][i] = madd(load(A + lda * (ii + i) + l), //- load(B + ldb * (jj + j) + l), //- Cv[j][i]);- TC Cd[RN][RM];- for (int j = 0; j < RN; ++j)- for (int i = 0; i < RM; ++i)- Cd[j][i] = hsum(Cv[j][i]);- for (int j = 0; j < RN; ++j)- for (int i = 0; i < RM; ++i)- C[ldc * (jj + j) + (ii + i)] = Cd[j][i];- }- }-- const TA *const A;- const TB *const B;- TC *const C;- const int k;- const int lda;- const int ldb;- const int ldc;- const int ith;- const int nth;-};--} // namespacediff --git a/llamafile/sgemmer0.inc b/llamafile/sgemmer0.incdeleted file mode 100644index 2475875e..00000000--- a/llamafile/sgemmer0.inc+++ /dev/null@@ -1,222 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#include <algorithm>--#include "llama.cpp/ggml-impl.h"-#include "llama.cpp/ggml.h"--#include "hsum.h"-#include "kernel.h"-#include "varith.h"--namespace {--class SGEMMER0 {- public:- SGEMMER0(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)- : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {- }-- void matmul(int m, int n) {- mnpack(0, m, 0, n);- }-- private:- void mnpack(int m0, int m, int n0, int n) {- int mc, nc, mp, np;- switch ((std::min(m - m0, 4) << 4) | std::min(n - n0, 4)) {-#if VECTOR_REGISTERS == 32- case 0x44:- mc = 4;- nc = 4;- gemm<4, 4>(m0, m, n0, n);- break;- case 0x43:- mc = 4;- nc = 3;- gemm<4, 3>(m0, m, n0, n);- break;- case 0x34:- mc = 3;- nc = 4;- gemm<3, 4>(m0, m, n0, n);- break;- case 0x33:- mc = 3;- nc = 3;- gemm<3, 3>(m0, m, n0, n);- break;- case 0x42:- mc = 4;- nc = 2;- gemm<4, 2>(m0, m, n0, n);- break;- case 0x24:- mc = 2;- nc = 4;- gemm<2, 4>(m0, m, n0, n);- break;-#else- case 0x44:- case 0x43:- case 0x42:- mc = 4;- nc = 2;- gemm<4, 2>(m0, m, n0, n);- break;- case 0x34:- case 0x24:- mc = 2;- nc = 4;- gemm<2, 4>(m0, m, n0, n);- break;- case 0x33:-#endif- case 0x32:- mc = 3;- nc = 2;- gemm<3, 2>(m0, m, n0, n);- break;- case 0x23:- mc = 2;- nc = 3;- gemm<2, 3>(m0, m, n0, n);- break;- case 0x41:- mc = 4;- nc = 1;- gemm<4, 1>(m0, m, n0, n);- break;- case 0x22:- mc = 2;- nc = 2;- gemm<2, 2>(m0, m, n0, n);- break;- case 0x14:- mc = 1;- nc = 4;- gemm<1, 4>(m0, m, n0, n);- break;- case 0x31:- mc = 3;- nc = 1;- gemm<3, 1>(m0, m, n0, n);- break;- case 0x13:- mc = 1;- nc = 3;- gemm<1, 3>(m0, m, n0, n);- break;- case 0x21:- mc = 2;- nc = 1;- gemm<2, 1>(m0, m, n0, n);- break;- case 0x12:- mc = 1;- nc = 2;- gemm<1, 2>(m0, m, n0, n);- break;- case 0x11:- mc = 1;- nc = 1;- gemm<1, 1>(m0, m, n0, n);- break;- default:- return;- }- mp = m0 + (m - m0) / mc * mc;- np = n0 + (n - n0) / nc * nc;- mnpack(mp, m, n0, np);- mnpack(m0, m, np, n);- }-- template <int RM, int RN> dontinline void gemm(int m0, int m, int n0, int n) {- int ytiles = (m - m0) / RM;- int xtiles = (n - n0) / RN;- int tiles = xtiles * ytiles;- int duty = (tiles + nth - 1) / nth;- int start = duty * ith;- int end = start + duty;- if (end > tiles)- end = tiles;- for (int job = start; job < end; ++job) {- int ii = m0 + job / xtiles * RM;- int jj = n0 + job % xtiles * RN;- __m256 Cv[RN][RM] = {0};- for (int l = 0; l < k; ++l)- for (int j = 0; j < RN; ++j)- for (int i = 0; i < RM; ++i)- Cv[j][i] = madd(_mm256_set1_ps(unhalf(A[lda * (ii + i) + l].d) *- unhalf(B[ldb * (jj + j) + l].d)),- updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),- load(A + lda * (ii + i) + l)),- _mm256_sign_epi8(load(B + ldb * (jj + j) + l),- load(A + lda * (ii + i) + l))),- Cv[j][i]);- TC Cd[RN][RM];- for (int j = 0; j < RN; ++j)- for (int i = 0; i < RM; ++i)- Cd[j][i] = hsum(Cv[j][i]);- for (int j = 0; j < RN; ++j)- for (int i = 0; i < RM; ++i)- C[ldc * (jj + j) + (ii + i)] = Cd[j][i];- }- }-- inline __m256i load(const block_q8_0 *b) {- return _mm256_loadu_si256((const __m256i *)b->qs);- }-- inline __m256i load(const block_q4_0 *b) {- return _mm256_sub_epi8(denibble(b->qs), _mm256_set1_epi8(8));- }-- inline __m256 updot(__m256i u, __m256i s) {- __m256i res;-#if defined(__AVXVNNI__) || defined(__AVX512VNNI__)- res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);-#else- res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));-#endif- return _mm256_cvtepi32_ps(res);- }-- static inline __m256i denibble(const uint8_t *p) {- const __m128i tmp = _mm_loadu_si128((const __m128i *)p);- const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);- const __m256i lowMask = _mm256_set1_epi8(15);- return _mm256_and_si256(lowMask, bytes);- }-- static inline float unhalf(unsigned short d) {- return GGML_FP16_TO_FP32(d);- }-- const TA *const A;- const TB *const B;- TC *const C;- const int k;- const int lda;- const int ldb;- const int ldc;- const int ith;- const int nth;-};--} // namespacediff --git a/llamafile/sgemmer1.inc b/llamafile/sgemmer1.incdeleted file mode 100644index f8966d8f..00000000--- a/llamafile/sgemmer1.inc+++ /dev/null@@ -1,215 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#include "llama.cpp/ggml-impl.h"-#include "llama.cpp/ggml.h"--#include "hsum.h"-#include "kernel.h"-#include "varith.h"--namespace {--class SGEMMER1 {- public:- SGEMMER1(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)- : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {- }-- void matmul(int m, int n) {- mnpack(0, m, 0, n);- }-- private:- dontinline void mnpack(int m0, int m, int n0, int n) {- if (m - m0 <= 0 || n - n0 <= 0)- return;- int mc, nc, mp, np;- if (VECTOR_REGISTERS >= 32 && m - m0 >= 4 && n - n0 >= 2) {- mc = 4;- nc = 2;- gemm4x2(m0, m, n0, n);- } else if (m - m0 >= 4 && n - n0 >= 1) {- mc = 4;- nc = 1;- gemm4x1(m0, m, n0, n);- } else {- mc = 1;- nc = 1;- gemm1x1(m0, m, n0, n);- }- mp = m0 + (m - m0) / mc * mc;- np = n0 + (n - n0) / nc * nc;- mnpack(mp, m, n0, np);- mnpack(m0, m, np, n);- }-- dontinline void gemm4x2(int m0, int m, int n0, int n) {- BEGIN_KERNEL(4, 2)- __m128 s0 = _mm_setzero_ps();- __m128 s1 = _mm_setzero_ps();- __m256 c00 = _mm256_setzero_ps();- __m256 c01 = _mm256_setzero_ps();- __m256 c02 = _mm256_setzero_ps();- __m256 c03 = _mm256_setzero_ps();- __m256 c10 = _mm256_setzero_ps();- __m256 c11 = _mm256_setzero_ps();- __m256 c12 = _mm256_setzero_ps();- __m256 c13 = _mm256_setzero_ps();- const TA *Ap0 = A + lda * (i + 0);- const TA *Ap1 = A + lda * (i + 1);- const TA *Ap2 = A + lda * (i + 2);- const TA *Ap3 = A + lda * (i + 3);- const TB *Bp0 = B + ldb * (j + 0);- const TB *Bp1 = B + ldb * (j + 1);- for (int l = 0; l < k; ++l) {- __m128 sa = _mm_set_ps(unhalf(Ap3[l].dm >> 16), //- unhalf(Ap2[l].dm >> 16), //- unhalf(Ap1[l].dm >> 16), //- unhalf(Ap0[l].dm >> 16));- s0 = madd(_mm_set1_ps(unhalf(Bp0[l].ds >> 16)), sa, s0);- s1 = madd(_mm_set1_ps(unhalf(Bp1[l].ds >> 16)), sa, s1);- __m256i f0 = load(Bp0 + l);- __m256 d0 = _mm256_set1_ps(unhalf(Bp0[l].ds));- c00 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap0[l].dm)), d0),- updot(load(Ap0 + l), f0), c00);- c01 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap1[l].dm)), d0),- updot(load(Ap1 + l), f0), c01);- c02 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap2[l].dm)), d0),- updot(load(Ap2 + l), f0), c02);- c03 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap3[l].dm)), d0),- updot(load(Ap3 + l), f0), c03);- __m256i f1 = load(Bp1 + l);- __m256 d1 = _mm256_set1_ps(unhalf(Bp1[l].ds));- c10 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap0[l].dm)), d1),- updot(load(Ap0 + l), f1), c10);- c11 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap1[l].dm)), d1),- updot(load(Ap1 + l), f1), c11);- c12 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap2[l].dm)), d1),- updot(load(Ap2 + l), f1), c12);- c13 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap3[l].dm)), d1),- updot(load(Ap3 + l), f1), c13);- }- C[ldc * (j + 0) + (i + 0)] = hsum(c00) + s0[0];- C[ldc * (j + 0) + (i + 1)] = hsum(c01) + s0[1];- C[ldc * (j + 0) + (i + 2)] = hsum(c02) + s0[2];- C[ldc * (j + 0) + (i + 3)] = hsum(c03) + s0[3];- C[ldc * (j + 1) + (i + 0)] = hsum(c10) + s1[0];- C[ldc * (j + 1) + (i + 1)] = hsum(c11) + s1[1];- C[ldc * (j + 1) + (i + 2)] = hsum(c12) + s1[2];- C[ldc * (j + 1) + (i + 3)] = hsum(c13) + s1[3];- END_KERNEL()- }-- dontinline void gemm4x1(int m0, int m, int n0, int n) {- BEGIN_KERNEL(4, 1)- __m128 s = _mm_setzero_ps();- __m256 c0 = _mm256_setzero_ps();- __m256 c1 = _mm256_setzero_ps();- __m256 c2 = _mm256_setzero_ps();- __m256 c3 = _mm256_setzero_ps();- const TA *Ap0 = A + lda * (i + 0);- const TA *Ap1 = A + lda * (i + 1);- const TA *Ap2 = A + lda * (i + 2);- const TA *Ap3 = A + lda * (i + 3);- const TB *Bp = B + ldb * j;- for (int l = 0; l < k; ++l) {- s = madd(_mm_set1_ps(unhalf(Bp[l].ds >> 16)),- _mm_set_ps(unhalf(Ap3[l].dm >> 16), //- unhalf(Ap2[l].dm >> 16), //- unhalf(Ap1[l].dm >> 16), //- unhalf(Ap0[l].dm >> 16)),- s);- __m256i f = load(Bp + l);- __m256 d1 = _mm256_set1_ps(unhalf(Bp[l].ds));- c0 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap0[l].dm)), d1), updot(load(Ap0 + l), f),- c0);- c1 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap1[l].dm)), d1), updot(load(Ap1 + l), f),- c1);- c2 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap2[l].dm)), d1), updot(load(Ap2 + l), f),- c2);- c3 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap3[l].dm)), d1), updot(load(Ap3 + l), f),- c3);- }- C[ldc * j + (i + 0)] = hsum(c0) + s[0];- C[ldc * j + (i + 1)] = hsum(c1) + s[1];- C[ldc * j + (i + 2)] = hsum(c2) + s[2];- C[ldc * j + (i + 3)] = hsum(c3) + s[3];- END_KERNEL()- }-- dontinline void gemm1x1(int m0, int m, int n0, int n) {- BEGIN_KERNEL(1, 1)- float s = 0;- __m256 c = _mm256_setzero_ps();- const TA *Ap = A + lda * i;- const TB *Bp = B + ldb * j;- for (int l = 0; l < k; ++l) {- unsigned dm = Ap[l].dm;- unsigned ds = Bp[l].ds;- __m256 d0 = _mm256_set1_ps(unhalf(dm));- __m256 d1 = _mm256_set1_ps(unhalf(ds));- s += unhalf(dm >> 16) * unhalf(ds >> 16);- __m256i e = load(Ap + l);- __m256i f = load(Bp + l);- c = madd(_mm256_mul_ps(d0, d1), updot(e, f), c);- }- C[ldc * j + i] = hsum(c) + s;- END_KERNEL()- }-- static inline nosideeffect __m256i load(const block_q8_1 *b) {- return _mm256_loadu_si256((const __m256i *)b->qs);- }-- static inline nosideeffect __m256i load(const block_q4_1 *b) {- return denibble(b->qs);- }-- static inline pureconst __m256 updot(__m256i u, __m256i s) {- __m256i res;-#if defined(__AVXVNNI__) || defined(__AVX512VNNI__)- res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);-#else- res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));-#endif- return _mm256_cvtepi32_ps(res);- }-- static inline nosideeffect __m256i denibble(const uint8_t *p) {- const __m128i tmp = _mm_loadu_si128((const __m128i *)p);- const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);- const __m256i lowMask = _mm256_set1_epi8(15);- return _mm256_and_si256(lowMask, bytes);- }-- static inline pureconst float unhalf(unsigned short d) {- return GGML_FP16_TO_FP32(d);- }-- const TA *const A;- const TB *const B;- TC *const C;- const int k;- const int lda;- const int ldb;- const int ldc;- const int ith;- const int nth;-};--} // namespacediff --git a/llamafile/tinyblas_cpu.inc b/llamafile/tinyblas_cpu.incnew file mode 100644index 00000000..abf05539--- /dev/null+++ b/llamafile/tinyblas_cpu.inc@@ -0,0 +1,1496 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++//+// _ _ ___ _ _ ___+// | |_(_)_ _ _ _| _ ) | /_\ / __|+// | _| | ' \ || | _ \ |__ / _ \\__ \.+// \__|_|_||_\_, |___/____/_/ \_\___/+// |__/+//+// BASIC LINEAR ALGEBRA SUBPROGRAMS+//+//+// This file implements multithreaded CPU matrix multiplication for the+// common contiguous use case C = Aᵀ * B. These kernels are designed to+// have excellent performance[1] for matrices that fit in the CPU cache+// without imposing any overhead such as cache filling or malloc calls.+//+// This implementation does not guarantee any upper bound with rounding+// errors, which grow along with k. Our goal's to maximally exploit the+// hardware for performance, and then use whatever resources remain for+// improving numerical accuracy.+//+// [1] J. Tunney, ‘LLaMA Now Goes Faster on CPUs’, Mar. 2024. [Online].+// Available: https://justine.lol/matmul/. [Accessed: 29-Mar-2024].++#include "llama.cpp/ggml-impl.h"+#include "llama.cpp/ggml-quants.h"+#include "llamafile.h"+#include <algorithm>++#pragma GCC diagnostic ignored "-Wpedantic"+#pragma GCC diagnostic ignored "-Wignored-attributes"++#define ROW_ALIGN 64+#define MATRIX_ALIGN 4096+#define MAX_ALIGN 4096++#ifdef _MSC_VER+#define NOINLINE __declspec(noinline)+#else+#define NOINLINE __attribute__((__noinline__))+#endif++#if defined(__ARM_NEON) || defined(__AVX512F__)+#define VECTOR_REGISTERS 32+#else+#define VECTOR_REGISTERS 16+#endif++namespace {++inline float unhalf(ggml_fp16_t d) {+ return GGML_FP16_TO_FP32(d);+}++////////////////////////////////////////////////////////////////////////////////////////////////////+// MATRIX MEMORY INDEXING++#define NCA 1+#define NCB 2+#define NCC 4++#define INDEX(A, lda, j, i) index<CONFIG & NC##A>(A, lda, j, i)++template <int NC, typename T>+inline T &index(T *A, int lda, int j, int i) {+ if (NC)+ return ((T **)A)[j][i];+ else+ return A[lda * j + i];+}++template <int NC, typename T>+inline const T &index(const T *A, int lda, int j, int i) {+ if (NC)+ return ((const T *const *)A)[j][i];+ else+ return A[lda * j + i];+}++////////////////////////////////////////////////////////////////////////////////////////////////////+// GGML TYPE TRAITS++template <typename T>+struct ggml_type_trait;+template <>+struct ggml_type_trait<float> {+ static constexpr ggml_type id = GGML_TYPE_F32;+};+template <>+struct ggml_type_trait<ggml_bf16_t> {+ static constexpr ggml_type id = GGML_TYPE_BF16;+};+template <>+struct ggml_type_trait<ggml_fp16_t> {+ static constexpr ggml_type id = GGML_TYPE_F16;+};+template <>+struct ggml_type_trait<block_q8_0> {+ static constexpr ggml_type id = GGML_TYPE_Q8_0;+};++////////////////////////////////////////////////////////////////////////////////////////////////////+// VECTORIZED ARITHMETIC OPERATIONS++#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)+inline __m128 add(__m128 x, __m128 y) {+ return _mm_add_ps(x, y);+}+inline __m128 sub(__m128 x, __m128 y) {+ return _mm_sub_ps(x, y);+}+inline __m128 mul(__m128 x, __m128 y) {+ return _mm_mul_ps(x, y);+}+#endif // __SSE__++#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)+inline __m256 add(__m256 x, __m256 y) {+ return _mm256_add_ps(x, y);+}+inline __m256 sub(__m256 x, __m256 y) {+ return _mm256_sub_ps(x, y);+}+inline __m256 mul(__m256 x, __m256 y) {+ return _mm256_mul_ps(x, y);+}+#endif // __AVX__++#if defined(__AVX512F__)+inline __m512 add(__m512 x, __m512 y) {+ return _mm512_add_ps(x, y);+}+inline __m512 sub(__m512 x, __m512 y) {+ return _mm512_sub_ps(x, y);+}+inline __m512 mul(__m512 x, __m512 y) {+ return _mm512_mul_ps(x, y);+}+#endif // __AVX512F__++#if defined(__ARM_NEON)+inline float32x4_t add(float32x4_t x, float32x4_t y) {+ return vaddq_f32(x, y);+}+inline float32x4_t sub(float32x4_t x, float32x4_t y) {+ return vsubq_f32(x, y);+}+inline float32x4_t mul(float32x4_t x, float32x4_t y) {+ return vmulq_f32(x, y);+}+#endif // __ARM_NEON++#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)+inline float16x8_t add(float16x8_t x, float16x8_t y) {+ return vaddq_f16(x, y);+}+inline float16x8_t sub(float16x8_t x, float16x8_t y) {+ return vsubq_f16(x, y);+}+inline float16x8_t mul(float16x8_t x, float16x8_t y) {+ return vmulq_f16(x, y);+}+#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC++////////////////////////////////////////////////////////////////////////////////////////////////////+// VECTORIZED FUSED MULTIPLY ADD++/**+ * Computes a * b + c.+ */+template <typename T, typename U>+inline U madd(T a, T b, U c) {+ return add(mul(a, b), c);+}++/**+ * Computes a * b + c with error correction.+ *+ * @see W. Kahan, "Further remarks on reducing truncation errors,"+ * Communications of the ACM, vol. 8, no. 1, p. 40, Jan. 1965,+ * doi: 10.1145/363707.363723.+ */+template <typename T, typename U>+inline U madder(T a, T b, U c, U *e) {+ U y = sub(mul(a, b), *e);+ U t = add(c, y);+ *e = sub(sub(t, c), y);+ return t;+}++#if defined(__FMA__)+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)+template <>+inline __m256 madd(__m256 a, __m256 b, __m256 c) {+ return _mm256_fmadd_ps(a, b, c);+}+#endif+#if defined(__AVX512F__)+template <>+inline __m512 madd(__m512 a, __m512 b, __m512 c) {+ return _mm512_fmadd_ps(a, b, c);+}+#endif+#endif++#if defined(__ARM_FEATURE_FMA)+template <>+inline float32x4_t madd(float32x4_t a, float32x4_t b, float32x4_t c) {+ return vfmaq_f32(c, b, a);+}+#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)+template <>+inline float16x8_t madd(float16x8_t a, float16x8_t b, float16x8_t c) {+ return vfmaq_f16(c, b, a);+}+#endif+#endif++#if defined(__AVX512BF16__)+template <>+inline __m512 madd(__m512bh x, __m512bh y, __m512 z) {+ return _mm512_dpbf16_ps(z, x, y);+}+template <>+inline __m512 madder(__m512bh x, __m512bh y, __m512 z, __m512 *_) {+ return _mm512_dpbf16_ps(z, x, y);+}+#endif++////////////////////////////////////////////////////////////////////////////////////////////////////+// VECTORIZED HORIZONTAL SUM++#if defined(__ARM_NEON)+inline float hsum(float32x4_t x) {+ return vaddvq_f32(x);+}+#endif // __ARM_NEON++#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)+inline float hsum(float16x8_t x) {+ return vaddvq_f32(vaddq_f32(vcvt_f32_f16(vget_low_f16(x)), vcvt_f32_f16(vget_high_f16(x))));+}+#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC++#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)+inline float hsum(__m128 x) {+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)+ x = _mm_add_ps(x, _mm_movehl_ps(x, x));+ x = _mm_add_ss(x, _mm_movehdup_ps(x));+#else+ __m128 t;+ t = _mm_shuffle_ps(x, x, _MM_SHUFFLE(2, 3, 0, 1));+ x = _mm_add_ps(x, t);+ t = _mm_movehl_ps(t, x);+ x = _mm_add_ss(x, t);+#endif+ return _mm_cvtss_f32(x);+}+#endif++#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)+inline float hsum(__m256 x) {+ return hsum(_mm_add_ps(_mm256_extractf128_ps(x, 1), _mm256_castps256_ps128(x)));+}+#endif // __AVX__++#if defined(__AVX512F__)+inline float hsum(__m512 x) {+ return _mm512_reduce_add_ps(x);+}+#endif // __AVX512F__++////////////////////////////////////////////////////////////////////////////////////////////////////+// VECTORIZED MEMORY LOADING++template <typename T, typename U>+T load(const U *);++#if defined(__ARM_NEON)+template <>+inline float32x4_t load(const float *p) {+ return vld1q_f32(p);+}+template <>+inline float32x4_t load(const ggml_bf16_t *p) {+ return vreinterpretq_f32_u32(vshll_n_u16(vld1_u16((const unsigned short *)p), 16));+}+#if !defined(_MSC_VER)+template <>+inline float16x8_t load(const ggml_fp16_t *p) {+ return vld1q_f16((const float16_t *)p);+}+template <>+inline float32x4_t load(const ggml_fp16_t *p) {+ return vcvt_f32_f16(vld1_f16((const float16_t *)p));+}+#endif // _MSC_VER+#endif // __ARM_NEON++#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)+template <>+inline __m128 load(const float *p) {+ return _mm_loadu_ps(p);+}+#endif // __SSE__++#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)+template <>+inline __m256 load(const float *p) {+ return _mm256_loadu_ps(p);+}+#endif // __AVX__++#if defined(__AVX2__) || defined(__AVX512F__)+template <>+inline __m256 load(const ggml_bf16_t *p) {+ return _mm256_castsi256_ps(+ _mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)p)), 16));+}+#endif // __AVX2__++#if defined(__F16C__)+template <>+inline __m256 load(const ggml_fp16_t *p) {+ return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));+}+#endif // __F16C__++#if defined(__AVX512F__)+template <>+inline __m512 load(const float *p) {+ return _mm512_loadu_ps(p);+}+template <>+inline __m512 load(const ggml_fp16_t *p) {+ return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)p));+}+template <>+inline __m512 load(const ggml_bf16_t *p) {+ return _mm512_castsi512_ps(+ _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)p)), 16));+}+#endif // __AVX512F__++#if defined(__AVX512BF16__)+template <>+inline __m512bh load(const ggml_bf16_t *p) {+ return (__m512bh)_mm512_loadu_ps((const float *)p);+}+#endif // __AVX512BF16__++////////////////////////////////////////////////////////////////////////////////////////////////////+// FLOATING POINT MATRIX MULTIPLICATION++template <int CONFIG, int KN, typename D, typename V, typename TA, typename TB, typename TC>+class tinyBLAS {+ public:+ tinyBLAS(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {+ }++ void matmul(int m, int n, int task) {+ if (task == GGML_TASK_TYPE_COMPUTE)+ mnpack(0, m, 0, n);+ }++ private:+ NOINLINE void mnpack(int m0, int m, int n0, int n) {+ int mc, nc, mp, np;+ switch ((std::min(m - m0, 5) << 4) | std::min(n - n0, 5)) {+#if VECTOR_REGISTERS == 32+ case 0x55:+ mc = 5;+ nc = 5;+ gemm<5, 5, false>(m0, m, n0, n);+ break;+ case 0x45:+ mc = 4;+ nc = 5;+ gemm<4, 5, false>(m0, m, n0, n);+ break;+ case 0x54:+ mc = 5;+ nc = 4;+ gemm<5, 4, false>(m0, m, n0, n);+ break;+ case 0x44:+ mc = 4;+ nc = 4;+ gemm<4, 4, false>(m0, m, n0, n);+ break;+ case 0x53:+ mc = 5;+ nc = 3;+ gemm<5, 3, false>(m0, m, n0, n);+ break;+ case 0x35:+ mc = 3;+ nc = 5;+ gemm<3, 5, false>(m0, m, n0, n);+ break;+ case 0x43:+ mc = 4;+ nc = 3;+ gemm<4, 3, false>(m0, m, n0, n);+ break;+#else+ case 0x55:+ case 0x54:+ case 0x53:+ case 0x45:+ case 0x44:+ case 0x43:+ mc = 4;+ nc = 3;+ gemm<4, 3, false>(m0, m, n0, n);+ break;+ case 0x35:+#endif+ case 0x34:+ mc = 3;+ nc = 4;+ gemm<3, 4, false>(m0, m, n0, n);+ break;+ case 0x52:+ mc = 5;+ nc = 2;+ gemm<5, 2, false>(m0, m, n0, n);+ break;+ case 0x33:+ mc = 3;+ nc = 3;+ gemm<3, 3, false>(m0, m, n0, n);+ break;+ case 0x25:+ mc = 2;+ nc = 5;+ gemm<2, 5, false>(m0, m, n0, n);+ break;+ case 0x42:+ mc = 4;+ nc = 2;+ gemm<4, 2, false>(m0, m, n0, n);+ break;+ case 0x24:+ mc = 2;+ nc = 4;+ gemm<2, 4, false>(m0, m, n0, n);+ break;+ case 0x32:+ mc = 3;+ nc = 2;+ gemm<3, 2, true>(m0, m, n0, n);+ break;+ case 0x23:+ mc = 2;+ nc = 3;+ gemm<2, 3, true>(m0, m, n0, n);+ break;+ case 0x51:+ mc = 5;+ nc = 1;+ gemm<5, 1, true>(m0, m, n0, n);+ break;+ case 0x41:+ mc = 4;+ nc = 1;+ gemm<4, 1, true>(m0, m, n0, n);+ break;+ case 0x22:+ mc = 2;+ nc = 2;+ gemm<2, 2, true>(m0, m, n0, n);+ break;+ case 0x15:+ mc = 1;+ nc = 5;+ gemm<1, 5, true>(m0, m, n0, n);+ break;+ case 0x14:+ mc = 1;+ nc = 4;+ gemm<1, 4, true>(m0, m, n0, n);+ break;+ case 0x31:+ mc = 3;+ nc = 1;+ gemm<3, 1, true>(m0, m, n0, n);+ break;+ case 0x13:+ mc = 1;+ nc = 3;+ gemm<1, 3, true>(m0, m, n0, n);+ break;+ case 0x21:+ mc = 2;+ nc = 1;+ gemm<2, 1, true>(m0, m, n0, n);+ break;+ case 0x12:+ mc = 1;+ nc = 2;+ gemm<1, 2, true>(m0, m, n0, n);+ break;+ case 0x11:+ mc = 1;+ nc = 1;+ gemm<1, 1, true>(m0, m, n0, n);+ break;+ default:+ return;+ }+ mp = m0 + (m - m0) / mc * mc;+ np = n0 + (n - n0) / nc * nc;+ mnpack(mp, m, n0, np);+ mnpack(m0, m, np, n);+ }++ template <int RM, int RN, int KAHAN>+ NOINLINE void gemm(int m0, int m, int n0, int n) {+ int ytiles = RM > 1 ? (m - m0) / RM : 1;+ int xtiles = RN > 1 ? (n - n0) / RN : 1;+ int tiles = xtiles * ytiles;+ int duty = (tiles + nth - 1) / nth;+ int start = duty * ith;+ int end = start + duty;+ if (end > tiles)+ end = tiles;+ for (int job = start; job < end; ++job) {+ int ii = m0 + job / xtiles * RM;+ int jj = n0 + job % xtiles * RN;+ D Cv[RN][RM] = {};+ D Ce[RN][RM] = {};+ for (int l = 0; l < k; l += KN)+ for (int j = 0; j < RN; ++j)+ for (int i = 0; i < RM; ++i)+ if (KAHAN)+ Cv[j][i] = madder(load<V>(&INDEX(A, lda, ii + i, l)), //+ load<V>(&INDEX(B, ldb, jj + j, l)), //+ Cv[j][i], &Ce[j][i]);+ else+ Cv[j][i] = madd(load<V>(&INDEX(A, lda, ii + i, l)), //+ load<V>(&INDEX(B, ldb, jj + j, l)), //+ Cv[j][i]);+ for (int j = 0; j < RN; ++j)+ for (int i = 0; i < RM; ++i)+ INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]);+ }+ }++ const TA *const A;+ const TB *const B;+ TC *const C;+ const int k;+ const int lda;+ const int ldb;+ const int ldc;+ const int ith;+ const int nth;+};++//////////////////////////////////////////////////////////////////////////////////////////+// QUANT ZERO MATRIX MULTIPLICATION++#if defined(__ARM_FEATURE_DOTPROD)+template <int CONFIG, typename TA>+class tinyBLAS_Q0_ARM {+ public:+ tinyBLAS_Q0_ARM(int k, const TA *A, int lda, const block_q8_0 *B, int ldb, float *C, int ldc,+ int ith, int nth)+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {+ }++ void matmul(int m, int n, int task) {+ if (task == GGML_TASK_TYPE_COMPUTE)+ mnpack(0, m, 0, n);+ }++ private:+ NOINLINE void mnpack(int m0, int m, int n0, int n) {+ int mc, nc, mp, np;+ switch ((std::min(m - m0, 3) << 4) | std::min(n - n0, 3)) {+ case 0x33:+ mc = 3;+ nc = 3;+ gemm<3, 3>(m0, m, n0, n);+ break;+ case 0x32:+ mc = 3;+ nc = 2;+ gemm<3, 2>(m0, m, n0, n);+ break;+ case 0x23:+ mc = 2;+ nc = 3;+ gemm<2, 3>(m0, m, n0, n);+ break;+ case 0x22:+ mc = 2;+ nc = 2;+ gemm<2, 2>(m0, m, n0, n);+ break;+ case 0x31:+ mc = 3;+ nc = 1;+ gemm<3, 1>(m0, m, n0, n);+ break;+ case 0x13:+ mc = 1;+ nc = 3;+ gemm<1, 3>(m0, m, n0, n);+ break;+ case 0x21:+ mc = 2;+ nc = 1;+ gemm<2, 1>(m0, m, n0, n);+ break;+ case 0x12:+ mc = 1;+ nc = 2;+ gemm<1, 2>(m0, m, n0, n);+ break;+ case 0x11:+ mc = 1;+ nc = 1;+ gemm<1, 1>(m0, m, n0, n);+ break;+ default:+ return;+ }+ mp = m0 + (m - m0) / mc * mc;+ np = n0 + (n - n0) / nc * nc;+ mnpack(mp, m, n0, np);+ mnpack(m0, m, np, n);+ }++ template <int RM, int RN>+ NOINLINE void gemm(int m0, int m, int n0, int n) {+ int ytiles = RM > 1 ? (m - m0) / RM : 1;+ int xtiles = RN > 1 ? (n - n0) / RN : 1;+ int tiles = xtiles * ytiles;+ int duty = (tiles + nth - 1) / nth;+ int start = duty * ith;+ int end = start + duty;+ if (end > tiles)+ end = tiles;+ for (int job = start; job < end; ++job) {+ int ii = m0 + job / xtiles * RM;+ int jj = n0 + job % xtiles * RN;+ float32x4_t Cv[RN][RM] = {};+ for (int l = 0; l < k; ++l)+ for (int j = 0; j < RN; ++j)+ for (int i = 0; i < RM; ++i)+ Cv[j][i] = vmlaq_n_f32(+ Cv[j][i],+ vcvtq_f32_s32(vdotq_s32(vdotq_s32(vdupq_n_s32(0),+ load_lo(&INDEX(A, lda, ii + i, l)),+ load_lo(&INDEX(B, ldb, jj + j, l))),+ load_hi(&INDEX(A, lda, ii + i, l)),+ load_hi(&INDEX(B, ldb, jj + j, l)))),+ (unhalf(INDEX(A, lda, ii + i, l).d) *+ unhalf(INDEX(B, ldb, jj + j, l).d)));+ for (int j = 0; j < RN; ++j)+ for (int i = 0; i < RM; ++i)+ INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]);+ }+ }++ inline int8x16_t load_lo(const block_q8_0 *b) {+ return vld1q_s8(b->qs);+ }++ inline int8x16_t load_hi(const block_q8_0 *b) {+ return vld1q_s8(b->qs + 16);+ }++ inline int8x16_t load_lo(const block_q4_0 *b) {+ return vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vld1q_u8(b->qs), vdupq_n_u8(0x0f))),+ vdupq_n_s8(0x8));+ }++ inline int8x16_t load_hi(const block_q4_0 *b) {+ return vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(vld1q_u8(b->qs), 4)), vdupq_n_s8(0x8));+ }++ const TA *const A;+ const block_q8_0 *const B;+ float *const C;+ const int k;+ const int lda;+ const int ldb;+ const int ldc;+ const int ith;+ const int nth;+};+#endif // __ARM_FEATURE_DOTPROD++#if defined(__AVX2__) || defined(__AVX512F__)+template <int CONFIG, typename TA, typename TB, typename TC>+class tinyBLAS_Q0_AVX2 {+ public:+ tinyBLAS_Q0_AVX2(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith,+ int nth)+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {+ }++ void matmul(int m, int n, int task) {+ if (task == GGML_TASK_TYPE_COMPUTE)+ mnpack(0, m, 0, n);+ }++ private:+ void mnpack(int m0, int m, int n0, int n) {+ int mc, nc, mp, np;+ switch ((std::min(m - m0, 4) << 4) | std::min(n - n0, 4)) {+#if VECTOR_REGISTERS == 32+ case 0x44:+ mc = 4;+ nc = 4;+ gemm<4, 4>(m0, m, n0, n);+ break;+ case 0x43:+ mc = 4;+ nc = 3;+ gemm<4, 3>(m0, m, n0, n);+ break;+ case 0x34:+ mc = 3;+ nc = 4;+ gemm<3, 4>(m0, m, n0, n);+ break;+ case 0x33:+ mc = 3;+ nc = 3;+ gemm<3, 3>(m0, m, n0, n);+ break;+ case 0x42:+ mc = 4;+ nc = 2;+ gemm<4, 2>(m0, m, n0, n);+ break;+ case 0x24:+ mc = 2;+ nc = 4;+ gemm<2, 4>(m0, m, n0, n);+ break;+#else+ case 0x44:+ case 0x43:+ case 0x42:+ mc = 4;+ nc = 2;+ gemm<4, 2>(m0, m, n0, n);+ break;+ case 0x34:+ case 0x24:+ mc = 2;+ nc = 4;+ gemm<2, 4>(m0, m, n0, n);+ break;+ case 0x33:+#endif+ case 0x32:+ mc = 3;+ nc = 2;+ gemm<3, 2>(m0, m, n0, n);+ break;+ case 0x23:+ mc = 2;+ nc = 3;+ gemm<2, 3>(m0, m, n0, n);+ break;+ case 0x41:+ mc = 4;+ nc = 1;+ gemm<4, 1>(m0, m, n0, n);+ break;+ case 0x22:+ mc = 2;+ nc = 2;+ gemm<2, 2>(m0, m, n0, n);+ break;+ case 0x14:+ mc = 1;+ nc = 4;+ gemm<1, 4>(m0, m, n0, n);+ break;+ case 0x31:+ mc = 3;+ nc = 1;+ gemm<3, 1>(m0, m, n0, n);+ break;+ case 0x13:+ mc = 1;+ nc = 3;+ gemm<1, 3>(m0, m, n0, n);+ break;+ case 0x21:+ mc = 2;+ nc = 1;+ gemm<2, 1>(m0, m, n0, n);+ break;+ case 0x12:+ mc = 1;+ nc = 2;+ gemm<1, 2>(m0, m, n0, n);+ break;+ case 0x11:+ mc = 1;+ nc = 1;+ gemm<1, 1>(m0, m, n0, n);+ break;+ default:+ return;+ }+ mp = m0 + (m - m0) / mc * mc;+ np = n0 + (n - n0) / nc * nc;+ mnpack(mp, m, n0, np);+ mnpack(m0, m, np, n);+ }++ template <int RM, int RN>+ NOINLINE void gemm(int m0, int m, int n0, int n) {+ int ytiles = RM > 1 ? (m - m0) / RM : 1;+ int xtiles = RN > 1 ? (n - n0) / RN : 1;+ int tiles = xtiles * ytiles;+ int duty = (tiles + nth - 1) / nth;+ int start = duty * ith;+ int end = start + duty;+ if (end > tiles)+ end = tiles;+ for (int job = start; job < end; ++job) {+ int ii = m0 + job / xtiles * RM;+ int jj = n0 + job % xtiles * RN;+ __m256 Cv[RN][RM] = {};+ for (int l = 0; l < k; ++l)+ for (int j = 0; j < RN; ++j)+ for (int i = 0; i < RM; ++i)+ Cv[j][i] = madd(_mm256_set1_ps(unhalf(INDEX(A, lda, ii + i, l).d) *+ unhalf(INDEX(B, ldb, jj + j, l).d)),+ updot(_mm256_sign_epi8(load(&INDEX(A, lda, ii + i, l)),+ load(&INDEX(A, lda, ii + i, l))),+ _mm256_sign_epi8(load(&INDEX(B, ldb, jj + j, l)),+ load(&INDEX(A, lda, ii + i, l)))),+ Cv[j][i]);+ for (int j = 0; j < RN; ++j)+ for (int i = 0; i < RM; ++i)+ INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]);+ }+ }++ inline __m256i load(const block_q8_0 *b) {+ return _mm256_loadu_si256((const __m256i *)b->qs);+ }++ inline __m256i load(const block_q4_0 *b) {+ return _mm256_sub_epi8(denibble(b->qs), _mm256_set1_epi8(8));+ }++ inline __m256 updot(__m256i u, __m256i s) {+ __m256i res;+#if defined(__AVXVNNI__) || (defined(__AVX512VNNI__) && defined(__AVX512VL__))+ res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);+#else+ res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));+#endif+ return _mm256_cvtepi32_ps(res);+ }++ static inline __m256i denibble(const uint8_t *p) {+ __m128i x = _mm_loadu_si128((const __m128i *)p);+ return _mm256_and_si256(+ _mm256_set1_epi8(15),+ _mm256_insertf128_si256(_mm256_castsi128_si256(x), _mm_srli_epi16(x, 4), 1));+ }++ const TA *const A;+ const TB *const B;+ TC *const C;+ const int k;+ const int lda;+ const int ldb;+ const int ldc;+ const int ith;+ const int nth;+};+#endif // __AVX2__++} // namespace++/**+ * Performs optimized matrix multiplication on CPU.+ *+ * This subroutine may compute C = Aᵀ * B with column major ordering.+ * Despite its name, this isn't a generalized implementation. Work is+ * only performed when a handwritten kernel is written and available.+ * Otherwise the caller should fall back to a general matmul routine.+ *+ * For example, for single-threaded single-precision GEMM you can say+ *+ * llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc,+ * 0, 1, GGML_TASK_TYPE_COMPUTE,+ * GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32);+ *+ * @param m is rows in `A` and `C`+ * @param n is cols in `B` and `C`+ * @param k is cols in `A` and rows in `B`+ * @param A is first input matrix (always transposed)+ * @param lda is row stride of `A`+ * @param B is second input matrix (never transposed)+ * @param ldb is row stride of `B`+ * @param C is input/output array of output matrices+ * @param ldc is row stride of `C`+ * @param ith is thread id (must be less than `nth`)+ * @param nth is number of threads (must be greater than zero)+ * @param task is GGML task type+ * @param Atype is GGML data type of `A`+ * @param Btype is GGML data type of `B`+ * @param Ctype is GGML data type of `C`+ * @return true if this function was able to service the matmul request+ */+bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B, int ldb, void *C,+ int ldc, int ith, int nth, int task, int Atype, int Btype, int Ctype) {++ assert(m >= 0);+ assert(n >= 0);+ assert(k >= 0);+ assert(lda >= k);+ assert(ldb >= k);+ assert(ldc >= m);+ assert(nth > 0);+ assert(ith < nth);+ assert(1ll * lda * m <= 0x7fffffff);+ assert(1ll * ldb * n <= 0x7fffffff);+ assert(1ll * ldc * n <= 0x7fffffff);++ if (Ctype != GGML_TYPE_F32)+ return false;++ switch (Atype) {++ case GGML_TYPE_F32: {+ if (Btype != GGML_TYPE_F32)+ return false;+#if defined(__AVX512F__)+ if (k % 16)+ return false;+ tinyBLAS<0, 16, __m512, __m512, float, float, float> tb{+ k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+#elif defined(__AVX__) || defined(__AVX2__)+ if (k % 8)+ return false;+ tinyBLAS<0, 8, __m256, __m256, float, float, float> tb{+ k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+#elif defined(__ARM_NEON)+ if (n < 4)+ return false;+ if (k % 4)+ return false;+ tinyBLAS<0, 4, float32x4_t, float32x4_t, float, float, float> tb{+ k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+#else+ return false;+#endif+ }++ case GGML_TYPE_BF16: {+#if defined(__AVX512BF16__)+ if (k % 32)+ return false;+ if (Btype != GGML_TYPE_BF16)+ return false;+ tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, float> tb{+ k, (const ggml_bf16_t *)A, lda, (const ggml_bf16_t *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+#elif defined(__AVX512F__)+ if (k % 16)+ return false;+ if (Btype != GGML_TYPE_F32)+ return false;+ tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, float, float> tb{+ k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+#elif defined(__AVX2__)+ // TODO(jart): Why is this the fastest on Zen4?+ if (k % 8)+ return false;+ if (Btype != GGML_TYPE_F32)+ return false;+ tinyBLAS<0, 8, __m256, __m256, ggml_bf16_t, float, float> tb{+ k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+#elif defined(__ARM_NEON) && !defined(_MSC_VER)+ if (k % 4)+ return false;+ if (Btype != GGML_TYPE_F32)+ return false;+ tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_bf16_t, float, float> tb{+ k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+#else+ return false;+#endif+ }++ case GGML_TYPE_F16: {+#if defined(__AVX512F__)+ if (k % 16)+ return false;+ if (Btype != GGML_TYPE_F32)+ return false;+ tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, float, float> tb{+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)+ if (k % 8)+ return false;+ if (Btype != GGML_TYPE_F32)+ return false;+ tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, float> tb{+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)+ if (n < 8)+ return false;+ if (k % 8)+ return false;+ if (Btype != GGML_TYPE_F16)+ return false;+ tinyBLAS<0, 8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, float> tb{+ k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+#elif defined(__ARM_NEON) && !defined(_MSC_VER)+ if (k % 4)+ return false;+ if (Btype != GGML_TYPE_F32)+ return false;+ tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, float> tb{+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+#else+ return false;+#endif+ }++ case GGML_TYPE_Q8_0: {+ if (Btype != GGML_TYPE_Q8_0)+ return false;+#if defined(__AVX2__) || defined(__AVX512F__)+ tinyBLAS_Q0_AVX2<0, block_q8_0, block_q8_0, float> tb{+ k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+#elif defined(__ARM_FEATURE_DOTPROD)+ tinyBLAS_Q0_ARM<0, block_q8_0> tb{+ k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+#else+ return false;+#endif+ }++ case GGML_TYPE_Q4_0: {+ if (Btype != GGML_TYPE_Q8_0)+ return false;+#if defined(__AVX2__) || defined(__AVX512F__)+ tinyBLAS_Q0_AVX2<0, block_q4_0, block_q8_0, float> tb{+ k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+#elif defined(__ARM_FEATURE_DOTPROD)+ tinyBLAS_Q0_ARM<0, block_q4_0> tb{+ k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};+ tb.matmul(m, n, task);+ return true;+#else+ return false;+#endif+ }++ default:+ return false;+ }++ (void)m;+ (void)n;+ (void)k;+ (void)A;+ (void)lda;+ (void)B;+ (void)ldb;+ (void)C;+ (void)ldc;+ (void)ith;+ (void)nth;+ (void)task;+ (void)Atype;+ (void)Btype;+ (void)Ctype;+}++//+// _ _ ___ _ _ ___+// | |_(_)_ _ _ _| _ ) | /_\ / __|+// | _| | ' \ || | _ \ |__ / _ \\__ \.+// \__|_|_||_\_, |___/____/_/ \_\___/+// |__/+//+// MIXTURE OF EXPERTS TENSOR MULTIPLICATION+//+//+// SHAPES+//+// - weights [cols, rows, experts]+// - thought [cols, tasks, tokens] w/ tasks ≤ thinkers+// - result [rows, thinkers, tokens] w/ thinkers ≤ experts+// - plan [thinkers, tokens] w/ i32 < experts+//+// DEFINITION+//+// for thinker in range(thinkers):+// for token in range(tokens):+// for row in range(rows):+// c = 0+// for col in range(cols):+// expert = plan[token][thinker]+// a = weights[expert][row][col]+// b = thought[token][thinker % tasks][col]+// c += a * b+// result[token][thinker][row] = c+//+// REGULARITIES+//+// - tokens can be odd+// - thinkers is usually 2+// - tasks is usually 1 or 2+// - cols should be a multiple of 64+// - rows should be a multiple of 64+// - experts is usually 8 but could be 60+// - tokens is always 1 for token generation+// - tokens can be huge for prompt processing+//+// EXAMPLE+//+// mixtral 8x7b w/ 217 token prompt+//+// | ne*0 ne*1 ne*2 ne*3 | nb*0 nb*1 nb*2 nb*3 | type+// =========================================================================+// weights | 16384 6144 8 1 | 18 0x2400 0x3600000 0x1b000000 | q4_0+// thought | 16384 2 217 1 | 4 0x10000 0x20000 0x1b20000 | f32+// result | 6144 2 217 1 | 4 0x6000 0xc000 0xa2c000 | f32+// plan | 2 217 1 1 | 4 0x20 0x1b20 0x1b20 | i32+//++namespace {+class MixMul {+ public:+ MixMul(const ggml_compute_params *params, const ggml_tensor *weights,+ const ggml_tensor *thought, const ggml_tensor *plan, ggml_tensor *result)+ : params(params),+ weights(weights),+ thought(thought),+ plan(plan),+ result(result),+ rows(weights->ne[1]),+ cols(weights->ne[0]),+ experts(weights->ne[2]),+ thinkers(plan->ne[0]),+ tasks(thought->ne[1]),+ tokens(thought->ne[2]),+ ldq((cols * 2 + ROW_ALIGN - 1) & -ROW_ALIGN),+ wdata_((char *)(((uintptr_t)params->wdata + MAX_ALIGN - 1) & -MAX_ALIGN)),+ allocated_(0) {+ }++ bool allocate_shared_memory() {+ if (!(quantized_thought_ = allocate<char>(MATRIX_ALIGN, tokens * tasks * ldq)))+ return false;+ if (!(rowptr_result_ = allocate<uintptr_t>(ROW_ALIGN, experts * tokens * thinkers)))+ return false;+ if (!(rowptr_thought_ = allocate<uintptr_t>(ROW_ALIGN, experts * tokens * thinkers)))+ return false;+ if (!(rowptr_count_ = allocate<int>(sizeof(int), experts)))+ return false;+ return true;+ }++ size_t get_allocated_bytes() {+ return (wdata_ - (char *)params->wdata) + allocated_;+ }++ bool mixmul() {++ // invariants+ assert(tasks <= thinkers);+ assert(thinkers <= experts);+ assert(tokens == plan->ne[1]);+ assert(rows == result->ne[0]);+ assert(cols == thought->ne[0]);+ assert(tokens == result->ne[2]);+ assert(thinkers == result->ne[1]);++ // dimensionality+ assert(plan->ne[2] == 1);+ assert(plan->ne[3] == 1);+ assert(result->ne[3] == 1);+ assert(weights->ne[3] == 1);+ assert(thought->ne[3] == 1);++ // miscellaneous+ assert(params->nth > 0);+ assert(params->ith < params->nth);+ assert(plan->type == GGML_TYPE_I32);++ // supported types+ if (result->type != GGML_TYPE_F32)+ return false;++ // check nb01 is convertible to lda+ if (weights->nb[1] % ggml_type_size(weights->type))+ return false;++ // no support for column strides+ if (result->nb[0] != ggml_type_size(result->type))+ return false;+ if (thought->nb[0] != ggml_type_size(thought->type))+ return false;+ if (weights->nb[0] != ggml_type_size(weights->type))+ return false;++ switch (weights->type) {++ case GGML_TYPE_F32:+ if (thought->type != GGML_TYPE_F32)+ return false;+#if defined(__AVX512F__)+ return mixmat<16, 1, tinyBLAS<NCB | NCC, 16, __m512, __m512, float, float, float>,+ float, float, float>();+#elif defined(__AVX__) || defined(__AVX2__)+ return mixmat<8, 1, tinyBLAS<NCB | NCC, 8, __m256, __m256, float, float, float>, float,+ float, float>();+#elif defined(__SSE__)+ return mixmat<4, 1, tinyBLAS<NCB | NCC, 4, __m128, __m128, float, float, float>, float,+ float, float>();+#elif defined(__ARM_NEON)+ return mixmat<4, 1,+ tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, float, float, float>,+ float, float, float>();+#else+ return false;+#endif++ case GGML_TYPE_BF16:+ if (thought->type != GGML_TYPE_F32 && thought->type != GGML_TYPE_BF16)+ return false;+#if defined(__AVX512BF16__)+ return mixmat<+ 32, 1, tinyBLAS<NCB | NCC, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, float>,+ ggml_bf16_t, ggml_bf16_t, float>();+#elif defined(__AVX512F__)+ return mixmat<16, 1,+ tinyBLAS<NCB | NCC, 16, __m512, __m512, ggml_bf16_t, ggml_bf16_t, float>,+ ggml_bf16_t, ggml_bf16_t, float>();+#elif defined(__AVX2__)+ return mixmat<8, 1,+ tinyBLAS<NCB | NCC, 8, __m256, __m256, ggml_bf16_t, ggml_bf16_t, float>,+ ggml_bf16_t, ggml_bf16_t, float>();+#elif defined(__ARM_NEON) && !defined(_MSC_VER)+ return mixmat<+ 4, 1,+ tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, ggml_bf16_t, ggml_bf16_t, float>,+ ggml_bf16_t, ggml_bf16_t, float>();+#else+ return false;+#endif++ case GGML_TYPE_F16:+ if (thought->type != GGML_TYPE_F32 && thought->type != GGML_TYPE_F16)+ return false;+#if defined(__AVX512F__)+ return mixmat<16, 1,+ tinyBLAS<NCB | NCC, 16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, float>,+ ggml_fp16_t, ggml_fp16_t, float>();+#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)+ return mixmat<8, 1,+ tinyBLAS<NCB | NCC, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float>,+ ggml_fp16_t, ggml_fp16_t, float>();+#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)+ return mixmat<+ 8, 1,+ tinyBLAS<NCB | NCC, 8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, float>,+ ggml_fp16_t, ggml_fp16_t, float>();+#elif defined(__ARM_NEON) && !defined(_MSC_VER)+ return mixmat<+ 4, 1,+ tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, ggml_fp16_t, ggml_fp16_t, float>,+ ggml_fp16_t, ggml_fp16_t, float>();+#else+ return false;+#endif++ case GGML_TYPE_Q4_0:+ if (thought->type != GGML_TYPE_F32 && thought->type != GGML_TYPE_Q8_0)+ return false;+#if defined(__AVX2__) || defined(__AVX512F__)+ return mixmat<32, 32, tinyBLAS_Q0_AVX2<NCB | NCC, block_q4_0, block_q8_0, float>,+ block_q4_0, block_q8_0, float>();+#elif defined(__ARM_FEATURE_DOTPROD)+ return mixmat<32, 32, tinyBLAS_Q0_ARM<NCB | NCC, block_q4_0>, block_q4_0, block_q8_0,+ float>();+#else+ return false;+#endif++ case GGML_TYPE_Q8_0:+ if (thought->type != GGML_TYPE_F32 && thought->type != GGML_TYPE_Q8_0)+ return false;+#if defined(__AVX2__) || defined(__AVX512F__)+ return mixmat<32, 32, tinyBLAS_Q0_AVX2<NCB | NCC, block_q8_0, block_q8_0, float>,+ block_q8_0, block_q8_0, float>();+#elif defined(__ARM_FEATURE_DOTPROD)+ return mixmat<32, 32, tinyBLAS_Q0_ARM<NCB | NCC, block_q8_0>, block_q8_0, block_q8_0,+ float>();+#else+ return false;+#endif++ default:+ return false;+ }+ }++ private:+ template <int KN, int BS, typename BLAS, typename TA, typename TB, typename TC>+ bool mixmat() {+ if (cols % KN)+ return false;+ switch (params->type) {+ case GGML_TASK_TYPE_INIT:+ if (thought->type != ggml_type_trait<TB>::id)+ quantize_thought(ggml_type_trait<TB>::id);+ build_row_pointers(ggml_type_trait<TB>::id);+ return true;+ case GGML_TASK_TYPE_COMPUTE:+ assert(!(cols % BS));+ assert(!(weights->nb[1] % sizeof(TA)));+ for (int expert = 0; expert < experts; ++expert) {+ BLAS tb{cols / BS,+ (const TA *)((const char *)weights->data + expert * weights->nb[2]),+ (int)(weights->nb[1] / sizeof(TA)),+ (const TB *)(rowptr_thought_ + expert * tokens * thinkers),+ 0,+ (TC *)(rowptr_result_ + expert * tokens * thinkers),+ 0,+ params->ith,+ params->nth};+ tb.matmul(rows, rowptr_count_[expert], GGML_TASK_TYPE_COMPUTE);+ }+ return true;+ default:+ return true;+ }+ }++ void build_row_pointers(ggml_type vec_dot_type) {+ for (int expert = params->ith; expert < experts; expert += params->nth) {+ int count = 0;+ for (int token = 0; token < tokens; ++token)+ for (int thinker = 0; thinker < thinkers; ++thinker)+ if (expert == *(const int *)((const char *)plan->data + token * plan->nb[1] ++ thinker * plan->nb[0])) {+ int row = count++;+ int idx = expert * thinkers * tokens + row;+ rowptr_result_[idx] =+ (uintptr_t)((char *)result->data + token * result->nb[2] ++ thinker * result->nb[1]);+ if (thought->type == vec_dot_type)+ rowptr_thought_[idx] =+ (uintptr_t)((char *)thought->data + token * thought->nb[2] ++ thinker % tasks * thought->nb[1]);+ else+ rowptr_thought_[idx] =+ (uintptr_t)((char *)quantized_thought_ + token * tasks * ldq ++ thinker % tasks * ldq);+ }+ rowptr_count_[expert] = count;+ }+ }++ void quantize_thought(ggml_type vec_dot_type) {+ int chore = 0;+ for (int token = 0; token < tokens; ++token)+ for (int task = 0; task < tasks; ++task)+ if (chore++ % params->nth == params->ith)+ quantize_row(quantized_thought_ + token * tasks * ldq + task * ldq,+ (const float *)((const char *)thought->data ++ token * thought->nb[2] + task * thought->nb[1]),+ vec_dot_type);+ }++ void quantize_row(void *dst, const float *src, ggml_type type) {+ assert((int)ggml_row_size(type, cols) <= ldq);+ switch (type) {+ case GGML_TYPE_F16:+ ggml_fp32_to_fp16_row(src, (ggml_fp16_t *)dst, cols);+ break;+ case GGML_TYPE_BF16:+ ggml_fp32_to_bf16_row(src, (ggml_bf16_t *)dst, cols);+ break;+ case GGML_TYPE_Q8_0:+ quantize_row_q8_0((const float *)src, (block_q8_0 *)dst, cols);+ break;+ default:+ GGML_UNREACHABLE();+ }+ }++ template <typename T>+ T *allocate(int align, int elems) {+ T *res = nullptr;+ size_t need = sizeof(T) * elems;+ size_t base = allocated_;+ base += align - 1;+ base &= -align;+ size_t toto = base + need;+ if (toto >= allocated_ && toto <= params->wsize && elems >= 0) {+ res = (T *)(wdata_ + base);+ allocated_ = toto;+ }+ return res;+ }++ const ggml_compute_params *const params;+ const ggml_tensor *const weights;+ const ggml_tensor *const thought;+ const ggml_tensor *const plan;+ ggml_tensor *const result;+ const int rows;+ const int cols;+ const int experts;+ const int thinkers;+ const int tasks;+ const int tokens;+ const int ldq;++ // variables+ char *const wdata_;+ size_t allocated_;++ // shared memory+ int *rowptr_count_ /*[experts]*/;+ char *quantized_thought_ /*[tokens][tasks][cols][2]*/;+ uintptr_t *rowptr_result_ /*[experts][tokens*thinkers]*/;+ uintptr_t *rowptr_thought_ /*[experts][tokens*thinkers]*/;+};+} // namespace++/**+ * Performs "mixture of experts" tensor multiplication on CPU.+ */+bool llamafile_mixmul(const ggml_compute_params *params, const ggml_tensor *weights,+ const ggml_tensor *thought, const ggml_tensor *plan, ggml_tensor *result) {+ MixMul mm{params, weights, thought, plan, result};+ return mm.allocate_shared_memory() && mm.mixmul();+}diff --git a/llamafile/tinyblas_cpu_amd_avx.cpp b/llamafile/tinyblas_cpu_amd_avx.cppnew file mode 100644index 00000000..75ea97e1--- /dev/null+++ b/llamafile/tinyblas_cpu_amd_avx.cpp@@ -0,0 +1,21 @@+#ifdef __x86_64__+#define llamafile_sgemm llamafile_sgemm_amd_avx+#define llamafile_mixmul llamafile_mixmul_amd_avx+#include "tinyblas_cpu.inc"++/**+ * Returns number of shared memory bytes llamafile_mixmul() needs.+ */+size_t llamafile_mixmul_needs(const ggml_tensor *weights, const ggml_tensor *thought,+ const ggml_tensor *plan) {+ ggml_compute_params params{};+ params.wsize = 0x7ffff000;+ params.wdata = (void *)0x1000;+ MixMul mm{¶ms, weights, thought, plan, 0};+ if (mm.allocate_shared_memory())+ return mm.get_allocated_bytes();+ else+ return 0;+}++#endif // __x86_64__diff --git a/llamafile/tinyblas_cpu_amd_avx2.cpp b/llamafile/tinyblas_cpu_amd_avx2.cppnew file mode 100644index 00000000..3457d3f6--- /dev/null+++ b/llamafile/tinyblas_cpu_amd_avx2.cpp@@ -0,0 +1,5 @@+#ifdef __x86_64__+#define llamafile_sgemm llamafile_sgemm_amd_avx2+#define llamafile_mixmul llamafile_mixmul_amd_avx2+#include "tinyblas_cpu.inc"+#endif // __x86_64__diff --git a/llamafile/tinyblas_cpu_amd_avx512f.cpp b/llamafile/tinyblas_cpu_amd_avx512f.cppnew file mode 100644index 00000000..e0af44fd--- /dev/null+++ b/llamafile/tinyblas_cpu_amd_avx512f.cpp@@ -0,0 +1,5 @@+#ifdef __x86_64__+#define llamafile_sgemm llamafile_sgemm_amd_avx512f+#define llamafile_mixmul llamafile_mixmul_amd_avx512f+#include "tinyblas_cpu.inc"+#endif // __x86_64__diff --git a/llamafile/tinyblas_cpu_amd_avxvnni.cpp b/llamafile/tinyblas_cpu_amd_avxvnni.cppnew file mode 100644index 00000000..84b6f608--- /dev/null+++ b/llamafile/tinyblas_cpu_amd_avxvnni.cpp@@ -0,0 +1,5 @@+#ifdef __x86_64__+#define llamafile_sgemm llamafile_sgemm_amd_avxvnni+#define llamafile_mixmul llamafile_mixmul_amd_avxvnni+#include "tinyblas_cpu.inc"+#endif // __x86_64__diff --git a/llamafile/tinyblas_cpu_amd_fma.cpp b/llamafile/tinyblas_cpu_amd_fma.cppnew file mode 100644index 00000000..6e42e4d5--- /dev/null+++ b/llamafile/tinyblas_cpu_amd_fma.cpp@@ -0,0 +1,5 @@+#ifdef __x86_64__+#define llamafile_sgemm llamafile_sgemm_amd_fma+#define llamafile_mixmul llamafile_mixmul_amd_fma+#include "tinyblas_cpu.inc"+#endif // __x86_64__diff --git a/llamafile/tinyblas_cpu_amd_zen4.cpp b/llamafile/tinyblas_cpu_amd_zen4.cppnew file mode 100644index 00000000..e905fe3e--- /dev/null+++ b/llamafile/tinyblas_cpu_amd_zen4.cpp@@ -0,0 +1,5 @@+#ifdef __x86_64__+#define llamafile_sgemm llamafile_sgemm_amd_zen4+#define llamafile_mixmul llamafile_mixmul_amd_zen4+#include "tinyblas_cpu.inc"+#endif // __x86_64__diff --git a/llamafile/tinyblas_cpu_arm80.cpp b/llamafile/tinyblas_cpu_arm80.cppnew file mode 100644index 00000000..a5268fd4--- /dev/null+++ b/llamafile/tinyblas_cpu_arm80.cpp@@ -0,0 +1,21 @@+#ifdef __aarch64__+#define llamafile_sgemm llamafile_sgemm_arm80+#define llamafile_mixmul llamafile_mixmul_arm80+#include "tinyblas_cpu.inc"++/**+ * Returns number of shared memory bytes llamafile_mixmul() needs.+ */+size_t llamafile_mixmul_needs(const ggml_tensor *weights, const ggml_tensor *thought,+ const ggml_tensor *plan) {+ ggml_compute_params params{};+ params.wsize = 0x7ffff000;+ params.wdata = (void *)0x1000;+ MixMul mm{¶ms, weights, thought, plan, 0};+ if (mm.allocate_shared_memory())+ return mm.get_allocated_bytes();+ else+ return 0;+}++#endif // __aarch64__diff --git a/llamafile/tinyblas_cpu_arm82.cpp b/llamafile/tinyblas_cpu_arm82.cppnew file mode 100644index 00000000..1246d825--- /dev/null+++ b/llamafile/tinyblas_cpu_arm82.cpp@@ -0,0 +1,5 @@+#ifdef __aarch64__+#define llamafile_sgemm llamafile_sgemm_arm82+#define llamafile_mixmul llamafile_mixmul_arm82+#include "tinyblas_cpu.inc"+#endif // __aarch64__diff --git a/llamafile/varith.h b/llamafile/varith.hdeleted file mode 100644index 8fb4d414..00000000--- a/llamafile/varith.h+++ /dev/null@@ -1,72 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-// clang-format off-#pragma once--#ifdef __ARM_NEON-#include <arm_neon.h>-#endif-#ifdef __x86_64__-#include <immintrin.h>-#endif--inline float add(float x, float y) { return x + y; }-inline float sub(float x, float y) { return x - y; }-inline float mul(float x, float y) { return x * y; }--#ifdef __SSE__-inline __m128 add(__m128 x, __m128 y) { return _mm_add_ps(x, y); }-inline __m128 sub(__m128 x, __m128 y) { return _mm_sub_ps(x, y); }-inline __m128 mul(__m128 x, __m128 y) { return _mm_mul_ps(x, y); }-#endif // __SSE__--#ifdef __AVX__-inline __m256 add(__m256 x, __m256 y) { return _mm256_add_ps(x, y); }-inline __m256 sub(__m256 x, __m256 y) { return _mm256_sub_ps(x, y); }-inline __m256 mul(__m256 x, __m256 y) { return _mm256_mul_ps(x, y); }-#endif // __AVX__--#ifdef __AVX512F__-inline __m512 add(__m512 x, __m512 y) { return _mm512_add_ps(x, y); }-inline __m512 sub(__m512 x, __m512 y) { return _mm512_sub_ps(x, y); }-inline __m512 mul(__m512 x, __m512 y) { return _mm512_mul_ps(x, y); }-#endif // __AVX512F__--#ifdef __ARM_NEON-inline float32x4_t add(float32x4_t x, float32x4_t y) { return vaddq_f32(x, y); }-inline float32x4_t sub(float32x4_t x, float32x4_t y) { return vsubq_f32(x, y); }-inline float32x4_t mul(float32x4_t x, float32x4_t y) { return vmulq_f32(x, y); }-#endif // __ARM_NEON--#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC-inline float16x8_t add(float16x8_t x, float16x8_t y) { return vaddq_f16(x, y); }-inline float16x8_t sub(float16x8_t x, float16x8_t y) { return vsubq_f16(x, y); }-inline float16x8_t mul(float16x8_t x, float16x8_t y) { return vmulq_f16(x, y); }-#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC--/**- * Computes a * b + c.- *- * This operation will become fused into a single arithmetic instruction- * if the hardware has support for this feature, e.g. Intel Haswell+ (c.- * 2013), AMD Bulldozer+ (c. 2011), etc.- */-template <typename T, typename U>-inline U madd(T a, T b, U c) {- return add(mul(a, b), c);-}--/**- * Computes a * b + c with error correction.- *- * @see W. Kahan, "Further remarks on reducing truncation errors,"- * Communications of the ACM, vol. 8, no. 1, p. 40, Jan. 1965,- * doi: 10.1145/363707.363723.- */-template <typename T, typename U>-inline U madder(T a, T b, U c, U *e) {- U y = sub(mul(a, b), *e);- U t = add(c, y);- *e = sub(sub(t, c), y);- return t;-}