Skip to content

Distribute and run LLMs with a single file.

Make MoE prompt eval faster on CPUs

Justine Tunney4db03a114362
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 2b2769e1..9a5332fc 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -298,8 +298,56 @@ void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) {
}
}
+#ifdef __x86_64__
+#pragma GCC push_options
+#pragma GCC target("avx512f")
+static void ggml_bf16_to_fp32_row_avx512f(const ggml_bf16_t * x, float * y, int64_t n) {
+ int64_t i = 0;
+ for (; i + 16 <= n; i += 16) {
+ _mm512_storeu_ps(y + i,
+ _mm512_castsi512_ps(
+ _mm512_slli_epi32(
+ _mm512_cvtepu16_epi32(
+ _mm256_loadu_si256(
+ (const __m256i *)(x + i))),
+ 16)));
+ }
+ for (; i < n; i++) {
+ y[i] = GGML_BF16_TO_FP32(x[i]);
+ }
+}
+#pragma GCC pop_options
+#endif
+
+#ifdef __x86_64__
+#pragma GCC push_options
+#pragma GCC target("avx2")
+static void ggml_bf16_to_fp32_row_avx2(const ggml_bf16_t * x, float * y, int64_t n) {
+ int64_t i = 0;
+ for (; i + 8 <= n; i += 8) {
+ _mm256_storeu_ps(y + i,
+ _mm256_castsi256_ps(
+ _mm256_slli_epi32(
+ _mm256_cvtepu16_epi32(
+ _mm_loadu_si128(
+ (const __m128i *)(x + i))),
+ 16)));
+ }
+ for (; i < n; i++) {
+ y[i] = GGML_BF16_TO_FP32(x[i]);
+ }
+}
+#pragma GCC pop_options
+#endif
+
void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) {
- for (int64_t i = 0; i < n; i++) {
+#ifdef __x86_64__
+ if (X86_HAVE(AVX512F))
+ return ggml_bf16_to_fp32_row_avx512f(x, y, n);
+ if (X86_HAVE(AVX2))
+ return ggml_bf16_to_fp32_row_avx2(x, y, n);
+#endif
+ for (int i = 0; i < n; i++) {
y[i] = GGML_BF16_TO_FP32(x[i]);
}
}
@@ -11010,11 +11058,14 @@ static void ggml_compute_forward_mul_mat_id(
const struct ggml_tensor * src1 = dst->src[1];
const struct ggml_tensor * ids = dst->src[2];
- GGML_TENSOR_BINARY_OP_LOCALS
+ if (llamafile_mixmul(params, src0, src1, ids, dst))
+ return;
const int ith = params->ith;
const int nth = params->nth;
+ GGML_TENSOR_BINARY_OP_LOCALS
+
const enum ggml_type type = src0->type;
const bool src1_cont = ggml_is_contiguous(src1);
@@ -18558,6 +18609,7 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa
cur = 0;
const struct ggml_tensor * src0 = node->src[0];
const struct ggml_tensor * src1 = node->src[1];
+ const struct ggml_tensor * src2 = node->src[2];
const enum ggml_type vec_dot_type = type_traits[src0->type].vec_dot_type;
if (src1->type != vec_dot_type) {
cur += ggml_row_size(vec_dot_type, ggml_nelements(src1));
@@ -18566,6 +18618,8 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa
cur += GGML_PAD(cur, sizeof(int64_t)); // align
cur += n_as * sizeof(int64_t); // matrix_row_counts
cur += n_as * src1->ne[2] * sizeof(int64_t); // matrix_rows
+ size_t cur2 = llamafile_mixmul_needs(src0, src1, src2);
+ cur = cur > cur2 ? cur : cur2;
} break;
case GGML_OP_OUT_PROD:
{
diff --git a/llamafile/.clang-format b/llamafile/.clang-format
index 3b1dd74d..e2f7fba0 100644
--- a/llamafile/.clang-format
+++ b/llamafile/.clang-format
@@ -7,4 +7,6 @@ Language: Cpp
AllowShortFunctionsOnASingleLine: false
AlignTrailingComments: false
AlignEscapedNewlines: DontAlign
+AlwaysBreakTemplateDeclarations: true
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
---
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index 34923f7f..3cab491d 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -43,28 +43,12 @@ o/$(MODE)/llamafile: \
################################################################################
# microarchitectures
-o/$(MODE)/llamafile/sgemm_sss_avx.o: private TARGET_ARCH += -Xx86_64-mavx
-o/$(MODE)/llamafile/sgemm_sss_fma.o: private TARGET_ARCH += -Xx86_64-mfma
-o/$(MODE)/llamafile/sgemm_bss_avx2.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma
-o/$(MODE)/llamafile/sgemm_bss_avx512.o: private TARGET_ARCH += -Xx86_64-mavx512f
-o/$(MODE)/llamafile/sgemm_bss_avx512bf16.o: private TARGET_ARCH += -Xx86_64-mavx512bf16 -Xx86_64-mavx512vl
-o/$(MODE)/llamafile/sgemm_sss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f
-o/$(MODE)/llamafile/sgemm_hss_f16c.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mf16c
-o/$(MODE)/llamafile/sgemm_hss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f
-o/$(MODE)/llamafile/sgemm_hhs_neon.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+fp16
-o/$(MODE)/llamafile/sgemm_hsh_f16c.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mf16c
-o/$(MODE)/llamafile/sgemm_hsh_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f
-o/$(MODE)/llamafile/sgemm_hhh_neon.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+fp16
-o/$(MODE)/llamafile/sgemm_q0q0s_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni
-o/$(MODE)/llamafile/sgemm_q0q0s_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni
-o/$(MODE)/llamafile/sgemm_q0q0s_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma
-o/$(MODE)/llamafile/sgemm_q0q0s_dotprod.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+dotprod
-o/$(MODE)/llamafile/sgemm_e0q0s_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni
-o/$(MODE)/llamafile/sgemm_e0q0s_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni
-o/$(MODE)/llamafile/sgemm_e0q0s_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma
-o/$(MODE)/llamafile/sgemm_e1q1s_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni
-o/$(MODE)/llamafile/sgemm_e1q1s_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni
-o/$(MODE)/llamafile/sgemm_e1q1s_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma
+o/$(MODE)/llamafile/tinyblas_cpu_amd_fma.o: private TARGET_ARCH += -Xx86_64-mfma
+o/$(MODE)/llamafile/tinyblas_cpu_amd_avx2.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma
+o/$(MODE)/llamafile/tinyblas_cpu_amd_avxvnni.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma -Xx86_64-mavxvnni
+o/$(MODE)/llamafile/tinyblas_cpu_amd_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f
+o/$(MODE)/llamafile/tinyblas_cpu_amd_zen4.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni -Xx86_64-mavx512bf16
+o/$(MODE)/llamafile/tinyblas_cpu_arm82.o: private TARGET_ARCH += -Xaarch64-march=armv8.2-a+dotprod+fp16
################################################################################
# testing
diff --git a/llamafile/bench.h b/llamafile/bench.h
index 461ff081..6e5a118c 100644
--- a/llamafile/bench.h
+++ b/llamafile/bench.h
@@ -16,5 +16,5 @@
x; \
__asm__ volatile("" ::: "memory"); \
} \
- printf("%8lld us %s\n", (micros() - start + ITERATIONS - 1) / ITERATIONS, #x); \
+ printf("%9lld us %s\n", (micros() - start + ITERATIONS - 1) / ITERATIONS, #x); \
} while (0)
diff --git a/llamafile/hsum.h b/llamafile/hsum.h
deleted file mode 100644
index 347e32c1..00000000
--- a/llamafile/hsum.h
+++ /dev/null
@@ -1,56 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-#pragma once
-
-#ifdef __ARM_NEON
-#include <arm_neon.h>
-#endif
-#ifdef __x86_64__
-#include <immintrin.h>
-#endif
-
-inline float hsum(float x) {
- return x;
-}
-
-#ifdef __ARM_NEON
-inline float hsum(float32x4_t x) {
- return vaddvq_f32(x);
-}
-#endif
-
-#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
-inline float hsum(float16x8_t x) {
- float32x4_t t = vcvt_f32_f16(vget_low_f16(x));
- float32x4_t u = vcvt_f32_f16(vget_high_f16(x));
- return vaddvq_f32(vaddq_f32(t, u));
-}
-#endif
-
-#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
-inline float hsum(__m128 x) {
-#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
- x = _mm_add_ps(x, _mm_movehl_ps(x, x));
- x = _mm_add_ss(x, _mm_movehdup_ps(x));
- return _mm_cvtss_f32(x);
-#else
- __m128 shuf = _mm_shuffle_ps(x, x, _MM_SHUFFLE(2, 3, 0, 1));
- __m128 sums = _mm_add_ps(x, shuf);
- shuf = _mm_movehl_ps(shuf, sums);
- sums = _mm_add_ss(sums, shuf);
- return _mm_cvtss_f32(sums);
-#endif
-}
-#endif
-
-#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
-inline float hsum(__m256 x) {
- return hsum(_mm_add_ps(_mm256_extractf128_ps(x, 1), _mm256_castps256_ps128(x)));
-}
-#endif
-
-#if defined(__AVX512F__)
-inline float hsum(__m512 x) {
- return _mm512_reduce_add_ps(x);
-}
-#endif
diff --git a/llamafile/kernel.h b/llamafile/kernel.h
deleted file mode 100644
index 66d5ff4f..00000000
--- a/llamafile/kernel.h
+++ /dev/null
@@ -1,32 +0,0 @@
-#pragma once
-
-#if defined(__ARM_NEON) || defined(__AVX512F__)
-#define VECTOR_REGISTERS 32
-#else
-#define VECTOR_REGISTERS 16
-#endif
-
-#if defined(__AVX512F__)
-#define VECTOR_WIDTH 64
-#elif defined(__AVX__) || defined(__AVX2__)
-#define VECTOR_WIDTH 32
-#else
-#define VECTOR_WIDTH 16
-#endif
-
-#define BEGIN_KERNEL(RM, RN) \
- int ytiles = (m - m0) / RM; \
- int xtiles = (n - n0) / RN; \
- int tiles = ytiles * xtiles; \
- int duty = (tiles + nth - 1) / nth; \
- int start = duty * ith; \
- int end = start + duty; \
- if (end > tiles) \
- end = tiles; \
- for (int job = start; job < end; ++job) { \
- int i = m0 + job / xtiles * RM; \
- int j = n0 + job % xtiles * RN;
-
-#define END_KERNEL() }
-
-#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)
diff --git a/llamafile/kernelcount.sh b/llamafile/kernelcount.sh
deleted file mode 100755
index 2bc4844e..00000000
--- a/llamafile/kernelcount.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/sh
-# prints number of llamafile_sgemm kernels
-
-echo $(( $(aarch64-unknown-cosmo-nm \
- -C o//llama.cpp/main/main.aarch64.elf |
- grep GEMMER.*::gemm |
- wc -l) +
- $(x86_64-unknown-cosmo-nm \
- -C o//llama.cpp/main/main.com.dbg |
- grep GEMMER.*::gemm |
- wc -l) ))
diff --git a/llamafile/llamafile.h b/llamafile/llamafile.h
index 71694858..c5b783e7 100644
--- a/llamafile/llamafile.h
+++ b/llamafile/llamafile.h
@@ -51,6 +51,13 @@ const char *llamafile_describe_gpu(void);
bool llamafile_sgemm(int, int, int, const void *, int, const void *, int, void *, int, int, int,
int, int, int, int);
+struct ggml_tensor;
+struct ggml_compute_params;
+bool llamafile_mixmul(const struct ggml_compute_params *, const struct ggml_tensor *,
+ const struct ggml_tensor *, const struct ggml_tensor *, struct ggml_tensor *);
+size_t llamafile_mixmul_needs(const struct ggml_tensor *, const struct ggml_tensor *,
+ const struct ggml_tensor *);
+
#ifdef __cplusplus
}
#endif
diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cpp
index 0a9b1a9a..d292bac7 100644
--- a/llamafile/sgemm.cpp
+++ b/llamafile/sgemm.cpp
@@ -52,189 +52,73 @@ static const long hwcap = getauxval(AT_HWCAP);
*/
bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B, int ldb, void *C,
int ldc, int ith, int nth, int task, int Atype, int Btype, int Ctype) {
-
- assert(m >= 0);
- assert(n >= 0);
- assert(k >= 0);
- assert(lda >= k);
- assert(ldb >= k);
- assert(ldc >= m);
- assert(nth > 0);
- assert(ith < nth);
-
- switch (Atype) {
-
- case GGML_TYPE_F32:
- if (Ctype != GGML_TYPE_F32)
- return false;
- if (Btype != GGML_TYPE_F32)
- return false;
#ifdef __x86_64__
- if (!X86_HAVE(AVX))
- return false;
- if (X86_HAVE(AVX512F) && !(k % 16))
- return llamafile_sgemm_sss_avx512f(m, n, k, (const float *)A, lda, (const float *)B,
- ldb, (float *)C, ldc, ith, nth, task);
- if (X86_HAVE(FMA) && !(k % 8))
- return llamafile_sgemm_sss_fma(m, n, k, (const float *)A, lda, (const float *)B, ldb,
- (float *)C, ldc, ith, nth, task);
- if (!(k % 8))
- return llamafile_sgemm_sss_avx(m, n, k, (const float *)A, lda, (const float *)B, ldb,
- (float *)C, ldc, ith, nth, task);
+ if (X86_HAVE(FMA)) {
+ if (X86_HAVE(AVX2)) {
+ if (X86_HAVE(AVX512F)) {
+ if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && X86_HAVE(AVX512_BF16)) {
+ return llamafile_sgemm_amd_zen4(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task,
+ Atype, Btype, Ctype);
+ } else {
+ return llamafile_sgemm_amd_avx512f(m, n, k, A, lda, B, ldb, C, ldc, ith, nth,
+ task, Atype, Btype, Ctype);
+ }
+ } else if (X86_HAVE(AVXVNNI)) {
+ return llamafile_sgemm_amd_avxvnni(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task,
+ Atype, Btype, Ctype);
+ } else {
+ return llamafile_sgemm_amd_avx2(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task,
+ Atype, Btype, Ctype);
+ }
+ } else {
+ return llamafile_sgemm_amd_fma(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype,
+ Btype, Ctype);
+ }
+ } else {
+ return llamafile_sgemm_amd_avx(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype,
+ Btype, Ctype);
+ }
#elif defined(__aarch64__)
- if (n > 1 && !(k % 4))
- return llamafile_sgemm_sss_neon(m, n, k, (const float *)A, lda, (const float *)B, ldb,
- (float *)C, ldc, ith, nth, task);
+ if (hwcap & HWCAP_FPHP) {
+ return llamafile_sgemm_arm82(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,
+ Ctype);
+ } else {
+ return llamafile_sgemm_arm80(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,
+ Ctype);
+ }
#endif
- return false;
+}
- case GGML_TYPE_F16:
- switch (Ctype) {
- case GGML_TYPE_F32:
-#ifdef __x86_64__
- if (!X86_HAVE(AVX))
- return false;
- if (Btype != GGML_TYPE_F32)
- return false;
- if (X86_HAVE(AVX512F) && !(k % 16))
- return llamafile_sgemm_hss_avx512f(m, n, k, (const unsigned short *)A, lda,
- (const float *)B, ldb, (float *)C, ldc, ith, nth,
- task);
- if (X86_HAVE(FMA) && X86_HAVE(F16C) && !(k % 8))
- return llamafile_sgemm_hss_f16c(m, n, k, (const unsigned short *)A, lda,
- (const float *)B, ldb, (float *)C, ldc, ith, nth,
- task);
-#elif defined(__aarch64__)
- if (n > 1 && !(k % 8) && (hwcap & HWCAP_FPHP) && Btype == GGML_TYPE_F16)
- return llamafile_sgemm_hhs_neon(m, n, k, (const unsigned short *)A, lda,
- (const unsigned short *)B, ldb, (float *)C, ldc,
- ith, nth, task);
- if (n > 1 && !(k % 4) && !(hwcap & HWCAP_FPHP) && Btype == GGML_TYPE_F32)
- return llamafile_sgemm_hss_neon(m, n, k, (const unsigned short *)A, lda,
- (const float *)B, ldb, (float *)C, ldc, ith, nth,
- task);
-#endif
- return false;
- case GGML_TYPE_F16:
+/**
+ * Performs "mixture of experts" tensor multiplication on CPU.
+ */
+bool llamafile_mixmul(const ggml_compute_params *params, const ggml_tensor *weights,
+ const ggml_tensor *thought, const ggml_tensor *plan, ggml_tensor *result) {
#ifdef __x86_64__
- if (!X86_HAVE(AVX))
- return false;
- if (Btype != GGML_TYPE_F32)
- return false;
- if (X86_HAVE(AVX512F) && !(k % 16))
- return llamafile_sgemm_hsh_avx512f(m, n, k, (const unsigned short *)A, lda,
- (const float *)B, ldb, (llamafile_fp16 *)C, ldc,
- ith, nth, task);
- if (X86_HAVE(FMA) && X86_HAVE(F16C) && !(k % 8))
- return llamafile_sgemm_hsh_f16c(m, n, k, (const unsigned short *)A, lda,
- (const float *)B, ldb, (llamafile_fp16 *)C, ldc,
- ith, nth, task);
-#elif defined(__aarch64__)
- if (n > 1 && !(k % 4) && !(hwcap & HWCAP_FPHP) && Btype == GGML_TYPE_F32)
- return llamafile_sgemm_hsh_neon(m, n, k, (const unsigned short *)A, lda,
- (const float *)B, ldb, (llamafile_fp16 *)C, ldc,
- ith, nth, task);
-#endif
- return false;
- default:
- return false;
+ if (X86_HAVE(FMA)) {
+ if (X86_HAVE(AVX2)) {
+ if (X86_HAVE(AVX512F)) {
+ if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && X86_HAVE(AVX512_BF16)) {
+ return llamafile_mixmul_amd_zen4(params, weights, thought, plan, result);
+ } else {
+ return llamafile_mixmul_amd_avx512f(params, weights, thought, plan, result);
+ }
+ } else if (X86_HAVE(AVXVNNI)) {
+ return llamafile_mixmul_amd_avxvnni(params, weights, thought, plan, result);
+ } else {
+ return llamafile_mixmul_amd_avx2(params, weights, thought, plan, result);
+ }
+ } else {
+ return llamafile_mixmul_amd_fma(params, weights, thought, plan, result);
}
-
- case GGML_TYPE_BF16:
- if (Ctype != GGML_TYPE_F32)
- return false;
-#ifdef __x86_64__
- if (Btype != GGML_TYPE_F32)
- return false;
- if (X86_HAVE(AVX512_BF16) && !(k % 32))
- return llamafile_sgemm_bss_avx512bf16(m, n, k, (const ggml_bf16_t *)A, lda,
- (const float *)B, ldb, (float *)C, ldc, ith, nth,
- task);
- if (X86_HAVE(AVX512F) && !(k % 16))
- return llamafile_sgemm_bss_avx512(m, n, k, (const ggml_bf16_t *)A, lda,
- (const float *)B, ldb, (float *)C, ldc, ith, nth,
- task);
- if (X86_HAVE(AVX2) && X86_HAVE(FMA) && !(k % 8))
- return llamafile_sgemm_bss_avx2(m, n, k, (const ggml_bf16_t *)A, lda, (const float *)B,
- ldb, (float *)C, ldc, ith, nth, task);
-#endif
- return false;
-
- case GGML_TYPE_Q8_0:
- if (Ctype != GGML_TYPE_F32)
- return false;
- if (Btype != GGML_TYPE_Q8_0)
- return false;
-#ifdef __x86_64__
- if (!X86_HAVE(AVX))
- return false;
- if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI))
- return llamafile_sgemm_q0q0s_avx512vnni(m, n, k, (const block_q8_0 *)A, lda,
- (const block_q8_0 *)B, ldb, (float *)C, ldc,
- ith, nth, task);
- if (X86_HAVE(FMA) && X86_HAVE(AVXVNNI))
- return llamafile_sgemm_q0q0s_avxvnni(m, n, k, (const block_q8_0 *)A, lda,
- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,
- nth, task);
- if (X86_HAVE(FMA) && X86_HAVE(AVX2))
- return llamafile_sgemm_q0q0s_fma(m, n, k, (const block_q8_0 *)A, lda,
- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,
- task);
+ } else {
+ return llamafile_mixmul_amd_avx(params, weights, thought, plan, result);
+ }
#elif defined(__aarch64__)
- if (hwcap & HWCAP_FPHP)
- return llamafile_sgemm_q0q0s_dotprod(m, n, k, (const block_q8_0 *)A, lda,
- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,
- nth, task);
-#endif
- return false;
-
- case GGML_TYPE_Q4_0:
- if (Ctype != GGML_TYPE_F32)
- return false;
- if (Btype != GGML_TYPE_Q8_0)
- return false;
-#ifdef __x86_64__
- if (!X86_HAVE(AVX))
- return false;
- if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI))
- return llamafile_sgemm_e0q0s_avx512vnni(m, n, k, (const block_q4_0 *)A, lda,
- (const block_q8_0 *)B, ldb, (float *)C, ldc,
- ith, nth, task);
- if (X86_HAVE(FMA) && X86_HAVE(AVXVNNI))
- return llamafile_sgemm_e0q0s_avxvnni(m, n, k, (const block_q4_0 *)A, lda,
- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,
- nth, task);
- if (X86_HAVE(FMA) && X86_HAVE(AVX2))
- return llamafile_sgemm_e0q0s_fma(m, n, k, (const block_q4_0 *)A, lda,
- (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,
- task);
-#endif
- return false;
-
- case GGML_TYPE_Q4_1:
- if (Ctype != GGML_TYPE_F32)
- return false;
- if (Btype != GGML_TYPE_Q8_1)
- return false;
-#ifdef __x86_64__
- if (!X86_HAVE(AVX))
- return false;
- if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI))
- return llamafile_sgemm_e1q1s_avx512vnni(m, n, k, (const block_q4_1 *)A, lda,
- (const block_q8_1 *)B, ldb, (float *)C, ldc,
- ith, nth, task);
- if (X86_HAVE(FMA) && X86_HAVE(AVXVNNI))
- return llamafile_sgemm_e1q1s_avxvnni(m, n, k, (const block_q4_1 *)A, lda,
- (const block_q8_1 *)B, ldb, (float *)C, ldc, ith,
- nth, task);
- if (X86_HAVE(FMA) && X86_HAVE(AVX2))
- return llamafile_sgemm_e1q1s_fma(m, n, k, (const block_q4_1 *)A, lda,
- (const block_q8_1 *)B, ldb, (float *)C, ldc, ith, nth,
- task);
-#endif
- return false;
-
- default:
- return false;
+ if (hwcap & HWCAP_FPHP) {
+ return llamafile_mixmul_arm82(params, weights, thought, plan, result);
+ } else {
+ return llamafile_mixmul_arm80(params, weights, thought, plan, result);
}
+#endif
}
diff --git a/llamafile/sgemm.h b/llamafile/sgemm.h
index 398e2672..f384f890 100644
--- a/llamafile/sgemm.h
+++ b/llamafile/sgemm.h
@@ -1,65 +1,52 @@
#pragma once
-#include "llama.cpp/ggml-quants.h"
-#include "llama.cpp/ggml.h"
-#include "llamafile/fp16.h"
#ifdef __cplusplus
extern "C" {
#endif
-bool llamafile_sgemm_sss_avx(int, int, int, const float *, int, const float *, int, float *, int,
- int, int, int);
-bool llamafile_sgemm_sss_fma(int, int, int, const float *, int, const float *, int, float *, int,
- int, int, int);
-bool llamafile_sgemm_sss_avx512f(int, int, int, const float *, int, const float *, int, float *,
- int, int, int, int);
-bool llamafile_sgemm_sss_neon(int, int, int, const float *, int, const float *, int, float *, int,
- int, int, int);
-
-bool llamafile_sgemm_hss_f16c(int, int, int, const unsigned short *, int, const float *, int,
- float *, int, int, int, int);
-bool llamafile_sgemm_hss_avx512f(int, int, int, const unsigned short *, int, const float *, int,
- float *, int, int, int, int);
-bool llamafile_sgemm_hss_neon(int, int, int, const unsigned short *, int, const float *, int,
- float *, int, int, int, int);
-bool llamafile_sgemm_hhs_neon(int, int, int, const unsigned short *, int, const unsigned short *,
- int, float *, int, int, int, int);
-
-bool llamafile_sgemm_hsh_f16c(int, int, int, const unsigned short *, int, const float *, int,
- llamafile_fp16 *, int, int, int, int);
-bool llamafile_sgemm_hsh_avx512f(int, int, int, const unsigned short *, int, const float *, int,
- llamafile_fp16 *, int, int, int, int);
-bool llamafile_sgemm_hsh_neon(int, int, int, const unsigned short *, int, const float *, int,
- llamafile_fp16 *, int, int, int, int);
-
-bool llamafile_sgemm_bss_avx2(int, int, int, const ggml_bf16_t *, int, const float *, int, float *,
- int, int, int, int);
-bool llamafile_sgemm_bss_avx512(int, int, int, const ggml_bf16_t *, int, const float *, int,
- float *, int, int, int, int);
-bool llamafile_sgemm_bss_avx512bf16(int, int, int, const ggml_bf16_t *, int, const float *, int,
- float *, int, int, int, int);
-
-bool llamafile_sgemm_q0q0s_avx512vnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *,
- int, float *, int, int, int, int);
-bool llamafile_sgemm_q0q0s_avxvnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,
- float *, int, int, int, int);
-bool llamafile_sgemm_q0q0s_fma(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,
- float *, int, int, int, int);
-bool llamafile_sgemm_q0q0s_dotprod(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,
- float *, int, int, int, int);
-
-bool llamafile_sgemm_e0q0s_avx512vnni(int, int, int, const block_q4_0 *, int, const block_q8_0 *,
- int, float *, int, int, int, int);
-bool llamafile_sgemm_e0q0s_avxvnni(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,
- float *, int, int, int, int);
-bool llamafile_sgemm_e0q0s_fma(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,
- float *, int, int, int, int);
-
-bool llamafile_sgemm_e1q1s_avx512vnni(int, int, int, const block_q4_1 *, int, const block_q8_1 *,
- int, float *, int, int, int, int);
-bool llamafile_sgemm_e1q1s_avxvnni(int, int, int, const block_q4_1 *, int, const block_q8_1 *, int,
- float *, int, int, int, int);
-bool llamafile_sgemm_e1q1s_fma(int, int, int, const block_q4_1 *, int, const block_q8_1 *, int,
- float *, int, int, int, int);
+struct ggml_tensor;
+struct ggml_compute_params;
+
+bool llamafile_sgemm_amd_avx(int, int, int, const void *, int, const void *, int, void *, int, int,
+ int, int, int, int, int);
+bool llamafile_sgemm_amd_fma(int, int, int, const void *, int, const void *, int, void *, int, int,
+ int, int, int, int, int);
+bool llamafile_sgemm_amd_avx2(int, int, int, const void *, int, const void *, int, void *, int, int,
+ int, int, int, int, int);
+bool llamafile_sgemm_amd_avxvnni(int, int, int, const void *, int, const void *, int, void *, int,
+ int, int, int, int, int, int);
+bool llamafile_sgemm_amd_avx512f(int, int, int, const void *, int, const void *, int, void *, int,
+ int, int, int, int, int, int);
+bool llamafile_sgemm_amd_zen4(int, int, int, const void *, int, const void *, int, void *, int, int,
+ int, int, int, int, int);
+bool llamafile_sgemm_arm80(int, int, int, const void *, int, const void *, int, void *, int, int,
+ int, int, int, int, int);
+bool llamafile_sgemm_arm82(int, int, int, const void *, int, const void *, int, void *, int, int,
+ int, int, int, int, int);
+
+bool llamafile_mixmul_amd_avx(const struct ggml_compute_params *, const struct ggml_tensor *,
+ const struct ggml_tensor *, const struct ggml_tensor *,
+ struct ggml_tensor *);
+bool llamafile_mixmul_amd_fma(const struct ggml_compute_params *, const struct ggml_tensor *,
+ const struct ggml_tensor *, const struct ggml_tensor *,
+ struct ggml_tensor *);
+bool llamafile_mixmul_amd_avx2(const struct ggml_compute_params *, const struct ggml_tensor *,
+ const struct ggml_tensor *, const struct ggml_tensor *,
+ struct ggml_tensor *);
+bool llamafile_mixmul_amd_avxvnni(const struct ggml_compute_params *, const struct ggml_tensor *,
+ const struct ggml_tensor *, const struct ggml_tensor *,
+ struct ggml_tensor *);
+bool llamafile_mixmul_amd_avx512f(const struct ggml_compute_params *, const struct ggml_tensor *,
+ const struct ggml_tensor *, const struct ggml_tensor *,
+ struct ggml_tensor *);
+bool llamafile_mixmul_amd_zen4(const struct ggml_compute_params *, const struct ggml_tensor *,
+ const struct ggml_tensor *, const struct ggml_tensor *,
+ struct ggml_tensor *);
+bool llamafile_mixmul_arm80(const struct ggml_compute_params *, const struct ggml_tensor *,
+ const struct ggml_tensor *, const struct ggml_tensor *,
+ struct ggml_tensor *);
+bool llamafile_mixmul_arm82(const struct ggml_compute_params *, const struct ggml_tensor *,
+ const struct ggml_tensor *, const struct ggml_tensor *,
+ struct ggml_tensor *);
#ifdef __cplusplus
}
diff --git a/llamafile/sgemm_bss_avx2.cpp b/llamafile/sgemm_bss_avx2.cpp
deleted file mode 100644
index 5630e34c..00000000
--- a/llamafile/sgemm_bss_avx2.cpp
+++ /dev/null
@@ -1,51 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-#include <immintrin.h>
-
-#define KN 8
-
-#define V __m256
-#define D __m256
-#define TA ggml_bf16_t
-#define TB float
-#define TC float
-
-static inline __m256 load(const float *p) {
- return _mm256_loadu_ps(p);
-}
-
-static inline __m256 load(const ggml_bf16_t *p) {
- return _mm256_castsi256_ps(
- _mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)p)), 16));
-}
-
-#include "sgemmer.inc"
-
-bool llamafile_sgemm_bss_avx2(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_bss_avx512.cpp b/llamafile/sgemm_bss_avx512.cpp
deleted file mode 100644
index 96c563a9..00000000
--- a/llamafile/sgemm_bss_avx512.cpp
+++ /dev/null
@@ -1,51 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-#include <immintrin.h>
-
-#define KN 16
-
-#define V __m512
-#define D __m512
-#define TA ggml_bf16_t
-#define TB float
-#define TC float
-
-static inline __m512 load(const float *p) {
- return _mm512_loadu_ps(p);
-}
-
-static inline __m512 load(const ggml_bf16_t *p) {
- return _mm512_castsi512_ps(
- _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)p)), 16));
-}
-
-#include "sgemmer.inc"
-
-bool llamafile_sgemm_bss_avx512(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_bss_avx512bf16.cpp b/llamafile/sgemm_bss_avx512bf16.cpp
deleted file mode 100644
index a130e3a8..00000000
--- a/llamafile/sgemm_bss_avx512bf16.cpp
+++ /dev/null
@@ -1,59 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-#include "varith.h"
-#include <immintrin.h>
-
-#define KN 32
-
-#define D __m512
-#define V __m512bh
-#define TA ggml_bf16_t
-#define TB float
-#define TC float
-
-static inline __m512bh load(const ggml_bf16_t *p) {
- return (__m512bh)_mm512_loadu_ps((const float *)p);
-}
-
-static inline __m512bh load(const float *p) {
- return _mm512_cvtne2ps_pbh(_mm512_loadu_ps(p + 16), _mm512_loadu_ps(p));
-}
-
-template <> inline __m512 madd(__m512bh x, __m512bh y, __m512 z) {
- return _mm512_dpbf16_ps(z, x, y);
-}
-
-template <> inline __m512 madder(__m512bh x, __m512bh y, __m512 z, __m512 *e) {
- return _mm512_dpbf16_ps(z, x, y);
-}
-
-#include "sgemmer.inc"
-
-bool llamafile_sgemm_bss_avx512bf16(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_e0q0s_avx512vnni.cpp b/llamafile/sgemm_e0q0s_avx512vnni.cpp
deleted file mode 100644
index 5107717a..00000000
--- a/llamafile/sgemm_e0q0s_avx512vnni.cpp
+++ /dev/null
@@ -1,37 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-
-#define TA block_q4_0
-#define TB block_q8_0
-#define TC float
-
-#include "sgemmer0.inc"
-
-bool llamafile_sgemm_e0q0s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,
- int ldb, TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_e0q0s_avxvnni.cpp b/llamafile/sgemm_e0q0s_avxvnni.cpp
deleted file mode 100644
index b5713bb7..00000000
--- a/llamafile/sgemm_e0q0s_avxvnni.cpp
+++ /dev/null
@@ -1,37 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-
-#define TA block_q4_0
-#define TB block_q8_0
-#define TC float
-
-#include "sgemmer0.inc"
-
-bool llamafile_sgemm_e0q0s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_e0q0s_fma.cpp b/llamafile/sgemm_e0q0s_fma.cpp
deleted file mode 100644
index 8b0ccee3..00000000
--- a/llamafile/sgemm_e0q0s_fma.cpp
+++ /dev/null
@@ -1,37 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-
-#define TA block_q4_0
-#define TB block_q8_0
-#define TC float
-
-#include "sgemmer0.inc"
-
-bool llamafile_sgemm_e0q0s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_e1q1s_avx512vnni.cpp b/llamafile/sgemm_e1q1s_avx512vnni.cpp
deleted file mode 100644
index 1aa2264c..00000000
--- a/llamafile/sgemm_e1q1s_avx512vnni.cpp
+++ /dev/null
@@ -1,37 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-
-#define TA block_q4_1
-#define TB block_q8_1
-#define TC float
-
-#include "sgemmer1.inc"
-
-bool llamafile_sgemm_e1q1s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,
- int ldb, TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER1 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_e1q1s_avxvnni.cpp b/llamafile/sgemm_e1q1s_avxvnni.cpp
deleted file mode 100644
index 49ced14d..00000000
--- a/llamafile/sgemm_e1q1s_avxvnni.cpp
+++ /dev/null
@@ -1,37 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-
-#define TA block_q4_1
-#define TB block_q8_1
-#define TC float
-
-#include "sgemmer1.inc"
-
-bool llamafile_sgemm_e1q1s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER1 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_e1q1s_fma.cpp b/llamafile/sgemm_e1q1s_fma.cpp
deleted file mode 100644
index 178e23ac..00000000
--- a/llamafile/sgemm_e1q1s_fma.cpp
+++ /dev/null
@@ -1,37 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-
-#define TA block_q4_1
-#define TB block_q8_1
-#define TC float
-
-#include "sgemmer1.inc"
-
-bool llamafile_sgemm_e1q1s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER1 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_hhs_neon.cpp b/llamafile/sgemm_hhs_neon.cpp
deleted file mode 100644
index 5507a045..00000000
--- a/llamafile/sgemm_hhs_neon.cpp
+++ /dev/null
@@ -1,46 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __aarch64__
-
-#include "sgemm.h"
-#include <arm_neon.h>
-
-#define KN 8
-
-#define V float16x8_t
-#define D float16x8_t
-#define TA unsigned short
-#define TB unsigned short
-#define TC float
-
-static inline V load(const unsigned short *p) {
- return vld1q_f16((const __fp16 *)p);
-}
-
-#include "sgemmer.inc"
-
-bool llamafile_sgemm_hhs_neon(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __aarch64__
diff --git a/llamafile/sgemm_hsh_avx512f.cpp b/llamafile/sgemm_hsh_avx512f.cpp
deleted file mode 100644
index 463dcdf8..00000000
--- a/llamafile/sgemm_hsh_avx512f.cpp
+++ /dev/null
@@ -1,50 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-#include <immintrin.h>
-
-#define KN 16
-
-#define V __m512
-#define D __m512
-#define TA unsigned short
-#define TB float
-#define TC llamafile_fp16
-
-static inline V load(const float *p) {
- return _mm512_loadu_ps(p);
-}
-
-static inline V load(const unsigned short *p) {
- return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)p));
-}
-
-#include "sgemmer.inc"
-
-bool llamafile_sgemm_hsh_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_hsh_f16c.cpp b/llamafile/sgemm_hsh_f16c.cpp
deleted file mode 100644
index 7bb09b4e..00000000
--- a/llamafile/sgemm_hsh_f16c.cpp
+++ /dev/null
@@ -1,50 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-#include <immintrin.h>
-
-#define KN 8
-
-#define V __m256
-#define D __m256
-#define TA unsigned short
-#define TB float
-#define TC llamafile_fp16
-
-static inline V load(const float *p) {
- return _mm256_loadu_ps(p);
-}
-
-static inline V load(const unsigned short *p) {
- return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));
-}
-
-#include "sgemmer.inc"
-
-bool llamafile_sgemm_hsh_f16c(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_hsh_neon.cpp b/llamafile/sgemm_hsh_neon.cpp
deleted file mode 100644
index 35db6a4b..00000000
--- a/llamafile/sgemm_hsh_neon.cpp
+++ /dev/null
@@ -1,50 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __aarch64__
-
-#include "sgemm.h"
-#include <arm_neon.h>
-
-#define KN 4
-
-#define V float32x4_t
-#define D float32x4_t
-#define TA unsigned short
-#define TB float
-#define TC llamafile_fp16
-
-static inline V load(const float *p) {
- return vld1q_f32(p);
-}
-
-static inline V load(const unsigned short *p) {
- return vcvt_f32_f16(vld1_f16((const __fp16 *)p));
-}
-
-#include "sgemmer.inc"
-
-bool llamafile_sgemm_hsh_neon(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __aarch64__
diff --git a/llamafile/sgemm_hss_avx512f.cpp b/llamafile/sgemm_hss_avx512f.cpp
deleted file mode 100644
index f0a8bf33..00000000
--- a/llamafile/sgemm_hss_avx512f.cpp
+++ /dev/null
@@ -1,50 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-#include <immintrin.h>
-
-#define KN 16
-
-#define V __m512
-#define D __m512
-#define TA unsigned short
-#define TB float
-#define TC float
-
-static inline V load(const float *p) {
- return _mm512_loadu_ps(p);
-}
-
-static inline V load(const unsigned short *p) {
- return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)p));
-}
-
-#include "sgemmer.inc"
-
-bool llamafile_sgemm_hss_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_hss_f16c.cpp b/llamafile/sgemm_hss_f16c.cpp
deleted file mode 100644
index e9ba461b..00000000
--- a/llamafile/sgemm_hss_f16c.cpp
+++ /dev/null
@@ -1,50 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-#include <immintrin.h>
-
-#define KN 8
-
-#define V __m256
-#define D __m256
-#define TA unsigned short
-#define TB float
-#define TC float
-
-static inline V load(const float *p) {
- return _mm256_loadu_ps(p);
-}
-
-static inline V load(const unsigned short *p) {
- return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));
-}
-
-#include "sgemmer.inc"
-
-bool llamafile_sgemm_hss_f16c(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_hss_neon.cpp b/llamafile/sgemm_hss_neon.cpp
deleted file mode 100644
index e10e37e9..00000000
--- a/llamafile/sgemm_hss_neon.cpp
+++ /dev/null
@@ -1,50 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __aarch64__
-
-#include "sgemm.h"
-#include <arm_neon.h>
-
-#define KN 4
-
-#define V float32x4_t
-#define D float32x4_t
-#define TA unsigned short
-#define TB float
-#define TC float
-
-static inline V load(const float *p) {
- return vld1q_f32(p);
-}
-
-static inline V load(const unsigned short *p) {
- return vcvt_f32_f16(vld1_f16((const __fp16 *)p));
-}
-
-#include "sgemmer.inc"
-
-bool llamafile_sgemm_hss_neon(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __aarch64__
diff --git a/llamafile/sgemm_q0q0s_avx512vnni.cpp b/llamafile/sgemm_q0q0s_avx512vnni.cpp
deleted file mode 100644
index e5b3ed24..00000000
--- a/llamafile/sgemm_q0q0s_avx512vnni.cpp
+++ /dev/null
@@ -1,37 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-
-#define TA block_q8_0
-#define TB block_q8_0
-#define TC float
-
-#include "sgemmer0.inc"
-
-bool llamafile_sgemm_q0q0s_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B,
- int ldb, TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_q0q0s_avxvnni.cpp b/llamafile/sgemm_q0q0s_avxvnni.cpp
deleted file mode 100644
index d53b2979..00000000
--- a/llamafile/sgemm_q0q0s_avxvnni.cpp
+++ /dev/null
@@ -1,37 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-
-#define TA block_q8_0
-#define TB block_q8_0
-#define TC float
-
-#include "sgemmer0.inc"
-
-bool llamafile_sgemm_q0q0s_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_q0q0s_dotprod.cpp b/llamafile/sgemm_q0q0s_dotprod.cpp
deleted file mode 100644
index c737f66c..00000000
--- a/llamafile/sgemm_q0q0s_dotprod.cpp
+++ /dev/null
@@ -1,182 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __aarch64__
-
-#include <arm_neon.h>
-
-#include "llama.cpp/ggml-impl.h"
-#include "llama.cpp/ggml.h"
-
-#include "hsum.h"
-#include "kernel.h"
-#include "sgemm.h"
-
-namespace {
-
-class GEMMERQ0ARM {
- public:
- GEMMERQ0ARM(int k, const block_q8_0 *A, int lda, const block_q8_0 *B, int ldb, float *C,
- int ldc, int ith, int nth)
- : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
- }
-
- void matmul(int m, int n) {
- mnpack(0, m, 0, n);
- }
-
- private:
- dontinline void mnpack(int m0, int m, int n0, int n) {
- if (m - m0 <= 0 || n - n0 <= 0)
- return;
- int mc, nc, mp, np;
- if (m - m0 >= 3 && n - n0 >= 3) {
- mc = 3;
- nc = 3;
- gemm3x3(m0, m, n0, n);
- } else {
- mc = 1;
- nc = 1;
- gemm1x1(m0, m, n0, n);
- }
- mp = m0 + (m - m0) / mc * mc;
- np = n0 + (n - n0) / nc * nc;
- mnpack(mp, m, n0, np);
- mnpack(m0, m, np, n);
- }
-
- dontinline void gemm3x3(int m0, int m, int n0, int n) {
- BEGIN_KERNEL(3, 3)
- int32x4_t zero = vdupq_n_s32(0);
- float32x4_t c00 = vdupq_n_f32(0.f);
- float32x4_t c01 = vdupq_n_f32(0.f);
- float32x4_t c02 = vdupq_n_f32(0.f);
- float32x4_t c10 = vdupq_n_f32(0.f);
- float32x4_t c11 = vdupq_n_f32(0.f);
- float32x4_t c12 = vdupq_n_f32(0.f);
- float32x4_t c20 = vdupq_n_f32(0.f);
- float32x4_t c21 = vdupq_n_f32(0.f);
- float32x4_t c22 = vdupq_n_f32(0.f);
- const block_q8_0 *Ap0 = A + lda * (i + 0);
- const block_q8_0 *Ap1 = A + lda * (i + 1);
- const block_q8_0 *Ap2 = A + lda * (i + 2);
- const block_q8_0 *Bp0 = B + ldb * (j + 0);
- const block_q8_0 *Bp1 = B + ldb * (j + 1);
- const block_q8_0 *Bp2 = B + ldb * (j + 2);
- for (int l = 0; l < k; ++l) {
- c00 = vmlaq_n_f32(
- c00,
- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap0[l].qs), vld1q_s8(Bp0[l].qs)),
- vld1q_s8(Ap0[l].qs + 16), vld1q_s8(Bp0[l].qs + 16))),
- unhalf(Ap0[l].d) * unhalf(Bp0[l].d));
- c01 = vmlaq_n_f32(
- c01,
- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap0[l].qs), vld1q_s8(Bp1[l].qs)),
- vld1q_s8(Ap0[l].qs + 16), vld1q_s8(Bp1[l].qs + 16))),
- unhalf(Ap0[l].d) * unhalf(Bp1[l].d));
- c02 = vmlaq_n_f32(
- c02,
- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap0[l].qs), vld1q_s8(Bp2[l].qs)),
- vld1q_s8(Ap0[l].qs + 16), vld1q_s8(Bp2[l].qs + 16))),
- unhalf(Ap0[l].d) * unhalf(Bp2[l].d));
- c10 = vmlaq_n_f32(
- c10,
- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap1[l].qs), vld1q_s8(Bp0[l].qs)),
- vld1q_s8(Ap1[l].qs + 16), vld1q_s8(Bp0[l].qs + 16))),
- unhalf(Ap1[l].d) * unhalf(Bp0[l].d));
- c11 = vmlaq_n_f32(
- c11,
- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap1[l].qs), vld1q_s8(Bp1[l].qs)),
- vld1q_s8(Ap1[l].qs + 16), vld1q_s8(Bp1[l].qs + 16))),
- unhalf(Ap1[l].d) * unhalf(Bp1[l].d));
- c12 = vmlaq_n_f32(
- c12,
- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap1[l].qs), vld1q_s8(Bp2[l].qs)),
- vld1q_s8(Ap1[l].qs + 16), vld1q_s8(Bp2[l].qs + 16))),
- unhalf(Ap1[l].d) * unhalf(Bp2[l].d));
- c20 = vmlaq_n_f32(
- c20,
- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap2[l].qs), vld1q_s8(Bp0[l].qs)),
- vld1q_s8(Ap2[l].qs + 16), vld1q_s8(Bp0[l].qs + 16))),
- unhalf(Ap2[l].d) * unhalf(Bp0[l].d));
- c21 = vmlaq_n_f32(
- c21,
- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap2[l].qs), vld1q_s8(Bp1[l].qs)),
- vld1q_s8(Ap2[l].qs + 16), vld1q_s8(Bp1[l].qs + 16))),
- unhalf(Ap2[l].d) * unhalf(Bp1[l].d));
- c22 = vmlaq_n_f32(
- c22,
- vcvtq_f32_s32(vdotq_s32(vdotq_s32(zero, vld1q_s8(Ap2[l].qs), vld1q_s8(Bp2[l].qs)),
- vld1q_s8(Ap2[l].qs + 16), vld1q_s8(Bp2[l].qs + 16))),
- unhalf(Ap2[l].d) * unhalf(Bp2[l].d));
- }
- C[ldc * (j + 0) + (i + 0)] = hsum(c00);
- C[ldc * (j + 0) + (i + 1)] = hsum(c10);
- C[ldc * (j + 0) + (i + 2)] = hsum(c20);
- C[ldc * (j + 1) + (i + 0)] = hsum(c01);
- C[ldc * (j + 1) + (i + 1)] = hsum(c11);
- C[ldc * (j + 1) + (i + 2)] = hsum(c21);
- C[ldc * (j + 2) + (i + 0)] = hsum(c02);
- C[ldc * (j + 2) + (i + 1)] = hsum(c12);
- C[ldc * (j + 2) + (i + 2)] = hsum(c22);
- END_KERNEL()
- }
-
- dontinline void gemm1x1(int m0, int m, int n0, int n) {
- BEGIN_KERNEL(1, 1)
- float32x4_t acc = vdupq_n_f32(0.f);
- const block_q8_0 *Ap = A + lda * i;
- const block_q8_0 *Bp = B + ldb * j;
- for (int l = 0; l < k; ++l) {
- acc = vmlaq_n_f32(acc,
- vcvtq_f32_s32(vdotq_s32(
- vdotq_s32(vdupq_n_s32(0), vld1q_s8(Ap[l].qs), vld1q_s8(Bp[l].qs)),
- vld1q_s8(Ap[l].qs + 16), vld1q_s8(Bp[l].qs + 16))),
- unhalf(Ap[l].d) * unhalf(Bp[l].d));
- }
- C[ldc * j + i] = hsum(acc);
- END_KERNEL()
- }
-
- static inline float unhalf(unsigned short d) {
- return GGML_FP16_TO_FP32(d);
- }
-
- const block_q8_0 *const A;
- const block_q8_0 *const B;
- float *const C;
- const int k;
- const int lda;
- const int ldb;
- const int ldc;
- const int ith;
- const int nth;
-};
-
-} // namespace
-
-bool llamafile_sgemm_q0q0s_dotprod(int m, int n, int k, const block_q8_0 *A, int lda,
- const block_q8_0 *B, int ldb, float *C, int ldc, int ith,
- int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- GEMMERQ0ARM tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __aarch64__
diff --git a/llamafile/sgemm_q0q0s_fma.cpp b/llamafile/sgemm_q0q0s_fma.cpp
deleted file mode 100644
index 437ec389..00000000
--- a/llamafile/sgemm_q0q0s_fma.cpp
+++ /dev/null
@@ -1,37 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-
-#define TA block_q8_0
-#define TB block_q8_0
-#define TC float
-
-#include "sgemmer0.inc"
-
-bool llamafile_sgemm_q0q0s_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_sss_avx.cpp b/llamafile/sgemm_sss_avx.cpp
deleted file mode 100644
index a49427c8..00000000
--- a/llamafile/sgemm_sss_avx.cpp
+++ /dev/null
@@ -1,46 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-#include <immintrin.h>
-
-#define KN 8
-
-#define V __m256
-#define D __m256
-#define TA float
-#define TB float
-#define TC float
-
-static inline V load(const float *p) {
- return _mm256_loadu_ps(p);
-}
-
-#include "sgemmer.inc"
-
-bool llamafile_sgemm_sss_avx(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,
- int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_sss_avx512f.cpp b/llamafile/sgemm_sss_avx512f.cpp
deleted file mode 100644
index a9dbc1a0..00000000
--- a/llamafile/sgemm_sss_avx512f.cpp
+++ /dev/null
@@ -1,46 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-#include <immintrin.h>
-
-#define KN 16
-
-#define V __m512
-#define D __m512
-#define TA float
-#define TB float
-#define TC float
-
-static inline V load(const float *p) {
- return _mm512_loadu_ps(p);
-}
-
-#include "sgemmer.inc"
-
-bool llamafile_sgemm_sss_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_sss_fma.cpp b/llamafile/sgemm_sss_fma.cpp
deleted file mode 100644
index f12ad7d6..00000000
--- a/llamafile/sgemm_sss_fma.cpp
+++ /dev/null
@@ -1,46 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __x86_64__
-
-#include "sgemm.h"
-#include <immintrin.h>
-
-#define KN 8
-
-#define V __m256
-#define D __m256
-#define TA float
-#define TB float
-#define TC float
-
-static inline V load(const float *p) {
- return _mm256_loadu_ps(p);
-}
-
-#include "sgemmer.inc"
-
-bool llamafile_sgemm_sss_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,
- int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __x86_64__
diff --git a/llamafile/sgemm_sss_neon.cpp b/llamafile/sgemm_sss_neon.cpp
deleted file mode 100644
index 88644221..00000000
--- a/llamafile/sgemm_sss_neon.cpp
+++ /dev/null
@@ -1,46 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifdef __aarch64__
-
-#include "sgemm.h"
-#include <arm_neon.h>
-
-#define KN 4
-
-#define V float32x4_t
-#define D float32x4_t
-#define TA float
-#define TB float
-#define TC float
-
-static inline V load(const float *p) {
- return vld1q_f32(p);
-}
-
-#include "sgemmer.inc"
-
-bool llamafile_sgemm_sss_neon(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
- TC *C, int ldc, int ith, int nth, int task) {
- if (task != GGML_TASK_TYPE_COMPUTE)
- return true;
- SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
- tb.matmul(m, n);
- return true;
-}
-
-#endif // __aarch64__
diff --git a/llamafile/sgemm_sss_test.cpp b/llamafile/sgemm_sss_test.cpp
deleted file mode 100644
index 79c00839..00000000
--- a/llamafile/sgemm_sss_test.cpp
+++ /dev/null
@@ -1,79 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "bench.h"
-#include "float.h"
-#include "gemm.h"
-#include "llama.cpp/ggml.h"
-#include "llamafile.h"
-#include "numba.h"
-
-#define ITERATIONS 5
-
-int main(int argc, char *argv[]) {
- float tolerance = 2e-5;
- int m = 510;
- int n = 513;
- int k = 512;
- int l = 3;
- int lda = k + l;
- int ldb = k + l;
- int ldc = m + l;
- float *A = new float[lda * m];
- float *B = new float[ldb * n];
- float *C = new float[ldc * n];
- float *G = new float[ldc * n];
- broadcast(A, lda * m, NAN);
- broadcast(B, ldb * n, NAN);
- broadcast(C, ldc * n, NAN);
- broadcast(G, ldc * n, NAN);
- randomize(k, m, A, lda);
- randomize(k, n, B, ldb);
-
- BENCH(gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc));
- BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,
- GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32));
-
- for (int i = 0; i < m; ++i)
- for (int j = 0; j < n; ++j) {
- auto g = G[ldc * j + i];
- auto c = C[ldc * j + i];
- if (flt::isnan(g)) {
- fprintf(stderr, "%s:%d: found nan in reference matrix: i=%d j=%d\n", __FILE__,
- __LINE__, i, j);
- return 3;
- }
- if (flt::isnan(c)) {
- fprintf(stderr, "%s:%d: found nan in output matrix: i=%d j=%d\n", __FILE__,
- __LINE__, i, j);
- return 4;
- }
- auto d = g - c;
- if (d < 0)
- d = -d;
- if (d > tolerance) {
- fprintf(stderr, "%s:%d: difference exceeds tolerance: %g vs. %g (%g) i=%d j=%d\n",
- __FILE__, __LINE__, g, c, d, i, j);
- return 5;
- }
- }
-
- delete[] G;
- delete[] C;
- delete[] B;
- delete[] A;
-}
diff --git a/llamafile/sgemm_vecdot_test.cpp b/llamafile/sgemm_vecdot_test.cpp
index 79507d41..78c7587b 100644
--- a/llamafile/sgemm_vecdot_test.cpp
+++ b/llamafile/sgemm_vecdot_test.cpp
@@ -74,24 +74,28 @@ int main(int argc, char *argv[]) {
}
double err_avg = err_sum / (m * n);
- fprintf(stderr, "note: %g average ulp\n", err_avg);
- fprintf(stderr, "note: %lld worst ulp\n", err_worst);
+ fprintf(stderr, "%9g ulp average\n", err_avg);
+ fprintf(stderr, "%9lld ulp worst\n", err_worst);
// using one accumulator
- // 52.7393 average ulp
- // 9052 worst ulp
+ // 40209 us gemm
+ // 2851 us llamafile_sgemm
+ // 42.0078 ulp average
+ // 6731 ulp worst
// using three accumulators
// 22.291 average ulp
// 1566 worst ulp
// using kahan summation
- // 2.14244 average ulp
- // 134 worst ulp
+ // 40190 us gemm
+ // 3028 us llamafile_sgemm
+ // 2.14244 ulp average
+ // 134 ulp worst
- if (err_avg > 30)
+ if (err_avg > 2.15)
return 5;
- if (err_worst > 3000)
+ if (err_worst > 134)
return 6;
delete[] G;
diff --git a/llamafile/sgemmer.inc b/llamafile/sgemmer.inc
deleted file mode 100644
index 762cafaa..00000000
--- a/llamafile/sgemmer.inc
+++ /dev/null
@@ -1,230 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <algorithm>
-
-#include "llama.cpp/ggml.h"
-
-#include "hsum.h"
-#include "kernel.h"
-#include "varith.h"
-
-namespace {
-
-class SGEMMER {
- public:
- SGEMMER(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)
- : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
- }
-
- void matmul(int m, int n) {
- mnpack(0, m, 0, n);
- }
-
- private:
- void mnpack(int m0, int m, int n0, int n) {
- int mc, nc, mp, np;
- switch ((std::min(m - m0, 5) << 4) | std::min(n - n0, 5)) {
-#if VECTOR_REGISTERS == 32
- case 0x55:
- mc = 5;
- nc = 5;
- gemm<5, 5>(m0, m, n0, n);
- break;
- case 0x45:
- mc = 4;
- nc = 5;
- gemm<4, 5>(m0, m, n0, n);
- break;
- case 0x54:
- mc = 5;
- nc = 4;
- gemm<5, 4>(m0, m, n0, n);
- break;
- case 0x44:
- mc = 4;
- nc = 4;
- gemm<4, 4>(m0, m, n0, n);
- break;
- case 0x53:
- mc = 5;
- nc = 3;
- gemm<5, 3>(m0, m, n0, n);
- break;
- case 0x35:
- mc = 3;
- nc = 5;
- gemm<3, 5>(m0, m, n0, n);
- break;
- case 0x43:
- mc = 4;
- nc = 3;
- gemm<4, 3>(m0, m, n0, n);
- break;
-#else
- case 0x55:
- case 0x54:
- case 0x53:
- case 0x45:
- case 0x44:
- case 0x43:
- mc = 4;
- nc = 3;
- gemm<4, 3>(m0, m, n0, n);
- break;
- case 0x35:
-#endif
- case 0x34:
- mc = 3;
- nc = 4;
- gemm<3, 4>(m0, m, n0, n);
- break;
- case 0x52:
- mc = 5;
- nc = 2;
- gemm<5, 2>(m0, m, n0, n);
- break;
- case 0x33:
- mc = 3;
- nc = 3;
- gemm<3, 3>(m0, m, n0, n);
- break;
- case 0x25:
- mc = 2;
- nc = 5;
- gemm<2, 5>(m0, m, n0, n);
- break;
- case 0x42:
- mc = 4;
- nc = 2;
- gemm<4, 2>(m0, m, n0, n);
- break;
- case 0x24:
- mc = 2;
- nc = 4;
- gemm<2, 4>(m0, m, n0, n);
- break;
- case 0x32:
- mc = 3;
- nc = 2;
- gemm<3, 2>(m0, m, n0, n);
- break;
- case 0x23:
- mc = 2;
- nc = 3;
- gemm<2, 3>(m0, m, n0, n);
- break;
- case 0x51:
- mc = 5;
- nc = 1;
- gemm<5, 1>(m0, m, n0, n);
- break;
- case 0x41:
- mc = 4;
- nc = 1;
- gemm<4, 1>(m0, m, n0, n);
- break;
- case 0x22:
- mc = 2;
- nc = 2;
- gemm<2, 2>(m0, m, n0, n);
- break;
- case 0x15:
- mc = 1;
- nc = 5;
- gemm<1, 5>(m0, m, n0, n);
- break;
- case 0x14:
- mc = 1;
- nc = 4;
- gemm<1, 4>(m0, m, n0, n);
- break;
- case 0x31:
- mc = 3;
- nc = 1;
- gemm<3, 1>(m0, m, n0, n);
- break;
- case 0x13:
- mc = 1;
- nc = 3;
- gemm<1, 3>(m0, m, n0, n);
- break;
- case 0x21:
- mc = 2;
- nc = 1;
- gemm<2, 1>(m0, m, n0, n);
- break;
- case 0x12:
- mc = 1;
- nc = 2;
- gemm<1, 2>(m0, m, n0, n);
- break;
- case 0x11:
- mc = 1;
- nc = 1;
- gemm<1, 1>(m0, m, n0, n);
- break;
- default:
- return;
- }
- mp = m0 + (m - m0) / mc * mc;
- np = n0 + (n - n0) / nc * nc;
- mnpack(mp, m, n0, np);
- mnpack(m0, m, np, n);
- }
-
- template <int RM, int RN> dontinline void gemm(int m0, int m, int n0, int n) {
- int ytiles = (m - m0) / RM;
- int xtiles = (n - n0) / RN;
- int tiles = xtiles * ytiles;
- int duty = (tiles + nth - 1) / nth;
- int start = duty * ith;
- int end = start + duty;
- if (end > tiles)
- end = tiles;
- for (int job = start; job < end; ++job) {
- int ii = m0 + job / xtiles * RM;
- int jj = n0 + job % xtiles * RN;
- D Cv[RN][RM] = {0};
- for (int l = 0; l < k; l += KN)
- for (int j = 0; j < RN; ++j)
- for (int i = 0; i < RM; ++i)
- Cv[j][i] = madd(load(A + lda * (ii + i) + l), //
- load(B + ldb * (jj + j) + l), //
- Cv[j][i]);
- TC Cd[RN][RM];
- for (int j = 0; j < RN; ++j)
- for (int i = 0; i < RM; ++i)
- Cd[j][i] = hsum(Cv[j][i]);
- for (int j = 0; j < RN; ++j)
- for (int i = 0; i < RM; ++i)
- C[ldc * (jj + j) + (ii + i)] = Cd[j][i];
- }
- }
-
- const TA *const A;
- const TB *const B;
- TC *const C;
- const int k;
- const int lda;
- const int ldb;
- const int ldc;
- const int ith;
- const int nth;
-};
-
-} // namespace
diff --git a/llamafile/sgemmer0.inc b/llamafile/sgemmer0.inc
deleted file mode 100644
index 2475875e..00000000
--- a/llamafile/sgemmer0.inc
+++ /dev/null
@@ -1,222 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <algorithm>
-
-#include "llama.cpp/ggml-impl.h"
-#include "llama.cpp/ggml.h"
-
-#include "hsum.h"
-#include "kernel.h"
-#include "varith.h"
-
-namespace {
-
-class SGEMMER0 {
- public:
- SGEMMER0(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)
- : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
- }
-
- void matmul(int m, int n) {
- mnpack(0, m, 0, n);
- }
-
- private:
- void mnpack(int m0, int m, int n0, int n) {
- int mc, nc, mp, np;
- switch ((std::min(m - m0, 4) << 4) | std::min(n - n0, 4)) {
-#if VECTOR_REGISTERS == 32
- case 0x44:
- mc = 4;
- nc = 4;
- gemm<4, 4>(m0, m, n0, n);
- break;
- case 0x43:
- mc = 4;
- nc = 3;
- gemm<4, 3>(m0, m, n0, n);
- break;
- case 0x34:
- mc = 3;
- nc = 4;
- gemm<3, 4>(m0, m, n0, n);
- break;
- case 0x33:
- mc = 3;
- nc = 3;
- gemm<3, 3>(m0, m, n0, n);
- break;
- case 0x42:
- mc = 4;
- nc = 2;
- gemm<4, 2>(m0, m, n0, n);
- break;
- case 0x24:
- mc = 2;
- nc = 4;
- gemm<2, 4>(m0, m, n0, n);
- break;
-#else
- case 0x44:
- case 0x43:
- case 0x42:
- mc = 4;
- nc = 2;
- gemm<4, 2>(m0, m, n0, n);
- break;
- case 0x34:
- case 0x24:
- mc = 2;
- nc = 4;
- gemm<2, 4>(m0, m, n0, n);
- break;
- case 0x33:
-#endif
- case 0x32:
- mc = 3;
- nc = 2;
- gemm<3, 2>(m0, m, n0, n);
- break;
- case 0x23:
- mc = 2;
- nc = 3;
- gemm<2, 3>(m0, m, n0, n);
- break;
- case 0x41:
- mc = 4;
- nc = 1;
- gemm<4, 1>(m0, m, n0, n);
- break;
- case 0x22:
- mc = 2;
- nc = 2;
- gemm<2, 2>(m0, m, n0, n);
- break;
- case 0x14:
- mc = 1;
- nc = 4;
- gemm<1, 4>(m0, m, n0, n);
- break;
- case 0x31:
- mc = 3;
- nc = 1;
- gemm<3, 1>(m0, m, n0, n);
- break;
- case 0x13:
- mc = 1;
- nc = 3;
- gemm<1, 3>(m0, m, n0, n);
- break;
- case 0x21:
- mc = 2;
- nc = 1;
- gemm<2, 1>(m0, m, n0, n);
- break;
- case 0x12:
- mc = 1;
- nc = 2;
- gemm<1, 2>(m0, m, n0, n);
- break;
- case 0x11:
- mc = 1;
- nc = 1;
- gemm<1, 1>(m0, m, n0, n);
- break;
- default:
- return;
- }
- mp = m0 + (m - m0) / mc * mc;
- np = n0 + (n - n0) / nc * nc;
- mnpack(mp, m, n0, np);
- mnpack(m0, m, np, n);
- }
-
- template <int RM, int RN> dontinline void gemm(int m0, int m, int n0, int n) {
- int ytiles = (m - m0) / RM;
- int xtiles = (n - n0) / RN;
- int tiles = xtiles * ytiles;
- int duty = (tiles + nth - 1) / nth;
- int start = duty * ith;
- int end = start + duty;
- if (end > tiles)
- end = tiles;
- for (int job = start; job < end; ++job) {
- int ii = m0 + job / xtiles * RM;
- int jj = n0 + job % xtiles * RN;
- __m256 Cv[RN][RM] = {0};
- for (int l = 0; l < k; ++l)
- for (int j = 0; j < RN; ++j)
- for (int i = 0; i < RM; ++i)
- Cv[j][i] = madd(_mm256_set1_ps(unhalf(A[lda * (ii + i) + l].d) *
- unhalf(B[ldb * (jj + j) + l].d)),
- updot(_mm256_sign_epi8(load(A + lda * (ii + i) + l),
- load(A + lda * (ii + i) + l)),
- _mm256_sign_epi8(load(B + ldb * (jj + j) + l),
- load(A + lda * (ii + i) + l))),
- Cv[j][i]);
- TC Cd[RN][RM];
- for (int j = 0; j < RN; ++j)
- for (int i = 0; i < RM; ++i)
- Cd[j][i] = hsum(Cv[j][i]);
- for (int j = 0; j < RN; ++j)
- for (int i = 0; i < RM; ++i)
- C[ldc * (jj + j) + (ii + i)] = Cd[j][i];
- }
- }
-
- inline __m256i load(const block_q8_0 *b) {
- return _mm256_loadu_si256((const __m256i *)b->qs);
- }
-
- inline __m256i load(const block_q4_0 *b) {
- return _mm256_sub_epi8(denibble(b->qs), _mm256_set1_epi8(8));
- }
-
- inline __m256 updot(__m256i u, __m256i s) {
- __m256i res;
-#if defined(__AVXVNNI__) || defined(__AVX512VNNI__)
- res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);
-#else
- res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));
-#endif
- return _mm256_cvtepi32_ps(res);
- }
-
- static inline __m256i denibble(const uint8_t *p) {
- const __m128i tmp = _mm_loadu_si128((const __m128i *)p);
- const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);
- const __m256i lowMask = _mm256_set1_epi8(15);
- return _mm256_and_si256(lowMask, bytes);
- }
-
- static inline float unhalf(unsigned short d) {
- return GGML_FP16_TO_FP32(d);
- }
-
- const TA *const A;
- const TB *const B;
- TC *const C;
- const int k;
- const int lda;
- const int ldb;
- const int ldc;
- const int ith;
- const int nth;
-};
-
-} // namespace
diff --git a/llamafile/sgemmer1.inc b/llamafile/sgemmer1.inc
deleted file mode 100644
index f8966d8f..00000000
--- a/llamafile/sgemmer1.inc
+++ /dev/null
@@ -1,215 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "llama.cpp/ggml-impl.h"
-#include "llama.cpp/ggml.h"
-
-#include "hsum.h"
-#include "kernel.h"
-#include "varith.h"
-
-namespace {
-
-class SGEMMER1 {
- public:
- SGEMMER1(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)
- : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
- }
-
- void matmul(int m, int n) {
- mnpack(0, m, 0, n);
- }
-
- private:
- dontinline void mnpack(int m0, int m, int n0, int n) {
- if (m - m0 <= 0 || n - n0 <= 0)
- return;
- int mc, nc, mp, np;
- if (VECTOR_REGISTERS >= 32 && m - m0 >= 4 && n - n0 >= 2) {
- mc = 4;
- nc = 2;
- gemm4x2(m0, m, n0, n);
- } else if (m - m0 >= 4 && n - n0 >= 1) {
- mc = 4;
- nc = 1;
- gemm4x1(m0, m, n0, n);
- } else {
- mc = 1;
- nc = 1;
- gemm1x1(m0, m, n0, n);
- }
- mp = m0 + (m - m0) / mc * mc;
- np = n0 + (n - n0) / nc * nc;
- mnpack(mp, m, n0, np);
- mnpack(m0, m, np, n);
- }
-
- dontinline void gemm4x2(int m0, int m, int n0, int n) {
- BEGIN_KERNEL(4, 2)
- __m128 s0 = _mm_setzero_ps();
- __m128 s1 = _mm_setzero_ps();
- __m256 c00 = _mm256_setzero_ps();
- __m256 c01 = _mm256_setzero_ps();
- __m256 c02 = _mm256_setzero_ps();
- __m256 c03 = _mm256_setzero_ps();
- __m256 c10 = _mm256_setzero_ps();
- __m256 c11 = _mm256_setzero_ps();
- __m256 c12 = _mm256_setzero_ps();
- __m256 c13 = _mm256_setzero_ps();
- const TA *Ap0 = A + lda * (i + 0);
- const TA *Ap1 = A + lda * (i + 1);
- const TA *Ap2 = A + lda * (i + 2);
- const TA *Ap3 = A + lda * (i + 3);
- const TB *Bp0 = B + ldb * (j + 0);
- const TB *Bp1 = B + ldb * (j + 1);
- for (int l = 0; l < k; ++l) {
- __m128 sa = _mm_set_ps(unhalf(Ap3[l].dm >> 16), //
- unhalf(Ap2[l].dm >> 16), //
- unhalf(Ap1[l].dm >> 16), //
- unhalf(Ap0[l].dm >> 16));
- s0 = madd(_mm_set1_ps(unhalf(Bp0[l].ds >> 16)), sa, s0);
- s1 = madd(_mm_set1_ps(unhalf(Bp1[l].ds >> 16)), sa, s1);
- __m256i f0 = load(Bp0 + l);
- __m256 d0 = _mm256_set1_ps(unhalf(Bp0[l].ds));
- c00 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap0[l].dm)), d0),
- updot(load(Ap0 + l), f0), c00);
- c01 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap1[l].dm)), d0),
- updot(load(Ap1 + l), f0), c01);
- c02 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap2[l].dm)), d0),
- updot(load(Ap2 + l), f0), c02);
- c03 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap3[l].dm)), d0),
- updot(load(Ap3 + l), f0), c03);
- __m256i f1 = load(Bp1 + l);
- __m256 d1 = _mm256_set1_ps(unhalf(Bp1[l].ds));
- c10 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap0[l].dm)), d1),
- updot(load(Ap0 + l), f1), c10);
- c11 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap1[l].dm)), d1),
- updot(load(Ap1 + l), f1), c11);
- c12 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap2[l].dm)), d1),
- updot(load(Ap2 + l), f1), c12);
- c13 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap3[l].dm)), d1),
- updot(load(Ap3 + l), f1), c13);
- }
- C[ldc * (j + 0) + (i + 0)] = hsum(c00) + s0[0];
- C[ldc * (j + 0) + (i + 1)] = hsum(c01) + s0[1];
- C[ldc * (j + 0) + (i + 2)] = hsum(c02) + s0[2];
- C[ldc * (j + 0) + (i + 3)] = hsum(c03) + s0[3];
- C[ldc * (j + 1) + (i + 0)] = hsum(c10) + s1[0];
- C[ldc * (j + 1) + (i + 1)] = hsum(c11) + s1[1];
- C[ldc * (j + 1) + (i + 2)] = hsum(c12) + s1[2];
- C[ldc * (j + 1) + (i + 3)] = hsum(c13) + s1[3];
- END_KERNEL()
- }
-
- dontinline void gemm4x1(int m0, int m, int n0, int n) {
- BEGIN_KERNEL(4, 1)
- __m128 s = _mm_setzero_ps();
- __m256 c0 = _mm256_setzero_ps();
- __m256 c1 = _mm256_setzero_ps();
- __m256 c2 = _mm256_setzero_ps();
- __m256 c3 = _mm256_setzero_ps();
- const TA *Ap0 = A + lda * (i + 0);
- const TA *Ap1 = A + lda * (i + 1);
- const TA *Ap2 = A + lda * (i + 2);
- const TA *Ap3 = A + lda * (i + 3);
- const TB *Bp = B + ldb * j;
- for (int l = 0; l < k; ++l) {
- s = madd(_mm_set1_ps(unhalf(Bp[l].ds >> 16)),
- _mm_set_ps(unhalf(Ap3[l].dm >> 16), //
- unhalf(Ap2[l].dm >> 16), //
- unhalf(Ap1[l].dm >> 16), //
- unhalf(Ap0[l].dm >> 16)),
- s);
- __m256i f = load(Bp + l);
- __m256 d1 = _mm256_set1_ps(unhalf(Bp[l].ds));
- c0 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap0[l].dm)), d1), updot(load(Ap0 + l), f),
- c0);
- c1 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap1[l].dm)), d1), updot(load(Ap1 + l), f),
- c1);
- c2 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap2[l].dm)), d1), updot(load(Ap2 + l), f),
- c2);
- c3 = madd(_mm256_mul_ps(_mm256_set1_ps(unhalf(Ap3[l].dm)), d1), updot(load(Ap3 + l), f),
- c3);
- }
- C[ldc * j + (i + 0)] = hsum(c0) + s[0];
- C[ldc * j + (i + 1)] = hsum(c1) + s[1];
- C[ldc * j + (i + 2)] = hsum(c2) + s[2];
- C[ldc * j + (i + 3)] = hsum(c3) + s[3];
- END_KERNEL()
- }
-
- dontinline void gemm1x1(int m0, int m, int n0, int n) {
- BEGIN_KERNEL(1, 1)
- float s = 0;
- __m256 c = _mm256_setzero_ps();
- const TA *Ap = A + lda * i;
- const TB *Bp = B + ldb * j;
- for (int l = 0; l < k; ++l) {
- unsigned dm = Ap[l].dm;
- unsigned ds = Bp[l].ds;
- __m256 d0 = _mm256_set1_ps(unhalf(dm));
- __m256 d1 = _mm256_set1_ps(unhalf(ds));
- s += unhalf(dm >> 16) * unhalf(ds >> 16);
- __m256i e = load(Ap + l);
- __m256i f = load(Bp + l);
- c = madd(_mm256_mul_ps(d0, d1), updot(e, f), c);
- }
- C[ldc * j + i] = hsum(c) + s;
- END_KERNEL()
- }
-
- static inline nosideeffect __m256i load(const block_q8_1 *b) {
- return _mm256_loadu_si256((const __m256i *)b->qs);
- }
-
- static inline nosideeffect __m256i load(const block_q4_1 *b) {
- return denibble(b->qs);
- }
-
- static inline pureconst __m256 updot(__m256i u, __m256i s) {
- __m256i res;
-#if defined(__AVXVNNI__) || defined(__AVX512VNNI__)
- res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);
-#else
- res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));
-#endif
- return _mm256_cvtepi32_ps(res);
- }
-
- static inline nosideeffect __m256i denibble(const uint8_t *p) {
- const __m128i tmp = _mm_loadu_si128((const __m128i *)p);
- const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);
- const __m256i lowMask = _mm256_set1_epi8(15);
- return _mm256_and_si256(lowMask, bytes);
- }
-
- static inline pureconst float unhalf(unsigned short d) {
- return GGML_FP16_TO_FP32(d);
- }
-
- const TA *const A;
- const TB *const B;
- TC *const C;
- const int k;
- const int lda;
- const int ldb;
- const int ldc;
- const int ith;
- const int nth;
-};
-
-} // namespace
diff --git a/llamafile/tinyblas_cpu.inc b/llamafile/tinyblas_cpu.inc
new file mode 100644
index 00000000..abf05539
--- /dev/null
+++ b/llamafile/tinyblas_cpu.inc
@@ -0,0 +1,1496 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//
+// _ _ ___ _ _ ___
+// | |_(_)_ _ _ _| _ ) | /_\ / __|
+// | _| | ' \ || | _ \ |__ / _ \\__ \.
+// \__|_|_||_\_, |___/____/_/ \_\___/
+// |__/
+//
+// BASIC LINEAR ALGEBRA SUBPROGRAMS
+//
+//
+// This file implements multithreaded CPU matrix multiplication for the
+// common contiguous use case C = Aᵀ * B. These kernels are designed to
+// have excellent performance[1] for matrices that fit in the CPU cache
+// without imposing any overhead such as cache filling or malloc calls.
+//
+// This implementation does not guarantee any upper bound with rounding
+// errors, which grow along with k. Our goal's to maximally exploit the
+// hardware for performance, and then use whatever resources remain for
+// improving numerical accuracy.
+//
+// [1] J. Tunney, ‘LLaMA Now Goes Faster on CPUs’, Mar. 2024. [Online].
+// Available: https://justine.lol/matmul/. [Accessed: 29-Mar-2024].
+
+#include "llama.cpp/ggml-impl.h"
+#include "llama.cpp/ggml-quants.h"
+#include "llamafile.h"
+#include <algorithm>
+
+#pragma GCC diagnostic ignored "-Wpedantic"
+#pragma GCC diagnostic ignored "-Wignored-attributes"
+
+#define ROW_ALIGN 64
+#define MATRIX_ALIGN 4096
+#define MAX_ALIGN 4096
+
+#ifdef _MSC_VER
+#define NOINLINE __declspec(noinline)
+#else
+#define NOINLINE __attribute__((__noinline__))
+#endif
+
+#if defined(__ARM_NEON) || defined(__AVX512F__)
+#define VECTOR_REGISTERS 32
+#else
+#define VECTOR_REGISTERS 16
+#endif
+
+namespace {
+
+inline float unhalf(ggml_fp16_t d) {
+ return GGML_FP16_TO_FP32(d);
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// MATRIX MEMORY INDEXING
+
+#define NCA 1
+#define NCB 2
+#define NCC 4
+
+#define INDEX(A, lda, j, i) index<CONFIG & NC##A>(A, lda, j, i)
+
+template <int NC, typename T>
+inline T &index(T *A, int lda, int j, int i) {
+ if (NC)
+ return ((T **)A)[j][i];
+ else
+ return A[lda * j + i];
+}
+
+template <int NC, typename T>
+inline const T &index(const T *A, int lda, int j, int i) {
+ if (NC)
+ return ((const T *const *)A)[j][i];
+ else
+ return A[lda * j + i];
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// GGML TYPE TRAITS
+
+template <typename T>
+struct ggml_type_trait;
+template <>
+struct ggml_type_trait<float> {
+ static constexpr ggml_type id = GGML_TYPE_F32;
+};
+template <>
+struct ggml_type_trait<ggml_bf16_t> {
+ static constexpr ggml_type id = GGML_TYPE_BF16;
+};
+template <>
+struct ggml_type_trait<ggml_fp16_t> {
+ static constexpr ggml_type id = GGML_TYPE_F16;
+};
+template <>
+struct ggml_type_trait<block_q8_0> {
+ static constexpr ggml_type id = GGML_TYPE_Q8_0;
+};
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// VECTORIZED ARITHMETIC OPERATIONS
+
+#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+inline __m128 add(__m128 x, __m128 y) {
+ return _mm_add_ps(x, y);
+}
+inline __m128 sub(__m128 x, __m128 y) {
+ return _mm_sub_ps(x, y);
+}
+inline __m128 mul(__m128 x, __m128 y) {
+ return _mm_mul_ps(x, y);
+}
+#endif // __SSE__
+
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+inline __m256 add(__m256 x, __m256 y) {
+ return _mm256_add_ps(x, y);
+}
+inline __m256 sub(__m256 x, __m256 y) {
+ return _mm256_sub_ps(x, y);
+}
+inline __m256 mul(__m256 x, __m256 y) {
+ return _mm256_mul_ps(x, y);
+}
+#endif // __AVX__
+
+#if defined(__AVX512F__)
+inline __m512 add(__m512 x, __m512 y) {
+ return _mm512_add_ps(x, y);
+}
+inline __m512 sub(__m512 x, __m512 y) {
+ return _mm512_sub_ps(x, y);
+}
+inline __m512 mul(__m512 x, __m512 y) {
+ return _mm512_mul_ps(x, y);
+}
+#endif // __AVX512F__
+
+#if defined(__ARM_NEON)
+inline float32x4_t add(float32x4_t x, float32x4_t y) {
+ return vaddq_f32(x, y);
+}
+inline float32x4_t sub(float32x4_t x, float32x4_t y) {
+ return vsubq_f32(x, y);
+}
+inline float32x4_t mul(float32x4_t x, float32x4_t y) {
+ return vmulq_f32(x, y);
+}
+#endif // __ARM_NEON
+
+#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
+inline float16x8_t add(float16x8_t x, float16x8_t y) {
+ return vaddq_f16(x, y);
+}
+inline float16x8_t sub(float16x8_t x, float16x8_t y) {
+ return vsubq_f16(x, y);
+}
+inline float16x8_t mul(float16x8_t x, float16x8_t y) {
+ return vmulq_f16(x, y);
+}
+#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// VECTORIZED FUSED MULTIPLY ADD
+
+/**
+ * Computes a * b + c.
+ */
+template <typename T, typename U>
+inline U madd(T a, T b, U c) {
+ return add(mul(a, b), c);
+}
+
+/**
+ * Computes a * b + c with error correction.
+ *
+ * @see W. Kahan, "Further remarks on reducing truncation errors,"
+ * Communications of the ACM, vol. 8, no. 1, p. 40, Jan. 1965,
+ * doi: 10.1145/363707.363723.
+ */
+template <typename T, typename U>
+inline U madder(T a, T b, U c, U *e) {
+ U y = sub(mul(a, b), *e);
+ U t = add(c, y);
+ *e = sub(sub(t, c), y);
+ return t;
+}
+
+#if defined(__FMA__)
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+template <>
+inline __m256 madd(__m256 a, __m256 b, __m256 c) {
+ return _mm256_fmadd_ps(a, b, c);
+}
+#endif
+#if defined(__AVX512F__)
+template <>
+inline __m512 madd(__m512 a, __m512 b, __m512 c) {
+ return _mm512_fmadd_ps(a, b, c);
+}
+#endif
+#endif
+
+#if defined(__ARM_FEATURE_FMA)
+template <>
+inline float32x4_t madd(float32x4_t a, float32x4_t b, float32x4_t c) {
+ return vfmaq_f32(c, b, a);
+}
+#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
+template <>
+inline float16x8_t madd(float16x8_t a, float16x8_t b, float16x8_t c) {
+ return vfmaq_f16(c, b, a);
+}
+#endif
+#endif
+
+#if defined(__AVX512BF16__)
+template <>
+inline __m512 madd(__m512bh x, __m512bh y, __m512 z) {
+ return _mm512_dpbf16_ps(z, x, y);
+}
+template <>
+inline __m512 madder(__m512bh x, __m512bh y, __m512 z, __m512 *_) {
+ return _mm512_dpbf16_ps(z, x, y);
+}
+#endif
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// VECTORIZED HORIZONTAL SUM
+
+#if defined(__ARM_NEON)
+inline float hsum(float32x4_t x) {
+ return vaddvq_f32(x);
+}
+#endif // __ARM_NEON
+
+#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
+inline float hsum(float16x8_t x) {
+ return vaddvq_f32(vaddq_f32(vcvt_f32_f16(vget_low_f16(x)), vcvt_f32_f16(vget_high_f16(x))));
+}
+#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+
+#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+inline float hsum(__m128 x) {
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+ x = _mm_add_ps(x, _mm_movehl_ps(x, x));
+ x = _mm_add_ss(x, _mm_movehdup_ps(x));
+#else
+ __m128 t;
+ t = _mm_shuffle_ps(x, x, _MM_SHUFFLE(2, 3, 0, 1));
+ x = _mm_add_ps(x, t);
+ t = _mm_movehl_ps(t, x);
+ x = _mm_add_ss(x, t);
+#endif
+ return _mm_cvtss_f32(x);
+}
+#endif
+
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+inline float hsum(__m256 x) {
+ return hsum(_mm_add_ps(_mm256_extractf128_ps(x, 1), _mm256_castps256_ps128(x)));
+}
+#endif // __AVX__
+
+#if defined(__AVX512F__)
+inline float hsum(__m512 x) {
+ return _mm512_reduce_add_ps(x);
+}
+#endif // __AVX512F__
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// VECTORIZED MEMORY LOADING
+
+template <typename T, typename U>
+T load(const U *);
+
+#if defined(__ARM_NEON)
+template <>
+inline float32x4_t load(const float *p) {
+ return vld1q_f32(p);
+}
+template <>
+inline float32x4_t load(const ggml_bf16_t *p) {
+ return vreinterpretq_f32_u32(vshll_n_u16(vld1_u16((const unsigned short *)p), 16));
+}
+#if !defined(_MSC_VER)
+template <>
+inline float16x8_t load(const ggml_fp16_t *p) {
+ return vld1q_f16((const float16_t *)p);
+}
+template <>
+inline float32x4_t load(const ggml_fp16_t *p) {
+ return vcvt_f32_f16(vld1_f16((const float16_t *)p));
+}
+#endif // _MSC_VER
+#endif // __ARM_NEON
+
+#if defined(__SSE__) || defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+template <>
+inline __m128 load(const float *p) {
+ return _mm_loadu_ps(p);
+}
+#endif // __SSE__
+
+#if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__)
+template <>
+inline __m256 load(const float *p) {
+ return _mm256_loadu_ps(p);
+}
+#endif // __AVX__
+
+#if defined(__AVX2__) || defined(__AVX512F__)
+template <>
+inline __m256 load(const ggml_bf16_t *p) {
+ return _mm256_castsi256_ps(
+ _mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)p)), 16));
+}
+#endif // __AVX2__
+
+#if defined(__F16C__)
+template <>
+inline __m256 load(const ggml_fp16_t *p) {
+ return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));
+}
+#endif // __F16C__
+
+#if defined(__AVX512F__)
+template <>
+inline __m512 load(const float *p) {
+ return _mm512_loadu_ps(p);
+}
+template <>
+inline __m512 load(const ggml_fp16_t *p) {
+ return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)p));
+}
+template <>
+inline __m512 load(const ggml_bf16_t *p) {
+ return _mm512_castsi512_ps(
+ _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)p)), 16));
+}
+#endif // __AVX512F__
+
+#if defined(__AVX512BF16__)
+template <>
+inline __m512bh load(const ggml_bf16_t *p) {
+ return (__m512bh)_mm512_loadu_ps((const float *)p);
+}
+#endif // __AVX512BF16__
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+// FLOATING POINT MATRIX MULTIPLICATION
+
+template <int CONFIG, int KN, typename D, typename V, typename TA, typename TB, typename TC>
+class tinyBLAS {
+ public:
+ tinyBLAS(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
+ }
+
+ void matmul(int m, int n, int task) {
+ if (task == GGML_TASK_TYPE_COMPUTE)
+ mnpack(0, m, 0, n);
+ }
+
+ private:
+ NOINLINE void mnpack(int m0, int m, int n0, int n) {
+ int mc, nc, mp, np;
+ switch ((std::min(m - m0, 5) << 4) | std::min(n - n0, 5)) {
+#if VECTOR_REGISTERS == 32
+ case 0x55:
+ mc = 5;
+ nc = 5;
+ gemm<5, 5, false>(m0, m, n0, n);
+ break;
+ case 0x45:
+ mc = 4;
+ nc = 5;
+ gemm<4, 5, false>(m0, m, n0, n);
+ break;
+ case 0x54:
+ mc = 5;
+ nc = 4;
+ gemm<5, 4, false>(m0, m, n0, n);
+ break;
+ case 0x44:
+ mc = 4;
+ nc = 4;
+ gemm<4, 4, false>(m0, m, n0, n);
+ break;
+ case 0x53:
+ mc = 5;
+ nc = 3;
+ gemm<5, 3, false>(m0, m, n0, n);
+ break;
+ case 0x35:
+ mc = 3;
+ nc = 5;
+ gemm<3, 5, false>(m0, m, n0, n);
+ break;
+ case 0x43:
+ mc = 4;
+ nc = 3;
+ gemm<4, 3, false>(m0, m, n0, n);
+ break;
+#else
+ case 0x55:
+ case 0x54:
+ case 0x53:
+ case 0x45:
+ case 0x44:
+ case 0x43:
+ mc = 4;
+ nc = 3;
+ gemm<4, 3, false>(m0, m, n0, n);
+ break;
+ case 0x35:
+#endif
+ case 0x34:
+ mc = 3;
+ nc = 4;
+ gemm<3, 4, false>(m0, m, n0, n);
+ break;
+ case 0x52:
+ mc = 5;
+ nc = 2;
+ gemm<5, 2, false>(m0, m, n0, n);
+ break;
+ case 0x33:
+ mc = 3;
+ nc = 3;
+ gemm<3, 3, false>(m0, m, n0, n);
+ break;
+ case 0x25:
+ mc = 2;
+ nc = 5;
+ gemm<2, 5, false>(m0, m, n0, n);
+ break;
+ case 0x42:
+ mc = 4;
+ nc = 2;
+ gemm<4, 2, false>(m0, m, n0, n);
+ break;
+ case 0x24:
+ mc = 2;
+ nc = 4;
+ gemm<2, 4, false>(m0, m, n0, n);
+ break;
+ case 0x32:
+ mc = 3;
+ nc = 2;
+ gemm<3, 2, true>(m0, m, n0, n);
+ break;
+ case 0x23:
+ mc = 2;
+ nc = 3;
+ gemm<2, 3, true>(m0, m, n0, n);
+ break;
+ case 0x51:
+ mc = 5;
+ nc = 1;
+ gemm<5, 1, true>(m0, m, n0, n);
+ break;
+ case 0x41:
+ mc = 4;
+ nc = 1;
+ gemm<4, 1, true>(m0, m, n0, n);
+ break;
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2, true>(m0, m, n0, n);
+ break;
+ case 0x15:
+ mc = 1;
+ nc = 5;
+ gemm<1, 5, true>(m0, m, n0, n);
+ break;
+ case 0x14:
+ mc = 1;
+ nc = 4;
+ gemm<1, 4, true>(m0, m, n0, n);
+ break;
+ case 0x31:
+ mc = 3;
+ nc = 1;
+ gemm<3, 1, true>(m0, m, n0, n);
+ break;
+ case 0x13:
+ mc = 1;
+ nc = 3;
+ gemm<1, 3, true>(m0, m, n0, n);
+ break;
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1, true>(m0, m, n0, n);
+ break;
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2, true>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1, true>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
+ mp = m0 + (m - m0) / mc * mc;
+ np = n0 + (n - n0) / nc * nc;
+ mnpack(mp, m, n0, np);
+ mnpack(m0, m, np, n);
+ }
+
+ template <int RM, int RN, int KAHAN>
+ NOINLINE void gemm(int m0, int m, int n0, int n) {
+ int ytiles = RM > 1 ? (m - m0) / RM : 1;
+ int xtiles = RN > 1 ? (n - n0) / RN : 1;
+ int tiles = xtiles * ytiles;
+ int duty = (tiles + nth - 1) / nth;
+ int start = duty * ith;
+ int end = start + duty;
+ if (end > tiles)
+ end = tiles;
+ for (int job = start; job < end; ++job) {
+ int ii = m0 + job / xtiles * RM;
+ int jj = n0 + job % xtiles * RN;
+ D Cv[RN][RM] = {};
+ D Ce[RN][RM] = {};
+ for (int l = 0; l < k; l += KN)
+ for (int j = 0; j < RN; ++j)
+ for (int i = 0; i < RM; ++i)
+ if (KAHAN)
+ Cv[j][i] = madder(load<V>(&INDEX(A, lda, ii + i, l)), //
+ load<V>(&INDEX(B, ldb, jj + j, l)), //
+ Cv[j][i], &Ce[j][i]);
+ else
+ Cv[j][i] = madd(load<V>(&INDEX(A, lda, ii + i, l)), //
+ load<V>(&INDEX(B, ldb, jj + j, l)), //
+ Cv[j][i]);
+ for (int j = 0; j < RN; ++j)
+ for (int i = 0; i < RM; ++i)
+ INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]);
+ }
+ }
+
+ const TA *const A;
+ const TB *const B;
+ TC *const C;
+ const int k;
+ const int lda;
+ const int ldb;
+ const int ldc;
+ const int ith;
+ const int nth;
+};
+
+//////////////////////////////////////////////////////////////////////////////////////////
+// QUANT ZERO MATRIX MULTIPLICATION
+
+#if defined(__ARM_FEATURE_DOTPROD)
+template <int CONFIG, typename TA>
+class tinyBLAS_Q0_ARM {
+ public:
+ tinyBLAS_Q0_ARM(int k, const TA *A, int lda, const block_q8_0 *B, int ldb, float *C, int ldc,
+ int ith, int nth)
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
+ }
+
+ void matmul(int m, int n, int task) {
+ if (task == GGML_TASK_TYPE_COMPUTE)
+ mnpack(0, m, 0, n);
+ }
+
+ private:
+ NOINLINE void mnpack(int m0, int m, int n0, int n) {
+ int mc, nc, mp, np;
+ switch ((std::min(m - m0, 3) << 4) | std::min(n - n0, 3)) {
+ case 0x33:
+ mc = 3;
+ nc = 3;
+ gemm<3, 3>(m0, m, n0, n);
+ break;
+ case 0x32:
+ mc = 3;
+ nc = 2;
+ gemm<3, 2>(m0, m, n0, n);
+ break;
+ case 0x23:
+ mc = 2;
+ nc = 3;
+ gemm<2, 3>(m0, m, n0, n);
+ break;
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2>(m0, m, n0, n);
+ break;
+ case 0x31:
+ mc = 3;
+ nc = 1;
+ gemm<3, 1>(m0, m, n0, n);
+ break;
+ case 0x13:
+ mc = 1;
+ nc = 3;
+ gemm<1, 3>(m0, m, n0, n);
+ break;
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1>(m0, m, n0, n);
+ break;
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
+ mp = m0 + (m - m0) / mc * mc;
+ np = n0 + (n - n0) / nc * nc;
+ mnpack(mp, m, n0, np);
+ mnpack(m0, m, np, n);
+ }
+
+ template <int RM, int RN>
+ NOINLINE void gemm(int m0, int m, int n0, int n) {
+ int ytiles = RM > 1 ? (m - m0) / RM : 1;
+ int xtiles = RN > 1 ? (n - n0) / RN : 1;
+ int tiles = xtiles * ytiles;
+ int duty = (tiles + nth - 1) / nth;
+ int start = duty * ith;
+ int end = start + duty;
+ if (end > tiles)
+ end = tiles;
+ for (int job = start; job < end; ++job) {
+ int ii = m0 + job / xtiles * RM;
+ int jj = n0 + job % xtiles * RN;
+ float32x4_t Cv[RN][RM] = {};
+ for (int l = 0; l < k; ++l)
+ for (int j = 0; j < RN; ++j)
+ for (int i = 0; i < RM; ++i)
+ Cv[j][i] = vmlaq_n_f32(
+ Cv[j][i],
+ vcvtq_f32_s32(vdotq_s32(vdotq_s32(vdupq_n_s32(0),
+ load_lo(&INDEX(A, lda, ii + i, l)),
+ load_lo(&INDEX(B, ldb, jj + j, l))),
+ load_hi(&INDEX(A, lda, ii + i, l)),
+ load_hi(&INDEX(B, ldb, jj + j, l)))),
+ (unhalf(INDEX(A, lda, ii + i, l).d) *
+ unhalf(INDEX(B, ldb, jj + j, l).d)));
+ for (int j = 0; j < RN; ++j)
+ for (int i = 0; i < RM; ++i)
+ INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]);
+ }
+ }
+
+ inline int8x16_t load_lo(const block_q8_0 *b) {
+ return vld1q_s8(b->qs);
+ }
+
+ inline int8x16_t load_hi(const block_q8_0 *b) {
+ return vld1q_s8(b->qs + 16);
+ }
+
+ inline int8x16_t load_lo(const block_q4_0 *b) {
+ return vsubq_s8(vreinterpretq_s8_u8(vandq_u8(vld1q_u8(b->qs), vdupq_n_u8(0x0f))),
+ vdupq_n_s8(0x8));
+ }
+
+ inline int8x16_t load_hi(const block_q4_0 *b) {
+ return vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(vld1q_u8(b->qs), 4)), vdupq_n_s8(0x8));
+ }
+
+ const TA *const A;
+ const block_q8_0 *const B;
+ float *const C;
+ const int k;
+ const int lda;
+ const int ldb;
+ const int ldc;
+ const int ith;
+ const int nth;
+};
+#endif // __ARM_FEATURE_DOTPROD
+
+#if defined(__AVX2__) || defined(__AVX512F__)
+template <int CONFIG, typename TA, typename TB, typename TC>
+class tinyBLAS_Q0_AVX2 {
+ public:
+ tinyBLAS_Q0_AVX2(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith,
+ int nth)
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
+ }
+
+ void matmul(int m, int n, int task) {
+ if (task == GGML_TASK_TYPE_COMPUTE)
+ mnpack(0, m, 0, n);
+ }
+
+ private:
+ void mnpack(int m0, int m, int n0, int n) {
+ int mc, nc, mp, np;
+ switch ((std::min(m - m0, 4) << 4) | std::min(n - n0, 4)) {
+#if VECTOR_REGISTERS == 32
+ case 0x44:
+ mc = 4;
+ nc = 4;
+ gemm<4, 4>(m0, m, n0, n);
+ break;
+ case 0x43:
+ mc = 4;
+ nc = 3;
+ gemm<4, 3>(m0, m, n0, n);
+ break;
+ case 0x34:
+ mc = 3;
+ nc = 4;
+ gemm<3, 4>(m0, m, n0, n);
+ break;
+ case 0x33:
+ mc = 3;
+ nc = 3;
+ gemm<3, 3>(m0, m, n0, n);
+ break;
+ case 0x42:
+ mc = 4;
+ nc = 2;
+ gemm<4, 2>(m0, m, n0, n);
+ break;
+ case 0x24:
+ mc = 2;
+ nc = 4;
+ gemm<2, 4>(m0, m, n0, n);
+ break;
+#else
+ case 0x44:
+ case 0x43:
+ case 0x42:
+ mc = 4;
+ nc = 2;
+ gemm<4, 2>(m0, m, n0, n);
+ break;
+ case 0x34:
+ case 0x24:
+ mc = 2;
+ nc = 4;
+ gemm<2, 4>(m0, m, n0, n);
+ break;
+ case 0x33:
+#endif
+ case 0x32:
+ mc = 3;
+ nc = 2;
+ gemm<3, 2>(m0, m, n0, n);
+ break;
+ case 0x23:
+ mc = 2;
+ nc = 3;
+ gemm<2, 3>(m0, m, n0, n);
+ break;
+ case 0x41:
+ mc = 4;
+ nc = 1;
+ gemm<4, 1>(m0, m, n0, n);
+ break;
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2>(m0, m, n0, n);
+ break;
+ case 0x14:
+ mc = 1;
+ nc = 4;
+ gemm<1, 4>(m0, m, n0, n);
+ break;
+ case 0x31:
+ mc = 3;
+ nc = 1;
+ gemm<3, 1>(m0, m, n0, n);
+ break;
+ case 0x13:
+ mc = 1;
+ nc = 3;
+ gemm<1, 3>(m0, m, n0, n);
+ break;
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1>(m0, m, n0, n);
+ break;
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
+ mp = m0 + (m - m0) / mc * mc;
+ np = n0 + (n - n0) / nc * nc;
+ mnpack(mp, m, n0, np);
+ mnpack(m0, m, np, n);
+ }
+
+ template <int RM, int RN>
+ NOINLINE void gemm(int m0, int m, int n0, int n) {
+ int ytiles = RM > 1 ? (m - m0) / RM : 1;
+ int xtiles = RN > 1 ? (n - n0) / RN : 1;
+ int tiles = xtiles * ytiles;
+ int duty = (tiles + nth - 1) / nth;
+ int start = duty * ith;
+ int end = start + duty;
+ if (end > tiles)
+ end = tiles;
+ for (int job = start; job < end; ++job) {
+ int ii = m0 + job / xtiles * RM;
+ int jj = n0 + job % xtiles * RN;
+ __m256 Cv[RN][RM] = {};
+ for (int l = 0; l < k; ++l)
+ for (int j = 0; j < RN; ++j)
+ for (int i = 0; i < RM; ++i)
+ Cv[j][i] = madd(_mm256_set1_ps(unhalf(INDEX(A, lda, ii + i, l).d) *
+ unhalf(INDEX(B, ldb, jj + j, l).d)),
+ updot(_mm256_sign_epi8(load(&INDEX(A, lda, ii + i, l)),
+ load(&INDEX(A, lda, ii + i, l))),
+ _mm256_sign_epi8(load(&INDEX(B, ldb, jj + j, l)),
+ load(&INDEX(A, lda, ii + i, l)))),
+ Cv[j][i]);
+ for (int j = 0; j < RN; ++j)
+ for (int i = 0; i < RM; ++i)
+ INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]);
+ }
+ }
+
+ inline __m256i load(const block_q8_0 *b) {
+ return _mm256_loadu_si256((const __m256i *)b->qs);
+ }
+
+ inline __m256i load(const block_q4_0 *b) {
+ return _mm256_sub_epi8(denibble(b->qs), _mm256_set1_epi8(8));
+ }
+
+ inline __m256 updot(__m256i u, __m256i s) {
+ __m256i res;
+#if defined(__AVXVNNI__) || (defined(__AVX512VNNI__) && defined(__AVX512VL__))
+ res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);
+#else
+ res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));
+#endif
+ return _mm256_cvtepi32_ps(res);
+ }
+
+ static inline __m256i denibble(const uint8_t *p) {
+ __m128i x = _mm_loadu_si128((const __m128i *)p);
+ return _mm256_and_si256(
+ _mm256_set1_epi8(15),
+ _mm256_insertf128_si256(_mm256_castsi128_si256(x), _mm_srli_epi16(x, 4), 1));
+ }
+
+ const TA *const A;
+ const TB *const B;
+ TC *const C;
+ const int k;
+ const int lda;
+ const int ldb;
+ const int ldc;
+ const int ith;
+ const int nth;
+};
+#endif // __AVX2__
+
+} // namespace
+
+/**
+ * Performs optimized matrix multiplication on CPU.
+ *
+ * This subroutine may compute C = Aᵀ * B with column major ordering.
+ * Despite its name, this isn't a generalized implementation. Work is
+ * only performed when a handwritten kernel is written and available.
+ * Otherwise the caller should fall back to a general matmul routine.
+ *
+ * For example, for single-threaded single-precision GEMM you can say
+ *
+ * llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc,
+ * 0, 1, GGML_TASK_TYPE_COMPUTE,
+ * GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32);
+ *
+ * @param m is rows in `A` and `C`
+ * @param n is cols in `B` and `C`
+ * @param k is cols in `A` and rows in `B`
+ * @param A is first input matrix (always transposed)
+ * @param lda is row stride of `A`
+ * @param B is second input matrix (never transposed)
+ * @param ldb is row stride of `B`
+ * @param C is input/output array of output matrices
+ * @param ldc is row stride of `C`
+ * @param ith is thread id (must be less than `nth`)
+ * @param nth is number of threads (must be greater than zero)
+ * @param task is GGML task type
+ * @param Atype is GGML data type of `A`
+ * @param Btype is GGML data type of `B`
+ * @param Ctype is GGML data type of `C`
+ * @return true if this function was able to service the matmul request
+ */
+bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B, int ldb, void *C,
+ int ldc, int ith, int nth, int task, int Atype, int Btype, int Ctype) {
+
+ assert(m >= 0);
+ assert(n >= 0);
+ assert(k >= 0);
+ assert(lda >= k);
+ assert(ldb >= k);
+ assert(ldc >= m);
+ assert(nth > 0);
+ assert(ith < nth);
+ assert(1ll * lda * m <= 0x7fffffff);
+ assert(1ll * ldb * n <= 0x7fffffff);
+ assert(1ll * ldc * n <= 0x7fffffff);
+
+ if (Ctype != GGML_TYPE_F32)
+ return false;
+
+ switch (Atype) {
+
+ case GGML_TYPE_F32: {
+ if (Btype != GGML_TYPE_F32)
+ return false;
+#if defined(__AVX512F__)
+ if (k % 16)
+ return false;
+ tinyBLAS<0, 16, __m512, __m512, float, float, float> tb{
+ k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+#elif defined(__AVX__) || defined(__AVX2__)
+ if (k % 8)
+ return false;
+ tinyBLAS<0, 8, __m256, __m256, float, float, float> tb{
+ k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+#elif defined(__ARM_NEON)
+ if (n < 4)
+ return false;
+ if (k % 4)
+ return false;
+ tinyBLAS<0, 4, float32x4_t, float32x4_t, float, float, float> tb{
+ k, (const float *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ case GGML_TYPE_BF16: {
+#if defined(__AVX512BF16__)
+ if (k % 32)
+ return false;
+ if (Btype != GGML_TYPE_BF16)
+ return false;
+ tinyBLAS<0, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, float> tb{
+ k, (const ggml_bf16_t *)A, lda, (const ggml_bf16_t *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+#elif defined(__AVX512F__)
+ if (k % 16)
+ return false;
+ if (Btype != GGML_TYPE_F32)
+ return false;
+ tinyBLAS<0, 16, __m512, __m512, ggml_bf16_t, float, float> tb{
+ k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+#elif defined(__AVX2__)
+ // TODO(jart): Why is this the fastest on Zen4?
+ if (k % 8)
+ return false;
+ if (Btype != GGML_TYPE_F32)
+ return false;
+ tinyBLAS<0, 8, __m256, __m256, ggml_bf16_t, float, float> tb{
+ k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+#elif defined(__ARM_NEON) && !defined(_MSC_VER)
+ if (k % 4)
+ return false;
+ if (Btype != GGML_TYPE_F32)
+ return false;
+ tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_bf16_t, float, float> tb{
+ k, (const ggml_bf16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ case GGML_TYPE_F16: {
+#if defined(__AVX512F__)
+ if (k % 16)
+ return false;
+ if (Btype != GGML_TYPE_F32)
+ return false;
+ tinyBLAS<0, 16, __m512, __m512, ggml_fp16_t, float, float> tb{
+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)
+ if (k % 8)
+ return false;
+ if (Btype != GGML_TYPE_F32)
+ return false;
+ tinyBLAS<0, 8, __m256, __m256, ggml_fp16_t, float, float> tb{
+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
+ if (n < 8)
+ return false;
+ if (k % 8)
+ return false;
+ if (Btype != GGML_TYPE_F16)
+ return false;
+ tinyBLAS<0, 8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, float> tb{
+ k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+#elif defined(__ARM_NEON) && !defined(_MSC_VER)
+ if (k % 4)
+ return false;
+ if (Btype != GGML_TYPE_F32)
+ return false;
+ tinyBLAS<0, 4, float32x4_t, float32x4_t, ggml_fp16_t, float, float> tb{
+ k, (const ggml_fp16_t *)A, lda, (const float *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ case GGML_TYPE_Q8_0: {
+ if (Btype != GGML_TYPE_Q8_0)
+ return false;
+#if defined(__AVX2__) || defined(__AVX512F__)
+ tinyBLAS_Q0_AVX2<0, block_q8_0, block_q8_0, float> tb{
+ k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+#elif defined(__ARM_FEATURE_DOTPROD)
+ tinyBLAS_Q0_ARM<0, block_q8_0> tb{
+ k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ case GGML_TYPE_Q4_0: {
+ if (Btype != GGML_TYPE_Q8_0)
+ return false;
+#if defined(__AVX2__) || defined(__AVX512F__)
+ tinyBLAS_Q0_AVX2<0, block_q4_0, block_q8_0, float> tb{
+ k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+#elif defined(__ARM_FEATURE_DOTPROD)
+ tinyBLAS_Q0_ARM<0, block_q4_0> tb{
+ k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};
+ tb.matmul(m, n, task);
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ default:
+ return false;
+ }
+
+ (void)m;
+ (void)n;
+ (void)k;
+ (void)A;
+ (void)lda;
+ (void)B;
+ (void)ldb;
+ (void)C;
+ (void)ldc;
+ (void)ith;
+ (void)nth;
+ (void)task;
+ (void)Atype;
+ (void)Btype;
+ (void)Ctype;
+}
+
+//
+// _ _ ___ _ _ ___
+// | |_(_)_ _ _ _| _ ) | /_\ / __|
+// | _| | ' \ || | _ \ |__ / _ \\__ \.
+// \__|_|_||_\_, |___/____/_/ \_\___/
+// |__/
+//
+// MIXTURE OF EXPERTS TENSOR MULTIPLICATION
+//
+//
+// SHAPES
+//
+// - weights [cols, rows, experts]
+// - thought [cols, tasks, tokens] w/ tasks ≤ thinkers
+// - result [rows, thinkers, tokens] w/ thinkers ≤ experts
+// - plan [thinkers, tokens] w/ i32 < experts
+//
+// DEFINITION
+//
+// for thinker in range(thinkers):
+// for token in range(tokens):
+// for row in range(rows):
+// c = 0
+// for col in range(cols):
+// expert = plan[token][thinker]
+// a = weights[expert][row][col]
+// b = thought[token][thinker % tasks][col]
+// c += a * b
+// result[token][thinker][row] = c
+//
+// REGULARITIES
+//
+// - tokens can be odd
+// - thinkers is usually 2
+// - tasks is usually 1 or 2
+// - cols should be a multiple of 64
+// - rows should be a multiple of 64
+// - experts is usually 8 but could be 60
+// - tokens is always 1 for token generation
+// - tokens can be huge for prompt processing
+//
+// EXAMPLE
+//
+// mixtral 8x7b w/ 217 token prompt
+//
+// | ne*0 ne*1 ne*2 ne*3 | nb*0 nb*1 nb*2 nb*3 | type
+// =========================================================================
+// weights | 16384 6144 8 1 | 18 0x2400 0x3600000 0x1b000000 | q4_0
+// thought | 16384 2 217 1 | 4 0x10000 0x20000 0x1b20000 | f32
+// result | 6144 2 217 1 | 4 0x6000 0xc000 0xa2c000 | f32
+// plan | 2 217 1 1 | 4 0x20 0x1b20 0x1b20 | i32
+//
+
+namespace {
+class MixMul {
+ public:
+ MixMul(const ggml_compute_params *params, const ggml_tensor *weights,
+ const ggml_tensor *thought, const ggml_tensor *plan, ggml_tensor *result)
+ : params(params),
+ weights(weights),
+ thought(thought),
+ plan(plan),
+ result(result),
+ rows(weights->ne[1]),
+ cols(weights->ne[0]),
+ experts(weights->ne[2]),
+ thinkers(plan->ne[0]),
+ tasks(thought->ne[1]),
+ tokens(thought->ne[2]),
+ ldq((cols * 2 + ROW_ALIGN - 1) & -ROW_ALIGN),
+ wdata_((char *)(((uintptr_t)params->wdata + MAX_ALIGN - 1) & -MAX_ALIGN)),
+ allocated_(0) {
+ }
+
+ bool allocate_shared_memory() {
+ if (!(quantized_thought_ = allocate<char>(MATRIX_ALIGN, tokens * tasks * ldq)))
+ return false;
+ if (!(rowptr_result_ = allocate<uintptr_t>(ROW_ALIGN, experts * tokens * thinkers)))
+ return false;
+ if (!(rowptr_thought_ = allocate<uintptr_t>(ROW_ALIGN, experts * tokens * thinkers)))
+ return false;
+ if (!(rowptr_count_ = allocate<int>(sizeof(int), experts)))
+ return false;
+ return true;
+ }
+
+ size_t get_allocated_bytes() {
+ return (wdata_ - (char *)params->wdata) + allocated_;
+ }
+
+ bool mixmul() {
+
+ // invariants
+ assert(tasks <= thinkers);
+ assert(thinkers <= experts);
+ assert(tokens == plan->ne[1]);
+ assert(rows == result->ne[0]);
+ assert(cols == thought->ne[0]);
+ assert(tokens == result->ne[2]);
+ assert(thinkers == result->ne[1]);
+
+ // dimensionality
+ assert(plan->ne[2] == 1);
+ assert(plan->ne[3] == 1);
+ assert(result->ne[3] == 1);
+ assert(weights->ne[3] == 1);
+ assert(thought->ne[3] == 1);
+
+ // miscellaneous
+ assert(params->nth > 0);
+ assert(params->ith < params->nth);
+ assert(plan->type == GGML_TYPE_I32);
+
+ // supported types
+ if (result->type != GGML_TYPE_F32)
+ return false;
+
+ // check nb01 is convertible to lda
+ if (weights->nb[1] % ggml_type_size(weights->type))
+ return false;
+
+ // no support for column strides
+ if (result->nb[0] != ggml_type_size(result->type))
+ return false;
+ if (thought->nb[0] != ggml_type_size(thought->type))
+ return false;
+ if (weights->nb[0] != ggml_type_size(weights->type))
+ return false;
+
+ switch (weights->type) {
+
+ case GGML_TYPE_F32:
+ if (thought->type != GGML_TYPE_F32)
+ return false;
+#if defined(__AVX512F__)
+ return mixmat<16, 1, tinyBLAS<NCB | NCC, 16, __m512, __m512, float, float, float>,
+ float, float, float>();
+#elif defined(__AVX__) || defined(__AVX2__)
+ return mixmat<8, 1, tinyBLAS<NCB | NCC, 8, __m256, __m256, float, float, float>, float,
+ float, float>();
+#elif defined(__SSE__)
+ return mixmat<4, 1, tinyBLAS<NCB | NCC, 4, __m128, __m128, float, float, float>, float,
+ float, float>();
+#elif defined(__ARM_NEON)
+ return mixmat<4, 1,
+ tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, float, float, float>,
+ float, float, float>();
+#else
+ return false;
+#endif
+
+ case GGML_TYPE_BF16:
+ if (thought->type != GGML_TYPE_F32 && thought->type != GGML_TYPE_BF16)
+ return false;
+#if defined(__AVX512BF16__)
+ return mixmat<
+ 32, 1, tinyBLAS<NCB | NCC, 32, __m512, __m512bh, ggml_bf16_t, ggml_bf16_t, float>,
+ ggml_bf16_t, ggml_bf16_t, float>();
+#elif defined(__AVX512F__)
+ return mixmat<16, 1,
+ tinyBLAS<NCB | NCC, 16, __m512, __m512, ggml_bf16_t, ggml_bf16_t, float>,
+ ggml_bf16_t, ggml_bf16_t, float>();
+#elif defined(__AVX2__)
+ return mixmat<8, 1,
+ tinyBLAS<NCB | NCC, 8, __m256, __m256, ggml_bf16_t, ggml_bf16_t, float>,
+ ggml_bf16_t, ggml_bf16_t, float>();
+#elif defined(__ARM_NEON) && !defined(_MSC_VER)
+ return mixmat<
+ 4, 1,
+ tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, ggml_bf16_t, ggml_bf16_t, float>,
+ ggml_bf16_t, ggml_bf16_t, float>();
+#else
+ return false;
+#endif
+
+ case GGML_TYPE_F16:
+ if (thought->type != GGML_TYPE_F32 && thought->type != GGML_TYPE_F16)
+ return false;
+#if defined(__AVX512F__)
+ return mixmat<16, 1,
+ tinyBLAS<NCB | NCC, 16, __m512, __m512, ggml_fp16_t, ggml_fp16_t, float>,
+ ggml_fp16_t, ggml_fp16_t, float>();
+#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)
+ return mixmat<8, 1,
+ tinyBLAS<NCB | NCC, 8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float>,
+ ggml_fp16_t, ggml_fp16_t, float>();
+#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
+ return mixmat<
+ 8, 1,
+ tinyBLAS<NCB | NCC, 8, float16x8_t, float16x8_t, ggml_fp16_t, ggml_fp16_t, float>,
+ ggml_fp16_t, ggml_fp16_t, float>();
+#elif defined(__ARM_NEON) && !defined(_MSC_VER)
+ return mixmat<
+ 4, 1,
+ tinyBLAS<NCB | NCC, 4, float32x4_t, float32x4_t, ggml_fp16_t, ggml_fp16_t, float>,
+ ggml_fp16_t, ggml_fp16_t, float>();
+#else
+ return false;
+#endif
+
+ case GGML_TYPE_Q4_0:
+ if (thought->type != GGML_TYPE_F32 && thought->type != GGML_TYPE_Q8_0)
+ return false;
+#if defined(__AVX2__) || defined(__AVX512F__)
+ return mixmat<32, 32, tinyBLAS_Q0_AVX2<NCB | NCC, block_q4_0, block_q8_0, float>,
+ block_q4_0, block_q8_0, float>();
+#elif defined(__ARM_FEATURE_DOTPROD)
+ return mixmat<32, 32, tinyBLAS_Q0_ARM<NCB | NCC, block_q4_0>, block_q4_0, block_q8_0,
+ float>();
+#else
+ return false;
+#endif
+
+ case GGML_TYPE_Q8_0:
+ if (thought->type != GGML_TYPE_F32 && thought->type != GGML_TYPE_Q8_0)
+ return false;
+#if defined(__AVX2__) || defined(__AVX512F__)
+ return mixmat<32, 32, tinyBLAS_Q0_AVX2<NCB | NCC, block_q8_0, block_q8_0, float>,
+ block_q8_0, block_q8_0, float>();
+#elif defined(__ARM_FEATURE_DOTPROD)
+ return mixmat<32, 32, tinyBLAS_Q0_ARM<NCB | NCC, block_q8_0>, block_q8_0, block_q8_0,
+ float>();
+#else
+ return false;
+#endif
+
+ default:
+ return false;
+ }
+ }
+
+ private:
+ template <int KN, int BS, typename BLAS, typename TA, typename TB, typename TC>
+ bool mixmat() {
+ if (cols % KN)
+ return false;
+ switch (params->type) {
+ case GGML_TASK_TYPE_INIT:
+ if (thought->type != ggml_type_trait<TB>::id)
+ quantize_thought(ggml_type_trait<TB>::id);
+ build_row_pointers(ggml_type_trait<TB>::id);
+ return true;
+ case GGML_TASK_TYPE_COMPUTE:
+ assert(!(cols % BS));
+ assert(!(weights->nb[1] % sizeof(TA)));
+ for (int expert = 0; expert < experts; ++expert) {
+ BLAS tb{cols / BS,
+ (const TA *)((const char *)weights->data + expert * weights->nb[2]),
+ (int)(weights->nb[1] / sizeof(TA)),
+ (const TB *)(rowptr_thought_ + expert * tokens * thinkers),
+ 0,
+ (TC *)(rowptr_result_ + expert * tokens * thinkers),
+ 0,
+ params->ith,
+ params->nth};
+ tb.matmul(rows, rowptr_count_[expert], GGML_TASK_TYPE_COMPUTE);
+ }
+ return true;
+ default:
+ return true;
+ }
+ }
+
+ void build_row_pointers(ggml_type vec_dot_type) {
+ for (int expert = params->ith; expert < experts; expert += params->nth) {
+ int count = 0;
+ for (int token = 0; token < tokens; ++token)
+ for (int thinker = 0; thinker < thinkers; ++thinker)
+ if (expert == *(const int *)((const char *)plan->data + token * plan->nb[1] +
+ thinker * plan->nb[0])) {
+ int row = count++;
+ int idx = expert * thinkers * tokens + row;
+ rowptr_result_[idx] =
+ (uintptr_t)((char *)result->data + token * result->nb[2] +
+ thinker * result->nb[1]);
+ if (thought->type == vec_dot_type)
+ rowptr_thought_[idx] =
+ (uintptr_t)((char *)thought->data + token * thought->nb[2] +
+ thinker % tasks * thought->nb[1]);
+ else
+ rowptr_thought_[idx] =
+ (uintptr_t)((char *)quantized_thought_ + token * tasks * ldq +
+ thinker % tasks * ldq);
+ }
+ rowptr_count_[expert] = count;
+ }
+ }
+
+ void quantize_thought(ggml_type vec_dot_type) {
+ int chore = 0;
+ for (int token = 0; token < tokens; ++token)
+ for (int task = 0; task < tasks; ++task)
+ if (chore++ % params->nth == params->ith)
+ quantize_row(quantized_thought_ + token * tasks * ldq + task * ldq,
+ (const float *)((const char *)thought->data +
+ token * thought->nb[2] + task * thought->nb[1]),
+ vec_dot_type);
+ }
+
+ void quantize_row(void *dst, const float *src, ggml_type type) {
+ assert((int)ggml_row_size(type, cols) <= ldq);
+ switch (type) {
+ case GGML_TYPE_F16:
+ ggml_fp32_to_fp16_row(src, (ggml_fp16_t *)dst, cols);
+ break;
+ case GGML_TYPE_BF16:
+ ggml_fp32_to_bf16_row(src, (ggml_bf16_t *)dst, cols);
+ break;
+ case GGML_TYPE_Q8_0:
+ quantize_row_q8_0((const float *)src, (block_q8_0 *)dst, cols);
+ break;
+ default:
+ GGML_UNREACHABLE();
+ }
+ }
+
+ template <typename T>
+ T *allocate(int align, int elems) {
+ T *res = nullptr;
+ size_t need = sizeof(T) * elems;
+ size_t base = allocated_;
+ base += align - 1;
+ base &= -align;
+ size_t toto = base + need;
+ if (toto >= allocated_ && toto <= params->wsize && elems >= 0) {
+ res = (T *)(wdata_ + base);
+ allocated_ = toto;
+ }
+ return res;
+ }
+
+ const ggml_compute_params *const params;
+ const ggml_tensor *const weights;
+ const ggml_tensor *const thought;
+ const ggml_tensor *const plan;
+ ggml_tensor *const result;
+ const int rows;
+ const int cols;
+ const int experts;
+ const int thinkers;
+ const int tasks;
+ const int tokens;
+ const int ldq;
+
+ // variables
+ char *const wdata_;
+ size_t allocated_;
+
+ // shared memory
+ int *rowptr_count_ /*[experts]*/;
+ char *quantized_thought_ /*[tokens][tasks][cols][2]*/;
+ uintptr_t *rowptr_result_ /*[experts][tokens*thinkers]*/;
+ uintptr_t *rowptr_thought_ /*[experts][tokens*thinkers]*/;
+};
+} // namespace
+
+/**
+ * Performs "mixture of experts" tensor multiplication on CPU.
+ */
+bool llamafile_mixmul(const ggml_compute_params *params, const ggml_tensor *weights,
+ const ggml_tensor *thought, const ggml_tensor *plan, ggml_tensor *result) {
+ MixMul mm{params, weights, thought, plan, result};
+ return mm.allocate_shared_memory() && mm.mixmul();
+}
diff --git a/llamafile/tinyblas_cpu_amd_avx.cpp b/llamafile/tinyblas_cpu_amd_avx.cpp
new file mode 100644
index 00000000..75ea97e1
--- /dev/null
+++ b/llamafile/tinyblas_cpu_amd_avx.cpp
@@ -0,0 +1,21 @@
+#ifdef __x86_64__
+#define llamafile_sgemm llamafile_sgemm_amd_avx
+#define llamafile_mixmul llamafile_mixmul_amd_avx
+#include "tinyblas_cpu.inc"
+
+/**
+ * Returns number of shared memory bytes llamafile_mixmul() needs.
+ */
+size_t llamafile_mixmul_needs(const ggml_tensor *weights, const ggml_tensor *thought,
+ const ggml_tensor *plan) {
+ ggml_compute_params params{};
+ params.wsize = 0x7ffff000;
+ params.wdata = (void *)0x1000;
+ MixMul mm{&params, weights, thought, plan, 0};
+ if (mm.allocate_shared_memory())
+ return mm.get_allocated_bytes();
+ else
+ return 0;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/tinyblas_cpu_amd_avx2.cpp b/llamafile/tinyblas_cpu_amd_avx2.cpp
new file mode 100644
index 00000000..3457d3f6
--- /dev/null
+++ b/llamafile/tinyblas_cpu_amd_avx2.cpp
@@ -0,0 +1,5 @@
+#ifdef __x86_64__
+#define llamafile_sgemm llamafile_sgemm_amd_avx2
+#define llamafile_mixmul llamafile_mixmul_amd_avx2
+#include "tinyblas_cpu.inc"
+#endif // __x86_64__
diff --git a/llamafile/tinyblas_cpu_amd_avx512f.cpp b/llamafile/tinyblas_cpu_amd_avx512f.cpp
new file mode 100644
index 00000000..e0af44fd
--- /dev/null
+++ b/llamafile/tinyblas_cpu_amd_avx512f.cpp
@@ -0,0 +1,5 @@
+#ifdef __x86_64__
+#define llamafile_sgemm llamafile_sgemm_amd_avx512f
+#define llamafile_mixmul llamafile_mixmul_amd_avx512f
+#include "tinyblas_cpu.inc"
+#endif // __x86_64__
diff --git a/llamafile/tinyblas_cpu_amd_avxvnni.cpp b/llamafile/tinyblas_cpu_amd_avxvnni.cpp
new file mode 100644
index 00000000..84b6f608
--- /dev/null
+++ b/llamafile/tinyblas_cpu_amd_avxvnni.cpp
@@ -0,0 +1,5 @@
+#ifdef __x86_64__
+#define llamafile_sgemm llamafile_sgemm_amd_avxvnni
+#define llamafile_mixmul llamafile_mixmul_amd_avxvnni
+#include "tinyblas_cpu.inc"
+#endif // __x86_64__
diff --git a/llamafile/tinyblas_cpu_amd_fma.cpp b/llamafile/tinyblas_cpu_amd_fma.cpp
new file mode 100644
index 00000000..6e42e4d5
--- /dev/null
+++ b/llamafile/tinyblas_cpu_amd_fma.cpp
@@ -0,0 +1,5 @@
+#ifdef __x86_64__
+#define llamafile_sgemm llamafile_sgemm_amd_fma
+#define llamafile_mixmul llamafile_mixmul_amd_fma
+#include "tinyblas_cpu.inc"
+#endif // __x86_64__
diff --git a/llamafile/tinyblas_cpu_amd_zen4.cpp b/llamafile/tinyblas_cpu_amd_zen4.cpp
new file mode 100644
index 00000000..e905fe3e
--- /dev/null
+++ b/llamafile/tinyblas_cpu_amd_zen4.cpp
@@ -0,0 +1,5 @@
+#ifdef __x86_64__
+#define llamafile_sgemm llamafile_sgemm_amd_zen4
+#define llamafile_mixmul llamafile_mixmul_amd_zen4
+#include "tinyblas_cpu.inc"
+#endif // __x86_64__
diff --git a/llamafile/tinyblas_cpu_arm80.cpp b/llamafile/tinyblas_cpu_arm80.cpp
new file mode 100644
index 00000000..a5268fd4
--- /dev/null
+++ b/llamafile/tinyblas_cpu_arm80.cpp
@@ -0,0 +1,21 @@
+#ifdef __aarch64__
+#define llamafile_sgemm llamafile_sgemm_arm80
+#define llamafile_mixmul llamafile_mixmul_arm80
+#include "tinyblas_cpu.inc"
+
+/**
+ * Returns number of shared memory bytes llamafile_mixmul() needs.
+ */
+size_t llamafile_mixmul_needs(const ggml_tensor *weights, const ggml_tensor *thought,
+ const ggml_tensor *plan) {
+ ggml_compute_params params{};
+ params.wsize = 0x7ffff000;
+ params.wdata = (void *)0x1000;
+ MixMul mm{&params, weights, thought, plan, 0};
+ if (mm.allocate_shared_memory())
+ return mm.get_allocated_bytes();
+ else
+ return 0;
+}
+
+#endif // __aarch64__
diff --git a/llamafile/tinyblas_cpu_arm82.cpp b/llamafile/tinyblas_cpu_arm82.cpp
new file mode 100644
index 00000000..1246d825
--- /dev/null
+++ b/llamafile/tinyblas_cpu_arm82.cpp
@@ -0,0 +1,5 @@
+#ifdef __aarch64__
+#define llamafile_sgemm llamafile_sgemm_arm82
+#define llamafile_mixmul llamafile_mixmul_arm82
+#include "tinyblas_cpu.inc"
+#endif // __aarch64__
diff --git a/llamafile/varith.h b/llamafile/varith.h
deleted file mode 100644
index 8fb4d414..00000000
--- a/llamafile/varith.h
+++ /dev/null
@@ -1,72 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-// clang-format off
-#pragma once
-
-#ifdef __ARM_NEON
-#include <arm_neon.h>
-#endif
-#ifdef __x86_64__
-#include <immintrin.h>
-#endif
-
-inline float add(float x, float y) { return x + y; }
-inline float sub(float x, float y) { return x - y; }
-inline float mul(float x, float y) { return x * y; }
-
-#ifdef __SSE__
-inline __m128 add(__m128 x, __m128 y) { return _mm_add_ps(x, y); }
-inline __m128 sub(__m128 x, __m128 y) { return _mm_sub_ps(x, y); }
-inline __m128 mul(__m128 x, __m128 y) { return _mm_mul_ps(x, y); }
-#endif // __SSE__
-
-#ifdef __AVX__
-inline __m256 add(__m256 x, __m256 y) { return _mm256_add_ps(x, y); }
-inline __m256 sub(__m256 x, __m256 y) { return _mm256_sub_ps(x, y); }
-inline __m256 mul(__m256 x, __m256 y) { return _mm256_mul_ps(x, y); }
-#endif // __AVX__
-
-#ifdef __AVX512F__
-inline __m512 add(__m512 x, __m512 y) { return _mm512_add_ps(x, y); }
-inline __m512 sub(__m512 x, __m512 y) { return _mm512_sub_ps(x, y); }
-inline __m512 mul(__m512 x, __m512 y) { return _mm512_mul_ps(x, y); }
-#endif // __AVX512F__
-
-#ifdef __ARM_NEON
-inline float32x4_t add(float32x4_t x, float32x4_t y) { return vaddq_f32(x, y); }
-inline float32x4_t sub(float32x4_t x, float32x4_t y) { return vsubq_f32(x, y); }
-inline float32x4_t mul(float32x4_t x, float32x4_t y) { return vmulq_f32(x, y); }
-#endif // __ARM_NEON
-
-#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
-inline float16x8_t add(float16x8_t x, float16x8_t y) { return vaddq_f16(x, y); }
-inline float16x8_t sub(float16x8_t x, float16x8_t y) { return vsubq_f16(x, y); }
-inline float16x8_t mul(float16x8_t x, float16x8_t y) { return vmulq_f16(x, y); }
-#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
-
-/**
- * Computes a * b + c.
- *
- * This operation will become fused into a single arithmetic instruction
- * if the hardware has support for this feature, e.g. Intel Haswell+ (c.
- * 2013), AMD Bulldozer+ (c. 2011), etc.
- */
-template <typename T, typename U>
-inline U madd(T a, T b, U c) {
- return add(mul(a, b), c);
-}
-
-/**
- * Computes a * b + c with error correction.
- *
- * @see W. Kahan, "Further remarks on reducing truncation errors,"
- * Communications of the ACM, vol. 8, no. 1, p. 40, Jan. 1965,
- * doi: 10.1145/363707.363723.
- */
-template <typename T, typename U>
-inline U madder(T a, T b, U c, U *e) {
- U y = sub(mul(a, b), *e);
- U t = add(c, y);
- *e = sub(sub(t, c), y);
- return t;
-}