Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Change build strategy for CPU optimizations
Justine Tunney47b4f2ff389d
diff --git a/build/cudacc b/build/cudaccindex 3bdb2f55..35d489b3 100755--- a/build/cudacc+++ b/build/cudacc@@ -44,7 +44,6 @@ for x; doset -- "$@" -lhipblas -lrocblascontinueelif [ x"$x" = x"--use_fast_math" ]; then- set -- "$@" -ffast-math -DHIP_FAST_MATHcontinuefifidiff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex 19acff7d..f9d54c05 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -929,7 +929,7 @@ static bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg,else { invalid_param = true; }return true;}- if (arg == "--verbose-prompt") {+ if (arg == "--verbose-prompt" || arg == "--silent-prompt") {params.verbose_prompt = true;return true;}diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 57ea4f4c..9a04d838 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -10716,6 +10716,26 @@ static void ggml_compute_forward_mul_mat(}#endif+#ifdef __x86_64__+ bool can_use_tinyblas = ggml_is_contiguous(src0) &&+ ggml_is_contiguous(src1) &&+ ((ne2 == 1 && ne3 == 1) &&+ (ne02 == 1 && ne03 == 1) &&+ (ne12 == 1 && ne13 == 1));+ if (can_use_tinyblas &&+ llamafile_sgemm(ne01, ne11, ne00,+ src0->data, nb01/ggml_type_size(src0->type),+ src1->data, nb11/ggml_type_size(src1->type),+ dst->data, nb1/4,+ ith, nth,+ params->type,+ src0->type,+ src1->type,+ dst->type)) {+ return;+ }+#endif+if (params->type == GGML_TASK_TYPE_INIT) {if (ith != 0) {return;@@ -10748,25 +10768,16 @@ static void ggml_compute_forward_mul_mat(const size_t row_size = ggml_row_size(vec_dot_type, ne10);#ifdef __x86_64__- if (X86_HAVE(FMA) &&- X86_HAVE(AVX2) &&- (dst->type == GGML_TYPE_F32 &&- src1->type == GGML_TYPE_F32 &&- (src0->type == GGML_TYPE_F32 ||- src0->type == GGML_TYPE_Q4_0 ||- src0->type == GGML_TYPE_Q8_0 ||- (src0->type == GGML_TYPE_F16 && X86_HAVE(F16C)))) &&- ne00 % 8 == 0 &&- ggml_is_contiguous(src0) &&- ggml_is_contiguous(src1) &&- ((ne2 == 1 && ne3 == 1) &&- (ne02 == 1 && ne03 == 1) &&- (ne12 == 1 && ne13 == 1))) {+ if (can_use_tinyblas &&llamafile_sgemm(ne01, ne11, ne00,- src0->type, src0->data, nb01/ggml_type_size(src0->type),+ src0->data, nb01/ggml_type_size(src0->type),wdata, row_size/ggml_type_size(vec_dot_type),dst->data, nb1/4,- ith, nth);+ ith, nth,+ params->type,+ src0->type,+ vec_dot_type,+ dst->type)) {return;}#endifdiff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex ba3f6f75..f2dc674b 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -1903,10 +1903,10 @@ struct llama_context {size_t logits_size = 0;float * logits = nullptr;-#ifndef NDEBUG- // guard against access to unset logits- std::vector<bool> logits_valid;-#endif+// #ifndef NDEBUG+// // guard against access to unset logits+// std::vector<bool> logits_valid;+// #endifbool logits_all = false;// embeddings output (2-dimensional array: [n_tokens][n_embd])@@ -8847,13 +8847,13 @@ static int llama_decode_internal(auto * logits_out = lctx.logits;-#ifndef NDEBUG- auto & logits_valid = lctx.logits_valid;- logits_valid.clear();- logits_valid.resize(n_tokens_all);+// #ifndef NDEBUG+// auto & logits_valid = lctx.logits_valid;+// logits_valid.clear();+// logits_valid.resize(n_tokens_all);- memset(logits_out, 0, lctx.logits_size*sizeof(float));-#endif+// memset(logits_out, 0, lctx.logits_size*sizeof(float));+// #endifconst auto n_ubatch = cparams.n_ubatch;@@ -9022,21 +9022,21 @@ static int llama_decode_internal(i_first = -1;}}-#ifndef NDEBUG- logits_valid[cur_token + i] = u_batch.logits[i] != 0;;-#endif+// #ifndef NDEBUG+// logits_valid[cur_token + i] = u_batch.logits[i] != 0;;+// #endif}} else if (lctx.logits_all) {ggml_backend_tensor_get_async(backend_res, res, logits_out + n_vocab*cur_token, 0, n_vocab*n_tokens*sizeof(float));-#ifndef NDEBUG- std::fill(logits_valid.begin() + cur_token, logits_valid.begin() + cur_token + n_tokens, true);-#endif+// #ifndef NDEBUG+// std::fill(logits_valid.begin() + cur_token, logits_valid.begin() + cur_token + n_tokens, true);+// #endif} else {if (cur_token + n_tokens >= n_tokens_all) {ggml_backend_tensor_get_async(backend_res, res, logits_out, n_vocab*(n_tokens - 1)*sizeof(float), n_vocab*sizeof(float));-#ifndef NDEBUG- logits_valid[0] = true;-#endif+// #ifndef NDEBUG+// logits_valid[0] = true;+// #endif}}}@@ -14096,7 +14096,7 @@ float * llama_get_logits(struct llama_context * ctx) {}float * llama_get_logits_ith(struct llama_context * ctx, int32_t i) {- assert(ctx->logits_valid.at(i));+ // assert(ctx->logits_valid.at(i));llama_synchronize(ctx);diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex 32161dfd..81e26a78 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -5,6 +5,7 @@ PKGS += LLAMAFILELLAMAFILE_FILES := $(wildcard llamafile/*.*)LLAMAFILE_HDRS = $(filter %.h,$(LLAMAFILE_FILES))+LLAMAFILE_INCS = $(filter %.inc,$(LLAMAFILE_FILES))LLAMAFILE_SRCS_C = $(filter %.c,$(LLAMAFILE_FILES))LLAMAFILE_SRCS_CU = $(filter %.cu,$(LLAMAFILE_FILES))LLAMAFILE_SRCS_CPP = $(filter %.cpp,$(LLAMAFILE_FILES))@@ -16,6 +17,18 @@ LLAMAFILE_OBJS = \$(LLAMAFILE_SRCS_CPP:%.cpp=o/$(MODE)/%.o) \$(LLAMAFILE_FILES:%=o/$(MODE)/%.zip.o)+o/$(MODE)/llamafile/sgemm_sss_avx.o: private TARGET_ARCH += -Xx86_64-mavx+o/$(MODE)/llamafile/sgemm_sss_fma.o: private TARGET_ARCH += -Xx86_64-mfma+o/$(MODE)/llamafile/sgemm_sss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f+o/$(MODE)/llamafile/sgemm_hss_f16c.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mf16c+o/$(MODE)/llamafile/sgemm_hss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f+o/$(MODE)/llamafile/sgemm_qqs_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni+o/$(MODE)/llamafile/sgemm_qqs_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni+o/$(MODE)/llamafile/sgemm_qqs_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma+o/$(MODE)/llamafile/sgemm_eqs_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni+o/$(MODE)/llamafile/sgemm_eqs_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni+o/$(MODE)/llamafile/sgemm_eqs_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma+o/$(MODE)/llamafile/zipalign: \o/$(MODE)/llamafile/zipalign.o \o/$(MODE)/llamafile/help.o \@@ -42,9 +55,13 @@ o/$(MODE)/llamafile: \################################################################################# testing+o/$(MODE)/llamafile/sgemm_test: \+ o/$(MODE)/llamafile/sgemm_test.o \+ o/$(MODE)/llama.cpp/llama.cpp.a+o/$(MODE)/llamafile/%.o: llamafile/%.cu llamafile/BUILD.mk@mkdir -p $(@D)- build/cudacc -fPIE -g -O3 -march=native -c -o $@ $<+ build/cudacc -fPIE -g -O3 -march=native -ffast-math --use_fast_math -c -o $@ $<o/$(MODE)/llamafile/tinyblas_test: \o/$(MODE)/llamafile/tinyblas_test.o \diff --git a/llamafile/hsum.h b/llamafile/hsum.hnew file mode 100644index 00000000..6295ebec--- /dev/null+++ b/llamafile/hsum.h@@ -0,0 +1,29 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+#pragma once+#ifdef __SSE__+#include <immintrin.h>++inline float hsum(__m128 x) {+ x = _mm_add_ps(x, _mm_movehl_ps(x, x));+ x = _mm_add_ss(x, _mm_movehdup_ps(x));+ return _mm_cvtss_f32(x);+}++#ifdef __AVX__++inline float hsum(__m256 x) {+ return hsum(_mm_add_ps(_mm256_extractf128_ps(x, 1), _mm256_castps256_ps128(x)));+}++#ifdef __AVX512F__++#include <immintrin.h>++inline float hsum(__m512 x) {+ return _mm512_reduce_add_ps(x);+}++#endif // __AVX512F__+#endif // __AVX__+#endif // __SSE__diff --git a/llamafile/kernel.h b/llamafile/kernel.hnew file mode 100644index 00000000..06777161--- /dev/null+++ b/llamafile/kernel.h@@ -0,0 +1,19 @@+#pragma once++#define BEGIN_KERNEL(RM, RN) \+ int ytiles = (m - m0) / RM; \+ int xtiles = (n - n0) / RN; \+ int tiles = ytiles * xtiles; \+ double duty = (double)tiles / nth; \+ if (duty < 1) \+ duty = 1; \+ double spot = duty * ith + .5; \+ int end = spot + duty; \+ int start = spot; \+ if (end > tiles) \+ end = tiles; \+ for (int job = start; job < end; ++job) { \+ int i = m0 + job / xtiles * RM; \+ int j = n0 + job % xtiles * RN;++#define END_KERNEL() }diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex d93e8e0a..71694858 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -48,8 +48,8 @@ bool llamafile_has_metal(void);int llamafile_gpu_parse(const char *);const char *llamafile_describe_gpu(void);-void llamafile_sgemm(long, long, long, int, const void *, long, const void *, long, float *, long,- long, long);+bool llamafile_sgemm(int, int, int, const void *, int, const void *, int, void *, int, int, int,+ int, int, int, int);#ifdef __cplusplus}diff --git a/llamafile/llmm3x4.inc b/llamafile/llmm3x4.incnew file mode 100644index 00000000..c0d6c66f--- /dev/null+++ b/llamafile/llmm3x4.inc@@ -0,0 +1,186 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include <algorithm>+#include <cassert>++#include "llama.cpp/ggml.h"++#include "hsum.h"+#include "kernel.h"+#include "madd.h"++namespace {++class LLMM3X4 {+ public:+ LLMM3X4(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {+ }++ void matmul(int m, int n) {+ mnpack(0, m, 0, n);+ }++ private:+ dontinline void mnpack(int m0, int m, int n0, int n) {+ if (m - m0 <= 0 || n - n0 <= 0)+ return;+ int mc, nc, mp, np;+ if (m - m0 >= 3 && n - n0 >= 4) {+ mc = 3;+ nc = 4;+ gemm3x4(m0, m, n0, n);+ } else if (m - m0 >= 4 && n - n0 >= 1) {+ mc = 4;+ nc = 1;+ gemm4x1(m0, m, n0, n);+ } else if (m - m0 >= 1 && n - n0 >= 4) {+ mc = 1;+ nc = 4;+ gemm1x4(m0, m, n0, n);+ } else {+ mc = 1;+ nc = 1;+ gemm1x1(m0, m, n0, n);+ }+ mp = m0 + (m - m0) / mc * mc;+ np = n0 + (n - n0) / nc * nc;+ mnpack(mp, m, n0, np);+ mnpack(m0, mp, np, n);+ mnpack(mp, m, np, n);+ }++ dontinline void gemm3x4(int m0, int m, int n0, int n) {+ BEGIN_KERNEL(3, 4)+ V c00 = zero();+ V c01 = zero();+ V c02 = zero();+ V c03 = zero();+ V c10 = zero();+ V c11 = zero();+ V c12 = zero();+ V c13 = zero();+ V c20 = zero();+ V c21 = zero();+ V c22 = zero();+ V c23 = zero();+ for (int l = 0; l < k; l += KN) {+ V k0 = load_b(B + ldb * (j + 0) + l);+ V k1 = load_b(B + ldb * (j + 1) + l);+ V k2 = load_b(B + ldb * (j + 2) + l);+ V k3 = load_b(B + ldb * (j + 3) + l);+ V a0 = load_a(A + lda * (i + 0) + l);+ c00 = madd(a0, k0, c00);+ c01 = madd(a0, k1, c01);+ c02 = madd(a0, k2, c02);+ c03 = madd(a0, k3, c03);+ V a1 = load_a(A + lda * (i + 1) + l);+ c10 = madd(a1, k0, c10);+ c11 = madd(a1, k1, c11);+ c12 = madd(a1, k2, c12);+ c13 = madd(a1, k3, c13);+ V a2 = load_a(A + lda * (i + 2) + l);+ c20 = madd(a2, k0, c20);+ c21 = madd(a2, k1, c21);+ c22 = madd(a2, k2, c22);+ c23 = madd(a2, k3, c23);+ }+ C[ldc * (j + 0) + (i + 0)] = hsum(c00);+ C[ldc * (j + 0) + (i + 1)] = hsum(c10);+ C[ldc * (j + 0) + (i + 2)] = hsum(c20);+ C[ldc * (j + 1) + (i + 0)] = hsum(c01);+ C[ldc * (j + 1) + (i + 1)] = hsum(c11);+ C[ldc * (j + 1) + (i + 2)] = hsum(c21);+ C[ldc * (j + 2) + (i + 0)] = hsum(c02);+ C[ldc * (j + 2) + (i + 1)] = hsum(c12);+ C[ldc * (j + 2) + (i + 2)] = hsum(c22);+ C[ldc * (j + 3) + (i + 0)] = hsum(c03);+ C[ldc * (j + 3) + (i + 1)] = hsum(c13);+ C[ldc * (j + 3) + (i + 2)] = hsum(c23);+ END_KERNEL()+ }++ dontinline void gemm1x4(int m0, int m, int n0, int n) {+ BEGIN_KERNEL(1, 4)+ V c00 = zero();+ V c01 = zero();+ V c02 = zero();+ V c03 = zero();+ for (int l = 0; l < k; l += KN) {+ V a0 = load_a(A + lda * (i + 0) + l);+ V k0 = load_b(B + ldb * (j + 0) + l);+ V k1 = load_b(B + ldb * (j + 1) + l);+ V k2 = load_b(B + ldb * (j + 2) + l);+ V k3 = load_b(B + ldb * (j + 3) + l);+ c00 = madd(a0, k0, c00);+ c01 = madd(a0, k1, c01);+ c02 = madd(a0, k2, c02);+ c03 = madd(a0, k3, c03);+ }+ C[ldc * (j + 0) + (i + 0)] = hsum(c00);+ C[ldc * (j + 1) + (i + 0)] = hsum(c01);+ C[ldc * (j + 2) + (i + 0)] = hsum(c02);+ C[ldc * (j + 3) + (i + 0)] = hsum(c03);+ END_KERNEL()+ }++ dontinline void gemm4x1(int m0, int m, int n0, int n) {+ BEGIN_KERNEL(4, 1)+ V c00 = zero();+ V c10 = zero();+ V c20 = zero();+ V c30 = zero();+ for (int l = 0; l < k; l += KN) {+ V k0 = load_b(B + ldb * (j + 0) + l);+ V a0 = load_a(A + lda * (i + 0) + l);+ c00 = madd(a0, k0, c00);+ V a1 = load_a(A + lda * (i + 1) + l);+ c10 = madd(a1, k0, c10);+ V a2 = load_a(A + lda * (i + 2) + l);+ c20 = madd(a2, k0, c20);+ V a3 = load_a(A + lda * (i + 3) + l);+ c30 = madd(a3, k0, c30);+ }+ C[ldc * (j + 0) + (i + 0)] = hsum(c00);+ C[ldc * (j + 0) + (i + 1)] = hsum(c10);+ C[ldc * (j + 0) + (i + 2)] = hsum(c20);+ C[ldc * (j + 0) + (i + 3)] = hsum(c30);+ END_KERNEL()+ }++ dontinline void gemm1x1(int m0, int m, int n0, int n) {+ BEGIN_KERNEL(1, 1)+ V c = zero();+ for (int l = 0; l < k; l += KN)+ c = madd(load_a(A + lda * i + l), load_b(B + ldb * j + l), c);+ C[ldc * j + i] = hsum(c);+ END_KERNEL()+ }++ const TA *const A;+ const TB *const B;+ TC *const C;+ const int k;+ const int lda;+ const int ldb;+ const int ldc;+ const int ith;+ const int nth;+};++} // namespacediff --git a/llamafile/llmmq0.inc b/llamafile/llmmq0.incnew file mode 100644index 00000000..87fd1eed--- /dev/null+++ b/llamafile/llmmq0.inc@@ -0,0 +1,204 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include <algorithm>+#include <cassert>++#include "llama.cpp/ggml-impl.h"+#include "llama.cpp/ggml.h"++#include "hsum.h"+#include "kernel.h"+#include "madd.h"++#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)++namespace {++class LLMMQ0 {+ public:+ LLMMQ0(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {+ }++ void matmul(int m, int n) {+ mnpack(0, m, 0, n);+ }++ private:+ dontinline void mnpack(int m0, int m, int n0, int n) {+ if (m - m0 <= 0 || n - n0 <= 0)+ return;+ int mc, nc, mp, np;+ if (m - m0 >= 1 && n - n0 >= 4) {+ mc = 1;+ nc = 4;+ gemm1x4(m0, m, n0, n);+ } else if (m - m0 >= 4 && n - n0 >= 1) {+ mc = 4;+ nc = 1;+ gemm4x1(m0, m, n0, n);+ } else {+ mc = 1;+ nc = 1;+ gemm1x1(m0, m, n0, n);+ }+ mp = m0 + (m - m0) / mc * mc;+ np = n0 + (n - n0) / nc * nc;+ mnpack(mp, m, n0, np);+ mnpack(m0, mp, np, n);+ mnpack(mp, m, np, n);+ }++ dontinline void gemm4x1(int m0, int m, int n0, int n) {+ BEGIN_KERNEL(4, 1)+ int k2 = k / 32;+ __m256 c0 = _mm256_setzero_ps();+ __m256 c1 = _mm256_setzero_ps();+ __m256 c2 = _mm256_setzero_ps();+ __m256 c3 = _mm256_setzero_ps();+ const TA *Ap0 = A + lda * (i + 0);+ const TA *Ap1 = A + lda * (i + 1);+ const TA *Ap2 = A + lda * (i + 2);+ const TA *Ap3 = A + lda * (i + 3);+ const TB *Bp = B + ldb * j;+ for (int l = 0; l < k2; ++l) {+ float db0 = unhalf(Bp[l].d);+ __m256i f = load(Bp + l);+ __m256i u = _mm256_sign_epi8(f, f);+ __m256 d0 = _mm256_set1_ps(unhalf(Ap0[l].d) * db0);+ __m256 d1 = _mm256_set1_ps(unhalf(Ap1[l].d) * db0);+ __m256 d2 = _mm256_set1_ps(unhalf(Ap2[l].d) * db0);+ __m256 d3 = _mm256_set1_ps(unhalf(Ap3[l].d) * db0);+ __m256i e0 = load(Ap0 + l);+ __m256i e1 = load(Ap1 + l);+ __m256i e2 = load(Ap2 + l);+ __m256i e3 = load(Ap3 + l);+ __m256i s0 = _mm256_sign_epi8(e0, f);+ __m256i s1 = _mm256_sign_epi8(e1, f);+ __m256i s2 = _mm256_sign_epi8(e2, f);+ __m256i s3 = _mm256_sign_epi8(e3, f);+ __m256 g0 = updot(u, s0);+ __m256 g1 = updot(u, s1);+ __m256 g2 = updot(u, s2);+ __m256 g3 = updot(u, s3);+ c0 = madd(d0, g0, c0);+ c1 = madd(d1, g1, c1);+ c2 = madd(d2, g2, c2);+ c3 = madd(d3, g3, c3);+ }+ C[ldc * j + (i + 0)] = hsum(c0);+ C[ldc * j + (i + 1)] = hsum(c1);+ C[ldc * j + (i + 2)] = hsum(c2);+ C[ldc * j + (i + 3)] = hsum(c3);+ END_KERNEL()+ }++ dontinline void gemm1x4(int m0, int m, int n0, int n) {+ BEGIN_KERNEL(1, 4)+ int k2 = k / 32;+ __m256 c0 = _mm256_setzero_ps();+ __m256 c1 = _mm256_setzero_ps();+ __m256 c2 = _mm256_setzero_ps();+ __m256 c3 = _mm256_setzero_ps();+ const TB *Bp0 = B + ldb * (j + 0);+ const TB *Bp1 = B + ldb * (j + 1);+ const TB *Bp2 = B + ldb * (j + 2);+ const TB *Bp3 = B + ldb * (j + 3);+ const TA *Ap = A + lda * i;+ for (int l = 0; l < k2; ++l) {+ float da0 = unhalf(Ap[l].d);+ __m256i f = load(Ap + l);+ __m256i u = _mm256_sign_epi8(f, f);+ __m256 d0 = _mm256_set1_ps(unhalf(Bp0[l].d) * da0);+ __m256 d1 = _mm256_set1_ps(unhalf(Bp1[l].d) * da0);+ __m256 d2 = _mm256_set1_ps(unhalf(Bp2[l].d) * da0);+ __m256 d3 = _mm256_set1_ps(unhalf(Bp3[l].d) * da0);+ __m256 g0 = updot(u, _mm256_sign_epi8(load(Bp0 + l), f));+ __m256 g1 = updot(u, _mm256_sign_epi8(load(Bp1 + l), f));+ __m256 g2 = updot(u, _mm256_sign_epi8(load(Bp2 + l), f));+ __m256 g3 = updot(u, _mm256_sign_epi8(load(Bp3 + l), f));+ c0 = madd(d0, g0, c0);+ c1 = madd(d1, g1, c1);+ c2 = madd(d2, g2, c2);+ c3 = madd(d3, g3, c3);+ }+ C[ldc * (j + 0) + i] = hsum(c0);+ C[ldc * (j + 1) + i] = hsum(c1);+ C[ldc * (j + 2) + i] = hsum(c2);+ C[ldc * (j + 3) + i] = hsum(c3);+ END_KERNEL()+ }++ dontinline void gemm1x1(int m0, int m, int n0, int n) {+ BEGIN_KERNEL(1, 1)+ int k2 = k / 32;+ __m256 c = _mm256_setzero_ps();+ const TA *Ap = A + lda * i;+ const TB *Bp = B + ldb * j;+ for (int l = 0; l < k2; ++l) {+ __m256 d = _mm256_set1_ps(unhalf(Ap[l].d) * unhalf(Bp[l].d));+ __m256i e = load(Ap + l);+ __m256i f = load(Bp + l);+ __m256 g = updot(_mm256_sign_epi8(e, e), _mm256_sign_epi8(f, e));+ c = madd(d, g, c);+ }+ C[ldc * j + i] = hsum(c);+ END_KERNEL()+ }++ inline __m256i load(const block_q8_0 *b) {+ return _mm256_loadu_si256((const __m256i *)b->qs);+ }++ inline __m256i load(const block_q4_0 *b) {+ return _mm256_sub_epi8(denibble(b->qs), _mm256_set1_epi8(8));+ }++ inline __m256 updot(__m256i u, __m256i s) {+ __m256i res;+#if defined(__AVXVNNI__) || defined(__AVX512VNNI__)+ res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);+#else+ res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));+#endif+ return _mm256_cvtepi32_ps(res);+ }++ static inline __m256i denibble(const uint8_t *p) {+ const __m128i tmp = _mm_loadu_si128((const __m128i *)p);+ const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);+ const __m256i lowMask = _mm256_set1_epi8(15);+ return _mm256_and_si256(lowMask, bytes);+ }++ static inline float unhalf(unsigned short d) {+ return GGML_FP16_TO_FP32(d);+ }++ const TA *const A;+ const TB *const B;+ TC *const C;+ const int k;+ const int lda;+ const int ldb;+ const int ldc;+ const int ith;+ const int nth;+};++} // namespacediff --git a/llamafile/madd.h b/llamafile/madd.hnew file mode 100644index 00000000..1777c883--- /dev/null+++ b/llamafile/madd.h@@ -0,0 +1,25 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+#pragma once+#ifdef __SSE__+#include <immintrin.h>++inline __m128 madd(__m128 x, __m128 y, __m128 z) {+ return _mm_add_ps(_mm_mul_ps(x, y), z);+}++#ifdef __AVX__++inline __m256 madd(__m256 x, __m256 y, __m256 z) {+ return _mm256_add_ps(_mm256_mul_ps(x, y), z);+}++#ifdef __AVX512F__++inline __m512 madd(__m512 x, __m512 y, __m512 z) {+ return _mm512_add_ps(_mm512_mul_ps(x, y), z);+}++#endif // __AVX512F__+#endif // __AVX__+#endif // __SSE__diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cppindex 36068f8c..10b539f0 100644--- a/llamafile/sgemm.cpp+++ b/llamafile/sgemm.cpp@@ -15,492 +15,88 @@// See the License for the specific language governing permissions and// limitations under the License.-#include "llama.cpp/ggml-impl.h"-#include "llama.cpp/ggml-quants.h"-#include "llamafile.h"-#include <assert.h>-#include <cosmo.h>-#include <immintrin.h>-#include <stdio.h>--//-// _ _ ___ _ _ ___-// | |_(_)_ _ _ _| _ ) | /_\ / __|-// | _| | ' \ || | _ \ |__ / _ \\__ \.-// \__|_|_||_\_, |___/____/_/ \_\___/-// |__/-//-// BASIC LINEAR ALGEBRA SUBPROGRAMS-//--// matrix multiplication for x86 microprocessors-//-// this file implements optimized cpu kernels for llama.cpp that support-// column major ordering with the A matrix transposed. they work by-// computing multiple dot products at once, because it exploits-// instruction level parallelism, in addition to minimizing register-// loads since the adjacent dot products can share register loads.-//-// this technique usually slightly outperforms mkl and blis at single-// threaded performance, and usually goes 2x faster than mkl when openmp-// threading is used. however please note that this code is adapted to-// use the llama.cpp synchronization model, which isn't as advanced.-//-// ┌───────────┬──────────────────┬───────────────────┐-// │ software │ skinny matrices │ fat matrices │-// │ │ token generation │ prompt evaluation │-// │ ───────── │ ──────────────── │ ───────────────── │-// │ llama.cpp │ fast │ slow │-// │ MKL │ slow │ fast │-// │ tinyBLAS │ fast │ fast │-// └───────────┴──────────────────┴───────────────────┘-//-// these kernels do not have any of the latency overhead typically-// associated with the blas, which means they can provide the same-// performance as llama.cpp when performing the most common type of-// matmul used by text generation, which is matrix-vector product.--#define END_KERNEL() }-#define BEGIN_KERNEL(RM, RN) \- long ytiles = (m - m0) / RM; \- long xtiles = (n - n0) / RN; \- long tiles = ytiles * xtiles; \- double duty = (double)tiles / nth; \- if (duty < 1) \- duty = 1; \- double spot = duty * ith + .5; \- long end = spot + duty; \- long start = spot; \- if (end > tiles) \- end = tiles; \- for (long job = start; job < end; ++job) { \- long i = m0 + job / xtiles * RM; \- long j = n0 + job % xtiles * RN;-#ifdef __x86_64__-#pragma GCC push_options-#pragma GCC target("avx2,fma,f16c,avxvnni")--#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)--static inline float unhalf(ggml_fp16_t d) {- return GGML_FP16_TO_FP32(d);-}-static dontinline float hsum8f(__m256 x) {- __m128 res = _mm256_extractf128_ps(x, 1);- res = _mm_add_ps(res, _mm256_castps256_ps128(x));- res = _mm_add_ps(res, _mm_movehl_ps(res, res));- res = _mm_add_ss(res, _mm_movehdup_ps(res));- return _mm_cvtss_f32(res);-}--static inline __m256i denibble32(const uint8_t *p) {- const __m128i tmp = _mm_loadu_si128((const __m128i *)p);- const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);- const __m256i lowMask = _mm256_set1_epi8(15);- return _mm256_and_si256(lowMask, bytes);-}--template <typename T> static inline __m256 load8f(const T *);-template <> inline __m256 load8f(const float *p) {- return _mm256_loadu_ps(p);-}-template <> inline __m256 load8f(const ggml_fp16_t *p) {- return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));-}--template <typename T> static inline __m256i load32i(const T *);-template <> inline __m256i load32i(const block_q8_0 *b) {- return _mm256_loadu_si256((const __m256i *)b->qs);-}-template <> inline __m256i load32i(const block_q4_0 *b) {- return _mm256_sub_epi8(denibble32(b->qs), _mm256_set1_epi8(8));-}--template <typename T> class tinyBLAS {- public:- tinyBLAS(long k, const T *A, long lda, const T *B, long ldb, float *C, long ldc, long ith,- long nth)- : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith), nth(nth) {- assert(A != nullptr);- assert(B != nullptr);- assert(C != nullptr);- assert(k >= 0 && k % 8 == 0);- assert(ith >= 0 && ith < nth);- }-- void gemm(long m, long n) {- assert(m >= 0);- assert(n >= 0);- assert(ldc >= m);- mnpack(0, m, 0, n);- }-- private:- void mnpack(long m0, long m, long n0, long n) {- if (m - m0 <= 0 || n - n0 <= 0)- return;- long mc, nc, mp, np;- if (m - m0 >= 3 && n - n0 >= 4) {- mc = 3;- nc = 4;- gemm3x4(m0, m, n0, n);- } else if (m - m0 >= 4 && n - n0 >= 1) {- mc = 4;- nc = 1;- gemm4x1(m0, m, n0, n);- } else if (m - m0 >= 1 && n - n0 >= 4) {- mc = 1;- nc = 4;- gemm1x4(m0, m, n0, n);- } else {- mc = 1;- nc = 1;- gemm1x1(m0, m, n0, n);- }- mp = m0 + (m - m0) / mc * mc;- np = n0 + (n - n0) / nc * nc;- mnpack(mp, m, n0, np);- mnpack(m0, mp, np, n);- mnpack(mp, m, np, n);- }-- dontinline void gemm3x4(long m0, long m, long n0, long n) {- BEGIN_KERNEL(3, 4)- __m256 c00 = _mm256_setzero_ps();- __m256 c01 = _mm256_setzero_ps();- __m256 c02 = _mm256_setzero_ps();- __m256 c03 = _mm256_setzero_ps();- __m256 c10 = _mm256_setzero_ps();- __m256 c11 = _mm256_setzero_ps();- __m256 c12 = _mm256_setzero_ps();- __m256 c13 = _mm256_setzero_ps();- __m256 c20 = _mm256_setzero_ps();- __m256 c21 = _mm256_setzero_ps();- __m256 c22 = _mm256_setzero_ps();- __m256 c23 = _mm256_setzero_ps();- for (long l = 0; l < k; l += 8) {- __m256 k0 = load8f(B + ldb * (j + 0) + l);- __m256 k1 = load8f(B + ldb * (j + 1) + l);- __m256 k2 = load8f(B + ldb * (j + 2) + l);- __m256 k3 = load8f(B + ldb * (j + 3) + l);- __m256 a0 = load8f(A + lda * (i + 0) + l);- c00 = _mm256_fmadd_ps(a0, k0, c00);- c01 = _mm256_fmadd_ps(a0, k1, c01);- c02 = _mm256_fmadd_ps(a0, k2, c02);- c03 = _mm256_fmadd_ps(a0, k3, c03);- __m256 a1 = load8f(A + lda * (i + 1) + l);- c10 = _mm256_fmadd_ps(a1, k0, c10);- c11 = _mm256_fmadd_ps(a1, k1, c11);- c12 = _mm256_fmadd_ps(a1, k2, c12);- c13 = _mm256_fmadd_ps(a1, k3, c13);- __m256 a2 = load8f(A + lda * (i + 2) + l);- c20 = _mm256_fmadd_ps(a2, k0, c20);- c21 = _mm256_fmadd_ps(a2, k1, c21);- c22 = _mm256_fmadd_ps(a2, k2, c22);- c23 = _mm256_fmadd_ps(a2, k3, c23);- }- C[ldc * (j + 0) + (i + 0)] = hsum8f(c00);- C[ldc * (j + 0) + (i + 1)] = hsum8f(c10);- C[ldc * (j + 0) + (i + 2)] = hsum8f(c20);- C[ldc * (j + 1) + (i + 0)] = hsum8f(c01);- C[ldc * (j + 1) + (i + 1)] = hsum8f(c11);- C[ldc * (j + 1) + (i + 2)] = hsum8f(c21);- C[ldc * (j + 2) + (i + 0)] = hsum8f(c02);- C[ldc * (j + 2) + (i + 1)] = hsum8f(c12);- C[ldc * (j + 2) + (i + 2)] = hsum8f(c22);- C[ldc * (j + 3) + (i + 0)] = hsum8f(c03);- C[ldc * (j + 3) + (i + 1)] = hsum8f(c13);- C[ldc * (j + 3) + (i + 2)] = hsum8f(c23);- END_KERNEL()- }-- dontinline void gemm1x4(long m0, long m, long n0, long n) {- BEGIN_KERNEL(1, 4)- __m256 c00 = _mm256_setzero_ps();- __m256 c01 = _mm256_setzero_ps();- __m256 c02 = _mm256_setzero_ps();- __m256 c03 = _mm256_setzero_ps();- for (long l = 0; l < k; l += 8) {- __m256 a0 = load8f(A + lda * (i + 0) + l);- __m256 k0 = load8f(B + ldb * (j + 0) + l);- __m256 k1 = load8f(B + ldb * (j + 1) + l);- __m256 k2 = load8f(B + ldb * (j + 2) + l);- __m256 k3 = load8f(B + ldb * (j + 3) + l);- c00 = _mm256_fmadd_ps(a0, k0, c00);- c01 = _mm256_fmadd_ps(a0, k1, c01);- c02 = _mm256_fmadd_ps(a0, k2, c02);- c03 = _mm256_fmadd_ps(a0, k3, c03);- }- C[ldc * (j + 0) + (i + 0)] = hsum8f(c00);- C[ldc * (j + 1) + (i + 0)] = hsum8f(c01);- C[ldc * (j + 2) + (i + 0)] = hsum8f(c02);- C[ldc * (j + 3) + (i + 0)] = hsum8f(c03);- END_KERNEL()- }-- dontinline void gemm4x1(long m0, long m, long n0, long n) {- BEGIN_KERNEL(4, 1)- __m256 c00 = _mm256_setzero_ps();- __m256 c10 = _mm256_setzero_ps();- __m256 c20 = _mm256_setzero_ps();- __m256 c30 = _mm256_setzero_ps();- for (long l = 0; l < k; l += 8) {- __m256 k0 = load8f(B + ldb * (j + 0) + l);- __m256 a0 = load8f(A + lda * (i + 0) + l);- c00 = _mm256_fmadd_ps(a0, k0, c00);- __m256 a1 = load8f(A + lda * (i + 1) + l);- c10 = _mm256_fmadd_ps(a1, k0, c10);- __m256 a2 = load8f(A + lda * (i + 2) + l);- c20 = _mm256_fmadd_ps(a2, k0, c20);- __m256 a3 = load8f(A + lda * (i + 3) + l);- c30 = _mm256_fmadd_ps(a3, k0, c30);- }- C[ldc * (j + 0) + (i + 0)] = hsum8f(c00);- C[ldc * (j + 0) + (i + 1)] = hsum8f(c10);- C[ldc * (j + 0) + (i + 2)] = hsum8f(c20);- C[ldc * (j + 0) + (i + 3)] = hsum8f(c30);- END_KERNEL()- }-- dontinline void gemm1x1(long m0, long m, long n0, long n) {- BEGIN_KERNEL(1, 1)- __m256 c = _mm256_setzero_ps();- for (long l = 0; l < k; l += 8) {- c = _mm256_fmadd_ps(load8f(A + lda * i + l), load8f(B + ldb * j + l), c);- }- C[ldc * j + i] = hsum8f(c);- END_KERNEL()- }-- const long k;- const T *const A;- const long lda;- const T *const B;- const long ldb;- float *const C;- const long ldc;- const long ith;- const long nth;-};--template <typename T, typename U, int avxvnni> class tinyBLASq {- public:- tinyBLASq(long k, const T *A, long lda, const U *B, long ldb, float *C, long ldc, long ith,- long nth)- : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith), nth(nth) {- assert(A != nullptr);- assert(B != nullptr);- assert(C != nullptr);- assert(k >= 0 && k % 32 == 0);- assert(ith >= 0 && ith < nth);- }-- void gemm(long m, long n) {- assert(m >= 0);- assert(n >= 0);- assert(ldc >= m);- mnpack(0, m, 0, n);- }-- private:- void mnpack(long m0, long m, long n0, long n) {- if (m - m0 <= 0 || n - n0 <= 0)- return;- long mc, nc, mp, np;- if (m - m0 >= 1 && n - n0 >= 4) {- mc = 1;- nc = 4;- gemm1x4(m0, m, n0, n);- } else if (m - m0 >= 4 && n - n0 >= 1) {- mc = 4;- nc = 1;- gemm4x1(m0, m, n0, n);- } else {- mc = 1;- nc = 1;- gemm1x1(m0, m, n0, n);- }- mp = m0 + (m - m0) / mc * mc;- np = n0 + (n - n0) / nc * nc;- mnpack(mp, m, n0, np);- mnpack(m0, mp, np, n);- mnpack(mp, m, np, n);- }-- dontinline void gemm4x1(long m0, long m, long n0, long n) {- BEGIN_KERNEL(4, 1)- long k2 = k / 32;- __m256 c0 = _mm256_setzero_ps();- __m256 c1 = _mm256_setzero_ps();- __m256 c2 = _mm256_setzero_ps();- __m256 c3 = _mm256_setzero_ps();- const T *Ap0 = A + lda * (i + 0);- const T *Ap1 = A + lda * (i + 1);- const T *Ap2 = A + lda * (i + 2);- const T *Ap3 = A + lda * (i + 3);- const U *Bp = B + ldb * j;- for (long l = 0; l < k2; ++l) {- float db0 = unhalf(Bp[l].d);- __m256i f = load32i(Bp + l);- __m256i u = _mm256_sign_epi8(f, f);- __m256 d0 = _mm256_set1_ps(unhalf(Ap0[l].d) * db0);- __m256 d1 = _mm256_set1_ps(unhalf(Ap1[l].d) * db0);- __m256 d2 = _mm256_set1_ps(unhalf(Ap2[l].d) * db0);- __m256 d3 = _mm256_set1_ps(unhalf(Ap3[l].d) * db0);- __m256i e0 = load32i(Ap0 + l);- __m256i e1 = load32i(Ap1 + l);- __m256i e2 = load32i(Ap2 + l);- __m256i e3 = load32i(Ap3 + l);- __m256i s0 = _mm256_sign_epi8(e0, f);- __m256i s1 = _mm256_sign_epi8(e1, f);- __m256i s2 = _mm256_sign_epi8(e2, f);- __m256i s3 = _mm256_sign_epi8(e3, f);- __m256 g0 = dotsome(u, s0);- __m256 g1 = dotsome(u, s1);- __m256 g2 = dotsome(u, s2);- __m256 g3 = dotsome(u, s3);- c0 = _mm256_fmadd_ps(d0, g0, c0);- c1 = _mm256_fmadd_ps(d1, g1, c1);- c2 = _mm256_fmadd_ps(d2, g2, c2);- c3 = _mm256_fmadd_ps(d3, g3, c3);- }- C[ldc * j + (i + 0)] = hsum8f(c0);- C[ldc * j + (i + 1)] = hsum8f(c1);- C[ldc * j + (i + 2)] = hsum8f(c2);- C[ldc * j + (i + 3)] = hsum8f(c3);- END_KERNEL()- }-- dontinline void gemm1x4(long m0, long m, long n0, long n) {- BEGIN_KERNEL(1, 4)- long k2 = k / 32;- __m256 c0 = _mm256_setzero_ps();- __m256 c1 = _mm256_setzero_ps();- __m256 c2 = _mm256_setzero_ps();- __m256 c3 = _mm256_setzero_ps();- const U *Bp0 = B + ldb * (j + 0);- const U *Bp1 = B + ldb * (j + 1);- const U *Bp2 = B + ldb * (j + 2);- const U *Bp3 = B + ldb * (j + 3);- const T *Ap = A + lda * i;- for (long l = 0; l < k2; ++l) {- float da0 = unhalf(Ap[l].d);- __m256i f = load32i(Ap + l);- __m256i u = _mm256_sign_epi8(f, f);- __m256 d0 = _mm256_set1_ps(unhalf(Bp0[l].d) * da0);- __m256 d1 = _mm256_set1_ps(unhalf(Bp1[l].d) * da0);- __m256 d2 = _mm256_set1_ps(unhalf(Bp2[l].d) * da0);- __m256 d3 = _mm256_set1_ps(unhalf(Bp3[l].d) * da0);- __m256 g0 = dotsome(u, _mm256_sign_epi8(load32i(Bp0 + l), f));- __m256 g1 = dotsome(u, _mm256_sign_epi8(load32i(Bp1 + l), f));- __m256 g2 = dotsome(u, _mm256_sign_epi8(load32i(Bp2 + l), f));- __m256 g3 = dotsome(u, _mm256_sign_epi8(load32i(Bp3 + l), f));- c0 = _mm256_fmadd_ps(d0, g0, c0);- c1 = _mm256_fmadd_ps(d1, g1, c1);- c2 = _mm256_fmadd_ps(d2, g2, c2);- c3 = _mm256_fmadd_ps(d3, g3, c3);- }- C[ldc * (j + 0) + i] = hsum8f(c0);- C[ldc * (j + 1) + i] = hsum8f(c1);- C[ldc * (j + 2) + i] = hsum8f(c2);- C[ldc * (j + 3) + i] = hsum8f(c3);- END_KERNEL()- }-- dontinline void gemm1x1(long m0, long m, long n0, long n) {- BEGIN_KERNEL(1, 1)- long k2 = k / 32;- __m256 c = _mm256_setzero_ps();- const T *Ap = A + lda * i;- const U *Bp = B + ldb * j;- for (long l = 0; l < k2; ++l) {- __m256 d = _mm256_set1_ps(unhalf(Ap[l].d) * unhalf(Bp[l].d));- __m256i e = load32i(Ap + l);- __m256i f = load32i(Bp + l);- __m256 g = dotsome(_mm256_sign_epi8(e, e), _mm256_sign_epi8(f, e));- c = _mm256_fmadd_ps(d, g, c);- }- C[ldc * j + i] = hsum8f(c);- END_KERNEL()- }-- inline __m256 dotsome(__m256i u, __m256i s) {- __m256i res;- if (avxvnni)- res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);- else- res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));- return _mm256_cvtepi32_ps(res);- }-- const long k;- const T *const A;- const long lda;- const U *const B;- const long ldb;- float *const C;- const long ldc;- const long ith;- const long nth;-};--#endif // __x86_64__--// computes m×k * n×k → n×m-void llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda, const void *B,- long ldb, float *C, long ldc, long ith, long nth) {- assert(X86_HAVE(FMA));- assert(X86_HAVE(AVX2));- switch (dtype) {-#ifdef __x86_64__+#include "llamafile.h"- case GGML_TYPE_F32: {- tinyBLAS<float> tb{k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};- tb.gemm(m, n);- break;- }+#include <cosmo.h>+#include <stdio.h>- case GGML_TYPE_F16: {- assert(X86_HAVE(F16C));- tinyBLAS<ggml_fp16_t> tb{- k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth,- };- tb.gemm(m, n);- break;- }+#include "sgemm.h"++bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B, int ldb, void *C,+ int ldc, int ith, int nth, int task, int Atype, int Btype, int Ctype) {++ if (!X86_HAVE(AVX))+ return false;+ if (Ctype != GGML_TYPE_F32)+ return false;++ switch (Atype) {++ case GGML_TYPE_F32:+ if (Btype != GGML_TYPE_F32)+ return false;+ if (X86_HAVE(AVX512F) && !(k % 16))+ return llamafile_sgemm_sss_avx512f(m, n, k, (const float *)A, lda, (const float *)B,+ ldb, (float *)C, ldc, ith, nth, task);+ if (X86_HAVE(FMA) && !(k % 8))+ return llamafile_sgemm_sss_fma(m, n, k, (const float *)A, lda, (const float *)B, ldb,+ (float *)C, ldc, ith, nth, task);+ if (!(k % 8))+ return llamafile_sgemm_sss_avx(m, n, k, (const float *)A, lda, (const float *)B, ldb,+ (float *)C, ldc, ith, nth, task);+ return false;++ case GGML_TYPE_F16:+ if (Btype != GGML_TYPE_F32)+ return false;+ if (X86_HAVE(AVX512F) && !(k % 16))+ return llamafile_sgemm_hss_avx512f(m, n, k, (const unsigned short *)A, lda,+ (const float *)B, ldb, (float *)C, ldc, ith, nth,+ task);+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && !(k % 8))+ return llamafile_sgemm_hss_f16c(m, n, k, (const unsigned short *)A, lda,+ (const float *)B, ldb, (float *)C, ldc, ith, nth, task);+ return false;case GGML_TYPE_Q8_0:- if (X86_HAVE(AVXVNNI)) {- tinyBLASq<block_q8_0, block_q8_0, true> tb{- k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth,- };- tb.gemm(m, n);- } else {- tinyBLASq<block_q8_0, block_q8_0, false> tb{- k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth,- };- tb.gemm(m, n);- }- break;+ if (Btype != GGML_TYPE_Q8_0)+ return false;+ if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && !(k % 32))+ return llamafile_sgemm_qqs_avx512vnni(m, n, k, (const block_q8_0 *)A, lda,+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,+ nth, task);+ if (X86_HAVE(FMA) && X86_HAVE(AVXVNNI) && !(k % 32))+ return llamafile_sgemm_qqs_avxvnni(m, n, k, (const block_q8_0 *)A, lda,+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,+ nth, task);+ if (X86_HAVE(FMA) && X86_HAVE(AVX2) && !(k % 32))+ return llamafile_sgemm_qqs_fma(m, n, k, (const block_q8_0 *)A, lda,+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,+ task);+ return false;case GGML_TYPE_Q4_0:- if (X86_HAVE(AVXVNNI)) {- tinyBLASq<block_q4_0, block_q8_0, true> tb{- k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth,- };- tb.gemm(m, n);- } else {- tinyBLASq<block_q4_0, block_q8_0, false> tb{- k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth,- };- tb.gemm(m, n);- }- break;+ if (Btype != GGML_TYPE_Q8_0)+ return false;+ if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && !(k % 32))+ return llamafile_sgemm_eqs_avx512vnni(m, n, k, (const block_q4_0 *)A, lda,+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,+ nth, task);+ if (X86_HAVE(FMA) && X86_HAVE(AVXVNNI) && !(k % 32))+ return llamafile_sgemm_eqs_avxvnni(m, n, k, (const block_q4_0 *)A, lda,+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,+ nth, task);+ if (X86_HAVE(FMA) && X86_HAVE(AVX2) && !(k % 32))+ return llamafile_sgemm_eqs_fma(m, n, k, (const block_q4_0 *)A, lda,+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,+ task);+ return false;-#endifdefault:- assert(!"unsupported dtype");+ return false;}}++#endif // __x86_64__diff --git a/llamafile/sgemm.h b/llamafile/sgemm.hnew file mode 100644index 00000000..14121d93--- /dev/null+++ b/llamafile/sgemm.h@@ -0,0 +1,36 @@+#pragma once+#include "llama.cpp/ggml-quants.h"+#include "llama.cpp/ggml.h"+#ifdef __cplusplus+extern "C" {+#endif++bool llamafile_sgemm_sss_avx(int, int, int, const float *, int, const float *, int, float *, int,+ int, int, int);+bool llamafile_sgemm_sss_fma(int, int, int, const float *, int, const float *, int, float *, int,+ int, int, int);+bool llamafile_sgemm_sss_avx512f(int, int, int, const float *, int, const float *, int, float *,+ int, int, int, int);++bool llamafile_sgemm_hss_f16c(int, int, int, const unsigned short *, int, const float *, int,+ float *, int, int, int, int);+bool llamafile_sgemm_hss_avx512f(int, int, int, const unsigned short *, int, const float *, int,+ float *, int, int, int, int);++bool llamafile_sgemm_qqs_avx512vnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,+ float *, int, int, int, int);+bool llamafile_sgemm_qqs_avxvnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,+ float *, int, int, int, int);+bool llamafile_sgemm_qqs_fma(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,+ float *, int, int, int, int);++bool llamafile_sgemm_eqs_avx512vnni(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,+ float *, int, int, int, int);+bool llamafile_sgemm_eqs_avxvnni(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,+ float *, int, int, int, int);+bool llamafile_sgemm_eqs_fma(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,+ float *, int, int, int, int);++#ifdef __cplusplus+}+#endifdiff --git a/llamafile/sgemm_eqs_avx512vnni.cpp b/llamafile/sgemm_eqs_avx512vnni.cppnew file mode 100644index 00000000..78c00eb9--- /dev/null+++ b/llamafile/sgemm_eqs_avx512vnni.cpp@@ -0,0 +1,36 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"++#define TA block_q4_0+#define TB block_q8_0+#define TC float+#include "llmmq0.inc"++bool llamafile_sgemm_eqs_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_eqs_avxvnni.cpp b/llamafile/sgemm_eqs_avxvnni.cppnew file mode 100644index 00000000..22a572d4--- /dev/null+++ b/llamafile/sgemm_eqs_avxvnni.cpp@@ -0,0 +1,36 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"++#define TA block_q4_0+#define TB block_q8_0+#define TC float+#include "llmmq0.inc"++bool llamafile_sgemm_eqs_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_eqs_fma.cpp b/llamafile/sgemm_eqs_fma.cppnew file mode 100644index 00000000..9bb95430--- /dev/null+++ b/llamafile/sgemm_eqs_fma.cpp@@ -0,0 +1,36 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"++#define TA block_q4_0+#define TB block_q8_0+#define TC float+#include "llmmq0.inc"++bool llamafile_sgemm_eqs_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,+ int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_hss_avx512f.cpp b/llamafile/sgemm_hss_avx512f.cppnew file mode 100644index 00000000..17d88bcc--- /dev/null+++ b/llamafile/sgemm_hss_avx512f.cpp@@ -0,0 +1,41 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"++#define KN 16+#define V __m512+#define TA unsigned short+#define TB float+#define TC float+#define zero() _mm512_setzero_ps()+#define load_a(p) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(p)))+#define load_b(p) _mm512_loadu_ps(p)+#include "llmm3x4.inc"++bool llamafile_sgemm_hss_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_hss_f16c.cpp b/llamafile/sgemm_hss_f16c.cppnew file mode 100644index 00000000..ed16d736--- /dev/null+++ b/llamafile/sgemm_hss_f16c.cpp@@ -0,0 +1,41 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"++#define KN 8+#define V __m256+#define TA unsigned short+#define TB float+#define TC float+#define zero() _mm256_setzero_ps()+#define load_a(p) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(p)))+#define load_b(p) _mm256_loadu_ps(p)+#include "llmm3x4.inc"++bool llamafile_sgemm_hss_f16c(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_qqs_avx512vnni.cpp b/llamafile/sgemm_qqs_avx512vnni.cppnew file mode 100644index 00000000..bd9254e7--- /dev/null+++ b/llamafile/sgemm_qqs_avx512vnni.cpp@@ -0,0 +1,36 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"++#define TA block_q8_0+#define TB block_q8_0+#define TC float+#include "llmmq0.inc"++bool llamafile_sgemm_qqs_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_qqs_avxvnni.cpp b/llamafile/sgemm_qqs_avxvnni.cppnew file mode 100644index 00000000..cdde2d96--- /dev/null+++ b/llamafile/sgemm_qqs_avxvnni.cpp@@ -0,0 +1,36 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"++#define TA block_q8_0+#define TB block_q8_0+#define TC float+#include "llmmq0.inc"++bool llamafile_sgemm_qqs_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_qqs_fma.cpp b/llamafile/sgemm_qqs_fma.cppnew file mode 100644index 00000000..32b82c33--- /dev/null+++ b/llamafile/sgemm_qqs_fma.cpp@@ -0,0 +1,36 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"++#define TA block_q8_0+#define TB block_q8_0+#define TC float+#include "llmmq0.inc"++bool llamafile_sgemm_qqs_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,+ int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_sss_avx.cpp b/llamafile/sgemm_sss_avx.cppnew file mode 100644index 00000000..6c73fc2f--- /dev/null+++ b/llamafile/sgemm_sss_avx.cpp@@ -0,0 +1,41 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"++#define KN 8+#define V __m256+#define TA float+#define TB float+#define TC float+#define zero() _mm256_setzero_ps()+#define load_a(p) _mm256_loadu_ps(p)+#define load_b(p) _mm256_loadu_ps(p)+#include "llmm3x4.inc"++bool llamafile_sgemm_sss_avx(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,+ int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_sss_avx512f.cpp b/llamafile/sgemm_sss_avx512f.cppnew file mode 100644index 00000000..1db08d29--- /dev/null+++ b/llamafile/sgemm_sss_avx512f.cpp@@ -0,0 +1,41 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"++#define KN 16+#define V __m512+#define TA float+#define TB float+#define TC float+#define zero() _mm512_setzero_ps()+#define load_a(p) _mm512_loadu_ps(p)+#define load_b(p) _mm512_loadu_ps(p)+#include "llmm3x4.inc"++bool llamafile_sgemm_sss_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_sss_fma.cpp b/llamafile/sgemm_sss_fma.cppnew file mode 100644index 00000000..84813ee7--- /dev/null+++ b/llamafile/sgemm_sss_fma.cpp@@ -0,0 +1,41 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"++#define KN 8+#define V __m256+#define TA float+#define TB float+#define TC float+#define zero() _mm256_setzero_ps()+#define load_a(p) _mm256_loadu_ps(p)+#define load_b(p) _mm256_loadu_ps(p)+#include "llmm3x4.inc"++bool llamafile_sgemm_sss_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,+ int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_test.cpp b/llamafile/sgemm_test.cppnew file mode 100644index 00000000..03978f51--- /dev/null+++ b/llamafile/sgemm_test.cpp@@ -0,0 +1,90 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "gemm.h"+#include "llama.cpp/ggml.h"+#include "llamafile.h"++int rand32(void) {+ static unsigned long long lcg = 1;+ lcg *= 6364136223846793005;+ lcg += 1442695040888963407;+ return lcg >> 32;+}++float float01(unsigned x) { // (0,1)+ return 1.f / 8388608 * ((x >> 9) + .5f);+}++float numba(void) { // (-1,1)+ return float01(rand32()) * 2 - 1;+}++template <typename T> void randomize(T *A, int n) {+ for (int i = 0; i < n; ++i)+ A[i] = numba();+}++template <typename T> void broadcast(T *A, int n, T x) {+ for (int i = 0; i < n; ++i)+ A[i] = x;+}++int main(int argc, char *argv[]) {+ float tolerance = 1e-5;+ int n = 32;+ int m = 32;+ int k = 1024;+ int l = 0;+ int lda = k + l;+ int ldb = k + l;+ int ldc = m + l;+ float *A = new float[lda * m];+ float *B = new float[ldb * n];+ float *C = new float[ldc * n];+ float *G = new float[ldc * n];+ randomize(A, lda * m);+ randomize(B, ldb * n);+ broadcast(C, ldc * n, NAN);+ broadcast(G, ldc * n, NAN);+ gemm<double>(true, false, m, n, k, 1.f, A, lda, B, ldb, 0.f, G, ldc);+ if (!llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,+ GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32))+ return 1;+ for (int i = 0; i < m; ++i) {+ for (int j = 0; j < n; ++j) {+ auto g = G[ldc * j + i];+ auto c = C[ldc * j + i];+ if (isnan(g))+ return 3;+ if (isnan(c))+ return 4;+ auto diff = g - c;+ if (diff < 0)+ diff = -diff;+ if (diff > tolerance)+ return 5;+ }+ }+}++#else+int main(int argc, char *argv[]) {+}+#endif // __x86_64__diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cuindex 49e4edcd..f988f2ad 100644--- a/llamafile/tinyblas.cu+++ b/llamafile/tinyblas.cu@@ -75,20 +75,6 @@ inline bool isone(float x) {return x == 1;}-template <typename T, typename TA, typename TB>-__device__ __forceinline__ void madd(T *tally, T *kahan, TA a, TB b) {- T x = a;- T y = b;-#ifdef __FAST_MATH__- *tally += x * y;-#else- T z = x * y - *kahan;- T t = *tally + z;- *kahan = (t - *tally) - z;- *tally = t;-#endif-}-////////////////////////////////////////////////////////////////////////////////////////////////////// tinyBLAS specialized matrix vector product kernel@@ -98,16 +84,24 @@ __forceinline__ __device__ float warpSum(float x) {return x;}+template <typename WORD, typename SRC>+__device__ __forceinline__ void madd(WORD *tally, WORD *kahan, SRC a, SRC b) {+ WORD x = a;+ WORD y = b;+ WORD z = x * y - *kahan;+ WORD t = *tally + z;+ *kahan = (t - *tally) - z;+ *tally = t;+}+template <typename WORD, typename SRC, typename DST>static __device__ void matvec(int m, int k, const SRC *A, int lda, const SRC *B, DST *C) {WORD Ct[WARPSIZE] = {0};WORD Ce[WARPSIZE] = {0};int i = blockIdx.y * WARPSIZE;- for (int l = threadIdx.x; l < k; l += WARPSIZE) {- WORD b = B[l];+ for (int l = threadIdx.x; l < k; l += WARPSIZE)for (int j = 0; j < WARPSIZE; ++j)- madd(&Ct[j], &Ce[j], A[lda * (i + j) + l], b);- }+ madd(&Ct[j], &Ce[j], A[lda * (i + j) + l], B[l]);for (int j = 0; j < WARPSIZE; ++j) {WORD c = warpSum(Ct[j]);if (!threadIdx.x)@@ -134,7 +128,7 @@ static tinyblasStatus_t matvec_launch(tinyblasHandle_t handle, int m, int k, contemplate <typename WORD>static bool can_use_matvec(tinyblasOperation_t aT, tinyblasOperation_t bT, int m, int n, int k,WORD alpha, WORD beta) {- return n == 1 && aT && !bT && //+ return n == 1 && k >= 4096 && aT && !bT && //!(m % WARPSIZE) && !(k % WARPSIZE) && //isone(alpha) && !beta;}@@ -171,7 +165,6 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, tinyblasOperatWORD At[TM];WORD Bt[TN];WORD Ct[TM * TN] = {0};- WORD Ce[TM * TN] = {0};if (CONFIG & ASSUME_A_OP_T)transa = TINYBLAS_OP_T;@@ -205,7 +198,7 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, tinyblasOperatBt[h] = Bs[BN * l + tj + h];for (int j = 0; j < TM; ++j)for (int h = 0; h < TN; ++h)- madd(&Ct[TN * j + h], &Ce[TN * j + h], At[j], Bt[h]);+ Ct[TN * j + h] += At[j] * Bt[h];}__syncthreads();@@ -277,7 +270,6 @@ static __device__ void matmul_warp2d(tinyblasOperation_t aT, //WORD Ar[WMI * TM] = {0};WORD Br[WNI * TN] = {0};WORD Ct[WMI * TM * WNI * TN] = {0};- WORD Ce[WMI * TM * WNI * TN] = {0};if (CONFIG & ASSUME_A_OP_T)aT = TINYBLAS_OP_T;@@ -327,9 +319,8 @@ static __device__ void matmul_warp2d(tinyblasOperation_t aT, //for (int jj = 0; jj < WNI; ++jj)for (int i = 0; i < TM; ++i)for (int j = 0; j < TN; ++j)- madd(&Ct[(WNI * TN) * (TM * ii + i) + (TN * jj) + j],- &Ce[(WNI * TN) * (TM * ii + i) + (TN * jj) + j], Ar[TM * ii + i],- Br[TN * jj + j]);+ Ct[(WNI * TN) * (TM * ii + i) + (TN * jj) + j] +=+ Ar[TM * ii + i] * Br[TN * jj + j];}__syncthreads();@@ -601,9 +592,9 @@ tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t handle, //case TINYBLAS_R_16F:switch (computeType) {case TINYBLAS_COMPUTE_16F:- return tinyblasGE_launch(handle, transa, transb, m, n, k, *(const half *)alpha,- (const half *)A, lda, (const half *)B, ldb,- *(const half *)beta, (half *)C, ldc);+ return tinyblasGE_launch(+ handle, transa, transb, m, n, k, (float)*(const half *)alpha, (const half *)A,+ lda, (const half *)B, ldb, (float)*(const half *)beta, (half *)C, ldc);case TINYBLAS_COMPUTE_32F:return tinyblasGE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,(const half *)A, lda, (const half *)B, ldb,@@ -752,10 +743,10 @@ tinyblasStatus_t tinyblasGemmBatchedEx(tinyblasHandle_t handle, tinyblasOperatiocase TINYBLAS_R_16F:switch (computeType) {case TINYBLAS_COMPUTE_16F:- return tinyblasGBE_launch(handle, transa, transb, m, n, k, *(const half *)alpha,- (const half *const *)Aarray, lda,- (const half *const *)Barray, ldb, *(const half *)beta,- (half *const *)Carray, ldc, batchCount);+ return tinyblasGBE_launch(+ handle, transa, transb, m, n, k, (float)*(const half *)alpha,+ (const half *const *)Aarray, lda, (const half *const *)Barray, ldb,+ (float)*(const half *)beta, (half *const *)Carray, ldc, batchCount);case TINYBLAS_COMPUTE_32F:return tinyblasGBE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,(const half *const *)Aarray, lda,@@ -936,10 +927,10 @@ tinyblasStatus_t tinyblasGemmStridedBatchedEx(tinyblasHandle_t handle, //case TINYBLAS_R_16F:switch (computeType) {case TINYBLAS_COMPUTE_16F:- return tinyblasGSBE_launch(handle, transa, transb, m, n, k, *(const half *)alpha,- (const half *)A, lda, strideA, (const half *)B, ldb,- strideB, *(const half *)beta, (half *)C, ldc, strideC,- batchCount);+ return tinyblasGSBE_launch(+ handle, transa, transb, m, n, k, (float)*(const half *)alpha, (const half *)A,+ lda, strideA, (const half *)B, ldb, strideB, (float)*(const half *)beta,+ (half *)C, ldc, strideC, batchCount);case TINYBLAS_COMPUTE_32F:return tinyblasGSBE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,(const half *)A, lda, strideA, (const half *)B, ldb,diff --git a/llamafile/tinyblas_test.cu b/llamafile/tinyblas_test.cuindex f52d701c..75755b47 100644--- a/llamafile/tinyblas_test.cu+++ b/llamafile/tinyblas_test.cu@@ -18,7 +18,7 @@#include "tester.h"void checkTinyblasWorksHHHH() {- double TOLERANCE = 1e-3;+ double TOLERANCE = 1e-2;half alpha = 1;half beta = 0;int m = 128;@@ -35,7 +35,7 @@ void checkTinyblasWorksHHHH() {randomize(k, n, B.p, ldb);broadcast(m, n, C.p, ldc, TOMBSTONE);broadcast(m, n, G.p, ldc, TOMBSTONE);- gemmref(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ gemmref<double>(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);tinyblas(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);@@ -51,14 +51,14 @@ void checkTinyblasWorksHHHH() {randomize(k, n, B.p, ldb);broadcast(m, n, G.p, ldc, TOMBSTONE);broadcast(m, n, C.p, ldc, TOMBSTONE);- gemmref(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ gemmref<double>(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);tinyblas(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);});}void checkTinyblasWorksHHHS() {- double TOLERANCE = 1e-5;+ double TOLERANCE = 1e-3;float alpha = 1;float beta = 0;int m = 4096;@@ -75,7 +75,7 @@ void checkTinyblasWorksHHHS() {randomize(k, n, B.p, ldb);broadcast(m, n, C.p, ldc, TOMBSTONE);broadcast(m, n, G.p, ldc, TOMBSTONE);- gemmref(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ gemmref<double>(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);tinyblas(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);@@ -91,14 +91,14 @@ void checkTinyblasWorksHHHS() {randomize(k, n, B.p, ldb);broadcast(m, n, G.p, ldc, TOMBSTONE);broadcast(m, n, C.p, ldc, TOMBSTONE);- gemmref(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ gemmref<double>(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);tinyblas(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);});}void checkTinyblasWorksHHSS() {- double TOLERANCE = 1e-5;+ double TOLERANCE = 1e-4;float alpha = 1;float beta = 0;int m = 4096;@@ -115,7 +115,7 @@ void checkTinyblasWorksHHSS() {randomize(k, n, B.p, ldb);broadcast(m, n, G.p, ldc, TOMBSTONE);broadcast(m, n, C.p, ldc, TOMBSTONE);- gemmref(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ gemmref<double>(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);tinyblas(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);@@ -131,14 +131,14 @@ void checkTinyblasWorksHHSS() {randomize(k, n, B.p, ldb);broadcast(m, n, G.p, ldc, TOMBSTONE);broadcast(m, n, C.p, ldc, TOMBSTONE);- gemmref(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ gemmref<double>(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);tinyblas(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);});}void checkTinyblasWorksSSSS() {- double TOLERANCE = 1e-5;+ double TOLERANCE = 1e-4;float alpha = 1;float beta = 0;int m = 4096;@@ -155,7 +155,7 @@ void checkTinyblasWorksSSSS() {randomize(k, n, B.p, ldb);broadcast(m, n, C.p, ldc, TOMBSTONE);broadcast(m, n, G.p, ldc, TOMBSTONE);- gemmref(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ gemmref<double>(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);tinyblas(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);@@ -171,7 +171,7 @@ void checkTinyblasWorksSSSS() {randomize(k, n, B.p, ldb);broadcast(m, n, G.p, ldc, TOMBSTONE);broadcast(m, n, C.p, ldc, TOMBSTONE);- gemmref(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ gemmref<double>(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);tinyblas(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);});@@ -192,13 +192,13 @@ void try_size(int m, int n, int k) {randomize(k, n, B.p, ldb);broadcast(m, n, C.p, ldc, TOMBSTONE);broadcast(m, n, G.p, ldc, TOMBSTONE);- gemmref(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);+ gemmref<double>(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);tinyblas(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);CHECK(1, m, n, k, G.p, ldc, C.p, ldc);}void test_gsbe() {- double TOLERANCE = 1e-5;+ double TOLERANCE = 1e-4;test_matmul([&](int m, int n, int k, int l, float alpha, float beta) {int lda = k;int ldb = k;@@ -213,7 +213,7 @@ void test_gsbe() {broadcast(m, n, G.p + 0 * ldc * n, ldc, TOMBSTONE);broadcast(m, n, C.p + 1 * ldc * n, ldc, TOMBSTONE);broadcast(m, n, G.p + 1 * ldc * n, ldc, TOMBSTONE);- gsberef(1, 0, m, n, k, alpha, A.p, lda, 0, B.p, ldb, 0, beta, G.p, ldc, ldc * n, 2);+ gsberef<double>(1, 0, m, n, k, alpha, A.p, lda, 0, B.p, ldb, 0, beta, G.p, ldc, ldc * n, 2);tinyblasGSBE(1, 0, m, n, k, alpha, A.p, lda, 0, B.p, ldb, 0, beta, C.p, ldc, ldc * n, 2);CHECK(TOLERANCE, m, n, k, G.p + 0 * ldc * n, ldc, C.p + 0 * ldc * n, ldc);CHECK(TOLERANCE, m, n, k, G.p + 0 * ldc * n, ldc, C.p + 0 * ldc * n, ldc);