Skip to content

Distribute and run LLMs with a single file.

Change build strategy for CPU optimizations

Justine Tunney47b4f2ff389d
diff --git a/build/cudacc b/build/cudacc
index 3bdb2f55..35d489b3 100755
--- a/build/cudacc
+++ b/build/cudacc
@@ -44,7 +44,6 @@ for x; do
set -- "$@" -lhipblas -lrocblas
continue
elif [ x"$x" = x"--use_fast_math" ]; then
- set -- "$@" -ffast-math -DHIP_FAST_MATH
continue
fi
fi
diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cpp
index 19acff7d..f9d54c05 100644
--- a/llama.cpp/common.cpp
+++ b/llama.cpp/common.cpp
@@ -929,7 +929,7 @@ static bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg,
else { invalid_param = true; }
return true;
}
- if (arg == "--verbose-prompt") {
+ if (arg == "--verbose-prompt" || arg == "--silent-prompt") {
params.verbose_prompt = true;
return true;
}
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 57ea4f4c..9a04d838 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -10716,6 +10716,26 @@ static void ggml_compute_forward_mul_mat(
}
#endif
+#ifdef __x86_64__
+ bool can_use_tinyblas = ggml_is_contiguous(src0) &&
+ ggml_is_contiguous(src1) &&
+ ((ne2 == 1 && ne3 == 1) &&
+ (ne02 == 1 && ne03 == 1) &&
+ (ne12 == 1 && ne13 == 1));
+ if (can_use_tinyblas &&
+ llamafile_sgemm(ne01, ne11, ne00,
+ src0->data, nb01/ggml_type_size(src0->type),
+ src1->data, nb11/ggml_type_size(src1->type),
+ dst->data, nb1/4,
+ ith, nth,
+ params->type,
+ src0->type,
+ src1->type,
+ dst->type)) {
+ return;
+ }
+#endif
+
if (params->type == GGML_TASK_TYPE_INIT) {
if (ith != 0) {
return;
@@ -10748,25 +10768,16 @@ static void ggml_compute_forward_mul_mat(
const size_t row_size = ggml_row_size(vec_dot_type, ne10);
#ifdef __x86_64__
- if (X86_HAVE(FMA) &&
- X86_HAVE(AVX2) &&
- (dst->type == GGML_TYPE_F32 &&
- src1->type == GGML_TYPE_F32 &&
- (src0->type == GGML_TYPE_F32 ||
- src0->type == GGML_TYPE_Q4_0 ||
- src0->type == GGML_TYPE_Q8_0 ||
- (src0->type == GGML_TYPE_F16 && X86_HAVE(F16C)))) &&
- ne00 % 8 == 0 &&
- ggml_is_contiguous(src0) &&
- ggml_is_contiguous(src1) &&
- ((ne2 == 1 && ne3 == 1) &&
- (ne02 == 1 && ne03 == 1) &&
- (ne12 == 1 && ne13 == 1))) {
+ if (can_use_tinyblas &&
llamafile_sgemm(ne01, ne11, ne00,
- src0->type, src0->data, nb01/ggml_type_size(src0->type),
+ src0->data, nb01/ggml_type_size(src0->type),
wdata, row_size/ggml_type_size(vec_dot_type),
dst->data, nb1/4,
- ith, nth);
+ ith, nth,
+ params->type,
+ src0->type,
+ vec_dot_type,
+ dst->type)) {
return;
}
#endif
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cpp
index ba3f6f75..f2dc674b 100644
--- a/llama.cpp/llama.cpp
+++ b/llama.cpp/llama.cpp
@@ -1903,10 +1903,10 @@ struct llama_context {
size_t logits_size = 0;
float * logits = nullptr;
-#ifndef NDEBUG
- // guard against access to unset logits
- std::vector<bool> logits_valid;
-#endif
+// #ifndef NDEBUG
+// // guard against access to unset logits
+// std::vector<bool> logits_valid;
+// #endif
bool logits_all = false;
// embeddings output (2-dimensional array: [n_tokens][n_embd])
@@ -8847,13 +8847,13 @@ static int llama_decode_internal(
auto * logits_out = lctx.logits;
-#ifndef NDEBUG
- auto & logits_valid = lctx.logits_valid;
- logits_valid.clear();
- logits_valid.resize(n_tokens_all);
+// #ifndef NDEBUG
+// auto & logits_valid = lctx.logits_valid;
+// logits_valid.clear();
+// logits_valid.resize(n_tokens_all);
- memset(logits_out, 0, lctx.logits_size*sizeof(float));
-#endif
+// memset(logits_out, 0, lctx.logits_size*sizeof(float));
+// #endif
const auto n_ubatch = cparams.n_ubatch;
@@ -9022,21 +9022,21 @@ static int llama_decode_internal(
i_first = -1;
}
}
-#ifndef NDEBUG
- logits_valid[cur_token + i] = u_batch.logits[i] != 0;;
-#endif
+// #ifndef NDEBUG
+// logits_valid[cur_token + i] = u_batch.logits[i] != 0;;
+// #endif
}
} else if (lctx.logits_all) {
ggml_backend_tensor_get_async(backend_res, res, logits_out + n_vocab*cur_token, 0, n_vocab*n_tokens*sizeof(float));
-#ifndef NDEBUG
- std::fill(logits_valid.begin() + cur_token, logits_valid.begin() + cur_token + n_tokens, true);
-#endif
+// #ifndef NDEBUG
+// std::fill(logits_valid.begin() + cur_token, logits_valid.begin() + cur_token + n_tokens, true);
+// #endif
} else {
if (cur_token + n_tokens >= n_tokens_all) {
ggml_backend_tensor_get_async(backend_res, res, logits_out, n_vocab*(n_tokens - 1)*sizeof(float), n_vocab*sizeof(float));
-#ifndef NDEBUG
- logits_valid[0] = true;
-#endif
+// #ifndef NDEBUG
+// logits_valid[0] = true;
+// #endif
}
}
}
@@ -14096,7 +14096,7 @@ float * llama_get_logits(struct llama_context * ctx) {
}
float * llama_get_logits_ith(struct llama_context * ctx, int32_t i) {
- assert(ctx->logits_valid.at(i));
+ // assert(ctx->logits_valid.at(i));
llama_synchronize(ctx);
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index 32161dfd..81e26a78 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -5,6 +5,7 @@ PKGS += LLAMAFILE
LLAMAFILE_FILES := $(wildcard llamafile/*.*)
LLAMAFILE_HDRS = $(filter %.h,$(LLAMAFILE_FILES))
+LLAMAFILE_INCS = $(filter %.inc,$(LLAMAFILE_FILES))
LLAMAFILE_SRCS_C = $(filter %.c,$(LLAMAFILE_FILES))
LLAMAFILE_SRCS_CU = $(filter %.cu,$(LLAMAFILE_FILES))
LLAMAFILE_SRCS_CPP = $(filter %.cpp,$(LLAMAFILE_FILES))
@@ -16,6 +17,18 @@ LLAMAFILE_OBJS = \
$(LLAMAFILE_SRCS_CPP:%.cpp=o/$(MODE)/%.o) \
$(LLAMAFILE_FILES:%=o/$(MODE)/%.zip.o)
+o/$(MODE)/llamafile/sgemm_sss_avx.o: private TARGET_ARCH += -Xx86_64-mavx
+o/$(MODE)/llamafile/sgemm_sss_fma.o: private TARGET_ARCH += -Xx86_64-mfma
+o/$(MODE)/llamafile/sgemm_sss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f
+o/$(MODE)/llamafile/sgemm_hss_f16c.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mf16c
+o/$(MODE)/llamafile/sgemm_hss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f
+o/$(MODE)/llamafile/sgemm_qqs_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni
+o/$(MODE)/llamafile/sgemm_qqs_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni
+o/$(MODE)/llamafile/sgemm_qqs_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma
+o/$(MODE)/llamafile/sgemm_eqs_avx512vnni.o: private TARGET_ARCH += -Xx86_64-mavx512vl -Xx86_64-mavx512vnni
+o/$(MODE)/llamafile/sgemm_eqs_avxvnni.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mavxvnni
+o/$(MODE)/llamafile/sgemm_eqs_fma.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma
+
o/$(MODE)/llamafile/zipalign: \
o/$(MODE)/llamafile/zipalign.o \
o/$(MODE)/llamafile/help.o \
@@ -42,9 +55,13 @@ o/$(MODE)/llamafile: \
################################################################################
# testing
+o/$(MODE)/llamafile/sgemm_test: \
+ o/$(MODE)/llamafile/sgemm_test.o \
+ o/$(MODE)/llama.cpp/llama.cpp.a
+
o/$(MODE)/llamafile/%.o: llamafile/%.cu llamafile/BUILD.mk
@mkdir -p $(@D)
- build/cudacc -fPIE -g -O3 -march=native -c -o $@ $<
+ build/cudacc -fPIE -g -O3 -march=native -ffast-math --use_fast_math -c -o $@ $<
o/$(MODE)/llamafile/tinyblas_test: \
o/$(MODE)/llamafile/tinyblas_test.o \
diff --git a/llamafile/hsum.h b/llamafile/hsum.h
new file mode 100644
index 00000000..6295ebec
--- /dev/null
+++ b/llamafile/hsum.h
@@ -0,0 +1,29 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#pragma once
+#ifdef __SSE__
+#include <immintrin.h>
+
+inline float hsum(__m128 x) {
+ x = _mm_add_ps(x, _mm_movehl_ps(x, x));
+ x = _mm_add_ss(x, _mm_movehdup_ps(x));
+ return _mm_cvtss_f32(x);
+}
+
+#ifdef __AVX__
+
+inline float hsum(__m256 x) {
+ return hsum(_mm_add_ps(_mm256_extractf128_ps(x, 1), _mm256_castps256_ps128(x)));
+}
+
+#ifdef __AVX512F__
+
+#include <immintrin.h>
+
+inline float hsum(__m512 x) {
+ return _mm512_reduce_add_ps(x);
+}
+
+#endif // __AVX512F__
+#endif // __AVX__
+#endif // __SSE__
diff --git a/llamafile/kernel.h b/llamafile/kernel.h
new file mode 100644
index 00000000..06777161
--- /dev/null
+++ b/llamafile/kernel.h
@@ -0,0 +1,19 @@
+#pragma once
+
+#define BEGIN_KERNEL(RM, RN) \
+ int ytiles = (m - m0) / RM; \
+ int xtiles = (n - n0) / RN; \
+ int tiles = ytiles * xtiles; \
+ double duty = (double)tiles / nth; \
+ if (duty < 1) \
+ duty = 1; \
+ double spot = duty * ith + .5; \
+ int end = spot + duty; \
+ int start = spot; \
+ if (end > tiles) \
+ end = tiles; \
+ for (int job = start; job < end; ++job) { \
+ int i = m0 + job / xtiles * RM; \
+ int j = n0 + job % xtiles * RN;
+
+#define END_KERNEL() }
diff --git a/llamafile/llamafile.h b/llamafile/llamafile.h
index d93e8e0a..71694858 100644
--- a/llamafile/llamafile.h
+++ b/llamafile/llamafile.h
@@ -48,8 +48,8 @@ bool llamafile_has_metal(void);
int llamafile_gpu_parse(const char *);
const char *llamafile_describe_gpu(void);
-void llamafile_sgemm(long, long, long, int, const void *, long, const void *, long, float *, long,
- long, long);
+bool llamafile_sgemm(int, int, int, const void *, int, const void *, int, void *, int, int, int,
+ int, int, int, int);
#ifdef __cplusplus
}
diff --git a/llamafile/llmm3x4.inc b/llamafile/llmm3x4.inc
new file mode 100644
index 00000000..c0d6c66f
--- /dev/null
+++ b/llamafile/llmm3x4.inc
@@ -0,0 +1,186 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <cassert>
+
+#include "llama.cpp/ggml.h"
+
+#include "hsum.h"
+#include "kernel.h"
+#include "madd.h"
+
+namespace {
+
+class LLMM3X4 {
+ public:
+ LLMM3X4(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
+ }
+
+ void matmul(int m, int n) {
+ mnpack(0, m, 0, n);
+ }
+
+ private:
+ dontinline void mnpack(int m0, int m, int n0, int n) {
+ if (m - m0 <= 0 || n - n0 <= 0)
+ return;
+ int mc, nc, mp, np;
+ if (m - m0 >= 3 && n - n0 >= 4) {
+ mc = 3;
+ nc = 4;
+ gemm3x4(m0, m, n0, n);
+ } else if (m - m0 >= 4 && n - n0 >= 1) {
+ mc = 4;
+ nc = 1;
+ gemm4x1(m0, m, n0, n);
+ } else if (m - m0 >= 1 && n - n0 >= 4) {
+ mc = 1;
+ nc = 4;
+ gemm1x4(m0, m, n0, n);
+ } else {
+ mc = 1;
+ nc = 1;
+ gemm1x1(m0, m, n0, n);
+ }
+ mp = m0 + (m - m0) / mc * mc;
+ np = n0 + (n - n0) / nc * nc;
+ mnpack(mp, m, n0, np);
+ mnpack(m0, mp, np, n);
+ mnpack(mp, m, np, n);
+ }
+
+ dontinline void gemm3x4(int m0, int m, int n0, int n) {
+ BEGIN_KERNEL(3, 4)
+ V c00 = zero();
+ V c01 = zero();
+ V c02 = zero();
+ V c03 = zero();
+ V c10 = zero();
+ V c11 = zero();
+ V c12 = zero();
+ V c13 = zero();
+ V c20 = zero();
+ V c21 = zero();
+ V c22 = zero();
+ V c23 = zero();
+ for (int l = 0; l < k; l += KN) {
+ V k0 = load_b(B + ldb * (j + 0) + l);
+ V k1 = load_b(B + ldb * (j + 1) + l);
+ V k2 = load_b(B + ldb * (j + 2) + l);
+ V k3 = load_b(B + ldb * (j + 3) + l);
+ V a0 = load_a(A + lda * (i + 0) + l);
+ c00 = madd(a0, k0, c00);
+ c01 = madd(a0, k1, c01);
+ c02 = madd(a0, k2, c02);
+ c03 = madd(a0, k3, c03);
+ V a1 = load_a(A + lda * (i + 1) + l);
+ c10 = madd(a1, k0, c10);
+ c11 = madd(a1, k1, c11);
+ c12 = madd(a1, k2, c12);
+ c13 = madd(a1, k3, c13);
+ V a2 = load_a(A + lda * (i + 2) + l);
+ c20 = madd(a2, k0, c20);
+ c21 = madd(a2, k1, c21);
+ c22 = madd(a2, k2, c22);
+ c23 = madd(a2, k3, c23);
+ }
+ C[ldc * (j + 0) + (i + 0)] = hsum(c00);
+ C[ldc * (j + 0) + (i + 1)] = hsum(c10);
+ C[ldc * (j + 0) + (i + 2)] = hsum(c20);
+ C[ldc * (j + 1) + (i + 0)] = hsum(c01);
+ C[ldc * (j + 1) + (i + 1)] = hsum(c11);
+ C[ldc * (j + 1) + (i + 2)] = hsum(c21);
+ C[ldc * (j + 2) + (i + 0)] = hsum(c02);
+ C[ldc * (j + 2) + (i + 1)] = hsum(c12);
+ C[ldc * (j + 2) + (i + 2)] = hsum(c22);
+ C[ldc * (j + 3) + (i + 0)] = hsum(c03);
+ C[ldc * (j + 3) + (i + 1)] = hsum(c13);
+ C[ldc * (j + 3) + (i + 2)] = hsum(c23);
+ END_KERNEL()
+ }
+
+ dontinline void gemm1x4(int m0, int m, int n0, int n) {
+ BEGIN_KERNEL(1, 4)
+ V c00 = zero();
+ V c01 = zero();
+ V c02 = zero();
+ V c03 = zero();
+ for (int l = 0; l < k; l += KN) {
+ V a0 = load_a(A + lda * (i + 0) + l);
+ V k0 = load_b(B + ldb * (j + 0) + l);
+ V k1 = load_b(B + ldb * (j + 1) + l);
+ V k2 = load_b(B + ldb * (j + 2) + l);
+ V k3 = load_b(B + ldb * (j + 3) + l);
+ c00 = madd(a0, k0, c00);
+ c01 = madd(a0, k1, c01);
+ c02 = madd(a0, k2, c02);
+ c03 = madd(a0, k3, c03);
+ }
+ C[ldc * (j + 0) + (i + 0)] = hsum(c00);
+ C[ldc * (j + 1) + (i + 0)] = hsum(c01);
+ C[ldc * (j + 2) + (i + 0)] = hsum(c02);
+ C[ldc * (j + 3) + (i + 0)] = hsum(c03);
+ END_KERNEL()
+ }
+
+ dontinline void gemm4x1(int m0, int m, int n0, int n) {
+ BEGIN_KERNEL(4, 1)
+ V c00 = zero();
+ V c10 = zero();
+ V c20 = zero();
+ V c30 = zero();
+ for (int l = 0; l < k; l += KN) {
+ V k0 = load_b(B + ldb * (j + 0) + l);
+ V a0 = load_a(A + lda * (i + 0) + l);
+ c00 = madd(a0, k0, c00);
+ V a1 = load_a(A + lda * (i + 1) + l);
+ c10 = madd(a1, k0, c10);
+ V a2 = load_a(A + lda * (i + 2) + l);
+ c20 = madd(a2, k0, c20);
+ V a3 = load_a(A + lda * (i + 3) + l);
+ c30 = madd(a3, k0, c30);
+ }
+ C[ldc * (j + 0) + (i + 0)] = hsum(c00);
+ C[ldc * (j + 0) + (i + 1)] = hsum(c10);
+ C[ldc * (j + 0) + (i + 2)] = hsum(c20);
+ C[ldc * (j + 0) + (i + 3)] = hsum(c30);
+ END_KERNEL()
+ }
+
+ dontinline void gemm1x1(int m0, int m, int n0, int n) {
+ BEGIN_KERNEL(1, 1)
+ V c = zero();
+ for (int l = 0; l < k; l += KN)
+ c = madd(load_a(A + lda * i + l), load_b(B + ldb * j + l), c);
+ C[ldc * j + i] = hsum(c);
+ END_KERNEL()
+ }
+
+ const TA *const A;
+ const TB *const B;
+ TC *const C;
+ const int k;
+ const int lda;
+ const int ldb;
+ const int ldc;
+ const int ith;
+ const int nth;
+};
+
+} // namespace
diff --git a/llamafile/llmmq0.inc b/llamafile/llmmq0.inc
new file mode 100644
index 00000000..87fd1eed
--- /dev/null
+++ b/llamafile/llmmq0.inc
@@ -0,0 +1,204 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <cassert>
+
+#include "llama.cpp/ggml-impl.h"
+#include "llama.cpp/ggml.h"
+
+#include "hsum.h"
+#include "kernel.h"
+#include "madd.h"
+
+#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)
+
+namespace {
+
+class LLMMQ0 {
+ public:
+ LLMMQ0(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith, int nth)
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
+ }
+
+ void matmul(int m, int n) {
+ mnpack(0, m, 0, n);
+ }
+
+ private:
+ dontinline void mnpack(int m0, int m, int n0, int n) {
+ if (m - m0 <= 0 || n - n0 <= 0)
+ return;
+ int mc, nc, mp, np;
+ if (m - m0 >= 1 && n - n0 >= 4) {
+ mc = 1;
+ nc = 4;
+ gemm1x4(m0, m, n0, n);
+ } else if (m - m0 >= 4 && n - n0 >= 1) {
+ mc = 4;
+ nc = 1;
+ gemm4x1(m0, m, n0, n);
+ } else {
+ mc = 1;
+ nc = 1;
+ gemm1x1(m0, m, n0, n);
+ }
+ mp = m0 + (m - m0) / mc * mc;
+ np = n0 + (n - n0) / nc * nc;
+ mnpack(mp, m, n0, np);
+ mnpack(m0, mp, np, n);
+ mnpack(mp, m, np, n);
+ }
+
+ dontinline void gemm4x1(int m0, int m, int n0, int n) {
+ BEGIN_KERNEL(4, 1)
+ int k2 = k / 32;
+ __m256 c0 = _mm256_setzero_ps();
+ __m256 c1 = _mm256_setzero_ps();
+ __m256 c2 = _mm256_setzero_ps();
+ __m256 c3 = _mm256_setzero_ps();
+ const TA *Ap0 = A + lda * (i + 0);
+ const TA *Ap1 = A + lda * (i + 1);
+ const TA *Ap2 = A + lda * (i + 2);
+ const TA *Ap3 = A + lda * (i + 3);
+ const TB *Bp = B + ldb * j;
+ for (int l = 0; l < k2; ++l) {
+ float db0 = unhalf(Bp[l].d);
+ __m256i f = load(Bp + l);
+ __m256i u = _mm256_sign_epi8(f, f);
+ __m256 d0 = _mm256_set1_ps(unhalf(Ap0[l].d) * db0);
+ __m256 d1 = _mm256_set1_ps(unhalf(Ap1[l].d) * db0);
+ __m256 d2 = _mm256_set1_ps(unhalf(Ap2[l].d) * db0);
+ __m256 d3 = _mm256_set1_ps(unhalf(Ap3[l].d) * db0);
+ __m256i e0 = load(Ap0 + l);
+ __m256i e1 = load(Ap1 + l);
+ __m256i e2 = load(Ap2 + l);
+ __m256i e3 = load(Ap3 + l);
+ __m256i s0 = _mm256_sign_epi8(e0, f);
+ __m256i s1 = _mm256_sign_epi8(e1, f);
+ __m256i s2 = _mm256_sign_epi8(e2, f);
+ __m256i s3 = _mm256_sign_epi8(e3, f);
+ __m256 g0 = updot(u, s0);
+ __m256 g1 = updot(u, s1);
+ __m256 g2 = updot(u, s2);
+ __m256 g3 = updot(u, s3);
+ c0 = madd(d0, g0, c0);
+ c1 = madd(d1, g1, c1);
+ c2 = madd(d2, g2, c2);
+ c3 = madd(d3, g3, c3);
+ }
+ C[ldc * j + (i + 0)] = hsum(c0);
+ C[ldc * j + (i + 1)] = hsum(c1);
+ C[ldc * j + (i + 2)] = hsum(c2);
+ C[ldc * j + (i + 3)] = hsum(c3);
+ END_KERNEL()
+ }
+
+ dontinline void gemm1x4(int m0, int m, int n0, int n) {
+ BEGIN_KERNEL(1, 4)
+ int k2 = k / 32;
+ __m256 c0 = _mm256_setzero_ps();
+ __m256 c1 = _mm256_setzero_ps();
+ __m256 c2 = _mm256_setzero_ps();
+ __m256 c3 = _mm256_setzero_ps();
+ const TB *Bp0 = B + ldb * (j + 0);
+ const TB *Bp1 = B + ldb * (j + 1);
+ const TB *Bp2 = B + ldb * (j + 2);
+ const TB *Bp3 = B + ldb * (j + 3);
+ const TA *Ap = A + lda * i;
+ for (int l = 0; l < k2; ++l) {
+ float da0 = unhalf(Ap[l].d);
+ __m256i f = load(Ap + l);
+ __m256i u = _mm256_sign_epi8(f, f);
+ __m256 d0 = _mm256_set1_ps(unhalf(Bp0[l].d) * da0);
+ __m256 d1 = _mm256_set1_ps(unhalf(Bp1[l].d) * da0);
+ __m256 d2 = _mm256_set1_ps(unhalf(Bp2[l].d) * da0);
+ __m256 d3 = _mm256_set1_ps(unhalf(Bp3[l].d) * da0);
+ __m256 g0 = updot(u, _mm256_sign_epi8(load(Bp0 + l), f));
+ __m256 g1 = updot(u, _mm256_sign_epi8(load(Bp1 + l), f));
+ __m256 g2 = updot(u, _mm256_sign_epi8(load(Bp2 + l), f));
+ __m256 g3 = updot(u, _mm256_sign_epi8(load(Bp3 + l), f));
+ c0 = madd(d0, g0, c0);
+ c1 = madd(d1, g1, c1);
+ c2 = madd(d2, g2, c2);
+ c3 = madd(d3, g3, c3);
+ }
+ C[ldc * (j + 0) + i] = hsum(c0);
+ C[ldc * (j + 1) + i] = hsum(c1);
+ C[ldc * (j + 2) + i] = hsum(c2);
+ C[ldc * (j + 3) + i] = hsum(c3);
+ END_KERNEL()
+ }
+
+ dontinline void gemm1x1(int m0, int m, int n0, int n) {
+ BEGIN_KERNEL(1, 1)
+ int k2 = k / 32;
+ __m256 c = _mm256_setzero_ps();
+ const TA *Ap = A + lda * i;
+ const TB *Bp = B + ldb * j;
+ for (int l = 0; l < k2; ++l) {
+ __m256 d = _mm256_set1_ps(unhalf(Ap[l].d) * unhalf(Bp[l].d));
+ __m256i e = load(Ap + l);
+ __m256i f = load(Bp + l);
+ __m256 g = updot(_mm256_sign_epi8(e, e), _mm256_sign_epi8(f, e));
+ c = madd(d, g, c);
+ }
+ C[ldc * j + i] = hsum(c);
+ END_KERNEL()
+ }
+
+ inline __m256i load(const block_q8_0 *b) {
+ return _mm256_loadu_si256((const __m256i *)b->qs);
+ }
+
+ inline __m256i load(const block_q4_0 *b) {
+ return _mm256_sub_epi8(denibble(b->qs), _mm256_set1_epi8(8));
+ }
+
+ inline __m256 updot(__m256i u, __m256i s) {
+ __m256i res;
+#if defined(__AVXVNNI__) || defined(__AVX512VNNI__)
+ res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);
+#else
+ res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));
+#endif
+ return _mm256_cvtepi32_ps(res);
+ }
+
+ static inline __m256i denibble(const uint8_t *p) {
+ const __m128i tmp = _mm_loadu_si128((const __m128i *)p);
+ const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);
+ const __m256i lowMask = _mm256_set1_epi8(15);
+ return _mm256_and_si256(lowMask, bytes);
+ }
+
+ static inline float unhalf(unsigned short d) {
+ return GGML_FP16_TO_FP32(d);
+ }
+
+ const TA *const A;
+ const TB *const B;
+ TC *const C;
+ const int k;
+ const int lda;
+ const int ldb;
+ const int ldc;
+ const int ith;
+ const int nth;
+};
+
+} // namespace
diff --git a/llamafile/madd.h b/llamafile/madd.h
new file mode 100644
index 00000000..1777c883
--- /dev/null
+++ b/llamafile/madd.h
@@ -0,0 +1,25 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+#pragma once
+#ifdef __SSE__
+#include <immintrin.h>
+
+inline __m128 madd(__m128 x, __m128 y, __m128 z) {
+ return _mm_add_ps(_mm_mul_ps(x, y), z);
+}
+
+#ifdef __AVX__
+
+inline __m256 madd(__m256 x, __m256 y, __m256 z) {
+ return _mm256_add_ps(_mm256_mul_ps(x, y), z);
+}
+
+#ifdef __AVX512F__
+
+inline __m512 madd(__m512 x, __m512 y, __m512 z) {
+ return _mm512_add_ps(_mm512_mul_ps(x, y), z);
+}
+
+#endif // __AVX512F__
+#endif // __AVX__
+#endif // __SSE__
diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cpp
index 36068f8c..10b539f0 100644
--- a/llamafile/sgemm.cpp
+++ b/llamafile/sgemm.cpp
@@ -15,492 +15,88 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-#include "llama.cpp/ggml-impl.h"
-#include "llama.cpp/ggml-quants.h"
-#include "llamafile.h"
-#include <assert.h>
-#include <cosmo.h>
-#include <immintrin.h>
-#include <stdio.h>
-
-//
-// _ _ ___ _ _ ___
-// | |_(_)_ _ _ _| _ ) | /_\ / __|
-// | _| | ' \ || | _ \ |__ / _ \\__ \.
-// \__|_|_||_\_, |___/____/_/ \_\___/
-// |__/
-//
-// BASIC LINEAR ALGEBRA SUBPROGRAMS
-//
-
-// matrix multiplication for x86 microprocessors
-//
-// this file implements optimized cpu kernels for llama.cpp that support
-// column major ordering with the A matrix transposed. they work by
-// computing multiple dot products at once, because it exploits
-// instruction level parallelism, in addition to minimizing register
-// loads since the adjacent dot products can share register loads.
-//
-// this technique usually slightly outperforms mkl and blis at single
-// threaded performance, and usually goes 2x faster than mkl when openmp
-// threading is used. however please note that this code is adapted to
-// use the llama.cpp synchronization model, which isn't as advanced.
-//
-// ┌───────────┬──────────────────┬───────────────────┐
-// │ software │ skinny matrices │ fat matrices │
-// │ │ token generation │ prompt evaluation │
-// │ ───────── │ ──────────────── │ ───────────────── │
-// │ llama.cpp │ fast │ slow │
-// │ MKL │ slow │ fast │
-// │ tinyBLAS │ fast │ fast │
-// └───────────┴──────────────────┴───────────────────┘
-//
-// these kernels do not have any of the latency overhead typically
-// associated with the blas, which means they can provide the same
-// performance as llama.cpp when performing the most common type of
-// matmul used by text generation, which is matrix-vector product.
-
-#define END_KERNEL() }
-#define BEGIN_KERNEL(RM, RN) \
- long ytiles = (m - m0) / RM; \
- long xtiles = (n - n0) / RN; \
- long tiles = ytiles * xtiles; \
- double duty = (double)tiles / nth; \
- if (duty < 1) \
- duty = 1; \
- double spot = duty * ith + .5; \
- long end = spot + duty; \
- long start = spot; \
- if (end > tiles) \
- end = tiles; \
- for (long job = start; job < end; ++job) { \
- long i = m0 + job / xtiles * RM; \
- long j = n0 + job % xtiles * RN;
-
#ifdef __x86_64__
-#pragma GCC push_options
-#pragma GCC target("avx2,fma,f16c,avxvnni")
-
-#define MM256_SET_M128I(a, b) _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)
-
-static inline float unhalf(ggml_fp16_t d) {
- return GGML_FP16_TO_FP32(d);
-}
-static dontinline float hsum8f(__m256 x) {
- __m128 res = _mm256_extractf128_ps(x, 1);
- res = _mm_add_ps(res, _mm256_castps256_ps128(x));
- res = _mm_add_ps(res, _mm_movehl_ps(res, res));
- res = _mm_add_ss(res, _mm_movehdup_ps(res));
- return _mm_cvtss_f32(res);
-}
-
-static inline __m256i denibble32(const uint8_t *p) {
- const __m128i tmp = _mm_loadu_si128((const __m128i *)p);
- const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);
- const __m256i lowMask = _mm256_set1_epi8(15);
- return _mm256_and_si256(lowMask, bytes);
-}
-
-template <typename T> static inline __m256 load8f(const T *);
-template <> inline __m256 load8f(const float *p) {
- return _mm256_loadu_ps(p);
-}
-template <> inline __m256 load8f(const ggml_fp16_t *p) {
- return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));
-}
-
-template <typename T> static inline __m256i load32i(const T *);
-template <> inline __m256i load32i(const block_q8_0 *b) {
- return _mm256_loadu_si256((const __m256i *)b->qs);
-}
-template <> inline __m256i load32i(const block_q4_0 *b) {
- return _mm256_sub_epi8(denibble32(b->qs), _mm256_set1_epi8(8));
-}
-
-template <typename T> class tinyBLAS {
- public:
- tinyBLAS(long k, const T *A, long lda, const T *B, long ldb, float *C, long ldc, long ith,
- long nth)
- : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith), nth(nth) {
- assert(A != nullptr);
- assert(B != nullptr);
- assert(C != nullptr);
- assert(k >= 0 && k % 8 == 0);
- assert(ith >= 0 && ith < nth);
- }
-
- void gemm(long m, long n) {
- assert(m >= 0);
- assert(n >= 0);
- assert(ldc >= m);
- mnpack(0, m, 0, n);
- }
-
- private:
- void mnpack(long m0, long m, long n0, long n) {
- if (m - m0 <= 0 || n - n0 <= 0)
- return;
- long mc, nc, mp, np;
- if (m - m0 >= 3 && n - n0 >= 4) {
- mc = 3;
- nc = 4;
- gemm3x4(m0, m, n0, n);
- } else if (m - m0 >= 4 && n - n0 >= 1) {
- mc = 4;
- nc = 1;
- gemm4x1(m0, m, n0, n);
- } else if (m - m0 >= 1 && n - n0 >= 4) {
- mc = 1;
- nc = 4;
- gemm1x4(m0, m, n0, n);
- } else {
- mc = 1;
- nc = 1;
- gemm1x1(m0, m, n0, n);
- }
- mp = m0 + (m - m0) / mc * mc;
- np = n0 + (n - n0) / nc * nc;
- mnpack(mp, m, n0, np);
- mnpack(m0, mp, np, n);
- mnpack(mp, m, np, n);
- }
-
- dontinline void gemm3x4(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(3, 4)
- __m256 c00 = _mm256_setzero_ps();
- __m256 c01 = _mm256_setzero_ps();
- __m256 c02 = _mm256_setzero_ps();
- __m256 c03 = _mm256_setzero_ps();
- __m256 c10 = _mm256_setzero_ps();
- __m256 c11 = _mm256_setzero_ps();
- __m256 c12 = _mm256_setzero_ps();
- __m256 c13 = _mm256_setzero_ps();
- __m256 c20 = _mm256_setzero_ps();
- __m256 c21 = _mm256_setzero_ps();
- __m256 c22 = _mm256_setzero_ps();
- __m256 c23 = _mm256_setzero_ps();
- for (long l = 0; l < k; l += 8) {
- __m256 k0 = load8f(B + ldb * (j + 0) + l);
- __m256 k1 = load8f(B + ldb * (j + 1) + l);
- __m256 k2 = load8f(B + ldb * (j + 2) + l);
- __m256 k3 = load8f(B + ldb * (j + 3) + l);
- __m256 a0 = load8f(A + lda * (i + 0) + l);
- c00 = _mm256_fmadd_ps(a0, k0, c00);
- c01 = _mm256_fmadd_ps(a0, k1, c01);
- c02 = _mm256_fmadd_ps(a0, k2, c02);
- c03 = _mm256_fmadd_ps(a0, k3, c03);
- __m256 a1 = load8f(A + lda * (i + 1) + l);
- c10 = _mm256_fmadd_ps(a1, k0, c10);
- c11 = _mm256_fmadd_ps(a1, k1, c11);
- c12 = _mm256_fmadd_ps(a1, k2, c12);
- c13 = _mm256_fmadd_ps(a1, k3, c13);
- __m256 a2 = load8f(A + lda * (i + 2) + l);
- c20 = _mm256_fmadd_ps(a2, k0, c20);
- c21 = _mm256_fmadd_ps(a2, k1, c21);
- c22 = _mm256_fmadd_ps(a2, k2, c22);
- c23 = _mm256_fmadd_ps(a2, k3, c23);
- }
- C[ldc * (j + 0) + (i + 0)] = hsum8f(c00);
- C[ldc * (j + 0) + (i + 1)] = hsum8f(c10);
- C[ldc * (j + 0) + (i + 2)] = hsum8f(c20);
- C[ldc * (j + 1) + (i + 0)] = hsum8f(c01);
- C[ldc * (j + 1) + (i + 1)] = hsum8f(c11);
- C[ldc * (j + 1) + (i + 2)] = hsum8f(c21);
- C[ldc * (j + 2) + (i + 0)] = hsum8f(c02);
- C[ldc * (j + 2) + (i + 1)] = hsum8f(c12);
- C[ldc * (j + 2) + (i + 2)] = hsum8f(c22);
- C[ldc * (j + 3) + (i + 0)] = hsum8f(c03);
- C[ldc * (j + 3) + (i + 1)] = hsum8f(c13);
- C[ldc * (j + 3) + (i + 2)] = hsum8f(c23);
- END_KERNEL()
- }
-
- dontinline void gemm1x4(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(1, 4)
- __m256 c00 = _mm256_setzero_ps();
- __m256 c01 = _mm256_setzero_ps();
- __m256 c02 = _mm256_setzero_ps();
- __m256 c03 = _mm256_setzero_ps();
- for (long l = 0; l < k; l += 8) {
- __m256 a0 = load8f(A + lda * (i + 0) + l);
- __m256 k0 = load8f(B + ldb * (j + 0) + l);
- __m256 k1 = load8f(B + ldb * (j + 1) + l);
- __m256 k2 = load8f(B + ldb * (j + 2) + l);
- __m256 k3 = load8f(B + ldb * (j + 3) + l);
- c00 = _mm256_fmadd_ps(a0, k0, c00);
- c01 = _mm256_fmadd_ps(a0, k1, c01);
- c02 = _mm256_fmadd_ps(a0, k2, c02);
- c03 = _mm256_fmadd_ps(a0, k3, c03);
- }
- C[ldc * (j + 0) + (i + 0)] = hsum8f(c00);
- C[ldc * (j + 1) + (i + 0)] = hsum8f(c01);
- C[ldc * (j + 2) + (i + 0)] = hsum8f(c02);
- C[ldc * (j + 3) + (i + 0)] = hsum8f(c03);
- END_KERNEL()
- }
-
- dontinline void gemm4x1(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(4, 1)
- __m256 c00 = _mm256_setzero_ps();
- __m256 c10 = _mm256_setzero_ps();
- __m256 c20 = _mm256_setzero_ps();
- __m256 c30 = _mm256_setzero_ps();
- for (long l = 0; l < k; l += 8) {
- __m256 k0 = load8f(B + ldb * (j + 0) + l);
- __m256 a0 = load8f(A + lda * (i + 0) + l);
- c00 = _mm256_fmadd_ps(a0, k0, c00);
- __m256 a1 = load8f(A + lda * (i + 1) + l);
- c10 = _mm256_fmadd_ps(a1, k0, c10);
- __m256 a2 = load8f(A + lda * (i + 2) + l);
- c20 = _mm256_fmadd_ps(a2, k0, c20);
- __m256 a3 = load8f(A + lda * (i + 3) + l);
- c30 = _mm256_fmadd_ps(a3, k0, c30);
- }
- C[ldc * (j + 0) + (i + 0)] = hsum8f(c00);
- C[ldc * (j + 0) + (i + 1)] = hsum8f(c10);
- C[ldc * (j + 0) + (i + 2)] = hsum8f(c20);
- C[ldc * (j + 0) + (i + 3)] = hsum8f(c30);
- END_KERNEL()
- }
-
- dontinline void gemm1x1(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(1, 1)
- __m256 c = _mm256_setzero_ps();
- for (long l = 0; l < k; l += 8) {
- c = _mm256_fmadd_ps(load8f(A + lda * i + l), load8f(B + ldb * j + l), c);
- }
- C[ldc * j + i] = hsum8f(c);
- END_KERNEL()
- }
-
- const long k;
- const T *const A;
- const long lda;
- const T *const B;
- const long ldb;
- float *const C;
- const long ldc;
- const long ith;
- const long nth;
-};
-
-template <typename T, typename U, int avxvnni> class tinyBLASq {
- public:
- tinyBLASq(long k, const T *A, long lda, const U *B, long ldb, float *C, long ldc, long ith,
- long nth)
- : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith), nth(nth) {
- assert(A != nullptr);
- assert(B != nullptr);
- assert(C != nullptr);
- assert(k >= 0 && k % 32 == 0);
- assert(ith >= 0 && ith < nth);
- }
-
- void gemm(long m, long n) {
- assert(m >= 0);
- assert(n >= 0);
- assert(ldc >= m);
- mnpack(0, m, 0, n);
- }
-
- private:
- void mnpack(long m0, long m, long n0, long n) {
- if (m - m0 <= 0 || n - n0 <= 0)
- return;
- long mc, nc, mp, np;
- if (m - m0 >= 1 && n - n0 >= 4) {
- mc = 1;
- nc = 4;
- gemm1x4(m0, m, n0, n);
- } else if (m - m0 >= 4 && n - n0 >= 1) {
- mc = 4;
- nc = 1;
- gemm4x1(m0, m, n0, n);
- } else {
- mc = 1;
- nc = 1;
- gemm1x1(m0, m, n0, n);
- }
- mp = m0 + (m - m0) / mc * mc;
- np = n0 + (n - n0) / nc * nc;
- mnpack(mp, m, n0, np);
- mnpack(m0, mp, np, n);
- mnpack(mp, m, np, n);
- }
-
- dontinline void gemm4x1(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(4, 1)
- long k2 = k / 32;
- __m256 c0 = _mm256_setzero_ps();
- __m256 c1 = _mm256_setzero_ps();
- __m256 c2 = _mm256_setzero_ps();
- __m256 c3 = _mm256_setzero_ps();
- const T *Ap0 = A + lda * (i + 0);
- const T *Ap1 = A + lda * (i + 1);
- const T *Ap2 = A + lda * (i + 2);
- const T *Ap3 = A + lda * (i + 3);
- const U *Bp = B + ldb * j;
- for (long l = 0; l < k2; ++l) {
- float db0 = unhalf(Bp[l].d);
- __m256i f = load32i(Bp + l);
- __m256i u = _mm256_sign_epi8(f, f);
- __m256 d0 = _mm256_set1_ps(unhalf(Ap0[l].d) * db0);
- __m256 d1 = _mm256_set1_ps(unhalf(Ap1[l].d) * db0);
- __m256 d2 = _mm256_set1_ps(unhalf(Ap2[l].d) * db0);
- __m256 d3 = _mm256_set1_ps(unhalf(Ap3[l].d) * db0);
- __m256i e0 = load32i(Ap0 + l);
- __m256i e1 = load32i(Ap1 + l);
- __m256i e2 = load32i(Ap2 + l);
- __m256i e3 = load32i(Ap3 + l);
- __m256i s0 = _mm256_sign_epi8(e0, f);
- __m256i s1 = _mm256_sign_epi8(e1, f);
- __m256i s2 = _mm256_sign_epi8(e2, f);
- __m256i s3 = _mm256_sign_epi8(e3, f);
- __m256 g0 = dotsome(u, s0);
- __m256 g1 = dotsome(u, s1);
- __m256 g2 = dotsome(u, s2);
- __m256 g3 = dotsome(u, s3);
- c0 = _mm256_fmadd_ps(d0, g0, c0);
- c1 = _mm256_fmadd_ps(d1, g1, c1);
- c2 = _mm256_fmadd_ps(d2, g2, c2);
- c3 = _mm256_fmadd_ps(d3, g3, c3);
- }
- C[ldc * j + (i + 0)] = hsum8f(c0);
- C[ldc * j + (i + 1)] = hsum8f(c1);
- C[ldc * j + (i + 2)] = hsum8f(c2);
- C[ldc * j + (i + 3)] = hsum8f(c3);
- END_KERNEL()
- }
-
- dontinline void gemm1x4(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(1, 4)
- long k2 = k / 32;
- __m256 c0 = _mm256_setzero_ps();
- __m256 c1 = _mm256_setzero_ps();
- __m256 c2 = _mm256_setzero_ps();
- __m256 c3 = _mm256_setzero_ps();
- const U *Bp0 = B + ldb * (j + 0);
- const U *Bp1 = B + ldb * (j + 1);
- const U *Bp2 = B + ldb * (j + 2);
- const U *Bp3 = B + ldb * (j + 3);
- const T *Ap = A + lda * i;
- for (long l = 0; l < k2; ++l) {
- float da0 = unhalf(Ap[l].d);
- __m256i f = load32i(Ap + l);
- __m256i u = _mm256_sign_epi8(f, f);
- __m256 d0 = _mm256_set1_ps(unhalf(Bp0[l].d) * da0);
- __m256 d1 = _mm256_set1_ps(unhalf(Bp1[l].d) * da0);
- __m256 d2 = _mm256_set1_ps(unhalf(Bp2[l].d) * da0);
- __m256 d3 = _mm256_set1_ps(unhalf(Bp3[l].d) * da0);
- __m256 g0 = dotsome(u, _mm256_sign_epi8(load32i(Bp0 + l), f));
- __m256 g1 = dotsome(u, _mm256_sign_epi8(load32i(Bp1 + l), f));
- __m256 g2 = dotsome(u, _mm256_sign_epi8(load32i(Bp2 + l), f));
- __m256 g3 = dotsome(u, _mm256_sign_epi8(load32i(Bp3 + l), f));
- c0 = _mm256_fmadd_ps(d0, g0, c0);
- c1 = _mm256_fmadd_ps(d1, g1, c1);
- c2 = _mm256_fmadd_ps(d2, g2, c2);
- c3 = _mm256_fmadd_ps(d3, g3, c3);
- }
- C[ldc * (j + 0) + i] = hsum8f(c0);
- C[ldc * (j + 1) + i] = hsum8f(c1);
- C[ldc * (j + 2) + i] = hsum8f(c2);
- C[ldc * (j + 3) + i] = hsum8f(c3);
- END_KERNEL()
- }
-
- dontinline void gemm1x1(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(1, 1)
- long k2 = k / 32;
- __m256 c = _mm256_setzero_ps();
- const T *Ap = A + lda * i;
- const U *Bp = B + ldb * j;
- for (long l = 0; l < k2; ++l) {
- __m256 d = _mm256_set1_ps(unhalf(Ap[l].d) * unhalf(Bp[l].d));
- __m256i e = load32i(Ap + l);
- __m256i f = load32i(Bp + l);
- __m256 g = dotsome(_mm256_sign_epi8(e, e), _mm256_sign_epi8(f, e));
- c = _mm256_fmadd_ps(d, g, c);
- }
- C[ldc * j + i] = hsum8f(c);
- END_KERNEL()
- }
-
- inline __m256 dotsome(__m256i u, __m256i s) {
- __m256i res;
- if (avxvnni)
- res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);
- else
- res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));
- return _mm256_cvtepi32_ps(res);
- }
-
- const long k;
- const T *const A;
- const long lda;
- const U *const B;
- const long ldb;
- float *const C;
- const long ldc;
- const long ith;
- const long nth;
-};
-
-#endif // __x86_64__
-
-// computes m×k * n×k → n×m
-void llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda, const void *B,
- long ldb, float *C, long ldc, long ith, long nth) {
- assert(X86_HAVE(FMA));
- assert(X86_HAVE(AVX2));
- switch (dtype) {
-#ifdef __x86_64__
+#include "llamafile.h"
- case GGML_TYPE_F32: {
- tinyBLAS<float> tb{k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
- tb.gemm(m, n);
- break;
- }
+#include <cosmo.h>
+#include <stdio.h>
- case GGML_TYPE_F16: {
- assert(X86_HAVE(F16C));
- tinyBLAS<ggml_fp16_t> tb{
- k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, C, ldc, ith, nth,
- };
- tb.gemm(m, n);
- break;
- }
+#include "sgemm.h"
+
+bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B, int ldb, void *C,
+ int ldc, int ith, int nth, int task, int Atype, int Btype, int Ctype) {
+
+ if (!X86_HAVE(AVX))
+ return false;
+ if (Ctype != GGML_TYPE_F32)
+ return false;
+
+ switch (Atype) {
+
+ case GGML_TYPE_F32:
+ if (Btype != GGML_TYPE_F32)
+ return false;
+ if (X86_HAVE(AVX512F) && !(k % 16))
+ return llamafile_sgemm_sss_avx512f(m, n, k, (const float *)A, lda, (const float *)B,
+ ldb, (float *)C, ldc, ith, nth, task);
+ if (X86_HAVE(FMA) && !(k % 8))
+ return llamafile_sgemm_sss_fma(m, n, k, (const float *)A, lda, (const float *)B, ldb,
+ (float *)C, ldc, ith, nth, task);
+ if (!(k % 8))
+ return llamafile_sgemm_sss_avx(m, n, k, (const float *)A, lda, (const float *)B, ldb,
+ (float *)C, ldc, ith, nth, task);
+ return false;
+
+ case GGML_TYPE_F16:
+ if (Btype != GGML_TYPE_F32)
+ return false;
+ if (X86_HAVE(AVX512F) && !(k % 16))
+ return llamafile_sgemm_hss_avx512f(m, n, k, (const unsigned short *)A, lda,
+ (const float *)B, ldb, (float *)C, ldc, ith, nth,
+ task);
+ if (X86_HAVE(FMA) && X86_HAVE(F16C) && !(k % 8))
+ return llamafile_sgemm_hss_f16c(m, n, k, (const unsigned short *)A, lda,
+ (const float *)B, ldb, (float *)C, ldc, ith, nth, task);
+ return false;
case GGML_TYPE_Q8_0:
- if (X86_HAVE(AVXVNNI)) {
- tinyBLASq<block_q8_0, block_q8_0, true> tb{
- k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth,
- };
- tb.gemm(m, n);
- } else {
- tinyBLASq<block_q8_0, block_q8_0, false> tb{
- k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth,
- };
- tb.gemm(m, n);
- }
- break;
+ if (Btype != GGML_TYPE_Q8_0)
+ return false;
+ if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && !(k % 32))
+ return llamafile_sgemm_qqs_avx512vnni(m, n, k, (const block_q8_0 *)A, lda,
+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,
+ nth, task);
+ if (X86_HAVE(FMA) && X86_HAVE(AVXVNNI) && !(k % 32))
+ return llamafile_sgemm_qqs_avxvnni(m, n, k, (const block_q8_0 *)A, lda,
+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,
+ nth, task);
+ if (X86_HAVE(FMA) && X86_HAVE(AVX2) && !(k % 32))
+ return llamafile_sgemm_qqs_fma(m, n, k, (const block_q8_0 *)A, lda,
+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,
+ task);
+ return false;
case GGML_TYPE_Q4_0:
- if (X86_HAVE(AVXVNNI)) {
- tinyBLASq<block_q4_0, block_q8_0, true> tb{
- k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth,
- };
- tb.gemm(m, n);
- } else {
- tinyBLASq<block_q4_0, block_q8_0, false> tb{
- k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth,
- };
- tb.gemm(m, n);
- }
- break;
+ if (Btype != GGML_TYPE_Q8_0)
+ return false;
+ if (X86_HAVE(AVX512VL) && X86_HAVE(AVX512_VNNI) && !(k % 32))
+ return llamafile_sgemm_eqs_avx512vnni(m, n, k, (const block_q4_0 *)A, lda,
+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,
+ nth, task);
+ if (X86_HAVE(FMA) && X86_HAVE(AVXVNNI) && !(k % 32))
+ return llamafile_sgemm_eqs_avxvnni(m, n, k, (const block_q4_0 *)A, lda,
+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith,
+ nth, task);
+ if (X86_HAVE(FMA) && X86_HAVE(AVX2) && !(k % 32))
+ return llamafile_sgemm_eqs_fma(m, n, k, (const block_q4_0 *)A, lda,
+ (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth,
+ task);
+ return false;
-#endif
default:
- assert(!"unsupported dtype");
+ return false;
}
}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm.h b/llamafile/sgemm.h
new file mode 100644
index 00000000..14121d93
--- /dev/null
+++ b/llamafile/sgemm.h
@@ -0,0 +1,36 @@
+#pragma once
+#include "llama.cpp/ggml-quants.h"
+#include "llama.cpp/ggml.h"
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+bool llamafile_sgemm_sss_avx(int, int, int, const float *, int, const float *, int, float *, int,
+ int, int, int);
+bool llamafile_sgemm_sss_fma(int, int, int, const float *, int, const float *, int, float *, int,
+ int, int, int);
+bool llamafile_sgemm_sss_avx512f(int, int, int, const float *, int, const float *, int, float *,
+ int, int, int, int);
+
+bool llamafile_sgemm_hss_f16c(int, int, int, const unsigned short *, int, const float *, int,
+ float *, int, int, int, int);
+bool llamafile_sgemm_hss_avx512f(int, int, int, const unsigned short *, int, const float *, int,
+ float *, int, int, int, int);
+
+bool llamafile_sgemm_qqs_avx512vnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,
+ float *, int, int, int, int);
+bool llamafile_sgemm_qqs_avxvnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,
+ float *, int, int, int, int);
+bool llamafile_sgemm_qqs_fma(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,
+ float *, int, int, int, int);
+
+bool llamafile_sgemm_eqs_avx512vnni(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,
+ float *, int, int, int, int);
+bool llamafile_sgemm_eqs_avxvnni(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,
+ float *, int, int, int, int);
+bool llamafile_sgemm_eqs_fma(int, int, int, const block_q4_0 *, int, const block_q8_0 *, int,
+ float *, int, int, int, int);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/llamafile/sgemm_eqs_avx512vnni.cpp b/llamafile/sgemm_eqs_avx512vnni.cpp
new file mode 100644
index 00000000..78c00eb9
--- /dev/null
+++ b/llamafile/sgemm_eqs_avx512vnni.cpp
@@ -0,0 +1,36 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+
+#define TA block_q4_0
+#define TB block_q8_0
+#define TC float
+#include "llmmq0.inc"
+
+bool llamafile_sgemm_eqs_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_eqs_avxvnni.cpp b/llamafile/sgemm_eqs_avxvnni.cpp
new file mode 100644
index 00000000..22a572d4
--- /dev/null
+++ b/llamafile/sgemm_eqs_avxvnni.cpp
@@ -0,0 +1,36 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+
+#define TA block_q4_0
+#define TB block_q8_0
+#define TC float
+#include "llmmq0.inc"
+
+bool llamafile_sgemm_eqs_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_eqs_fma.cpp b/llamafile/sgemm_eqs_fma.cpp
new file mode 100644
index 00000000..9bb95430
--- /dev/null
+++ b/llamafile/sgemm_eqs_fma.cpp
@@ -0,0 +1,36 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+
+#define TA block_q4_0
+#define TB block_q8_0
+#define TC float
+#include "llmmq0.inc"
+
+bool llamafile_sgemm_eqs_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,
+ int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_hss_avx512f.cpp b/llamafile/sgemm_hss_avx512f.cpp
new file mode 100644
index 00000000..17d88bcc
--- /dev/null
+++ b/llamafile/sgemm_hss_avx512f.cpp
@@ -0,0 +1,41 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+
+#define KN 16
+#define V __m512
+#define TA unsigned short
+#define TB float
+#define TC float
+#define zero() _mm512_setzero_ps()
+#define load_a(p) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(p)))
+#define load_b(p) _mm512_loadu_ps(p)
+#include "llmm3x4.inc"
+
+bool llamafile_sgemm_hss_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_hss_f16c.cpp b/llamafile/sgemm_hss_f16c.cpp
new file mode 100644
index 00000000..ed16d736
--- /dev/null
+++ b/llamafile/sgemm_hss_f16c.cpp
@@ -0,0 +1,41 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+
+#define KN 8
+#define V __m256
+#define TA unsigned short
+#define TB float
+#define TC float
+#define zero() _mm256_setzero_ps()
+#define load_a(p) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(p)))
+#define load_b(p) _mm256_loadu_ps(p)
+#include "llmm3x4.inc"
+
+bool llamafile_sgemm_hss_f16c(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_qqs_avx512vnni.cpp b/llamafile/sgemm_qqs_avx512vnni.cpp
new file mode 100644
index 00000000..bd9254e7
--- /dev/null
+++ b/llamafile/sgemm_qqs_avx512vnni.cpp
@@ -0,0 +1,36 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+
+#define TA block_q8_0
+#define TB block_q8_0
+#define TC float
+#include "llmmq0.inc"
+
+bool llamafile_sgemm_qqs_avx512vnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_qqs_avxvnni.cpp b/llamafile/sgemm_qqs_avxvnni.cpp
new file mode 100644
index 00000000..cdde2d96
--- /dev/null
+++ b/llamafile/sgemm_qqs_avxvnni.cpp
@@ -0,0 +1,36 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+
+#define TA block_q8_0
+#define TB block_q8_0
+#define TC float
+#include "llmmq0.inc"
+
+bool llamafile_sgemm_qqs_avxvnni(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_qqs_fma.cpp b/llamafile/sgemm_qqs_fma.cpp
new file mode 100644
index 00000000..32b82c33
--- /dev/null
+++ b/llamafile/sgemm_qqs_fma.cpp
@@ -0,0 +1,36 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+
+#define TA block_q8_0
+#define TB block_q8_0
+#define TC float
+#include "llmmq0.inc"
+
+bool llamafile_sgemm_qqs_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,
+ int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ LLMMQ0 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_sss_avx.cpp b/llamafile/sgemm_sss_avx.cpp
new file mode 100644
index 00000000..6c73fc2f
--- /dev/null
+++ b/llamafile/sgemm_sss_avx.cpp
@@ -0,0 +1,41 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+
+#define KN 8
+#define V __m256
+#define TA float
+#define TB float
+#define TC float
+#define zero() _mm256_setzero_ps()
+#define load_a(p) _mm256_loadu_ps(p)
+#define load_b(p) _mm256_loadu_ps(p)
+#include "llmm3x4.inc"
+
+bool llamafile_sgemm_sss_avx(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,
+ int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_sss_avx512f.cpp b/llamafile/sgemm_sss_avx512f.cpp
new file mode 100644
index 00000000..1db08d29
--- /dev/null
+++ b/llamafile/sgemm_sss_avx512f.cpp
@@ -0,0 +1,41 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+
+#define KN 16
+#define V __m512
+#define TA float
+#define TB float
+#define TC float
+#define zero() _mm512_setzero_ps()
+#define load_a(p) _mm512_loadu_ps(p)
+#define load_b(p) _mm512_loadu_ps(p)
+#include "llmm3x4.inc"
+
+bool llamafile_sgemm_sss_avx512f(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_sss_fma.cpp b/llamafile/sgemm_sss_fma.cpp
new file mode 100644
index 00000000..84813ee7
--- /dev/null
+++ b/llamafile/sgemm_sss_fma.cpp
@@ -0,0 +1,41 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+
+#define KN 8
+#define V __m256
+#define TA float
+#define TB float
+#define TC float
+#define zero() _mm256_setzero_ps()
+#define load_a(p) _mm256_loadu_ps(p)
+#define load_b(p) _mm256_loadu_ps(p)
+#include "llmm3x4.inc"
+
+bool llamafile_sgemm_sss_fma(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb, TC *C,
+ int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ LLMM3X4 tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_test.cpp b/llamafile/sgemm_test.cpp
new file mode 100644
index 00000000..03978f51
--- /dev/null
+++ b/llamafile/sgemm_test.cpp
@@ -0,0 +1,90 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "gemm.h"
+#include "llama.cpp/ggml.h"
+#include "llamafile.h"
+
+int rand32(void) {
+ static unsigned long long lcg = 1;
+ lcg *= 6364136223846793005;
+ lcg += 1442695040888963407;
+ return lcg >> 32;
+}
+
+float float01(unsigned x) { // (0,1)
+ return 1.f / 8388608 * ((x >> 9) + .5f);
+}
+
+float numba(void) { // (-1,1)
+ return float01(rand32()) * 2 - 1;
+}
+
+template <typename T> void randomize(T *A, int n) {
+ for (int i = 0; i < n; ++i)
+ A[i] = numba();
+}
+
+template <typename T> void broadcast(T *A, int n, T x) {
+ for (int i = 0; i < n; ++i)
+ A[i] = x;
+}
+
+int main(int argc, char *argv[]) {
+ float tolerance = 1e-5;
+ int n = 32;
+ int m = 32;
+ int k = 1024;
+ int l = 0;
+ int lda = k + l;
+ int ldb = k + l;
+ int ldc = m + l;
+ float *A = new float[lda * m];
+ float *B = new float[ldb * n];
+ float *C = new float[ldc * n];
+ float *G = new float[ldc * n];
+ randomize(A, lda * m);
+ randomize(B, ldb * n);
+ broadcast(C, ldc * n, NAN);
+ broadcast(G, ldc * n, NAN);
+ gemm<double>(true, false, m, n, k, 1.f, A, lda, B, ldb, 0.f, G, ldc);
+ if (!llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,
+ GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32))
+ return 1;
+ for (int i = 0; i < m; ++i) {
+ for (int j = 0; j < n; ++j) {
+ auto g = G[ldc * j + i];
+ auto c = C[ldc * j + i];
+ if (isnan(g))
+ return 3;
+ if (isnan(c))
+ return 4;
+ auto diff = g - c;
+ if (diff < 0)
+ diff = -diff;
+ if (diff > tolerance)
+ return 5;
+ }
+ }
+}
+
+#else
+int main(int argc, char *argv[]) {
+}
+#endif // __x86_64__
diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cu
index 49e4edcd..f988f2ad 100644
--- a/llamafile/tinyblas.cu
+++ b/llamafile/tinyblas.cu
@@ -75,20 +75,6 @@ inline bool isone(float x) {
return x == 1;
}
-template <typename T, typename TA, typename TB>
-__device__ __forceinline__ void madd(T *tally, T *kahan, TA a, TB b) {
- T x = a;
- T y = b;
-#ifdef __FAST_MATH__
- *tally += x * y;
-#else
- T z = x * y - *kahan;
- T t = *tally + z;
- *kahan = (t - *tally) - z;
- *tally = t;
-#endif
-}
-
////////////////////////////////////////////////////////////////////////////////////////////////////
// tinyBLAS specialized matrix vector product kernel
@@ -98,16 +84,24 @@ __forceinline__ __device__ float warpSum(float x) {
return x;
}
+template <typename WORD, typename SRC>
+__device__ __forceinline__ void madd(WORD *tally, WORD *kahan, SRC a, SRC b) {
+ WORD x = a;
+ WORD y = b;
+ WORD z = x * y - *kahan;
+ WORD t = *tally + z;
+ *kahan = (t - *tally) - z;
+ *tally = t;
+}
+
template <typename WORD, typename SRC, typename DST>
static __device__ void matvec(int m, int k, const SRC *A, int lda, const SRC *B, DST *C) {
WORD Ct[WARPSIZE] = {0};
WORD Ce[WARPSIZE] = {0};
int i = blockIdx.y * WARPSIZE;
- for (int l = threadIdx.x; l < k; l += WARPSIZE) {
- WORD b = B[l];
+ for (int l = threadIdx.x; l < k; l += WARPSIZE)
for (int j = 0; j < WARPSIZE; ++j)
- madd(&Ct[j], &Ce[j], A[lda * (i + j) + l], b);
- }
+ madd(&Ct[j], &Ce[j], A[lda * (i + j) + l], B[l]);
for (int j = 0; j < WARPSIZE; ++j) {
WORD c = warpSum(Ct[j]);
if (!threadIdx.x)
@@ -134,7 +128,7 @@ static tinyblasStatus_t matvec_launch(tinyblasHandle_t handle, int m, int k, con
template <typename WORD>
static bool can_use_matvec(tinyblasOperation_t aT, tinyblasOperation_t bT, int m, int n, int k,
WORD alpha, WORD beta) {
- return n == 1 && aT && !bT && //
+ return n == 1 && k >= 4096 && aT && !bT && //
!(m % WARPSIZE) && !(k % WARPSIZE) && //
isone(alpha) && !beta;
}
@@ -171,7 +165,6 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, tinyblasOperat
WORD At[TM];
WORD Bt[TN];
WORD Ct[TM * TN] = {0};
- WORD Ce[TM * TN] = {0};
if (CONFIG & ASSUME_A_OP_T)
transa = TINYBLAS_OP_T;
@@ -205,7 +198,7 @@ static __device__ void matmul_block2d(tinyblasOperation_t transa, tinyblasOperat
Bt[h] = Bs[BN * l + tj + h];
for (int j = 0; j < TM; ++j)
for (int h = 0; h < TN; ++h)
- madd(&Ct[TN * j + h], &Ce[TN * j + h], At[j], Bt[h]);
+ Ct[TN * j + h] += At[j] * Bt[h];
}
__syncthreads();
@@ -277,7 +270,6 @@ static __device__ void matmul_warp2d(tinyblasOperation_t aT, //
WORD Ar[WMI * TM] = {0};
WORD Br[WNI * TN] = {0};
WORD Ct[WMI * TM * WNI * TN] = {0};
- WORD Ce[WMI * TM * WNI * TN] = {0};
if (CONFIG & ASSUME_A_OP_T)
aT = TINYBLAS_OP_T;
@@ -327,9 +319,8 @@ static __device__ void matmul_warp2d(tinyblasOperation_t aT, //
for (int jj = 0; jj < WNI; ++jj)
for (int i = 0; i < TM; ++i)
for (int j = 0; j < TN; ++j)
- madd(&Ct[(WNI * TN) * (TM * ii + i) + (TN * jj) + j],
- &Ce[(WNI * TN) * (TM * ii + i) + (TN * jj) + j], Ar[TM * ii + i],
- Br[TN * jj + j]);
+ Ct[(WNI * TN) * (TM * ii + i) + (TN * jj) + j] +=
+ Ar[TM * ii + i] * Br[TN * jj + j];
}
__syncthreads();
@@ -601,9 +592,9 @@ tinyblasStatus_t tinyblasGemmEx(tinyblasHandle_t handle, //
case TINYBLAS_R_16F:
switch (computeType) {
case TINYBLAS_COMPUTE_16F:
- return tinyblasGE_launch(handle, transa, transb, m, n, k, *(const half *)alpha,
- (const half *)A, lda, (const half *)B, ldb,
- *(const half *)beta, (half *)C, ldc);
+ return tinyblasGE_launch(
+ handle, transa, transb, m, n, k, (float)*(const half *)alpha, (const half *)A,
+ lda, (const half *)B, ldb, (float)*(const half *)beta, (half *)C, ldc);
case TINYBLAS_COMPUTE_32F:
return tinyblasGE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,
(const half *)A, lda, (const half *)B, ldb,
@@ -752,10 +743,10 @@ tinyblasStatus_t tinyblasGemmBatchedEx(tinyblasHandle_t handle, tinyblasOperatio
case TINYBLAS_R_16F:
switch (computeType) {
case TINYBLAS_COMPUTE_16F:
- return tinyblasGBE_launch(handle, transa, transb, m, n, k, *(const half *)alpha,
- (const half *const *)Aarray, lda,
- (const half *const *)Barray, ldb, *(const half *)beta,
- (half *const *)Carray, ldc, batchCount);
+ return tinyblasGBE_launch(
+ handle, transa, transb, m, n, k, (float)*(const half *)alpha,
+ (const half *const *)Aarray, lda, (const half *const *)Barray, ldb,
+ (float)*(const half *)beta, (half *const *)Carray, ldc, batchCount);
case TINYBLAS_COMPUTE_32F:
return tinyblasGBE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,
(const half *const *)Aarray, lda,
@@ -936,10 +927,10 @@ tinyblasStatus_t tinyblasGemmStridedBatchedEx(tinyblasHandle_t handle, //
case TINYBLAS_R_16F:
switch (computeType) {
case TINYBLAS_COMPUTE_16F:
- return tinyblasGSBE_launch(handle, transa, transb, m, n, k, *(const half *)alpha,
- (const half *)A, lda, strideA, (const half *)B, ldb,
- strideB, *(const half *)beta, (half *)C, ldc, strideC,
- batchCount);
+ return tinyblasGSBE_launch(
+ handle, transa, transb, m, n, k, (float)*(const half *)alpha, (const half *)A,
+ lda, strideA, (const half *)B, ldb, strideB, (float)*(const half *)beta,
+ (half *)C, ldc, strideC, batchCount);
case TINYBLAS_COMPUTE_32F:
return tinyblasGSBE_launch(handle, transa, transb, m, n, k, *(const float *)alpha,
(const half *)A, lda, strideA, (const half *)B, ldb,
diff --git a/llamafile/tinyblas_test.cu b/llamafile/tinyblas_test.cu
index f52d701c..75755b47 100644
--- a/llamafile/tinyblas_test.cu
+++ b/llamafile/tinyblas_test.cu
@@ -18,7 +18,7 @@
#include "tester.h"
void checkTinyblasWorksHHHH() {
- double TOLERANCE = 1e-3;
+ double TOLERANCE = 1e-2;
half alpha = 1;
half beta = 0;
int m = 128;
@@ -35,7 +35,7 @@ void checkTinyblasWorksHHHH() {
randomize(k, n, B.p, ldb);
broadcast(m, n, C.p, ldc, TOMBSTONE);
broadcast(m, n, G.p, ldc, TOMBSTONE);
- gemmref(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ gemmref<double>(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
tinyblas(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
@@ -51,14 +51,14 @@ void checkTinyblasWorksHHHH() {
randomize(k, n, B.p, ldb);
broadcast(m, n, G.p, ldc, TOMBSTONE);
broadcast(m, n, C.p, ldc, TOMBSTONE);
- gemmref(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ gemmref<double>(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
tinyblas(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
});
}
void checkTinyblasWorksHHHS() {
- double TOLERANCE = 1e-5;
+ double TOLERANCE = 1e-3;
float alpha = 1;
float beta = 0;
int m = 4096;
@@ -75,7 +75,7 @@ void checkTinyblasWorksHHHS() {
randomize(k, n, B.p, ldb);
broadcast(m, n, C.p, ldc, TOMBSTONE);
broadcast(m, n, G.p, ldc, TOMBSTONE);
- gemmref(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ gemmref<double>(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
tinyblas(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
@@ -91,14 +91,14 @@ void checkTinyblasWorksHHHS() {
randomize(k, n, B.p, ldb);
broadcast(m, n, G.p, ldc, TOMBSTONE);
broadcast(m, n, C.p, ldc, TOMBSTONE);
- gemmref(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ gemmref<double>(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
tinyblas(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
});
}
void checkTinyblasWorksHHSS() {
- double TOLERANCE = 1e-5;
+ double TOLERANCE = 1e-4;
float alpha = 1;
float beta = 0;
int m = 4096;
@@ -115,7 +115,7 @@ void checkTinyblasWorksHHSS() {
randomize(k, n, B.p, ldb);
broadcast(m, n, G.p, ldc, TOMBSTONE);
broadcast(m, n, C.p, ldc, TOMBSTONE);
- gemmref(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ gemmref<double>(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
tinyblas(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
@@ -131,14 +131,14 @@ void checkTinyblasWorksHHSS() {
randomize(k, n, B.p, ldb);
broadcast(m, n, G.p, ldc, TOMBSTONE);
broadcast(m, n, C.p, ldc, TOMBSTONE);
- gemmref(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ gemmref<double>(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
tinyblas(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
});
}
void checkTinyblasWorksSSSS() {
- double TOLERANCE = 1e-5;
+ double TOLERANCE = 1e-4;
float alpha = 1;
float beta = 0;
int m = 4096;
@@ -155,7 +155,7 @@ void checkTinyblasWorksSSSS() {
randomize(k, n, B.p, ldb);
broadcast(m, n, C.p, ldc, TOMBSTONE);
broadcast(m, n, G.p, ldc, TOMBSTONE);
- gemmref(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ gemmref<double>(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
tinyblas(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
@@ -171,7 +171,7 @@ void checkTinyblasWorksSSSS() {
randomize(k, n, B.p, ldb);
broadcast(m, n, G.p, ldc, TOMBSTONE);
broadcast(m, n, C.p, ldc, TOMBSTONE);
- gemmref(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ gemmref<double>(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
tinyblas(0, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(TOLERANCE, m, n, k, G.p, ldc, C.p, ldc);
});
@@ -192,13 +192,13 @@ void try_size(int m, int n, int k) {
randomize(k, n, B.p, ldb);
broadcast(m, n, C.p, ldc, TOMBSTONE);
broadcast(m, n, G.p, ldc, TOMBSTONE);
- gemmref(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
+ gemmref<double>(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, G.p, ldc);
tinyblas(1, 0, m, n, k, alpha, A.p, lda, B.p, ldb, beta, C.p, ldc);
CHECK(1, m, n, k, G.p, ldc, C.p, ldc);
}
void test_gsbe() {
- double TOLERANCE = 1e-5;
+ double TOLERANCE = 1e-4;
test_matmul([&](int m, int n, int k, int l, float alpha, float beta) {
int lda = k;
int ldb = k;
@@ -213,7 +213,7 @@ void test_gsbe() {
broadcast(m, n, G.p + 0 * ldc * n, ldc, TOMBSTONE);
broadcast(m, n, C.p + 1 * ldc * n, ldc, TOMBSTONE);
broadcast(m, n, G.p + 1 * ldc * n, ldc, TOMBSTONE);
- gsberef(1, 0, m, n, k, alpha, A.p, lda, 0, B.p, ldb, 0, beta, G.p, ldc, ldc * n, 2);
+ gsberef<double>(1, 0, m, n, k, alpha, A.p, lda, 0, B.p, ldb, 0, beta, G.p, ldc, ldc * n, 2);
tinyblasGSBE(1, 0, m, n, k, alpha, A.p, lda, 0, B.p, ldb, 0, beta, C.p, ldc, ldc * n, 2);
CHECK(TOLERANCE, m, n, k, G.p + 0 * ldc * n, ldc, C.p + 0 * ldc * n, ldc);
CHECK(TOLERANCE, m, n, k, G.p + 0 * ldc * n, ldc, C.p + 0 * ldc * n, ldc);