Skip to content

Distribute and run LLMs with a single file.

Upgrade to Cosmopolitan 3.3

Justine Tunney5b109da5d27c
diff --git a/build/config.mk b/build/config.mk
index bf25ea30..a4b82a08 100644
--- a/build/config.mk
+++ b/build/config.mk
@@ -2,7 +2,7 @@
#── vi: set noet ft=make ts=8 sw=8 fenc=utf-8 :vi ────────────────────┘
PREFIX = /usr/local
-COSMOCC = .cosmocc/3.2.4
+COSMOCC = .cosmocc/3.3
TOOLCHAIN = $(COSMOCC)/bin/cosmo
AR = $(TOOLCHAIN)ar
@@ -50,5 +50,5 @@ clean:; rm -rf o
.PHONY: distclean
distclean:; rm -rf o cosmocc
-.cosmocc/3.2.4:
- build/download-cosmocc.sh $@ 3.2.4 d2fa6dbf6f987310494581deff5b915dbdc5ca701f20f7613bb0dcf1de2ee511
+.cosmocc/3.3:
+ build/download-cosmocc.sh $@ 3.3 d26ec8f4e48f6db004fc6a9677c7ff3b50c3b21e936e9393158aa2ed51b0b549
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 65ff90f5..053bb44e 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -33,7 +33,6 @@ SOFTWARE.\"");
#include "ggml-metal.h"
#include "ggml-cuda.h"
#include "llamafile/log.h"
-#include "libc/log/backtrace.internal.h"
#include <alloca.h>
#include <assert.h>
@@ -51,6 +50,7 @@ SOFTWARE.\"");
#include <signal.h>
#include <unistd.h>
#include <cosmo.h>
+#include <libc/log/backtrace.internal.h>
#include <pthread.h>
#include <stdatomic.h>
@@ -10106,8 +10106,9 @@ static void ggml_compute_forward_mul_mat(
(dst->type == GGML_TYPE_F32 &&
src1->type == GGML_TYPE_F32 &&
(src0->type == GGML_TYPE_F32 ||
- (src0->type == GGML_TYPE_F16 && X86_HAVE(F16C)) ||
- (src0->type == GGML_TYPE_Q8_0 && X86_HAVE(AVXVNNI)))) &&
+ src0->type == GGML_TYPE_Q4_0 ||
+ src0->type == GGML_TYPE_Q8_0 ||
+ (src0->type == GGML_TYPE_F16 && X86_HAVE(F16C)))) &&
ne00 % 8 == 0 &&
ggml_is_contiguous(src0) &&
ggml_is_contiguous(src1) &&
diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cpp
index 560b510f..4fbf4289 100644
--- a/llama.cpp/main/main.cpp
+++ b/llama.cpp/main/main.cpp
@@ -15,8 +15,8 @@
#include <string>
#include <unistd.h>
#include <vector>
+#include <tool/args/args.h>
-#include "tool/args/args.h"
#include "llamafile/version.h"
#include "llama.cpp/llama.h"
#include "llama.cpp/common.h"
diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cpp
index e10358b3..3f6a7be4 100644
--- a/llama.cpp/server/server.cpp
+++ b/llama.cpp/server/server.cpp
@@ -8,6 +8,9 @@
#include <condition_variable>
#include <cstddef>
#include <thread>
+#include <libc/calls/pledge.h>
+#include <tool/args/args.h>
+#include <libc/dce.h>
#include "llama.cpp/common.h"
#include "llama.cpp/ggml-cuda.h"
@@ -19,9 +22,6 @@
#include "llamafile/llamafile.h"
#include "llamafile/log.h"
#include "llamafile/version.h"
-#include "libc/calls/pledge.h"
-#include "tool/args/args.h"
-#include "libc/dce.h"
#include "oai.h"
#include "macsandbox.h"
diff --git a/llamafile/check_cpu.c b/llamafile/check_cpu.c
index 975c901b..f79b9bed 100644
--- a/llamafile/check_cpu.c
+++ b/llamafile/check_cpu.c
@@ -15,21 +15,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-#include <errno.h>
#include <cosmo.h>
+#include <errno.h>
#include <stdlib.h>
-static int on_missing_feature(const char *name) {
- tinyprint(2, program_invocation_name, ": fatal error: the cpu feature ", name,
- " was required at build time but isn't available on this system\n", NULL);
-#if defined(__AVX2__) && !defined(__AVX512F__)
- tinyprint(2, "note: amd microprocessors made after 2017 usually work\n"
- "note: intel microprocessors made after 2013 usually work\n", NULL);
-#endif
- tinyprint(2, "exiting process.\n", NULL);
- exit(1);
-}
-
/**
* Dies if CPU doesn't have mandatory features.
*
@@ -39,6 +28,29 @@ static int on_missing_feature(const char *name) {
* microarchitecture flags only get passed to that specific object
*/
void llamafile_check_cpu(void) {
+ // side effect: the constructor below has now been linked
+}
+
+static int on_missing_feature(const char *name) {
+ tinyprint(
+ 2, program_invocation_name, ": fatal error: the cpu feature ", name,
+ " was required at build time but isn't available on this system\n",
+ NULL);
+#if defined(__AVX2__) && !defined(__AVX512F__)
+ tinyprint(2,
+ "note: amd microprocessors made after 2017 usually work\n"
+ "note: intel microprocessors made after 2013 usually work\n",
+ NULL);
+#endif
+ tinyprint(2, "exiting process.\n", NULL);
+ _Exit(1);
+}
+
+// We need to perform this early in the initialization process, before
+// C++ codes built with -mavx has a chance to start allocating dynamic
+// memory that would otherwise crash the cpu before this could be done
+__attribute__((__constructor__(101))) static void
+llamafile_actually_check_cpu(void) {
if (X86_NEED(SSE3) && !X86_CHECK(SSE3)) {
on_missing_feature("SSE3");
}
diff --git a/llamafile/gemm_avx2.cpp b/llamafile/gemm_avx2.cpp
deleted file mode 100644
index f4d73e1a..00000000
--- a/llamafile/gemm_avx2.cpp
+++ /dev/null
@@ -1,639 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "llama.cpp/ggml-quants.h"
-#include "llamafile.h"
-#include <immintrin.h>
-#include <stdio.h>
-
-//
-// _ _ ___ _ _ ___
-// | |_(_)_ _ _ _| _ ) | /_\ / __|
-// | _| | ' \ || | _ \ |__ / _ \\__ \.
-// \__|_|_||_\_, |___/____/_/ \_\___/
-// |__/
-//
-// BASIC LINEAR ALGEBRA SUBPROGRAMS
-//
-
-#define ASSERT(x) \
- if (!(x)) { \
- fprintf(stderr, "%s:%d: assertion failed: %s\n", __FILE__, __LINE__, \
- #x); \
- __builtin_trap(); \
- }
-
-#define BEGIN_KERNEL(RM, RN) \
- long ytiles = (m - m0) / RM; \
- long xtiles = (n - n0) / RN; \
- long tiles = ytiles * xtiles; \
- double duty = (double)tiles / nth; \
- if (duty < 1) \
- duty = 1; \
- double spot = duty * ith + .5; \
- long end = spot + duty; \
- long start = spot; \
- if (end > tiles) \
- end = tiles; \
- for (long job = start; job < end; ++job) { \
- long i = m0 + job / xtiles * RM; \
- long j = n0 + job % xtiles * RN;
-
-#define END_KERNEL() }
-
-#ifdef __x86_64__
-#pragma GCC push_options
-#pragma GCC target("avx2,fma,f16c")
-
-dontinline static float hsum_float_8(__m256 x) {
- __m128 res = _mm256_extractf128_ps(x, 1);
- res = _mm_add_ps(res, _mm256_castps256_ps128(x));
- res = _mm_add_ps(res, _mm_movehl_ps(res, res));
- res = _mm_add_ss(res, _mm_movehdup_ps(res));
- return _mm_cvtss_f32(res);
-}
-
-static inline __m256 loadf16x8(const ggml_fp16_t *p) {
- return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));
-}
-
-static inline __m256i vpdpbusd(__m256i x, __m256i y, __m256i s) {
- register __m256i ymm2 asm("ymm2") = x;
- register __m256i ymm1 asm("ymm1") = y;
- register __m256i ymm0 asm("ymm0") = s;
- asm(".byte\t0xc4,0xe2,0x6d,0x50,0xc1" : "+x"(ymm0) : "x"(ymm1), "x"(ymm2));
- return ymm0;
-}
-
-static inline __m256 mul_sum_i8_pairs_float(__m256i x, __m256i y) {
- return _mm256_cvtepi32_ps(
- vpdpbusd(_mm256_sign_epi8(x, x), // make unsigned
- _mm256_sign_epi8(y, x), // make signed
- _mm256_setzero_si256())); // no accumulator
-}
-
-static inline __m256 mul_sum_signed_i8_pairs_float(__m256i u, __m256i s) {
- return _mm256_cvtepi32_ps(vpdpbusd(u, s, _mm256_setzero_si256()));
-}
-
-class tinyBLAS_F32_AVX2 {
- public:
- tinyBLAS_F32_AVX2(long k, const float *A, long lda, const float *B,
- long ldb, float *C, long ldc, long ith, long nth)
- : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith),
- nth(nth) {
- ASSERT(A != nullptr);
- ASSERT(B != nullptr);
- ASSERT(C != nullptr);
- ASSERT(k >= 0 && k % 8 == 0);
- ASSERT(ith >= 0 && ith < nth);
- }
-
- void gemm(long m, long n) {
- ASSERT(m >= 0);
- ASSERT(n >= 0);
- ASSERT(lda >= k);
- ASSERT(ldb >= k);
- ASSERT(ldc >= m);
- mnpack(0, m, 0, n);
- }
-
- private:
- void mnpack(long m0, long m, long n0, long n) {
- if (m - m0 <= 0 || n - n0 <= 0)
- return;
- long mc, nc, mp, np;
- if (m - m0 >= 3 && n - n0 >= 4) {
- mc = 3;
- nc = 4;
- gemm3x4(m0, m, n0, n);
- } else if (m - m0 >= 4 && n - n0 >= 1) {
- mc = 4;
- nc = 1;
- gemm4x1(m0, m, n0, n);
- } else if (m - m0 >= 1 && n - n0 >= 4) {
- mc = 1;
- nc = 4;
- gemm1x4(m0, m, n0, n);
- } else {
- mc = 1;
- nc = 1;
- gemm1x1(m0, m, n0, n);
- }
- mp = m0 + (m - m0) / mc * mc;
- np = n0 + (n - n0) / nc * nc;
- mnpack(mp, m, n0, np);
- mnpack(m0, mp, np, n);
- mnpack(mp, m, np, n);
- }
-
- dontinline void gemm3x4(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(3, 4)
- __m256 c00 = _mm256_setzero_ps();
- __m256 c01 = _mm256_setzero_ps();
- __m256 c02 = _mm256_setzero_ps();
- __m256 c03 = _mm256_setzero_ps();
- __m256 c10 = _mm256_setzero_ps();
- __m256 c11 = _mm256_setzero_ps();
- __m256 c12 = _mm256_setzero_ps();
- __m256 c13 = _mm256_setzero_ps();
- __m256 c20 = _mm256_setzero_ps();
- __m256 c21 = _mm256_setzero_ps();
- __m256 c22 = _mm256_setzero_ps();
- __m256 c23 = _mm256_setzero_ps();
- for (long l = 0; l < k; l += 8) {
- __m256 k0 = _mm256_loadu_ps(B + ldb * (j + 0) + l);
- __m256 k1 = _mm256_loadu_ps(B + ldb * (j + 1) + l);
- __m256 k2 = _mm256_loadu_ps(B + ldb * (j + 2) + l);
- __m256 k3 = _mm256_loadu_ps(B + ldb * (j + 3) + l);
- __m256 a0 = _mm256_loadu_ps(A + lda * (i + 0) + l);
- c00 = _mm256_fmadd_ps(a0, k0, c00);
- c01 = _mm256_fmadd_ps(a0, k1, c01);
- c02 = _mm256_fmadd_ps(a0, k2, c02);
- c03 = _mm256_fmadd_ps(a0, k3, c03);
- __m256 a1 = _mm256_loadu_ps(A + lda * (i + 1) + l);
- c10 = _mm256_fmadd_ps(a1, k0, c10);
- c11 = _mm256_fmadd_ps(a1, k1, c11);
- c12 = _mm256_fmadd_ps(a1, k2, c12);
- c13 = _mm256_fmadd_ps(a1, k3, c13);
- __m256 a2 = _mm256_loadu_ps(A + lda * (i + 2) + l);
- c20 = _mm256_fmadd_ps(a2, k0, c20);
- c21 = _mm256_fmadd_ps(a2, k1, c21);
- c22 = _mm256_fmadd_ps(a2, k2, c22);
- c23 = _mm256_fmadd_ps(a2, k3, c23);
- }
- C[ldc * (j + 0) + (i + 0)] = hsum_float_8(c00);
- C[ldc * (j + 0) + (i + 1)] = hsum_float_8(c10);
- C[ldc * (j + 0) + (i + 2)] = hsum_float_8(c20);
- C[ldc * (j + 1) + (i + 0)] = hsum_float_8(c01);
- C[ldc * (j + 1) + (i + 1)] = hsum_float_8(c11);
- C[ldc * (j + 1) + (i + 2)] = hsum_float_8(c21);
- C[ldc * (j + 2) + (i + 0)] = hsum_float_8(c02);
- C[ldc * (j + 2) + (i + 1)] = hsum_float_8(c12);
- C[ldc * (j + 2) + (i + 2)] = hsum_float_8(c22);
- C[ldc * (j + 3) + (i + 0)] = hsum_float_8(c03);
- C[ldc * (j + 3) + (i + 1)] = hsum_float_8(c13);
- C[ldc * (j + 3) + (i + 2)] = hsum_float_8(c23);
- END_KERNEL()
- }
-
- dontinline void gemm1x4(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(1, 4)
- __m256 c00 = _mm256_setzero_ps();
- __m256 c01 = _mm256_setzero_ps();
- __m256 c02 = _mm256_setzero_ps();
- __m256 c03 = _mm256_setzero_ps();
- for (long l = 0; l < k; l += 8) {
- __m256 a0 = _mm256_loadu_ps(A + lda * (i + 0) + l);
- __m256 k0 = _mm256_loadu_ps(B + ldb * (j + 0) + l);
- __m256 k1 = _mm256_loadu_ps(B + ldb * (j + 1) + l);
- __m256 k2 = _mm256_loadu_ps(B + ldb * (j + 2) + l);
- __m256 k3 = _mm256_loadu_ps(B + ldb * (j + 3) + l);
- c00 = _mm256_fmadd_ps(a0, k0, c00);
- c01 = _mm256_fmadd_ps(a0, k1, c01);
- c02 = _mm256_fmadd_ps(a0, k2, c02);
- c03 = _mm256_fmadd_ps(a0, k3, c03);
- }
- C[ldc * (j + 0) + (i + 0)] = hsum_float_8(c00);
- C[ldc * (j + 1) + (i + 0)] = hsum_float_8(c01);
- C[ldc * (j + 2) + (i + 0)] = hsum_float_8(c02);
- C[ldc * (j + 3) + (i + 0)] = hsum_float_8(c03);
- END_KERNEL()
- }
-
- dontinline void gemm4x1(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(4, 1)
- __m256 c00 = _mm256_setzero_ps();
- __m256 c10 = _mm256_setzero_ps();
- __m256 c20 = _mm256_setzero_ps();
- __m256 c30 = _mm256_setzero_ps();
- for (long l = 0; l < k; l += 8) {
- __m256 k0 = _mm256_loadu_ps(B + ldb * (j + 0) + l);
- __m256 a0 = _mm256_loadu_ps(A + lda * (i + 0) + l);
- c00 = _mm256_fmadd_ps(a0, k0, c00);
- __m256 a1 = _mm256_loadu_ps(A + lda * (i + 1) + l);
- c10 = _mm256_fmadd_ps(a1, k0, c10);
- __m256 a2 = _mm256_loadu_ps(A + lda * (i + 2) + l);
- c20 = _mm256_fmadd_ps(a2, k0, c20);
- __m256 a3 = _mm256_loadu_ps(A + lda * (i + 3) + l);
- c30 = _mm256_fmadd_ps(a3, k0, c30);
- }
- C[ldc * (j + 0) + (i + 0)] = hsum_float_8(c00);
- C[ldc * (j + 0) + (i + 1)] = hsum_float_8(c10);
- C[ldc * (j + 0) + (i + 2)] = hsum_float_8(c20);
- C[ldc * (j + 0) + (i + 3)] = hsum_float_8(c30);
- END_KERNEL()
- }
-
- dontinline void gemm1x1(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(1, 1)
- __m256 c = _mm256_setzero_ps();
- for (long l = 0; l < k; l += 8) {
- c = _mm256_fmadd_ps(_mm256_loadu_ps(A + lda * i + l),
- _mm256_loadu_ps(B + ldb * j + l), c);
- }
- C[ldc * j + i] = hsum_float_8(c);
- END_KERNEL()
- }
-
- const long k;
- const float *const A;
- const long lda;
- const float *const B;
- const long ldb;
- float *const C;
- const long ldc;
- const long ith;
- const long nth;
-};
-
-class tinyBLAS_F16_AVX2 {
- public:
- tinyBLAS_F16_AVX2(long k, const ggml_fp16_t *A, long lda,
- const ggml_fp16_t *B, long ldb, float *C, long ldc,
- long ith, long nth)
- : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith),
- nth(nth) {
- ASSERT(A != nullptr);
- ASSERT(B != nullptr);
- ASSERT(C != nullptr);
- ASSERT(k >= 0 && k % 16 == 0);
- ASSERT(ith >= 0 && ith < nth);
- }
-
- void gemm(long m, long n) {
- ASSERT(m >= 0);
- ASSERT(n >= 0);
- ASSERT(ldc >= m);
- mnpack(0, m, 0, n);
- }
-
- private:
- void mnpack(long m0, long m, long n0, long n) {
- if (m - m0 <= 0 || n - n0 <= 0)
- return;
- long mc, nc, mp, np;
- if (m - m0 >= 3 && n - n0 >= 4) {
- mc = 3;
- nc = 4;
- gemm3x4(m0, m, n0, n);
- } else if (m - m0 >= 4 && n - n0 >= 1) {
- mc = 4;
- nc = 1;
- gemm4x1(m0, m, n0, n);
- } else if (m - m0 >= 1 && n - n0 >= 4) {
- mc = 1;
- nc = 4;
- gemm1x4(m0, m, n0, n);
- } else {
- mc = 1;
- nc = 1;
- gemm1x1(m0, m, n0, n);
- }
- mp = m0 + (m - m0) / mc * mc;
- np = n0 + (n - n0) / nc * nc;
- mnpack(mp, m, n0, np);
- mnpack(m0, mp, np, n);
- mnpack(mp, m, np, n);
- }
-
- dontinline void gemm3x4(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(3, 4)
- __m256 c00 = _mm256_setzero_ps();
- __m256 c01 = _mm256_setzero_ps();
- __m256 c02 = _mm256_setzero_ps();
- __m256 c03 = _mm256_setzero_ps();
- __m256 c10 = _mm256_setzero_ps();
- __m256 c11 = _mm256_setzero_ps();
- __m256 c12 = _mm256_setzero_ps();
- __m256 c13 = _mm256_setzero_ps();
- __m256 c20 = _mm256_setzero_ps();
- __m256 c21 = _mm256_setzero_ps();
- __m256 c22 = _mm256_setzero_ps();
- __m256 c23 = _mm256_setzero_ps();
- const ggml_fp16_t *Ap0 = A + lda * (i + 0);
- const ggml_fp16_t *Ap1 = A + lda * (i + 1);
- const ggml_fp16_t *Ap2 = A + lda * (i + 2);
- const ggml_fp16_t *Bp0 = B + ldb * (j + 0);
- const ggml_fp16_t *Bp1 = B + ldb * (j + 1);
- const ggml_fp16_t *Bp2 = B + ldb * (j + 2);
- const ggml_fp16_t *Bp3 = B + ldb * (j + 3);
- for (long l = 0; l < k; l += 8) {
- __m256 k0 = loadf16x8(Bp0 + l);
- __m256 k1 = loadf16x8(Bp1 + l);
- __m256 k2 = loadf16x8(Bp2 + l);
- __m256 k3 = loadf16x8(Bp3 + l);
- __m256 a0 = loadf16x8(Ap0 + l);
- c00 = _mm256_fmadd_ps(a0, k0, c00);
- c01 = _mm256_fmadd_ps(a0, k1, c01);
- c02 = _mm256_fmadd_ps(a0, k2, c02);
- c03 = _mm256_fmadd_ps(a0, k3, c03);
- __m256 a1 = loadf16x8(Ap1 + l);
- c10 = _mm256_fmadd_ps(a1, k0, c10);
- c11 = _mm256_fmadd_ps(a1, k1, c11);
- c12 = _mm256_fmadd_ps(a1, k2, c12);
- c13 = _mm256_fmadd_ps(a1, k3, c13);
- __m256 a2 = loadf16x8(Ap2 + l);
- c20 = _mm256_fmadd_ps(a2, k0, c20);
- c21 = _mm256_fmadd_ps(a2, k1, c21);
- c22 = _mm256_fmadd_ps(a2, k2, c22);
- c23 = _mm256_fmadd_ps(a2, k3, c23);
- }
- C[ldc * (j + 0) + (i + 0)] = hsum_float_8(c00);
- C[ldc * (j + 0) + (i + 1)] = hsum_float_8(c10);
- C[ldc * (j + 0) + (i + 2)] = hsum_float_8(c20);
- C[ldc * (j + 1) + (i + 0)] = hsum_float_8(c01);
- C[ldc * (j + 1) + (i + 1)] = hsum_float_8(c11);
- C[ldc * (j + 1) + (i + 2)] = hsum_float_8(c21);
- C[ldc * (j + 2) + (i + 0)] = hsum_float_8(c02);
- C[ldc * (j + 2) + (i + 1)] = hsum_float_8(c12);
- C[ldc * (j + 2) + (i + 2)] = hsum_float_8(c22);
- C[ldc * (j + 3) + (i + 0)] = hsum_float_8(c03);
- C[ldc * (j + 3) + (i + 1)] = hsum_float_8(c13);
- C[ldc * (j + 3) + (i + 2)] = hsum_float_8(c23);
- END_KERNEL()
- }
-
- dontinline void gemm1x4(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(1, 4)
- __m256 c00 = _mm256_setzero_ps();
- __m256 c01 = _mm256_setzero_ps();
- __m256 c02 = _mm256_setzero_ps();
- __m256 c03 = _mm256_setzero_ps();
- for (long l = 0; l < k; l += 8) {
- __m256 a0 = loadf16x8(A + lda * (i + 0) + l);
- __m256 k0 = loadf16x8(B + ldb * (j + 0) + l);
- __m256 k1 = loadf16x8(B + ldb * (j + 1) + l);
- __m256 k2 = loadf16x8(B + ldb * (j + 2) + l);
- __m256 k3 = loadf16x8(B + ldb * (j + 3) + l);
- c00 = _mm256_fmadd_ps(a0, k0, c00);
- c01 = _mm256_fmadd_ps(a0, k1, c01);
- c02 = _mm256_fmadd_ps(a0, k2, c02);
- c03 = _mm256_fmadd_ps(a0, k3, c03);
- }
- C[ldc * (j + 0) + (i + 0)] = hsum_float_8(c00);
- C[ldc * (j + 1) + (i + 0)] = hsum_float_8(c01);
- C[ldc * (j + 2) + (i + 0)] = hsum_float_8(c02);
- C[ldc * (j + 3) + (i + 0)] = hsum_float_8(c03);
- END_KERNEL()
- }
-
- dontinline void gemm4x1(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(4, 1)
- __m256 c00 = _mm256_setzero_ps();
- __m256 c10 = _mm256_setzero_ps();
- __m256 c20 = _mm256_setzero_ps();
- __m256 c30 = _mm256_setzero_ps();
- for (long l = 0; l < k; l += 8) {
- __m256 k0 = loadf16x8(B + ldb * (j + 0) + l);
- __m256 a0 = loadf16x8(A + lda * (i + 0) + l);
- c00 = _mm256_fmadd_ps(a0, k0, c00);
- __m256 a1 = loadf16x8(A + lda * (i + 1) + l);
- c10 = _mm256_fmadd_ps(a1, k0, c10);
- __m256 a2 = loadf16x8(A + lda * (i + 2) + l);
- c20 = _mm256_fmadd_ps(a2, k0, c20);
- __m256 a3 = loadf16x8(A + lda * (i + 3) + l);
- c30 = _mm256_fmadd_ps(a3, k0, c30);
- }
- C[ldc * (j + 0) + (i + 0)] = hsum_float_8(c00);
- C[ldc * (j + 0) + (i + 1)] = hsum_float_8(c10);
- C[ldc * (j + 0) + (i + 2)] = hsum_float_8(c20);
- C[ldc * (j + 0) + (i + 3)] = hsum_float_8(c30);
- END_KERNEL()
- }
-
- dontinline void gemm1x1(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(1, 1)
- __m256 c = _mm256_setzero_ps();
- for (long l = 0; l < k; l += 8) {
- c = _mm256_fmadd_ps(loadf16x8(A + lda * i + l),
- loadf16x8(B + ldb * j + l), c);
- }
- C[ldc * j + i] = hsum_float_8(c);
- END_KERNEL()
- }
-
- const long k;
- const ggml_fp16_t *const A;
- const long lda;
- const ggml_fp16_t *const B;
- const long ldb;
- float *const C;
- const long ldc;
- const long ith;
- const long nth;
-};
-
-class tinyBLAS_Q8_VNNI {
- public:
- tinyBLAS_Q8_VNNI(long k, const block_q8_0 *A, long lda, const block_q8_0 *B,
- long ldb, float *C, long ldc, long ith, long nth)
- : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith),
- nth(nth) {
- ASSERT(A != nullptr);
- ASSERT(B != nullptr);
- ASSERT(C != nullptr);
- ASSERT(k >= 0 && k % 32 == 0);
- ASSERT(ith >= 0 && ith < nth);
- }
-
- void gemm(long m, long n) {
- ASSERT(m >= 0);
- ASSERT(n >= 0);
- ASSERT(ldc >= m);
- mnpack(0, m, 0, n);
- }
-
- private:
- void mnpack(long m0, long m, long n0, long n) {
- if (m - m0 <= 0 || n - n0 <= 0)
- return;
- long mc, nc, mp, np;
- if (m - m0 >= 4 && n - n0 >= 1) {
- mc = 4;
- nc = 1;
- gemm4x1(m0, m, n0, n);
- } else if (m - m0 >= 1 && n - n0 >= 4) {
- mc = 1;
- nc = 4;
- gemm1x4(m0, m, n0, n);
- } else {
- mc = 1;
- nc = 1;
- gemm1x1(m0, m, n0, n);
- }
- mp = m0 + (m - m0) / mc * mc;
- np = n0 + (n - n0) / nc * nc;
- mnpack(mp, m, n0, np);
- mnpack(m0, mp, np, n);
- mnpack(mp, m, np, n);
- }
-
- dontinline void gemm4x1(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(4, 1)
- long k2 = k / 32;
- __m256 c0 = _mm256_setzero_ps();
- __m256 c1 = _mm256_setzero_ps();
- __m256 c2 = _mm256_setzero_ps();
- __m256 c3 = _mm256_setzero_ps();
- const block_q8_0 *Ap0 = A + lda * (i + 0);
- const block_q8_0 *Ap1 = A + lda * (i + 1);
- const block_q8_0 *Ap2 = A + lda * (i + 2);
- const block_q8_0 *Ap3 = A + lda * (i + 3);
- const block_q8_0 *Bp = B + ldb * j;
- for (long l = 0; l < k2; ++l) {
- float db0 = GGML_FP16_TO_FP32(Bp[l].d);
- __m256i f = _mm256_loadu_si256((const __m256i *)Bp[l].qs);
- __m256i u = _mm256_sign_epi8(f, f);
- __m256 d0 = _mm256_set1_ps(GGML_FP16_TO_FP32(Ap0[l].d) * db0);
- __m256 d1 = _mm256_set1_ps(GGML_FP16_TO_FP32(Ap1[l].d) * db0);
- __m256 d2 = _mm256_set1_ps(GGML_FP16_TO_FP32(Ap2[l].d) * db0);
- __m256 d3 = _mm256_set1_ps(GGML_FP16_TO_FP32(Ap3[l].d) * db0);
- __m256i e0 = _mm256_loadu_si256((const __m256i *)Ap0[l].qs);
- __m256i e1 = _mm256_loadu_si256((const __m256i *)Ap1[l].qs);
- __m256i e2 = _mm256_loadu_si256((const __m256i *)Ap2[l].qs);
- __m256i e3 = _mm256_loadu_si256((const __m256i *)Ap3[l].qs);
- __m256i s0 = _mm256_sign_epi8(e0, f);
- __m256i s1 = _mm256_sign_epi8(e1, f);
- __m256i s2 = _mm256_sign_epi8(e2, f);
- __m256i s3 = _mm256_sign_epi8(e3, f);
- __m256 g0 = mul_sum_signed_i8_pairs_float(u, s0);
- __m256 g1 = mul_sum_signed_i8_pairs_float(u, s1);
- __m256 g2 = mul_sum_signed_i8_pairs_float(u, s2);
- __m256 g3 = mul_sum_signed_i8_pairs_float(u, s3);
- c0 = _mm256_fmadd_ps(d0, g0, c0);
- c1 = _mm256_fmadd_ps(d1, g1, c1);
- c2 = _mm256_fmadd_ps(d2, g2, c2);
- c3 = _mm256_fmadd_ps(d3, g3, c3);
- }
- C[ldc * j + (i + 0)] = hsum_float_8(c0);
- C[ldc * j + (i + 1)] = hsum_float_8(c1);
- C[ldc * j + (i + 2)] = hsum_float_8(c2);
- C[ldc * j + (i + 3)] = hsum_float_8(c3);
- END_KERNEL()
- }
-
- dontinline void gemm1x4(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(1, 4)
- long k2 = k / 32;
- __m256 c0 = _mm256_setzero_ps();
- __m256 c1 = _mm256_setzero_ps();
- __m256 c2 = _mm256_setzero_ps();
- __m256 c3 = _mm256_setzero_ps();
- const block_q8_0 *Bp0 = B + ldb * (j + 0);
- const block_q8_0 *Bp1 = B + ldb * (j + 1);
- const block_q8_0 *Bp2 = B + ldb * (j + 2);
- const block_q8_0 *Bp3 = B + ldb * (j + 3);
- const block_q8_0 *Ap = A + lda * i;
- for (long l = 0; l < k2; ++l) {
- float da0 = GGML_FP16_TO_FP32(Ap[l].d);
- __m256i f = _mm256_loadu_si256((const __m256i *)Ap[l].qs);
- __m256i u = _mm256_sign_epi8(f, f);
- __m256 d0 = _mm256_set1_ps(GGML_FP16_TO_FP32(Bp0[l].d) * da0);
- __m256 d1 = _mm256_set1_ps(GGML_FP16_TO_FP32(Bp1[l].d) * da0);
- __m256 d2 = _mm256_set1_ps(GGML_FP16_TO_FP32(Bp2[l].d) * da0);
- __m256 d3 = _mm256_set1_ps(GGML_FP16_TO_FP32(Bp3[l].d) * da0);
- __m256i e0 = _mm256_loadu_si256((const __m256i *)Bp0[l].qs);
- __m256i e1 = _mm256_loadu_si256((const __m256i *)Bp1[l].qs);
- __m256i e2 = _mm256_loadu_si256((const __m256i *)Bp2[l].qs);
- __m256i e3 = _mm256_loadu_si256((const __m256i *)Bp3[l].qs);
- __m256i s0 = _mm256_sign_epi8(e0, f);
- __m256i s1 = _mm256_sign_epi8(e1, f);
- __m256i s2 = _mm256_sign_epi8(e2, f);
- __m256i s3 = _mm256_sign_epi8(e3, f);
- __m256 g0 = mul_sum_signed_i8_pairs_float(u, s0);
- __m256 g1 = mul_sum_signed_i8_pairs_float(u, s1);
- __m256 g2 = mul_sum_signed_i8_pairs_float(u, s2);
- __m256 g3 = mul_sum_signed_i8_pairs_float(u, s3);
- c0 = _mm256_fmadd_ps(d0, g0, c0);
- c1 = _mm256_fmadd_ps(d1, g1, c1);
- c2 = _mm256_fmadd_ps(d2, g2, c2);
- c3 = _mm256_fmadd_ps(d3, g3, c3);
- }
- C[ldc * (j + 0) + i] = hsum_float_8(c0);
- C[ldc * (j + 1) + i] = hsum_float_8(c1);
- C[ldc * (j + 2) + i] = hsum_float_8(c2);
- C[ldc * (j + 3) + i] = hsum_float_8(c3);
- END_KERNEL()
- }
-
- dontinline void gemm1x1(long m0, long m, long n0, long n) {
- BEGIN_KERNEL(1, 1)
- long k2 = k / 32;
- __m256 c0 = _mm256_setzero_ps();
- const block_q8_0 *Ap = A + lda * i;
- const block_q8_0 *Bp = B + ldb * j;
- for (long l = 0; l < k2; ++l) {
- __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(Ap[l].d) *
- GGML_FP16_TO_FP32(Bp[l].d));
- __m256i e = _mm256_loadu_si256((const __m256i *)Ap[l].qs);
- __m256i f = _mm256_loadu_si256((const __m256i *)Bp[l].qs);
- __m256 g = mul_sum_i8_pairs_float(e, f);
- c0 = _mm256_fmadd_ps(d, g, c0);
- }
- C[ldc * j + i] = hsum_float_8(c0);
- END_KERNEL()
- }
-
- const long k;
- const block_q8_0 *const A;
- const long lda;
- const block_q8_0 *const B;
- const long ldb;
- float *const C;
- const long ldc;
- const long ith;
- const long nth;
-};
-
-#endif // __x86_64__
-
-// computes m×k * n×k → n×m
-void llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda,
- const void *B, long ldb, float *C, long ldc, long ith,
- long nth) {
- switch (dtype) {
-#ifdef __x86_64__
- case GGML_TYPE_F32: {
- tinyBLAS_F32_AVX2 tb{
- k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};
- tb.gemm(m, n);
- break;
- }
- case GGML_TYPE_F16: {
- tinyBLAS_F16_AVX2 tb{k, (const ggml_fp16_t *)A,
- lda, (const ggml_fp16_t *)B,
- ldb, C,
- ldc, ith,
- nth};
- tb.gemm(m, n);
- break;
- }
- case GGML_TYPE_Q8_0: {
- tinyBLAS_Q8_VNNI tb{k, (const block_q8_0 *)A,
- lda, (const block_q8_0 *)B,
- ldb, C,
- ldc, ith,
- nth};
- tb.gemm(m, n);
- break;
- }
-#endif
- default:
- ASSERT(!"unsupported dtype");
- }
-}
diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cpp
new file mode 100644
index 00000000..3b8f83d1
--- /dev/null
+++ b/llamafile/sgemm.cpp
@@ -0,0 +1,512 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "llama.cpp/ggml-quants.h"
+#include "llamafile.h"
+#include <cosmo.h>
+#include <immintrin.h>
+#include <stdio.h>
+
+//
+// _ _ ___ _ _ ___
+// | |_(_)_ _ _ _| _ ) | /_\ / __|
+// | _| | ' \ || | _ \ |__ / _ \\__ \.
+// \__|_|_||_\_, |___/____/_/ \_\___/
+// |__/
+//
+// BASIC LINEAR ALGEBRA SUBPROGRAMS
+//
+
+#define ASSERT(x) \
+ if (!(x)) { \
+ fprintf(stderr, "%s:%d: assertion failed: %s\n", __FILE__, __LINE__, \
+ #x); \
+ __builtin_trap(); \
+ }
+
+#define END_KERNEL() }
+#define BEGIN_KERNEL(RM, RN) \
+ long ytiles = (m - m0) / RM; \
+ long xtiles = (n - n0) / RN; \
+ long tiles = ytiles * xtiles; \
+ double duty = (double)tiles / nth; \
+ if (duty < 1) \
+ duty = 1; \
+ double spot = duty * ith + .5; \
+ long end = spot + duty; \
+ long start = spot; \
+ if (end > tiles) \
+ end = tiles; \
+ for (long job = start; job < end; ++job) { \
+ long i = m0 + job / xtiles * RM; \
+ long j = n0 + job % xtiles * RN;
+
+#ifdef __x86_64__
+#pragma GCC push_options
+#pragma GCC target("avx2,fma,f16c,avxvnni")
+
+#define MM256_SET_M128I(a, b) \
+ _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)
+
+static inline float unhalf(ggml_fp16_t d) {
+ return GGML_FP16_TO_FP32(d);
+}
+
+static dontinline float hsum8f(__m256 x) {
+ __m128 res = _mm256_extractf128_ps(x, 1);
+ res = _mm_add_ps(res, _mm256_castps256_ps128(x));
+ res = _mm_add_ps(res, _mm_movehl_ps(res, res));
+ res = _mm_add_ss(res, _mm_movehdup_ps(res));
+ return _mm_cvtss_f32(res);
+}
+
+static inline __m256i denibble32(const uint8_t *p) {
+ const __m128i tmp = _mm_loadu_si128((const __m128i *)p);
+ const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);
+ const __m256i lowMask = _mm256_set1_epi8(15);
+ return _mm256_and_si256(lowMask, bytes);
+}
+
+template <typename T> static inline __m256 load8f(const T *);
+template <> inline __m256 load8f(const float *p) {
+ return _mm256_loadu_ps(p);
+}
+template <> inline __m256 load8f(const ggml_fp16_t *p) {
+ return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));
+}
+
+template <typename T> static inline __m256i load32i(const T *);
+template <> inline __m256i load32i(const block_q8_0 *b) {
+ return _mm256_loadu_si256((const __m256i *)b->qs);
+}
+template <> inline __m256i load32i(const block_q4_0 *b) {
+ return _mm256_sub_epi8(denibble32(b->qs), _mm256_set1_epi8(8));
+}
+
+template <typename T> class tinyBLAS {
+ public:
+ tinyBLAS(long k, const T *A, long lda, const T *B, long ldb, float *C,
+ long ldc, long ith, long nth)
+ : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith),
+ nth(nth) {
+ ASSERT(A != nullptr);
+ ASSERT(B != nullptr);
+ ASSERT(C != nullptr);
+ ASSERT(k >= 0 && k % 8 == 0);
+ ASSERT(ith >= 0 && ith < nth);
+ }
+
+ void gemm(long m, long n) {
+ ASSERT(m >= 0);
+ ASSERT(n >= 0);
+ ASSERT(ldc >= m);
+ mnpack(0, m, 0, n);
+ }
+
+ private:
+ void mnpack(long m0, long m, long n0, long n) {
+ if (m - m0 <= 0 || n - n0 <= 0)
+ return;
+ long mc, nc, mp, np;
+ if (m - m0 >= 3 && n - n0 >= 4) {
+ mc = 3;
+ nc = 4;
+ gemm3x4(m0, m, n0, n);
+ } else if (m - m0 >= 4 && n - n0 >= 1) {
+ mc = 4;
+ nc = 1;
+ gemm4x1(m0, m, n0, n);
+ } else if (m - m0 >= 1 && n - n0 >= 4) {
+ mc = 1;
+ nc = 4;
+ gemm1x4(m0, m, n0, n);
+ } else {
+ mc = 1;
+ nc = 1;
+ gemm1x1(m0, m, n0, n);
+ }
+ mp = m0 + (m - m0) / mc * mc;
+ np = n0 + (n - n0) / nc * nc;
+ mnpack(mp, m, n0, np);
+ mnpack(m0, mp, np, n);
+ mnpack(mp, m, np, n);
+ }
+
+ dontinline void gemm3x4(long m0, long m, long n0, long n) {
+ BEGIN_KERNEL(3, 4)
+ __m256 c00 = _mm256_setzero_ps();
+ __m256 c01 = _mm256_setzero_ps();
+ __m256 c02 = _mm256_setzero_ps();
+ __m256 c03 = _mm256_setzero_ps();
+ __m256 c10 = _mm256_setzero_ps();
+ __m256 c11 = _mm256_setzero_ps();
+ __m256 c12 = _mm256_setzero_ps();
+ __m256 c13 = _mm256_setzero_ps();
+ __m256 c20 = _mm256_setzero_ps();
+ __m256 c21 = _mm256_setzero_ps();
+ __m256 c22 = _mm256_setzero_ps();
+ __m256 c23 = _mm256_setzero_ps();
+ for (long l = 0; l < k; l += 8) {
+ __m256 k0 = load8f(B + ldb * (j + 0) + l);
+ __m256 k1 = load8f(B + ldb * (j + 1) + l);
+ __m256 k2 = load8f(B + ldb * (j + 2) + l);
+ __m256 k3 = load8f(B + ldb * (j + 3) + l);
+ __m256 a0 = load8f(A + lda * (i + 0) + l);
+ c00 = _mm256_fmadd_ps(a0, k0, c00);
+ c01 = _mm256_fmadd_ps(a0, k1, c01);
+ c02 = _mm256_fmadd_ps(a0, k2, c02);
+ c03 = _mm256_fmadd_ps(a0, k3, c03);
+ __m256 a1 = load8f(A + lda * (i + 1) + l);
+ c10 = _mm256_fmadd_ps(a1, k0, c10);
+ c11 = _mm256_fmadd_ps(a1, k1, c11);
+ c12 = _mm256_fmadd_ps(a1, k2, c12);
+ c13 = _mm256_fmadd_ps(a1, k3, c13);
+ __m256 a2 = load8f(A + lda * (i + 2) + l);
+ c20 = _mm256_fmadd_ps(a2, k0, c20);
+ c21 = _mm256_fmadd_ps(a2, k1, c21);
+ c22 = _mm256_fmadd_ps(a2, k2, c22);
+ c23 = _mm256_fmadd_ps(a2, k3, c23);
+ }
+ C[ldc * (j + 0) + (i + 0)] = hsum8f(c00);
+ C[ldc * (j + 0) + (i + 1)] = hsum8f(c10);
+ C[ldc * (j + 0) + (i + 2)] = hsum8f(c20);
+ C[ldc * (j + 1) + (i + 0)] = hsum8f(c01);
+ C[ldc * (j + 1) + (i + 1)] = hsum8f(c11);
+ C[ldc * (j + 1) + (i + 2)] = hsum8f(c21);
+ C[ldc * (j + 2) + (i + 0)] = hsum8f(c02);
+ C[ldc * (j + 2) + (i + 1)] = hsum8f(c12);
+ C[ldc * (j + 2) + (i + 2)] = hsum8f(c22);
+ C[ldc * (j + 3) + (i + 0)] = hsum8f(c03);
+ C[ldc * (j + 3) + (i + 1)] = hsum8f(c13);
+ C[ldc * (j + 3) + (i + 2)] = hsum8f(c23);
+ END_KERNEL()
+ }
+
+ dontinline void gemm1x4(long m0, long m, long n0, long n) {
+ BEGIN_KERNEL(1, 4)
+ __m256 c00 = _mm256_setzero_ps();
+ __m256 c01 = _mm256_setzero_ps();
+ __m256 c02 = _mm256_setzero_ps();
+ __m256 c03 = _mm256_setzero_ps();
+ for (long l = 0; l < k; l += 8) {
+ __m256 a0 = load8f(A + lda * (i + 0) + l);
+ __m256 k0 = load8f(B + ldb * (j + 0) + l);
+ __m256 k1 = load8f(B + ldb * (j + 1) + l);
+ __m256 k2 = load8f(B + ldb * (j + 2) + l);
+ __m256 k3 = load8f(B + ldb * (j + 3) + l);
+ c00 = _mm256_fmadd_ps(a0, k0, c00);
+ c01 = _mm256_fmadd_ps(a0, k1, c01);
+ c02 = _mm256_fmadd_ps(a0, k2, c02);
+ c03 = _mm256_fmadd_ps(a0, k3, c03);
+ }
+ C[ldc * (j + 0) + (i + 0)] = hsum8f(c00);
+ C[ldc * (j + 1) + (i + 0)] = hsum8f(c01);
+ C[ldc * (j + 2) + (i + 0)] = hsum8f(c02);
+ C[ldc * (j + 3) + (i + 0)] = hsum8f(c03);
+ END_KERNEL()
+ }
+
+ dontinline void gemm4x1(long m0, long m, long n0, long n) {
+ BEGIN_KERNEL(4, 1)
+ __m256 c00 = _mm256_setzero_ps();
+ __m256 c10 = _mm256_setzero_ps();
+ __m256 c20 = _mm256_setzero_ps();
+ __m256 c30 = _mm256_setzero_ps();
+ for (long l = 0; l < k; l += 8) {
+ __m256 k0 = load8f(B + ldb * (j + 0) + l);
+ __m256 a0 = load8f(A + lda * (i + 0) + l);
+ c00 = _mm256_fmadd_ps(a0, k0, c00);
+ __m256 a1 = load8f(A + lda * (i + 1) + l);
+ c10 = _mm256_fmadd_ps(a1, k0, c10);
+ __m256 a2 = load8f(A + lda * (i + 2) + l);
+ c20 = _mm256_fmadd_ps(a2, k0, c20);
+ __m256 a3 = load8f(A + lda * (i + 3) + l);
+ c30 = _mm256_fmadd_ps(a3, k0, c30);
+ }
+ C[ldc * (j + 0) + (i + 0)] = hsum8f(c00);
+ C[ldc * (j + 0) + (i + 1)] = hsum8f(c10);
+ C[ldc * (j + 0) + (i + 2)] = hsum8f(c20);
+ C[ldc * (j + 0) + (i + 3)] = hsum8f(c30);
+ END_KERNEL()
+ }
+
+ dontinline void gemm1x1(long m0, long m, long n0, long n) {
+ BEGIN_KERNEL(1, 1)
+ __m256 c = _mm256_setzero_ps();
+ for (long l = 0; l < k; l += 8) {
+ c = _mm256_fmadd_ps(load8f(A + lda * i + l),
+ load8f(B + ldb * j + l), c);
+ }
+ C[ldc * j + i] = hsum8f(c);
+ END_KERNEL()
+ }
+
+ const long k;
+ const T *const A;
+ const long lda;
+ const T *const B;
+ const long ldb;
+ float *const C;
+ const long ldc;
+ const long ith;
+ const long nth;
+};
+
+template <typename T, typename U, int avxvnni> class tinyBLASq {
+ public:
+ tinyBLASq(long k, const T *A, long lda, const U *B, long ldb, float *C,
+ long ldc, long ith, long nth)
+ : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith),
+ nth(nth) {
+ ASSERT(A != nullptr);
+ ASSERT(B != nullptr);
+ ASSERT(C != nullptr);
+ ASSERT(k >= 0 && k % 32 == 0);
+ ASSERT(ith >= 0 && ith < nth);
+ }
+
+ void gemm(long m, long n) {
+ ASSERT(m >= 0);
+ ASSERT(n >= 0);
+ ASSERT(ldc >= m);
+ mnpack(0, m, 0, n);
+ }
+
+ private:
+ void mnpack(long m0, long m, long n0, long n) {
+ if (m - m0 <= 0 || n - n0 <= 0)
+ return;
+ long mc, nc, mp, np;
+ if (m - m0 >= 1 && n - n0 >= 4) {
+ mc = 1;
+ nc = 4;
+ gemm1x4(m0, m, n0, n);
+ } else if (m - m0 >= 4 && n - n0 >= 1) {
+ mc = 4;
+ nc = 1;
+ gemm4x1(m0, m, n0, n);
+ } else {
+ mc = 1;
+ nc = 1;
+ gemm1x1(m0, m, n0, n);
+ }
+ mp = m0 + (m - m0) / mc * mc;
+ np = n0 + (n - n0) / nc * nc;
+ mnpack(mp, m, n0, np);
+ mnpack(m0, mp, np, n);
+ mnpack(mp, m, np, n);
+ }
+
+ dontinline void gemm4x1(long m0, long m, long n0, long n) {
+ BEGIN_KERNEL(4, 1)
+ long k2 = k / 32;
+ __m256 c0 = _mm256_setzero_ps();
+ __m256 c1 = _mm256_setzero_ps();
+ __m256 c2 = _mm256_setzero_ps();
+ __m256 c3 = _mm256_setzero_ps();
+ const T *Ap0 = A + lda * (i + 0);
+ const T *Ap1 = A + lda * (i + 1);
+ const T *Ap2 = A + lda * (i + 2);
+ const T *Ap3 = A + lda * (i + 3);
+ const U *Bp = B + ldb * j;
+ for (long l = 0; l < k2; ++l) {
+ float db0 = unhalf(Bp[l].d);
+ __m256i f = load32i(Bp + l);
+ __m256i u = _mm256_sign_epi8(f, f);
+ __m256 d0 = _mm256_set1_ps(unhalf(Ap0[l].d) * db0);
+ __m256 d1 = _mm256_set1_ps(unhalf(Ap1[l].d) * db0);
+ __m256 d2 = _mm256_set1_ps(unhalf(Ap2[l].d) * db0);
+ __m256 d3 = _mm256_set1_ps(unhalf(Ap3[l].d) * db0);
+ __m256i e0 = load32i(Ap0 + l);
+ __m256i e1 = load32i(Ap1 + l);
+ __m256i e2 = load32i(Ap2 + l);
+ __m256i e3 = load32i(Ap3 + l);
+ __m256i s0 = _mm256_sign_epi8(e0, f);
+ __m256i s1 = _mm256_sign_epi8(e1, f);
+ __m256i s2 = _mm256_sign_epi8(e2, f);
+ __m256i s3 = _mm256_sign_epi8(e3, f);
+ __m256 g0 = dotsome(u, s0);
+ __m256 g1 = dotsome(u, s1);
+ __m256 g2 = dotsome(u, s2);
+ __m256 g3 = dotsome(u, s3);
+ c0 = _mm256_fmadd_ps(d0, g0, c0);
+ c1 = _mm256_fmadd_ps(d1, g1, c1);
+ c2 = _mm256_fmadd_ps(d2, g2, c2);
+ c3 = _mm256_fmadd_ps(d3, g3, c3);
+ }
+ C[ldc * j + (i + 0)] = hsum8f(c0);
+ C[ldc * j + (i + 1)] = hsum8f(c1);
+ C[ldc * j + (i + 2)] = hsum8f(c2);
+ C[ldc * j + (i + 3)] = hsum8f(c3);
+ END_KERNEL()
+ }
+
+ dontinline void gemm1x4(long m0, long m, long n0, long n) {
+ BEGIN_KERNEL(1, 4)
+ long k2 = k / 32;
+ __m256 c0 = _mm256_setzero_ps();
+ __m256 c1 = _mm256_setzero_ps();
+ __m256 c2 = _mm256_setzero_ps();
+ __m256 c3 = _mm256_setzero_ps();
+ const U *Bp0 = B + ldb * (j + 0);
+ const U *Bp1 = B + ldb * (j + 1);
+ const U *Bp2 = B + ldb * (j + 2);
+ const U *Bp3 = B + ldb * (j + 3);
+ const T *Ap = A + lda * i;
+ for (long l = 0; l < k2; ++l) {
+ float da0 = unhalf(Ap[l].d);
+ __m256i f = load32i(Ap + l);
+ __m256i u = _mm256_sign_epi8(f, f);
+ __m256 d0 = _mm256_set1_ps(unhalf(Bp0[l].d) * da0);
+ __m256 d1 = _mm256_set1_ps(unhalf(Bp1[l].d) * da0);
+ __m256 d2 = _mm256_set1_ps(unhalf(Bp2[l].d) * da0);
+ __m256 d3 = _mm256_set1_ps(unhalf(Bp3[l].d) * da0);
+ __m256 g0 = dotsome(u, _mm256_sign_epi8(load32i(Bp0 + l), f));
+ __m256 g1 = dotsome(u, _mm256_sign_epi8(load32i(Bp1 + l), f));
+ __m256 g2 = dotsome(u, _mm256_sign_epi8(load32i(Bp2 + l), f));
+ __m256 g3 = dotsome(u, _mm256_sign_epi8(load32i(Bp3 + l), f));
+ c0 = _mm256_fmadd_ps(d0, g0, c0);
+ c1 = _mm256_fmadd_ps(d1, g1, c1);
+ c2 = _mm256_fmadd_ps(d2, g2, c2);
+ c3 = _mm256_fmadd_ps(d3, g3, c3);
+ }
+ C[ldc * (j + 0) + i] = hsum8f(c0);
+ C[ldc * (j + 1) + i] = hsum8f(c1);
+ C[ldc * (j + 2) + i] = hsum8f(c2);
+ C[ldc * (j + 3) + i] = hsum8f(c3);
+ END_KERNEL()
+ }
+
+ dontinline void gemm1x1(long m0, long m, long n0, long n) {
+ BEGIN_KERNEL(1, 1)
+ long k2 = k / 32;
+ __m256 c = _mm256_setzero_ps();
+ const T *Ap = A + lda * i;
+ const U *Bp = B + ldb * j;
+ for (long l = 0; l < k2; ++l) {
+ __m256 d = _mm256_set1_ps(unhalf(Ap[l].d) * unhalf(Bp[l].d));
+ __m256i e = load32i(Ap + l);
+ __m256i f = load32i(Bp + l);
+ __m256 g = dotsome(_mm256_sign_epi8(e, e), _mm256_sign_epi8(f, e));
+ c = _mm256_fmadd_ps(d, g, c);
+ }
+ C[ldc * j + i] = hsum8f(c);
+ END_KERNEL()
+ }
+
+ inline __m256 dotsome(__m256i u, __m256i s) {
+ __m256i res;
+ if (avxvnni)
+ res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);
+ else
+ res = _mm256_madd_epi16(_mm256_set1_epi16(1),
+ _mm256_maddubs_epi16(u, s));
+ return _mm256_cvtepi32_ps(res);
+ }
+
+ const long k;
+ const T *const A;
+ const long lda;
+ const U *const B;
+ const long ldb;
+ float *const C;
+ const long ldc;
+ const long ith;
+ const long nth;
+};
+
+#endif // __x86_64__
+
+// computes m×k * n×k → n×m
+void llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda,
+ const void *B, long ldb, float *C, long ldc, long ith,
+ long nth) {
+ ASSERT(X86_HAVE(FMA));
+ ASSERT(X86_HAVE(AVX2));
+ switch (dtype) {
+#ifdef __x86_64__
+
+ case GGML_TYPE_F32: {
+ tinyBLAS<float> tb{
+ k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth,
+ };
+ tb.gemm(m, n);
+ break;
+ }
+
+ case GGML_TYPE_F16: {
+ ASSERT(X86_HAVE(F16C));
+ tinyBLAS<ggml_fp16_t> tb{
+ k, (const ggml_fp16_t *)A,
+ lda, (const ggml_fp16_t *)B,
+ ldb, C,
+ ldc, ith,
+ nth,
+ };
+ tb.gemm(m, n);
+ break;
+ }
+
+ case GGML_TYPE_Q8_0:
+ if (X86_HAVE(AVXVNNI)) {
+ tinyBLASq<block_q8_0, block_q8_0, true> tb{
+ k, (const block_q8_0 *)A,
+ lda, (const block_q8_0 *)B,
+ ldb, C,
+ ldc, ith,
+ nth,
+ };
+ tb.gemm(m, n);
+ } else {
+ tinyBLASq<block_q8_0, block_q8_0, false> tb{
+ k, (const block_q8_0 *)A,
+ lda, (const block_q8_0 *)B,
+ ldb, C,
+ ldc, ith,
+ nth,
+ };
+ tb.gemm(m, n);
+ }
+ break;
+
+ case GGML_TYPE_Q4_0:
+ if (X86_HAVE(AVXVNNI)) {
+ tinyBLASq<block_q4_0, block_q8_0, true> tb{
+ k, (const block_q4_0 *)A,
+ lda, (const block_q8_0 *)B,
+ ldb, C,
+ ldc, ith,
+ nth,
+ };
+ tb.gemm(m, n);
+ } else {
+ tinyBLASq<block_q4_0, block_q8_0, false> tb{
+ k, (const block_q4_0 *)A,
+ lda, (const block_q8_0 *)B,
+ ldb, C,
+ ldc, ith,
+ nth,
+ };
+ tb.gemm(m, n);
+ }
+ break;
+
+#endif
+ default:
+ ASSERT(!"unsupported dtype");
+ }
+}