Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Upgrade to Cosmopolitan 3.3
Justine Tunney5b109da5d27c
diff --git a/build/config.mk b/build/config.mkindex bf25ea30..a4b82a08 100644--- a/build/config.mk+++ b/build/config.mk@@ -2,7 +2,7 @@#── vi: set noet ft=make ts=8 sw=8 fenc=utf-8 :vi ────────────────────┘PREFIX = /usr/local-COSMOCC = .cosmocc/3.2.4+COSMOCC = .cosmocc/3.3TOOLCHAIN = $(COSMOCC)/bin/cosmoAR = $(TOOLCHAIN)ar@@ -50,5 +50,5 @@ clean:; rm -rf o.PHONY: distcleandistclean:; rm -rf o cosmocc-.cosmocc/3.2.4:- build/download-cosmocc.sh $@ 3.2.4 d2fa6dbf6f987310494581deff5b915dbdc5ca701f20f7613bb0dcf1de2ee511+.cosmocc/3.3:+ build/download-cosmocc.sh $@ 3.3 d26ec8f4e48f6db004fc6a9677c7ff3b50c3b21e936e9393158aa2ed51b0b549diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 65ff90f5..053bb44e 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -33,7 +33,6 @@ SOFTWARE.\"");#include "ggml-metal.h"#include "ggml-cuda.h"#include "llamafile/log.h"-#include "libc/log/backtrace.internal.h"#include <alloca.h>#include <assert.h>@@ -51,6 +50,7 @@ SOFTWARE.\"");#include <signal.h>#include <unistd.h>#include <cosmo.h>+#include <libc/log/backtrace.internal.h>#include <pthread.h>#include <stdatomic.h>@@ -10106,8 +10106,9 @@ static void ggml_compute_forward_mul_mat((dst->type == GGML_TYPE_F32 &&src1->type == GGML_TYPE_F32 &&(src0->type == GGML_TYPE_F32 ||- (src0->type == GGML_TYPE_F16 && X86_HAVE(F16C)) ||- (src0->type == GGML_TYPE_Q8_0 && X86_HAVE(AVXVNNI)))) &&+ src0->type == GGML_TYPE_Q4_0 ||+ src0->type == GGML_TYPE_Q8_0 ||+ (src0->type == GGML_TYPE_F16 && X86_HAVE(F16C)))) &&ne00 % 8 == 0 &&ggml_is_contiguous(src0) &&ggml_is_contiguous(src1) &&diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cppindex 560b510f..4fbf4289 100644--- a/llama.cpp/main/main.cpp+++ b/llama.cpp/main/main.cpp@@ -15,8 +15,8 @@#include <string>#include <unistd.h>#include <vector>+#include <tool/args/args.h>-#include "tool/args/args.h"#include "llamafile/version.h"#include "llama.cpp/llama.h"#include "llama.cpp/common.h"diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex e10358b3..3f6a7be4 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -8,6 +8,9 @@#include <condition_variable>#include <cstddef>#include <thread>+#include <libc/calls/pledge.h>+#include <tool/args/args.h>+#include <libc/dce.h>#include "llama.cpp/common.h"#include "llama.cpp/ggml-cuda.h"@@ -19,9 +22,6 @@#include "llamafile/llamafile.h"#include "llamafile/log.h"#include "llamafile/version.h"-#include "libc/calls/pledge.h"-#include "tool/args/args.h"-#include "libc/dce.h"#include "oai.h"#include "macsandbox.h"diff --git a/llamafile/check_cpu.c b/llamafile/check_cpu.cindex 975c901b..f79b9bed 100644--- a/llamafile/check_cpu.c+++ b/llamafile/check_cpu.c@@ -15,21 +15,10 @@// See the License for the specific language governing permissions and// limitations under the License.-#include <errno.h>#include <cosmo.h>+#include <errno.h>#include <stdlib.h>-static int on_missing_feature(const char *name) {- tinyprint(2, program_invocation_name, ": fatal error: the cpu feature ", name,- " was required at build time but isn't available on this system\n", NULL);-#if defined(__AVX2__) && !defined(__AVX512F__)- tinyprint(2, "note: amd microprocessors made after 2017 usually work\n"- "note: intel microprocessors made after 2013 usually work\n", NULL);-#endif- tinyprint(2, "exiting process.\n", NULL);- exit(1);-}-/*** Dies if CPU doesn't have mandatory features.*@@ -39,6 +28,29 @@ static int on_missing_feature(const char *name) {* microarchitecture flags only get passed to that specific object*/void llamafile_check_cpu(void) {+ // side effect: the constructor below has now been linked+}++static int on_missing_feature(const char *name) {+ tinyprint(+ 2, program_invocation_name, ": fatal error: the cpu feature ", name,+ " was required at build time but isn't available on this system\n",+ NULL);+#if defined(__AVX2__) && !defined(__AVX512F__)+ tinyprint(2,+ "note: amd microprocessors made after 2017 usually work\n"+ "note: intel microprocessors made after 2013 usually work\n",+ NULL);+#endif+ tinyprint(2, "exiting process.\n", NULL);+ _Exit(1);+}++// We need to perform this early in the initialization process, before+// C++ codes built with -mavx has a chance to start allocating dynamic+// memory that would otherwise crash the cpu before this could be done+__attribute__((__constructor__(101))) static void+llamafile_actually_check_cpu(void) {if (X86_NEED(SSE3) && !X86_CHECK(SSE3)) {on_missing_feature("SSE3");}diff --git a/llamafile/gemm_avx2.cpp b/llamafile/gemm_avx2.cppdeleted file mode 100644index f4d73e1a..00000000--- a/llamafile/gemm_avx2.cpp+++ /dev/null@@ -1,639 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#include "llama.cpp/ggml-quants.h"-#include "llamafile.h"-#include <immintrin.h>-#include <stdio.h>--//-// _ _ ___ _ _ ___-// | |_(_)_ _ _ _| _ ) | /_\ / __|-// | _| | ' \ || | _ \ |__ / _ \\__ \.-// \__|_|_||_\_, |___/____/_/ \_\___/-// |__/-//-// BASIC LINEAR ALGEBRA SUBPROGRAMS-//--#define ASSERT(x) \- if (!(x)) { \- fprintf(stderr, "%s:%d: assertion failed: %s\n", __FILE__, __LINE__, \- #x); \- __builtin_trap(); \- }--#define BEGIN_KERNEL(RM, RN) \- long ytiles = (m - m0) / RM; \- long xtiles = (n - n0) / RN; \- long tiles = ytiles * xtiles; \- double duty = (double)tiles / nth; \- if (duty < 1) \- duty = 1; \- double spot = duty * ith + .5; \- long end = spot + duty; \- long start = spot; \- if (end > tiles) \- end = tiles; \- for (long job = start; job < end; ++job) { \- long i = m0 + job / xtiles * RM; \- long j = n0 + job % xtiles * RN;--#define END_KERNEL() }--#ifdef __x86_64__-#pragma GCC push_options-#pragma GCC target("avx2,fma,f16c")--dontinline static float hsum_float_8(__m256 x) {- __m128 res = _mm256_extractf128_ps(x, 1);- res = _mm_add_ps(res, _mm256_castps256_ps128(x));- res = _mm_add_ps(res, _mm_movehl_ps(res, res));- res = _mm_add_ss(res, _mm_movehdup_ps(res));- return _mm_cvtss_f32(res);-}--static inline __m256 loadf16x8(const ggml_fp16_t *p) {- return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));-}--static inline __m256i vpdpbusd(__m256i x, __m256i y, __m256i s) {- register __m256i ymm2 asm("ymm2") = x;- register __m256i ymm1 asm("ymm1") = y;- register __m256i ymm0 asm("ymm0") = s;- asm(".byte\t0xc4,0xe2,0x6d,0x50,0xc1" : "+x"(ymm0) : "x"(ymm1), "x"(ymm2));- return ymm0;-}--static inline __m256 mul_sum_i8_pairs_float(__m256i x, __m256i y) {- return _mm256_cvtepi32_ps(- vpdpbusd(_mm256_sign_epi8(x, x), // make unsigned- _mm256_sign_epi8(y, x), // make signed- _mm256_setzero_si256())); // no accumulator-}--static inline __m256 mul_sum_signed_i8_pairs_float(__m256i u, __m256i s) {- return _mm256_cvtepi32_ps(vpdpbusd(u, s, _mm256_setzero_si256()));-}--class tinyBLAS_F32_AVX2 {- public:- tinyBLAS_F32_AVX2(long k, const float *A, long lda, const float *B,- long ldb, float *C, long ldc, long ith, long nth)- : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith),- nth(nth) {- ASSERT(A != nullptr);- ASSERT(B != nullptr);- ASSERT(C != nullptr);- ASSERT(k >= 0 && k % 8 == 0);- ASSERT(ith >= 0 && ith < nth);- }-- void gemm(long m, long n) {- ASSERT(m >= 0);- ASSERT(n >= 0);- ASSERT(lda >= k);- ASSERT(ldb >= k);- ASSERT(ldc >= m);- mnpack(0, m, 0, n);- }-- private:- void mnpack(long m0, long m, long n0, long n) {- if (m - m0 <= 0 || n - n0 <= 0)- return;- long mc, nc, mp, np;- if (m - m0 >= 3 && n - n0 >= 4) {- mc = 3;- nc = 4;- gemm3x4(m0, m, n0, n);- } else if (m - m0 >= 4 && n - n0 >= 1) {- mc = 4;- nc = 1;- gemm4x1(m0, m, n0, n);- } else if (m - m0 >= 1 && n - n0 >= 4) {- mc = 1;- nc = 4;- gemm1x4(m0, m, n0, n);- } else {- mc = 1;- nc = 1;- gemm1x1(m0, m, n0, n);- }- mp = m0 + (m - m0) / mc * mc;- np = n0 + (n - n0) / nc * nc;- mnpack(mp, m, n0, np);- mnpack(m0, mp, np, n);- mnpack(mp, m, np, n);- }-- dontinline void gemm3x4(long m0, long m, long n0, long n) {- BEGIN_KERNEL(3, 4)- __m256 c00 = _mm256_setzero_ps();- __m256 c01 = _mm256_setzero_ps();- __m256 c02 = _mm256_setzero_ps();- __m256 c03 = _mm256_setzero_ps();- __m256 c10 = _mm256_setzero_ps();- __m256 c11 = _mm256_setzero_ps();- __m256 c12 = _mm256_setzero_ps();- __m256 c13 = _mm256_setzero_ps();- __m256 c20 = _mm256_setzero_ps();- __m256 c21 = _mm256_setzero_ps();- __m256 c22 = _mm256_setzero_ps();- __m256 c23 = _mm256_setzero_ps();- for (long l = 0; l < k; l += 8) {- __m256 k0 = _mm256_loadu_ps(B + ldb * (j + 0) + l);- __m256 k1 = _mm256_loadu_ps(B + ldb * (j + 1) + l);- __m256 k2 = _mm256_loadu_ps(B + ldb * (j + 2) + l);- __m256 k3 = _mm256_loadu_ps(B + ldb * (j + 3) + l);- __m256 a0 = _mm256_loadu_ps(A + lda * (i + 0) + l);- c00 = _mm256_fmadd_ps(a0, k0, c00);- c01 = _mm256_fmadd_ps(a0, k1, c01);- c02 = _mm256_fmadd_ps(a0, k2, c02);- c03 = _mm256_fmadd_ps(a0, k3, c03);- __m256 a1 = _mm256_loadu_ps(A + lda * (i + 1) + l);- c10 = _mm256_fmadd_ps(a1, k0, c10);- c11 = _mm256_fmadd_ps(a1, k1, c11);- c12 = _mm256_fmadd_ps(a1, k2, c12);- c13 = _mm256_fmadd_ps(a1, k3, c13);- __m256 a2 = _mm256_loadu_ps(A + lda * (i + 2) + l);- c20 = _mm256_fmadd_ps(a2, k0, c20);- c21 = _mm256_fmadd_ps(a2, k1, c21);- c22 = _mm256_fmadd_ps(a2, k2, c22);- c23 = _mm256_fmadd_ps(a2, k3, c23);- }- C[ldc * (j + 0) + (i + 0)] = hsum_float_8(c00);- C[ldc * (j + 0) + (i + 1)] = hsum_float_8(c10);- C[ldc * (j + 0) + (i + 2)] = hsum_float_8(c20);- C[ldc * (j + 1) + (i + 0)] = hsum_float_8(c01);- C[ldc * (j + 1) + (i + 1)] = hsum_float_8(c11);- C[ldc * (j + 1) + (i + 2)] = hsum_float_8(c21);- C[ldc * (j + 2) + (i + 0)] = hsum_float_8(c02);- C[ldc * (j + 2) + (i + 1)] = hsum_float_8(c12);- C[ldc * (j + 2) + (i + 2)] = hsum_float_8(c22);- C[ldc * (j + 3) + (i + 0)] = hsum_float_8(c03);- C[ldc * (j + 3) + (i + 1)] = hsum_float_8(c13);- C[ldc * (j + 3) + (i + 2)] = hsum_float_8(c23);- END_KERNEL()- }-- dontinline void gemm1x4(long m0, long m, long n0, long n) {- BEGIN_KERNEL(1, 4)- __m256 c00 = _mm256_setzero_ps();- __m256 c01 = _mm256_setzero_ps();- __m256 c02 = _mm256_setzero_ps();- __m256 c03 = _mm256_setzero_ps();- for (long l = 0; l < k; l += 8) {- __m256 a0 = _mm256_loadu_ps(A + lda * (i + 0) + l);- __m256 k0 = _mm256_loadu_ps(B + ldb * (j + 0) + l);- __m256 k1 = _mm256_loadu_ps(B + ldb * (j + 1) + l);- __m256 k2 = _mm256_loadu_ps(B + ldb * (j + 2) + l);- __m256 k3 = _mm256_loadu_ps(B + ldb * (j + 3) + l);- c00 = _mm256_fmadd_ps(a0, k0, c00);- c01 = _mm256_fmadd_ps(a0, k1, c01);- c02 = _mm256_fmadd_ps(a0, k2, c02);- c03 = _mm256_fmadd_ps(a0, k3, c03);- }- C[ldc * (j + 0) + (i + 0)] = hsum_float_8(c00);- C[ldc * (j + 1) + (i + 0)] = hsum_float_8(c01);- C[ldc * (j + 2) + (i + 0)] = hsum_float_8(c02);- C[ldc * (j + 3) + (i + 0)] = hsum_float_8(c03);- END_KERNEL()- }-- dontinline void gemm4x1(long m0, long m, long n0, long n) {- BEGIN_KERNEL(4, 1)- __m256 c00 = _mm256_setzero_ps();- __m256 c10 = _mm256_setzero_ps();- __m256 c20 = _mm256_setzero_ps();- __m256 c30 = _mm256_setzero_ps();- for (long l = 0; l < k; l += 8) {- __m256 k0 = _mm256_loadu_ps(B + ldb * (j + 0) + l);- __m256 a0 = _mm256_loadu_ps(A + lda * (i + 0) + l);- c00 = _mm256_fmadd_ps(a0, k0, c00);- __m256 a1 = _mm256_loadu_ps(A + lda * (i + 1) + l);- c10 = _mm256_fmadd_ps(a1, k0, c10);- __m256 a2 = _mm256_loadu_ps(A + lda * (i + 2) + l);- c20 = _mm256_fmadd_ps(a2, k0, c20);- __m256 a3 = _mm256_loadu_ps(A + lda * (i + 3) + l);- c30 = _mm256_fmadd_ps(a3, k0, c30);- }- C[ldc * (j + 0) + (i + 0)] = hsum_float_8(c00);- C[ldc * (j + 0) + (i + 1)] = hsum_float_8(c10);- C[ldc * (j + 0) + (i + 2)] = hsum_float_8(c20);- C[ldc * (j + 0) + (i + 3)] = hsum_float_8(c30);- END_KERNEL()- }-- dontinline void gemm1x1(long m0, long m, long n0, long n) {- BEGIN_KERNEL(1, 1)- __m256 c = _mm256_setzero_ps();- for (long l = 0; l < k; l += 8) {- c = _mm256_fmadd_ps(_mm256_loadu_ps(A + lda * i + l),- _mm256_loadu_ps(B + ldb * j + l), c);- }- C[ldc * j + i] = hsum_float_8(c);- END_KERNEL()- }-- const long k;- const float *const A;- const long lda;- const float *const B;- const long ldb;- float *const C;- const long ldc;- const long ith;- const long nth;-};--class tinyBLAS_F16_AVX2 {- public:- tinyBLAS_F16_AVX2(long k, const ggml_fp16_t *A, long lda,- const ggml_fp16_t *B, long ldb, float *C, long ldc,- long ith, long nth)- : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith),- nth(nth) {- ASSERT(A != nullptr);- ASSERT(B != nullptr);- ASSERT(C != nullptr);- ASSERT(k >= 0 && k % 16 == 0);- ASSERT(ith >= 0 && ith < nth);- }-- void gemm(long m, long n) {- ASSERT(m >= 0);- ASSERT(n >= 0);- ASSERT(ldc >= m);- mnpack(0, m, 0, n);- }-- private:- void mnpack(long m0, long m, long n0, long n) {- if (m - m0 <= 0 || n - n0 <= 0)- return;- long mc, nc, mp, np;- if (m - m0 >= 3 && n - n0 >= 4) {- mc = 3;- nc = 4;- gemm3x4(m0, m, n0, n);- } else if (m - m0 >= 4 && n - n0 >= 1) {- mc = 4;- nc = 1;- gemm4x1(m0, m, n0, n);- } else if (m - m0 >= 1 && n - n0 >= 4) {- mc = 1;- nc = 4;- gemm1x4(m0, m, n0, n);- } else {- mc = 1;- nc = 1;- gemm1x1(m0, m, n0, n);- }- mp = m0 + (m - m0) / mc * mc;- np = n0 + (n - n0) / nc * nc;- mnpack(mp, m, n0, np);- mnpack(m0, mp, np, n);- mnpack(mp, m, np, n);- }-- dontinline void gemm3x4(long m0, long m, long n0, long n) {- BEGIN_KERNEL(3, 4)- __m256 c00 = _mm256_setzero_ps();- __m256 c01 = _mm256_setzero_ps();- __m256 c02 = _mm256_setzero_ps();- __m256 c03 = _mm256_setzero_ps();- __m256 c10 = _mm256_setzero_ps();- __m256 c11 = _mm256_setzero_ps();- __m256 c12 = _mm256_setzero_ps();- __m256 c13 = _mm256_setzero_ps();- __m256 c20 = _mm256_setzero_ps();- __m256 c21 = _mm256_setzero_ps();- __m256 c22 = _mm256_setzero_ps();- __m256 c23 = _mm256_setzero_ps();- const ggml_fp16_t *Ap0 = A + lda * (i + 0);- const ggml_fp16_t *Ap1 = A + lda * (i + 1);- const ggml_fp16_t *Ap2 = A + lda * (i + 2);- const ggml_fp16_t *Bp0 = B + ldb * (j + 0);- const ggml_fp16_t *Bp1 = B + ldb * (j + 1);- const ggml_fp16_t *Bp2 = B + ldb * (j + 2);- const ggml_fp16_t *Bp3 = B + ldb * (j + 3);- for (long l = 0; l < k; l += 8) {- __m256 k0 = loadf16x8(Bp0 + l);- __m256 k1 = loadf16x8(Bp1 + l);- __m256 k2 = loadf16x8(Bp2 + l);- __m256 k3 = loadf16x8(Bp3 + l);- __m256 a0 = loadf16x8(Ap0 + l);- c00 = _mm256_fmadd_ps(a0, k0, c00);- c01 = _mm256_fmadd_ps(a0, k1, c01);- c02 = _mm256_fmadd_ps(a0, k2, c02);- c03 = _mm256_fmadd_ps(a0, k3, c03);- __m256 a1 = loadf16x8(Ap1 + l);- c10 = _mm256_fmadd_ps(a1, k0, c10);- c11 = _mm256_fmadd_ps(a1, k1, c11);- c12 = _mm256_fmadd_ps(a1, k2, c12);- c13 = _mm256_fmadd_ps(a1, k3, c13);- __m256 a2 = loadf16x8(Ap2 + l);- c20 = _mm256_fmadd_ps(a2, k0, c20);- c21 = _mm256_fmadd_ps(a2, k1, c21);- c22 = _mm256_fmadd_ps(a2, k2, c22);- c23 = _mm256_fmadd_ps(a2, k3, c23);- }- C[ldc * (j + 0) + (i + 0)] = hsum_float_8(c00);- C[ldc * (j + 0) + (i + 1)] = hsum_float_8(c10);- C[ldc * (j + 0) + (i + 2)] = hsum_float_8(c20);- C[ldc * (j + 1) + (i + 0)] = hsum_float_8(c01);- C[ldc * (j + 1) + (i + 1)] = hsum_float_8(c11);- C[ldc * (j + 1) + (i + 2)] = hsum_float_8(c21);- C[ldc * (j + 2) + (i + 0)] = hsum_float_8(c02);- C[ldc * (j + 2) + (i + 1)] = hsum_float_8(c12);- C[ldc * (j + 2) + (i + 2)] = hsum_float_8(c22);- C[ldc * (j + 3) + (i + 0)] = hsum_float_8(c03);- C[ldc * (j + 3) + (i + 1)] = hsum_float_8(c13);- C[ldc * (j + 3) + (i + 2)] = hsum_float_8(c23);- END_KERNEL()- }-- dontinline void gemm1x4(long m0, long m, long n0, long n) {- BEGIN_KERNEL(1, 4)- __m256 c00 = _mm256_setzero_ps();- __m256 c01 = _mm256_setzero_ps();- __m256 c02 = _mm256_setzero_ps();- __m256 c03 = _mm256_setzero_ps();- for (long l = 0; l < k; l += 8) {- __m256 a0 = loadf16x8(A + lda * (i + 0) + l);- __m256 k0 = loadf16x8(B + ldb * (j + 0) + l);- __m256 k1 = loadf16x8(B + ldb * (j + 1) + l);- __m256 k2 = loadf16x8(B + ldb * (j + 2) + l);- __m256 k3 = loadf16x8(B + ldb * (j + 3) + l);- c00 = _mm256_fmadd_ps(a0, k0, c00);- c01 = _mm256_fmadd_ps(a0, k1, c01);- c02 = _mm256_fmadd_ps(a0, k2, c02);- c03 = _mm256_fmadd_ps(a0, k3, c03);- }- C[ldc * (j + 0) + (i + 0)] = hsum_float_8(c00);- C[ldc * (j + 1) + (i + 0)] = hsum_float_8(c01);- C[ldc * (j + 2) + (i + 0)] = hsum_float_8(c02);- C[ldc * (j + 3) + (i + 0)] = hsum_float_8(c03);- END_KERNEL()- }-- dontinline void gemm4x1(long m0, long m, long n0, long n) {- BEGIN_KERNEL(4, 1)- __m256 c00 = _mm256_setzero_ps();- __m256 c10 = _mm256_setzero_ps();- __m256 c20 = _mm256_setzero_ps();- __m256 c30 = _mm256_setzero_ps();- for (long l = 0; l < k; l += 8) {- __m256 k0 = loadf16x8(B + ldb * (j + 0) + l);- __m256 a0 = loadf16x8(A + lda * (i + 0) + l);- c00 = _mm256_fmadd_ps(a0, k0, c00);- __m256 a1 = loadf16x8(A + lda * (i + 1) + l);- c10 = _mm256_fmadd_ps(a1, k0, c10);- __m256 a2 = loadf16x8(A + lda * (i + 2) + l);- c20 = _mm256_fmadd_ps(a2, k0, c20);- __m256 a3 = loadf16x8(A + lda * (i + 3) + l);- c30 = _mm256_fmadd_ps(a3, k0, c30);- }- C[ldc * (j + 0) + (i + 0)] = hsum_float_8(c00);- C[ldc * (j + 0) + (i + 1)] = hsum_float_8(c10);- C[ldc * (j + 0) + (i + 2)] = hsum_float_8(c20);- C[ldc * (j + 0) + (i + 3)] = hsum_float_8(c30);- END_KERNEL()- }-- dontinline void gemm1x1(long m0, long m, long n0, long n) {- BEGIN_KERNEL(1, 1)- __m256 c = _mm256_setzero_ps();- for (long l = 0; l < k; l += 8) {- c = _mm256_fmadd_ps(loadf16x8(A + lda * i + l),- loadf16x8(B + ldb * j + l), c);- }- C[ldc * j + i] = hsum_float_8(c);- END_KERNEL()- }-- const long k;- const ggml_fp16_t *const A;- const long lda;- const ggml_fp16_t *const B;- const long ldb;- float *const C;- const long ldc;- const long ith;- const long nth;-};--class tinyBLAS_Q8_VNNI {- public:- tinyBLAS_Q8_VNNI(long k, const block_q8_0 *A, long lda, const block_q8_0 *B,- long ldb, float *C, long ldc, long ith, long nth)- : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith),- nth(nth) {- ASSERT(A != nullptr);- ASSERT(B != nullptr);- ASSERT(C != nullptr);- ASSERT(k >= 0 && k % 32 == 0);- ASSERT(ith >= 0 && ith < nth);- }-- void gemm(long m, long n) {- ASSERT(m >= 0);- ASSERT(n >= 0);- ASSERT(ldc >= m);- mnpack(0, m, 0, n);- }-- private:- void mnpack(long m0, long m, long n0, long n) {- if (m - m0 <= 0 || n - n0 <= 0)- return;- long mc, nc, mp, np;- if (m - m0 >= 4 && n - n0 >= 1) {- mc = 4;- nc = 1;- gemm4x1(m0, m, n0, n);- } else if (m - m0 >= 1 && n - n0 >= 4) {- mc = 1;- nc = 4;- gemm1x4(m0, m, n0, n);- } else {- mc = 1;- nc = 1;- gemm1x1(m0, m, n0, n);- }- mp = m0 + (m - m0) / mc * mc;- np = n0 + (n - n0) / nc * nc;- mnpack(mp, m, n0, np);- mnpack(m0, mp, np, n);- mnpack(mp, m, np, n);- }-- dontinline void gemm4x1(long m0, long m, long n0, long n) {- BEGIN_KERNEL(4, 1)- long k2 = k / 32;- __m256 c0 = _mm256_setzero_ps();- __m256 c1 = _mm256_setzero_ps();- __m256 c2 = _mm256_setzero_ps();- __m256 c3 = _mm256_setzero_ps();- const block_q8_0 *Ap0 = A + lda * (i + 0);- const block_q8_0 *Ap1 = A + lda * (i + 1);- const block_q8_0 *Ap2 = A + lda * (i + 2);- const block_q8_0 *Ap3 = A + lda * (i + 3);- const block_q8_0 *Bp = B + ldb * j;- for (long l = 0; l < k2; ++l) {- float db0 = GGML_FP16_TO_FP32(Bp[l].d);- __m256i f = _mm256_loadu_si256((const __m256i *)Bp[l].qs);- __m256i u = _mm256_sign_epi8(f, f);- __m256 d0 = _mm256_set1_ps(GGML_FP16_TO_FP32(Ap0[l].d) * db0);- __m256 d1 = _mm256_set1_ps(GGML_FP16_TO_FP32(Ap1[l].d) * db0);- __m256 d2 = _mm256_set1_ps(GGML_FP16_TO_FP32(Ap2[l].d) * db0);- __m256 d3 = _mm256_set1_ps(GGML_FP16_TO_FP32(Ap3[l].d) * db0);- __m256i e0 = _mm256_loadu_si256((const __m256i *)Ap0[l].qs);- __m256i e1 = _mm256_loadu_si256((const __m256i *)Ap1[l].qs);- __m256i e2 = _mm256_loadu_si256((const __m256i *)Ap2[l].qs);- __m256i e3 = _mm256_loadu_si256((const __m256i *)Ap3[l].qs);- __m256i s0 = _mm256_sign_epi8(e0, f);- __m256i s1 = _mm256_sign_epi8(e1, f);- __m256i s2 = _mm256_sign_epi8(e2, f);- __m256i s3 = _mm256_sign_epi8(e3, f);- __m256 g0 = mul_sum_signed_i8_pairs_float(u, s0);- __m256 g1 = mul_sum_signed_i8_pairs_float(u, s1);- __m256 g2 = mul_sum_signed_i8_pairs_float(u, s2);- __m256 g3 = mul_sum_signed_i8_pairs_float(u, s3);- c0 = _mm256_fmadd_ps(d0, g0, c0);- c1 = _mm256_fmadd_ps(d1, g1, c1);- c2 = _mm256_fmadd_ps(d2, g2, c2);- c3 = _mm256_fmadd_ps(d3, g3, c3);- }- C[ldc * j + (i + 0)] = hsum_float_8(c0);- C[ldc * j + (i + 1)] = hsum_float_8(c1);- C[ldc * j + (i + 2)] = hsum_float_8(c2);- C[ldc * j + (i + 3)] = hsum_float_8(c3);- END_KERNEL()- }-- dontinline void gemm1x4(long m0, long m, long n0, long n) {- BEGIN_KERNEL(1, 4)- long k2 = k / 32;- __m256 c0 = _mm256_setzero_ps();- __m256 c1 = _mm256_setzero_ps();- __m256 c2 = _mm256_setzero_ps();- __m256 c3 = _mm256_setzero_ps();- const block_q8_0 *Bp0 = B + ldb * (j + 0);- const block_q8_0 *Bp1 = B + ldb * (j + 1);- const block_q8_0 *Bp2 = B + ldb * (j + 2);- const block_q8_0 *Bp3 = B + ldb * (j + 3);- const block_q8_0 *Ap = A + lda * i;- for (long l = 0; l < k2; ++l) {- float da0 = GGML_FP16_TO_FP32(Ap[l].d);- __m256i f = _mm256_loadu_si256((const __m256i *)Ap[l].qs);- __m256i u = _mm256_sign_epi8(f, f);- __m256 d0 = _mm256_set1_ps(GGML_FP16_TO_FP32(Bp0[l].d) * da0);- __m256 d1 = _mm256_set1_ps(GGML_FP16_TO_FP32(Bp1[l].d) * da0);- __m256 d2 = _mm256_set1_ps(GGML_FP16_TO_FP32(Bp2[l].d) * da0);- __m256 d3 = _mm256_set1_ps(GGML_FP16_TO_FP32(Bp3[l].d) * da0);- __m256i e0 = _mm256_loadu_si256((const __m256i *)Bp0[l].qs);- __m256i e1 = _mm256_loadu_si256((const __m256i *)Bp1[l].qs);- __m256i e2 = _mm256_loadu_si256((const __m256i *)Bp2[l].qs);- __m256i e3 = _mm256_loadu_si256((const __m256i *)Bp3[l].qs);- __m256i s0 = _mm256_sign_epi8(e0, f);- __m256i s1 = _mm256_sign_epi8(e1, f);- __m256i s2 = _mm256_sign_epi8(e2, f);- __m256i s3 = _mm256_sign_epi8(e3, f);- __m256 g0 = mul_sum_signed_i8_pairs_float(u, s0);- __m256 g1 = mul_sum_signed_i8_pairs_float(u, s1);- __m256 g2 = mul_sum_signed_i8_pairs_float(u, s2);- __m256 g3 = mul_sum_signed_i8_pairs_float(u, s3);- c0 = _mm256_fmadd_ps(d0, g0, c0);- c1 = _mm256_fmadd_ps(d1, g1, c1);- c2 = _mm256_fmadd_ps(d2, g2, c2);- c3 = _mm256_fmadd_ps(d3, g3, c3);- }- C[ldc * (j + 0) + i] = hsum_float_8(c0);- C[ldc * (j + 1) + i] = hsum_float_8(c1);- C[ldc * (j + 2) + i] = hsum_float_8(c2);- C[ldc * (j + 3) + i] = hsum_float_8(c3);- END_KERNEL()- }-- dontinline void gemm1x1(long m0, long m, long n0, long n) {- BEGIN_KERNEL(1, 1)- long k2 = k / 32;- __m256 c0 = _mm256_setzero_ps();- const block_q8_0 *Ap = A + lda * i;- const block_q8_0 *Bp = B + ldb * j;- for (long l = 0; l < k2; ++l) {- __m256 d = _mm256_set1_ps(GGML_FP16_TO_FP32(Ap[l].d) *- GGML_FP16_TO_FP32(Bp[l].d));- __m256i e = _mm256_loadu_si256((const __m256i *)Ap[l].qs);- __m256i f = _mm256_loadu_si256((const __m256i *)Bp[l].qs);- __m256 g = mul_sum_i8_pairs_float(e, f);- c0 = _mm256_fmadd_ps(d, g, c0);- }- C[ldc * j + i] = hsum_float_8(c0);- END_KERNEL()- }-- const long k;- const block_q8_0 *const A;- const long lda;- const block_q8_0 *const B;- const long ldb;- float *const C;- const long ldc;- const long ith;- const long nth;-};--#endif // __x86_64__--// computes m×k * n×k → n×m-void llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda,- const void *B, long ldb, float *C, long ldc, long ith,- long nth) {- switch (dtype) {-#ifdef __x86_64__- case GGML_TYPE_F32: {- tinyBLAS_F32_AVX2 tb{- k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth};- tb.gemm(m, n);- break;- }- case GGML_TYPE_F16: {- tinyBLAS_F16_AVX2 tb{k, (const ggml_fp16_t *)A,- lda, (const ggml_fp16_t *)B,- ldb, C,- ldc, ith,- nth};- tb.gemm(m, n);- break;- }- case GGML_TYPE_Q8_0: {- tinyBLAS_Q8_VNNI tb{k, (const block_q8_0 *)A,- lda, (const block_q8_0 *)B,- ldb, C,- ldc, ith,- nth};- tb.gemm(m, n);- break;- }-#endif- default:- ASSERT(!"unsupported dtype");- }-}diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cppnew file mode 100644index 00000000..3b8f83d1--- /dev/null+++ b/llamafile/sgemm.cpp@@ -0,0 +1,512 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "llama.cpp/ggml-quants.h"+#include "llamafile.h"+#include <cosmo.h>+#include <immintrin.h>+#include <stdio.h>++//+// _ _ ___ _ _ ___+// | |_(_)_ _ _ _| _ ) | /_\ / __|+// | _| | ' \ || | _ \ |__ / _ \\__ \.+// \__|_|_||_\_, |___/____/_/ \_\___/+// |__/+//+// BASIC LINEAR ALGEBRA SUBPROGRAMS+//++#define ASSERT(x) \+ if (!(x)) { \+ fprintf(stderr, "%s:%d: assertion failed: %s\n", __FILE__, __LINE__, \+ #x); \+ __builtin_trap(); \+ }++#define END_KERNEL() }+#define BEGIN_KERNEL(RM, RN) \+ long ytiles = (m - m0) / RM; \+ long xtiles = (n - n0) / RN; \+ long tiles = ytiles * xtiles; \+ double duty = (double)tiles / nth; \+ if (duty < 1) \+ duty = 1; \+ double spot = duty * ith + .5; \+ long end = spot + duty; \+ long start = spot; \+ if (end > tiles) \+ end = tiles; \+ for (long job = start; job < end; ++job) { \+ long i = m0 + job / xtiles * RM; \+ long j = n0 + job % xtiles * RN;++#ifdef __x86_64__+#pragma GCC push_options+#pragma GCC target("avx2,fma,f16c,avxvnni")++#define MM256_SET_M128I(a, b) \+ _mm256_insertf128_si256(_mm256_castsi128_si256(b), (a), 1)++static inline float unhalf(ggml_fp16_t d) {+ return GGML_FP16_TO_FP32(d);+}++static dontinline float hsum8f(__m256 x) {+ __m128 res = _mm256_extractf128_ps(x, 1);+ res = _mm_add_ps(res, _mm256_castps256_ps128(x));+ res = _mm_add_ps(res, _mm_movehl_ps(res, res));+ res = _mm_add_ss(res, _mm_movehdup_ps(res));+ return _mm_cvtss_f32(res);+}++static inline __m256i denibble32(const uint8_t *p) {+ const __m128i tmp = _mm_loadu_si128((const __m128i *)p);+ const __m256i bytes = MM256_SET_M128I(_mm_srli_epi16(tmp, 4), tmp);+ const __m256i lowMask = _mm256_set1_epi8(15);+ return _mm256_and_si256(lowMask, bytes);+}++template <typename T> static inline __m256 load8f(const T *);+template <> inline __m256 load8f(const float *p) {+ return _mm256_loadu_ps(p);+}+template <> inline __m256 load8f(const ggml_fp16_t *p) {+ return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));+}++template <typename T> static inline __m256i load32i(const T *);+template <> inline __m256i load32i(const block_q8_0 *b) {+ return _mm256_loadu_si256((const __m256i *)b->qs);+}+template <> inline __m256i load32i(const block_q4_0 *b) {+ return _mm256_sub_epi8(denibble32(b->qs), _mm256_set1_epi8(8));+}++template <typename T> class tinyBLAS {+ public:+ tinyBLAS(long k, const T *A, long lda, const T *B, long ldb, float *C,+ long ldc, long ith, long nth)+ : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith),+ nth(nth) {+ ASSERT(A != nullptr);+ ASSERT(B != nullptr);+ ASSERT(C != nullptr);+ ASSERT(k >= 0 && k % 8 == 0);+ ASSERT(ith >= 0 && ith < nth);+ }++ void gemm(long m, long n) {+ ASSERT(m >= 0);+ ASSERT(n >= 0);+ ASSERT(ldc >= m);+ mnpack(0, m, 0, n);+ }++ private:+ void mnpack(long m0, long m, long n0, long n) {+ if (m - m0 <= 0 || n - n0 <= 0)+ return;+ long mc, nc, mp, np;+ if (m - m0 >= 3 && n - n0 >= 4) {+ mc = 3;+ nc = 4;+ gemm3x4(m0, m, n0, n);+ } else if (m - m0 >= 4 && n - n0 >= 1) {+ mc = 4;+ nc = 1;+ gemm4x1(m0, m, n0, n);+ } else if (m - m0 >= 1 && n - n0 >= 4) {+ mc = 1;+ nc = 4;+ gemm1x4(m0, m, n0, n);+ } else {+ mc = 1;+ nc = 1;+ gemm1x1(m0, m, n0, n);+ }+ mp = m0 + (m - m0) / mc * mc;+ np = n0 + (n - n0) / nc * nc;+ mnpack(mp, m, n0, np);+ mnpack(m0, mp, np, n);+ mnpack(mp, m, np, n);+ }++ dontinline void gemm3x4(long m0, long m, long n0, long n) {+ BEGIN_KERNEL(3, 4)+ __m256 c00 = _mm256_setzero_ps();+ __m256 c01 = _mm256_setzero_ps();+ __m256 c02 = _mm256_setzero_ps();+ __m256 c03 = _mm256_setzero_ps();+ __m256 c10 = _mm256_setzero_ps();+ __m256 c11 = _mm256_setzero_ps();+ __m256 c12 = _mm256_setzero_ps();+ __m256 c13 = _mm256_setzero_ps();+ __m256 c20 = _mm256_setzero_ps();+ __m256 c21 = _mm256_setzero_ps();+ __m256 c22 = _mm256_setzero_ps();+ __m256 c23 = _mm256_setzero_ps();+ for (long l = 0; l < k; l += 8) {+ __m256 k0 = load8f(B + ldb * (j + 0) + l);+ __m256 k1 = load8f(B + ldb * (j + 1) + l);+ __m256 k2 = load8f(B + ldb * (j + 2) + l);+ __m256 k3 = load8f(B + ldb * (j + 3) + l);+ __m256 a0 = load8f(A + lda * (i + 0) + l);+ c00 = _mm256_fmadd_ps(a0, k0, c00);+ c01 = _mm256_fmadd_ps(a0, k1, c01);+ c02 = _mm256_fmadd_ps(a0, k2, c02);+ c03 = _mm256_fmadd_ps(a0, k3, c03);+ __m256 a1 = load8f(A + lda * (i + 1) + l);+ c10 = _mm256_fmadd_ps(a1, k0, c10);+ c11 = _mm256_fmadd_ps(a1, k1, c11);+ c12 = _mm256_fmadd_ps(a1, k2, c12);+ c13 = _mm256_fmadd_ps(a1, k3, c13);+ __m256 a2 = load8f(A + lda * (i + 2) + l);+ c20 = _mm256_fmadd_ps(a2, k0, c20);+ c21 = _mm256_fmadd_ps(a2, k1, c21);+ c22 = _mm256_fmadd_ps(a2, k2, c22);+ c23 = _mm256_fmadd_ps(a2, k3, c23);+ }+ C[ldc * (j + 0) + (i + 0)] = hsum8f(c00);+ C[ldc * (j + 0) + (i + 1)] = hsum8f(c10);+ C[ldc * (j + 0) + (i + 2)] = hsum8f(c20);+ C[ldc * (j + 1) + (i + 0)] = hsum8f(c01);+ C[ldc * (j + 1) + (i + 1)] = hsum8f(c11);+ C[ldc * (j + 1) + (i + 2)] = hsum8f(c21);+ C[ldc * (j + 2) + (i + 0)] = hsum8f(c02);+ C[ldc * (j + 2) + (i + 1)] = hsum8f(c12);+ C[ldc * (j + 2) + (i + 2)] = hsum8f(c22);+ C[ldc * (j + 3) + (i + 0)] = hsum8f(c03);+ C[ldc * (j + 3) + (i + 1)] = hsum8f(c13);+ C[ldc * (j + 3) + (i + 2)] = hsum8f(c23);+ END_KERNEL()+ }++ dontinline void gemm1x4(long m0, long m, long n0, long n) {+ BEGIN_KERNEL(1, 4)+ __m256 c00 = _mm256_setzero_ps();+ __m256 c01 = _mm256_setzero_ps();+ __m256 c02 = _mm256_setzero_ps();+ __m256 c03 = _mm256_setzero_ps();+ for (long l = 0; l < k; l += 8) {+ __m256 a0 = load8f(A + lda * (i + 0) + l);+ __m256 k0 = load8f(B + ldb * (j + 0) + l);+ __m256 k1 = load8f(B + ldb * (j + 1) + l);+ __m256 k2 = load8f(B + ldb * (j + 2) + l);+ __m256 k3 = load8f(B + ldb * (j + 3) + l);+ c00 = _mm256_fmadd_ps(a0, k0, c00);+ c01 = _mm256_fmadd_ps(a0, k1, c01);+ c02 = _mm256_fmadd_ps(a0, k2, c02);+ c03 = _mm256_fmadd_ps(a0, k3, c03);+ }+ C[ldc * (j + 0) + (i + 0)] = hsum8f(c00);+ C[ldc * (j + 1) + (i + 0)] = hsum8f(c01);+ C[ldc * (j + 2) + (i + 0)] = hsum8f(c02);+ C[ldc * (j + 3) + (i + 0)] = hsum8f(c03);+ END_KERNEL()+ }++ dontinline void gemm4x1(long m0, long m, long n0, long n) {+ BEGIN_KERNEL(4, 1)+ __m256 c00 = _mm256_setzero_ps();+ __m256 c10 = _mm256_setzero_ps();+ __m256 c20 = _mm256_setzero_ps();+ __m256 c30 = _mm256_setzero_ps();+ for (long l = 0; l < k; l += 8) {+ __m256 k0 = load8f(B + ldb * (j + 0) + l);+ __m256 a0 = load8f(A + lda * (i + 0) + l);+ c00 = _mm256_fmadd_ps(a0, k0, c00);+ __m256 a1 = load8f(A + lda * (i + 1) + l);+ c10 = _mm256_fmadd_ps(a1, k0, c10);+ __m256 a2 = load8f(A + lda * (i + 2) + l);+ c20 = _mm256_fmadd_ps(a2, k0, c20);+ __m256 a3 = load8f(A + lda * (i + 3) + l);+ c30 = _mm256_fmadd_ps(a3, k0, c30);+ }+ C[ldc * (j + 0) + (i + 0)] = hsum8f(c00);+ C[ldc * (j + 0) + (i + 1)] = hsum8f(c10);+ C[ldc * (j + 0) + (i + 2)] = hsum8f(c20);+ C[ldc * (j + 0) + (i + 3)] = hsum8f(c30);+ END_KERNEL()+ }++ dontinline void gemm1x1(long m0, long m, long n0, long n) {+ BEGIN_KERNEL(1, 1)+ __m256 c = _mm256_setzero_ps();+ for (long l = 0; l < k; l += 8) {+ c = _mm256_fmadd_ps(load8f(A + lda * i + l),+ load8f(B + ldb * j + l), c);+ }+ C[ldc * j + i] = hsum8f(c);+ END_KERNEL()+ }++ const long k;+ const T *const A;+ const long lda;+ const T *const B;+ const long ldb;+ float *const C;+ const long ldc;+ const long ith;+ const long nth;+};++template <typename T, typename U, int avxvnni> class tinyBLASq {+ public:+ tinyBLASq(long k, const T *A, long lda, const U *B, long ldb, float *C,+ long ldc, long ith, long nth)+ : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith),+ nth(nth) {+ ASSERT(A != nullptr);+ ASSERT(B != nullptr);+ ASSERT(C != nullptr);+ ASSERT(k >= 0 && k % 32 == 0);+ ASSERT(ith >= 0 && ith < nth);+ }++ void gemm(long m, long n) {+ ASSERT(m >= 0);+ ASSERT(n >= 0);+ ASSERT(ldc >= m);+ mnpack(0, m, 0, n);+ }++ private:+ void mnpack(long m0, long m, long n0, long n) {+ if (m - m0 <= 0 || n - n0 <= 0)+ return;+ long mc, nc, mp, np;+ if (m - m0 >= 1 && n - n0 >= 4) {+ mc = 1;+ nc = 4;+ gemm1x4(m0, m, n0, n);+ } else if (m - m0 >= 4 && n - n0 >= 1) {+ mc = 4;+ nc = 1;+ gemm4x1(m0, m, n0, n);+ } else {+ mc = 1;+ nc = 1;+ gemm1x1(m0, m, n0, n);+ }+ mp = m0 + (m - m0) / mc * mc;+ np = n0 + (n - n0) / nc * nc;+ mnpack(mp, m, n0, np);+ mnpack(m0, mp, np, n);+ mnpack(mp, m, np, n);+ }++ dontinline void gemm4x1(long m0, long m, long n0, long n) {+ BEGIN_KERNEL(4, 1)+ long k2 = k / 32;+ __m256 c0 = _mm256_setzero_ps();+ __m256 c1 = _mm256_setzero_ps();+ __m256 c2 = _mm256_setzero_ps();+ __m256 c3 = _mm256_setzero_ps();+ const T *Ap0 = A + lda * (i + 0);+ const T *Ap1 = A + lda * (i + 1);+ const T *Ap2 = A + lda * (i + 2);+ const T *Ap3 = A + lda * (i + 3);+ const U *Bp = B + ldb * j;+ for (long l = 0; l < k2; ++l) {+ float db0 = unhalf(Bp[l].d);+ __m256i f = load32i(Bp + l);+ __m256i u = _mm256_sign_epi8(f, f);+ __m256 d0 = _mm256_set1_ps(unhalf(Ap0[l].d) * db0);+ __m256 d1 = _mm256_set1_ps(unhalf(Ap1[l].d) * db0);+ __m256 d2 = _mm256_set1_ps(unhalf(Ap2[l].d) * db0);+ __m256 d3 = _mm256_set1_ps(unhalf(Ap3[l].d) * db0);+ __m256i e0 = load32i(Ap0 + l);+ __m256i e1 = load32i(Ap1 + l);+ __m256i e2 = load32i(Ap2 + l);+ __m256i e3 = load32i(Ap3 + l);+ __m256i s0 = _mm256_sign_epi8(e0, f);+ __m256i s1 = _mm256_sign_epi8(e1, f);+ __m256i s2 = _mm256_sign_epi8(e2, f);+ __m256i s3 = _mm256_sign_epi8(e3, f);+ __m256 g0 = dotsome(u, s0);+ __m256 g1 = dotsome(u, s1);+ __m256 g2 = dotsome(u, s2);+ __m256 g3 = dotsome(u, s3);+ c0 = _mm256_fmadd_ps(d0, g0, c0);+ c1 = _mm256_fmadd_ps(d1, g1, c1);+ c2 = _mm256_fmadd_ps(d2, g2, c2);+ c3 = _mm256_fmadd_ps(d3, g3, c3);+ }+ C[ldc * j + (i + 0)] = hsum8f(c0);+ C[ldc * j + (i + 1)] = hsum8f(c1);+ C[ldc * j + (i + 2)] = hsum8f(c2);+ C[ldc * j + (i + 3)] = hsum8f(c3);+ END_KERNEL()+ }++ dontinline void gemm1x4(long m0, long m, long n0, long n) {+ BEGIN_KERNEL(1, 4)+ long k2 = k / 32;+ __m256 c0 = _mm256_setzero_ps();+ __m256 c1 = _mm256_setzero_ps();+ __m256 c2 = _mm256_setzero_ps();+ __m256 c3 = _mm256_setzero_ps();+ const U *Bp0 = B + ldb * (j + 0);+ const U *Bp1 = B + ldb * (j + 1);+ const U *Bp2 = B + ldb * (j + 2);+ const U *Bp3 = B + ldb * (j + 3);+ const T *Ap = A + lda * i;+ for (long l = 0; l < k2; ++l) {+ float da0 = unhalf(Ap[l].d);+ __m256i f = load32i(Ap + l);+ __m256i u = _mm256_sign_epi8(f, f);+ __m256 d0 = _mm256_set1_ps(unhalf(Bp0[l].d) * da0);+ __m256 d1 = _mm256_set1_ps(unhalf(Bp1[l].d) * da0);+ __m256 d2 = _mm256_set1_ps(unhalf(Bp2[l].d) * da0);+ __m256 d3 = _mm256_set1_ps(unhalf(Bp3[l].d) * da0);+ __m256 g0 = dotsome(u, _mm256_sign_epi8(load32i(Bp0 + l), f));+ __m256 g1 = dotsome(u, _mm256_sign_epi8(load32i(Bp1 + l), f));+ __m256 g2 = dotsome(u, _mm256_sign_epi8(load32i(Bp2 + l), f));+ __m256 g3 = dotsome(u, _mm256_sign_epi8(load32i(Bp3 + l), f));+ c0 = _mm256_fmadd_ps(d0, g0, c0);+ c1 = _mm256_fmadd_ps(d1, g1, c1);+ c2 = _mm256_fmadd_ps(d2, g2, c2);+ c3 = _mm256_fmadd_ps(d3, g3, c3);+ }+ C[ldc * (j + 0) + i] = hsum8f(c0);+ C[ldc * (j + 1) + i] = hsum8f(c1);+ C[ldc * (j + 2) + i] = hsum8f(c2);+ C[ldc * (j + 3) + i] = hsum8f(c3);+ END_KERNEL()+ }++ dontinline void gemm1x1(long m0, long m, long n0, long n) {+ BEGIN_KERNEL(1, 1)+ long k2 = k / 32;+ __m256 c = _mm256_setzero_ps();+ const T *Ap = A + lda * i;+ const U *Bp = B + ldb * j;+ for (long l = 0; l < k2; ++l) {+ __m256 d = _mm256_set1_ps(unhalf(Ap[l].d) * unhalf(Bp[l].d));+ __m256i e = load32i(Ap + l);+ __m256i f = load32i(Bp + l);+ __m256 g = dotsome(_mm256_sign_epi8(e, e), _mm256_sign_epi8(f, e));+ c = _mm256_fmadd_ps(d, g, c);+ }+ C[ldc * j + i] = hsum8f(c);+ END_KERNEL()+ }++ inline __m256 dotsome(__m256i u, __m256i s) {+ __m256i res;+ if (avxvnni)+ res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);+ else+ res = _mm256_madd_epi16(_mm256_set1_epi16(1),+ _mm256_maddubs_epi16(u, s));+ return _mm256_cvtepi32_ps(res);+ }++ const long k;+ const T *const A;+ const long lda;+ const U *const B;+ const long ldb;+ float *const C;+ const long ldc;+ const long ith;+ const long nth;+};++#endif // __x86_64__++// computes m×k * n×k → n×m+void llamafile_sgemm(long m, long n, long k, int dtype, const void *A, long lda,+ const void *B, long ldb, float *C, long ldc, long ith,+ long nth) {+ ASSERT(X86_HAVE(FMA));+ ASSERT(X86_HAVE(AVX2));+ switch (dtype) {+#ifdef __x86_64__++ case GGML_TYPE_F32: {+ tinyBLAS<float> tb{+ k, (const float *)A, lda, (const float *)B, ldb, C, ldc, ith, nth,+ };+ tb.gemm(m, n);+ break;+ }++ case GGML_TYPE_F16: {+ ASSERT(X86_HAVE(F16C));+ tinyBLAS<ggml_fp16_t> tb{+ k, (const ggml_fp16_t *)A,+ lda, (const ggml_fp16_t *)B,+ ldb, C,+ ldc, ith,+ nth,+ };+ tb.gemm(m, n);+ break;+ }++ case GGML_TYPE_Q8_0:+ if (X86_HAVE(AVXVNNI)) {+ tinyBLASq<block_q8_0, block_q8_0, true> tb{+ k, (const block_q8_0 *)A,+ lda, (const block_q8_0 *)B,+ ldb, C,+ ldc, ith,+ nth,+ };+ tb.gemm(m, n);+ } else {+ tinyBLASq<block_q8_0, block_q8_0, false> tb{+ k, (const block_q8_0 *)A,+ lda, (const block_q8_0 *)B,+ ldb, C,+ ldc, ith,+ nth,+ };+ tb.gemm(m, n);+ }+ break;++ case GGML_TYPE_Q4_0:+ if (X86_HAVE(AVXVNNI)) {+ tinyBLASq<block_q4_0, block_q8_0, true> tb{+ k, (const block_q4_0 *)A,+ lda, (const block_q8_0 *)B,+ ldb, C,+ ldc, ith,+ nth,+ };+ tb.gemm(m, n);+ } else {+ tinyBLASq<block_q4_0, block_q8_0, false> tb{+ k, (const block_q4_0 *)A,+ lda, (const block_q8_0 *)B,+ ldb, C,+ ldc, ith,+ nth,+ };+ tb.gemm(m, n);+ }+ break;++#endif+ default:+ ASSERT(!"unsupported dtype");+ }+}