Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Restore Raspberry Pi performance
Justine Tunney0d5a05ab4c4d
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 9a5332fc..360d8892 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -374,11 +374,25 @@ static void ggml_fp32_to_fp16_row_f16c(const float * x, ggml_fp16_t * y, int64_t#pragma GCC pop_options#endif+#ifdef __aarch64__+static void ggml_fp32_to_fp16_row_neon(const float *x, ggml_fp16_t *y, int64_t n) {+ int64_t i = 0;+ for (; i + 3 < n; i += 4) {+ vst1_f16((ggml_fp16_t *)(y + i), vcvt_f16_f32(vld1q_f32(x + i)));+ }+ for (; i < n; i++) {+ y[i] = GGML_FP32_TO_FP16(x[i]);+ }+}+#endif+void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n) {#ifdef __x86_64__if (X86_HAVE(F16C)) {return ggml_fp32_to_fp16_row_f16c(x, y, n);}+#elif defined(__aarch64__)+ return ggml_fp32_to_fp16_row_neon(x, y, n);#endiffor (int64_t i = 0; i < n; i++) {y[i] = GGML_FP32_TO_FP16(x[i]);diff --git a/llamafile/tinyblas_cpu.inc b/llamafile/tinyblas_cpu.incindex abf05539..1ff3225e 100644--- a/llamafile/tinyblas_cpu.inc+++ b/llamafile/tinyblas_cpu.inc@@ -75,23 +75,7 @@ inline float unhalf(ggml_fp16_t d) {#define NCB 2#define NCC 4-#define INDEX(A, lda, j, i) index<CONFIG & NC##A>(A, lda, j, i)--template <int NC, typename T>-inline T &index(T *A, int lda, int j, int i) {- if (NC)- return ((T **)A)[j][i];- else- return A[lda * j + i];-}--template <int NC, typename T>-inline const T &index(const T *A, int lda, int j, int i) {- if (NC)- return ((const T *const *)A)[j][i];- else- return A[lda * j + i];-}+#define INDEX(A, lda, j, i) (CONFIG & NC##A ? ((T##A **)A)[j] + i : A + lda * (j) + i)////////////////////////////////////////////////////////////////////////////////////////////////////// GGML TYPE TRAITS@@ -222,15 +206,17 @@ inline __m512 madd(__m512 a, __m512 b, __m512 c) {#if defined(__ARM_FEATURE_FMA)template <>inline float32x4_t madd(float32x4_t a, float32x4_t b, float32x4_t c) {- return vfmaq_f32(c, b, a);+ return vfmaq_f32(c, a, b);}-#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)+#if 0 // todo: this specialization chops gcc 12.3 performance in half+#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER) && 0template <>inline float16x8_t madd(float16x8_t a, float16x8_t b, float16x8_t c) {return vfmaq_f16(c, b, a);}#endif#endif+#endif#if defined(__AVX512BF16__)template <>@@ -254,6 +240,7 @@ inline float hsum(float32x4_t x) {#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)inline float hsum(float16x8_t x) {+ // todo: this works great on clang but it produces terrible code on gcc 12.3return vaddvq_f32(vaddq_f32(vcvt_f32_f16(vget_low_f16(x)), vcvt_f32_f16(vget_high_f16(x))));}#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC@@ -386,20 +373,16 @@ class tinyBLAS {switch ((std::min(m - m0, 5) << 4) | std::min(n - n0, 5)) {#if VECTOR_REGISTERS == 32case 0x55:+ case 0x54:mc = 5;- nc = 5;- gemm<5, 5, false>(m0, m, n0, n);+ nc = 4;+ gemm<5, 4, false>(m0, m, n0, n);break;case 0x45:mc = 4;nc = 5;gemm<4, 5, false>(m0, m, n0, n);break;- case 0x54:- mc = 5;- nc = 4;- gemm<5, 4, false>(m0, m, n0, n);- break;case 0x44:mc = 4;nc = 4;@@ -551,16 +534,16 @@ class tinyBLAS {for (int j = 0; j < RN; ++j)for (int i = 0; i < RM; ++i)if (KAHAN)- Cv[j][i] = madder(load<V>(&INDEX(A, lda, ii + i, l)), //- load<V>(&INDEX(B, ldb, jj + j, l)), //+ Cv[j][i] = madder(load<V>(INDEX(A, lda, ii + i, l)), //+ load<V>(INDEX(B, ldb, jj + j, l)), //Cv[j][i], &Ce[j][i]);else- Cv[j][i] = madd(load<V>(&INDEX(A, lda, ii + i, l)), //- load<V>(&INDEX(B, ldb, jj + j, l)), //+ Cv[j][i] = madd(load<V>(INDEX(A, lda, ii + i, l)), //+ load<V>(INDEX(B, ldb, jj + j, l)), //Cv[j][i]);for (int j = 0; j < RN; ++j)for (int i = 0; i < RM; ++i)- INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]);+ *INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]);}}@@ -579,11 +562,11 @@ class tinyBLAS {// QUANT ZERO MATRIX MULTIPLICATION#if defined(__ARM_FEATURE_DOTPROD)-template <int CONFIG, typename TA>+template <int CONFIG, typename TA, typename TB, typename TC>class tinyBLAS_Q0_ARM {public:- tinyBLAS_Q0_ARM(int k, const TA *A, int lda, const block_q8_0 *B, int ldb, float *C, int ldc,- int ith, int nth)+ tinyBLAS_Q0_ARM(int k, const TA *A, int lda, const TB *B, int ldb, TC *C, int ldc, int ith,+ int nth): A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {}@@ -670,15 +653,15 @@ class tinyBLAS_Q0_ARM {Cv[j][i] = vmlaq_n_f32(Cv[j][i],vcvtq_f32_s32(vdotq_s32(vdotq_s32(vdupq_n_s32(0),- load_lo(&INDEX(A, lda, ii + i, l)),- load_lo(&INDEX(B, ldb, jj + j, l))),- load_hi(&INDEX(A, lda, ii + i, l)),- load_hi(&INDEX(B, ldb, jj + j, l)))),- (unhalf(INDEX(A, lda, ii + i, l).d) *- unhalf(INDEX(B, ldb, jj + j, l).d)));+ load_lo(INDEX(A, lda, ii + i, l)),+ load_lo(INDEX(B, ldb, jj + j, l))),+ load_hi(INDEX(A, lda, ii + i, l)),+ load_hi(INDEX(B, ldb, jj + j, l)))),+ (unhalf(INDEX(A, lda, ii + i, l)->d) *+ unhalf(INDEX(B, ldb, jj + j, l)->d)));for (int j = 0; j < RN; ++j)for (int i = 0; i < RM; ++i)- INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]);+ *INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]);}}@@ -700,8 +683,8 @@ class tinyBLAS_Q0_ARM {}const TA *const A;- const block_q8_0 *const B;- float *const C;+ const TB *const B;+ TC *const C;const int k;const int lda;const int ldb;@@ -852,16 +835,16 @@ class tinyBLAS_Q0_AVX2 {for (int l = 0; l < k; ++l)for (int j = 0; j < RN; ++j)for (int i = 0; i < RM; ++i)- Cv[j][i] = madd(_mm256_set1_ps(unhalf(INDEX(A, lda, ii + i, l).d) *- unhalf(INDEX(B, ldb, jj + j, l).d)),- updot(_mm256_sign_epi8(load(&INDEX(A, lda, ii + i, l)),- load(&INDEX(A, lda, ii + i, l))),- _mm256_sign_epi8(load(&INDEX(B, ldb, jj + j, l)),- load(&INDEX(A, lda, ii + i, l)))),+ Cv[j][i] = madd(_mm256_set1_ps(unhalf(INDEX(A, lda, ii + i, l)->d) *+ unhalf(INDEX(B, ldb, jj + j, l)->d)),+ updot(_mm256_sign_epi8(load(INDEX(A, lda, ii + i, l)),+ load(INDEX(A, lda, ii + i, l))),+ _mm256_sign_epi8(load(INDEX(B, ldb, jj + j, l)),+ load(INDEX(A, lda, ii + i, l)))),Cv[j][i]);for (int j = 0; j < RN; ++j)for (int i = 0; i < RM; ++i)- INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]);+ *INDEX(C, ldc, jj + j, ii + i) = hsum(Cv[j][i]);}}@@ -1006,7 +989,6 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,tb.matmul(m, n, task);return true;#elif defined(__AVX2__)- // TODO(jart): Why is this the fastest on Zen4?if (k % 8)return false;if (Btype != GGML_TYPE_F32)@@ -1082,7 +1064,7 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,tb.matmul(m, n, task);return true;#elif defined(__ARM_FEATURE_DOTPROD)- tinyBLAS_Q0_ARM<0, block_q8_0> tb{+ tinyBLAS_Q0_ARM<0, block_q8_0, block_q8_0, float> tb{k, (const block_q8_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};tb.matmul(m, n, task);return true;@@ -1100,7 +1082,7 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,tb.matmul(m, n, task);return true;#elif defined(__ARM_FEATURE_DOTPROD)- tinyBLAS_Q0_ARM<0, block_q4_0> tb{+ tinyBLAS_Q0_ARM<0, block_q4_0, block_q8_0, float> tb{k, (const block_q4_0 *)A, lda, (const block_q8_0 *)B, ldb, (float *)C, ldc, ith, nth};tb.matmul(m, n, task);return true;@@ -1337,8 +1319,8 @@ class MixMul {return mixmat<32, 32, tinyBLAS_Q0_AVX2<NCB | NCC, block_q4_0, block_q8_0, float>,block_q4_0, block_q8_0, float>();#elif defined(__ARM_FEATURE_DOTPROD)- return mixmat<32, 32, tinyBLAS_Q0_ARM<NCB | NCC, block_q4_0>, block_q4_0, block_q8_0,- float>();+ return mixmat<32, 32, tinyBLAS_Q0_ARM<NCB | NCC, block_q4_0, block_q8_0, float>,+ block_q4_0, block_q8_0, float>();#elsereturn false;#endif@@ -1350,8 +1332,8 @@ class MixMul {return mixmat<32, 32, tinyBLAS_Q0_AVX2<NCB | NCC, block_q8_0, block_q8_0, float>,block_q8_0, block_q8_0, float>();#elif defined(__ARM_FEATURE_DOTPROD)- return mixmat<32, 32, tinyBLAS_Q0_ARM<NCB | NCC, block_q8_0>, block_q8_0, block_q8_0,- float>();+ return mixmat<32, 32, tinyBLAS_Q0_ARM<NCB | NCC, block_q8_0, block_q8_0, float>,+ block_q8_0, block_q8_0, float>();#elsereturn false;#endif