Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Do some code cleanup
Justine Tunney538985dde224
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 3a51c987..a1b67a66 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -10733,8 +10733,6 @@ static void ggml_compute_forward_mul_mat(src1->type,dst->type))goto UseGgmlGemm1;- if (!ith && ne11 > 1 && params->type == GGML_TASK_TYPE_COMPUTE)- llamafile_sgemm_was_used(ne01 * ne11 * ne00);return;}UseGgmlGemm1:@@ -10774,10 +10772,10 @@ UseGgmlGemm1:for (int64_t j = 0; j < ne13; j++)for (int64_t i = 0; i < ne12; i++)if (!llamafile_sgemm(ne01, ne11, ne00/ggml_blck_size(src0->type),- (char *)src0->data + i/r2*nb02 + j/r3*nb03,+ (const char *)src0->data + i/r2*nb02 + j/r3*nb03,nb01/ggml_type_size(src0->type),- (char *)wdata + (nb12/ggml_type_size(src1->type)*ggml_type_size(vec_dot_type)*i +- nb13/ggml_type_size(src1->type)*ggml_type_size(vec_dot_type)*j),+ (const char *)wdata + (nb12/ggml_type_size(src1->type)*ggml_type_size(vec_dot_type)*i ++ nb13/ggml_type_size(src1->type)*ggml_type_size(vec_dot_type)*j),row_size/ggml_type_size(vec_dot_type),(char *)dst->data + i*nb2 + j*nb3,nb1/ggml_type_size(dst->type),@@ -10787,15 +10785,10 @@ UseGgmlGemm1:vec_dot_type,dst->type))goto UseGgmlGemm2;- if (!ith && ne11 > 1 && params->type == GGML_TASK_TYPE_COMPUTE)- llamafile_sgemm_was_used(ne01 * ne11 * ne00);return;}UseGgmlGemm2:- if (!ith && ne11 > 1)- llamafile_sgemm_was_not_used(ne01 * ne11 * ne00 * ne13 * ne12, src1_cont);-const int64_t nr0 = ne01; // src0 rowsconst int64_t nr1 = ne1*ne12*ne13; // src1 rowsdiff --git a/llamafile/alderlake.c b/llamafile/alderlake.cindex b7e8eb32..c7418e5a 100644--- a/llamafile/alderlake.c+++ b/llamafile/alderlake.c@@ -48,7 +48,7 @@ static int count_math_cpus(int cpu_count) {int result = 0;for (int cpu = 0; cpu < cpu_count; ++cpu) {if (pin_cpu(cpu))- return cpu_count / 2; // xnu and openbsd don't support affinity+ return -1; // xnu and openbsd don't support affinityif (is_running_on_efficiency_core())continue; // efficiency cores harm lockstep threading++cpu; // hyperthreading isn't useful for linear algebra@@ -69,16 +69,19 @@ static void *count_math_cpus_worker(void *arg) {int llamafile_get_math_cpu_count(void) {int cpu_count = __get_cpu_count();if (cpu_count < 1)- return 1;+ return 4;#ifdef __x86_64__if (is_hybrid_cpu()) {pthread_t th; // some OSes don't support getaffinityif (!pthread_create(&th, 0, count_math_cpus_worker, (void *)(intptr_t)cpu_count)) {void *result;- if (!pthread_join(th, &result))+ if (!pthread_join(th, &result) && (intptr_t)result > 0)return (intptr_t)result;}}#endif- return cpu_count / 2;+ if (cpu_count <= 4)+ return cpu_count;+ else+ return cpu_count / 2;}diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex 58d97624..1f5694a4 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -52,14 +52,6 @@ const char *llamafile_describe_gpu(void);bool llamafile_sgemm(int, int, int, const void *, int, const void *, int, void *, int, int, int,int, int, int, int);-#if 0-void llamafile_sgemm_was_used(long long);-void llamafile_sgemm_was_not_used(long long, bool);-#else-#define llamafile_sgemm_was_used(x)-#define llamafile_sgemm_was_not_used(x, y)-#endif-#ifdef __cplusplus}#endifdiff --git a/llamafile/sgemm_accountant.cpp b/llamafile/sgemm_accountant.cppdeleted file mode 100644index e4e2248d..00000000--- a/llamafile/sgemm_accountant.cpp+++ /dev/null@@ -1,61 +0,0 @@-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*--// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-//-// Copyright 2024 Mozilla Foundation-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--#include "llamafile.h"-#include "log.h"-#include <cassert>--namespace {--struct SgemmAccountant {-- long long wont;-- long long ggml_ops;- long long llamafile_ops;-- long long ggml_flops;- long long llamafile_flops;-- SgemmAccountant() = default;-- ~SgemmAccountant() {- if (!ggml_ops && !llamafile_ops)- return;- double total_ops = ggml_ops + llamafile_ops;- double total_flops = ggml_flops + llamafile_flops;- tinylogf("llamafile_sgemm: successfully accelerated %g%% of flops and %g%% of ops on cpu"- " (%lld cant %lld wont)\n",- llamafile_ops / total_ops * 100, llamafile_flops / total_flops * 100,- ggml_ops - wont, wont);- }-};--struct SgemmAccountant g_sgemm_accountant;--} // namespace--void(llamafile_sgemm_was_used)(long long flops) {- g_sgemm_accountant.llamafile_ops++;- g_sgemm_accountant.llamafile_flops += flops;-}--void(llamafile_sgemm_was_not_used)(long long flops, bool wont) {- g_sgemm_accountant.ggml_ops++;- g_sgemm_accountant.ggml_flops += flops;- g_sgemm_accountant.wont += wont;-}