Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Increase tokens per second on tiny models
Justine Tunney584a327308e9
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 736d055d..7a3ea652 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -52,6 +52,7 @@ SOFTWARE.");#include <time.h>#include <math.h>#include <stdlib.h>+#include <stdalign.h>#include <stdatomic.h>#include <string.h>#include <stdint.h>@@ -65,6 +66,8 @@ SOFTWARE.");#include <pthread.h>#include <cosmo.h>#include <sys/stat.h>+#include <sys/auxv.h>+#include <alloca.h>typedef void * thread_ret_t;typedef llamafile_task_t ggml_thread_t;@@ -1529,6 +1532,10 @@ struct ggml_context_container {struct ggml_context context;};+struct ggml_phaser {+ alignas(64) atomic_uint i;+};+struct ggml_compute_state_shared {const struct ggml_cgraph * cgraph;const struct ggml_cplan * cplan;@@ -1537,7 +1544,7 @@ struct ggml_compute_state_shared {// synchronization primitivesatomic_int n_barrier;- atomic_uint n_barrier_passed;+ struct ggml_phaser *n_barrier_passed;ggml_abort_callback abort_callback; // abort ggml_graph_compute when truevoid * abort_callback_data;@@ -1803,23 +1810,60 @@ inline static void ggml_critical_section_start(void) {}}-void ggml_barrier(struct ggml_compute_state_shared * shared) {- if (shared->n_threads == 1)- return;- int n = shared->n_threads;- atomic_int * count = &shared->n_barrier;- atomic_uint * phase = &shared->n_barrier_passed;- unsigned i = atomic_load_explicit(phase, memory_order_relaxed);- if (atomic_fetch_add_explicit(count, 1, memory_order_acq_rel) == n - 1) {- atomic_store_explicit(count, 0, memory_order_relaxed);- atomic_store_explicit(phase, i + 1, memory_order_release);- } else {- while (atomic_load_explicit(phase, memory_order_acquire) == i)- while (atomic_load_explicit(phase, memory_order_relaxed) == i)- pthread_pause_np();+#define GGML_BARRIER_IMPL \+ if (params->shared->n_threads == 1) \+ return; \+ int n = params->shared->n_threads; \+ atomic_int * count = ¶ms->shared->n_barrier; \+ atomic_uint * phase = ¶ms->shared->n_barrier_passed[params->ith].i; \+ unsigned i = atomic_load_explicit(phase, memory_order_relaxed); \+ if (atomic_fetch_add_explicit(count, 1, memory_order_acq_rel) == n - 1) { \+ atomic_store_explicit(count, 0, memory_order_relaxed); \+ for (int j = 0; j < n; ++j) { \+ atomic_store_explicit(¶ms->shared->n_barrier_passed[j].i, \+ i + 1, memory_order_relaxed); \+ } \+ atomic_thread_fence(memory_order_release); \+ } else { \+ while (atomic_load_explicit(phase, memory_order_relaxed) == i) \+ pthread_pause_np(); \+ atomic_thread_fence(memory_order_acquire); \+ }++#ifndef __aarch64__++void ggml_barrier(const struct ggml_compute_params * params) {+ GGML_BARRIER_IMPL+}++#else++static void ggml_barrier_default(const struct ggml_compute_params * params) {+ GGML_BARRIER_IMPL+}++#pragma GCC push_options+#pragma GCC target ("arch=armv8.1-a") // shaves microseconds per call on m2+static void ggml_barrier_armv81a(const struct ggml_compute_params * params) {+ GGML_BARRIER_IMPL+}+#pragma GCC pop_options++void ggml_barrier(const struct ggml_compute_params * params) {+ static void (*impl)(const struct ggml_compute_params *);+ if (!impl) {+ long hwcap = getauxval(AT_HWCAP);+ if (hwcap & HWCAP_ATOMICS) {+ impl = ggml_barrier_armv81a;+ } else {+ impl = ggml_barrier_default;+ }}+ return impl(params);}+#endif // __aarch64__+// TODO: make this somehow automatically executed// some sort of "sentry" mechanisminline static void ggml_critical_section_end(void) {@@ -8740,7 +8784,7 @@ static void ggml_compute_forward_acc_f32(((char *) src0->data),ggml_nbytes(dst));}- ggml_barrier(params->shared);+ ggml_barrier(params);}const int ith = params->ith;@@ -11112,7 +11156,7 @@ UseGgmlGemm1:;}}- ggml_barrier(params->shared);+ ggml_barrier(params);}#if GGML_USE_LLAMAFILE@@ -11325,7 +11369,7 @@ static void ggml_compute_forward_mul_mat_id(}}- ggml_barrier(params->shared);+ ggml_barrier(params);// compute each matrix multiplication in sequencefor (int cur_a = 0; cur_a < n_as; ++cur_a) {@@ -11466,7 +11510,7 @@ static void ggml_compute_forward_out_prod_f32(if (ith == 0) {ggml_vec_set_f32(ne0*ne1*ne2*ne3, dst->data, 0);}- ggml_barrier(params->shared);+ ggml_barrier(params);// dst[:,:,:,:] = 0// for i2,i3:@@ -11584,7 +11628,7 @@ static void ggml_compute_forward_out_prod_q_f32(if (ith == 0) {ggml_vec_set_f32(ne0*ne1*ne2*ne3, dst->data, 0);}- ggml_barrier(params->shared);+ ggml_barrier(params);// parallelize by last three dimensions@@ -11771,7 +11815,7 @@ static void ggml_compute_forward_set_f32(((char *) src0->data),ggml_nbytes(dst));}- ggml_barrier(params->shared);+ ggml_barrier(params);}const int ith = params->ith;@@ -12353,7 +12397,7 @@ static void ggml_compute_forward_diag_mask_f32(((char *) src0->data),ggml_nbytes(dst));}- ggml_barrier(params->shared);+ ggml_barrier(params);}// TODO: handle transposed/permuted matrices@@ -13129,7 +13173,7 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(// need to zero dst since we are accumulating into itmemset(dst->data, 0, ggml_nbytes(dst));}- ggml_barrier(params->shared);+ ggml_barrier(params);const int32_t s0 = ((const int32_t*)(dst->op_params))[0];@@ -13217,7 +13261,7 @@ static void ggml_compute_forward_conv_transpose_1d_f32(// need to zero dst since we are accumulating into itmemset(dst->data, 0, ggml_nbytes(dst));}- ggml_barrier(params->shared);+ ggml_barrier(params);const int32_t s0 = ((const int32_t*)(dst->op_params))[0];@@ -13506,7 +13550,7 @@ static void ggml_compute_forward_conv_transpose_2d(memset(dst->data, 0, ggml_nbytes(dst));}- ggml_barrier(params->shared);+ ggml_barrier(params);const int32_t stride = ggml_get_op_params_i32(dst, 0);@@ -14240,7 +14284,7 @@ static void ggml_compute_forward_flash_attn_back_f32(if (ith == 0) {memset(dst->data, 0, nb0*ne0*ne1*ne2*ne3);}- ggml_barrier(params->shared);+ ggml_barrier(params);const int64_t elem_q = ggml_nelements(q);const int64_t elem_k = ggml_nelements(k);@@ -15013,7 +15057,7 @@ static void ggml_compute_forward_add_rel_pos_f32(if (params->ith == 0) {memcpy((char *) dst->data, (char *) src0->data, ggml_nbytes(dst));}- ggml_barrier(params->shared);+ ggml_barrier(params);}// ref: https://github.com/facebookresearch/segment-anything/blob/main/segment_anything/modeling/image_encoder.py#L357-L359@@ -15298,7 +15342,7 @@ static void ggml_compute_forward_cross_entropy_loss_f32(if (ith == 0) {memset(sums, 0, sizeof(float) * (nth + nth * nc));}- ggml_barrier(params->shared);+ ggml_barrier(params);const double eps = 1e-9;@@ -15346,7 +15390,7 @@ static void ggml_compute_forward_cross_entropy_loss_f32(}#endif}- ggml_barrier(params->shared);+ ggml_barrier(params);if (ith == 0) {float * dp = (float *) dst->data;@@ -17662,7 +17706,7 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {state->shared->ec = GGML_STATUS_ABORTED;}- ggml_barrier(state->shared);+ ggml_barrier(¶ms);if (state->shared->ec != GGML_STATUS_SUCCESS) {break;@@ -17704,12 +17748,19 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplint n_threads = cplan->n_threads;+ size_t pz = sizeof(struct ggml_phaser);+ size_t az = alignof(struct ggml_phaser);+ char *mem = alloca(az + pz * n_threads);+ struct ggml_phaser *n_barrier_passed =+ (struct ggml_phaser *)(((uintptr_t)mem + az) & -az);+ memset(n_barrier_passed, 0, pz * n_threads);+struct ggml_compute_state_shared state_shared = {/*.cgraph =*/ cgraph,/*.cgraph_plan =*/ cplan,/*.n_threads =*/ n_threads,/*.n_barrier =*/ 0,- /*.n_barrier_passed =*/ 0,+ /*.n_barrier_passed =*/ n_barrier_passed,/*.abort_callback =*/ NULL,/*.abort_callback_data =*/ NULL,/*.current_chunk =*/ 0,diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex 19406cc3..1184065c 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -705,7 +705,7 @@ extern "C" {struct ggml_compute_state_shared * shared;};- void ggml_barrier(struct ggml_compute_state_shared * shared);+ void ggml_barrier(const struct ggml_compute_params * params);//// GUIDdiff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex 6c69bbd1..249966ba 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -14355,14 +14355,6 @@ static int llama_decode_internal(llama_set_inputs(lctx, u_batch);- // [jart] On CPUs with many cores (e.g. EPYC, Threadripper)- // using more than twenty threads for token prediction- // never helps. This number appears to be optimal for all- // models ranging from TinyLLaMA 1.1B to mighty Mixtral 8x22B.- if (n_tokens <= 2) {- n_threads = std::min(32, n_threads);- }-llama_graph_compute(lctx, gf, n_threads);// update the kv ring bufferdiff --git a/llamafile/flags.cpp b/llamafile/flags.cppindex d2f0a703..9adb53e9 100644--- a/llamafile/flags.cpp+++ b/llamafile/flags.cpp@@ -290,6 +290,7 @@ void llamafile_get_flags(int argc, char **argv) {if (!strcmp(flag, "--trace")) {FLAG_trace = true;+ FLAG_unsecure = true;continue;}diff --git a/llamafile/server/doc/technical_details.md b/llamafile/server/doc/technical_details.mdindex f58f9450..9885a520 100644--- a/llamafile/server/doc/technical_details.md+++ b/llamafile/server/doc/technical_details.md@@ -9,7 +9,7 @@ technical aspects of LLaMAfiler.This server can serve embeddings an order of a magnitude faster than thellama.cpp example server. We measured this on Threadripper PRO 7995WX w/all-MiniLM-L6-v2 Q6\_K. Using a prompt with 50 tokens this server served-1,916 `/embedding` requests per second, whereas+2,015 `/embedding` requests per second, whereasllama.cpp/build/bin/server built with glibc served 771 per second.The following wrk script was used:@@ -34,16 +34,16 @@ The results for llamafiler were:```Thread Stats Avg Stdev Max +/- Stdev- Latency 16.50ms 3.01ms 31.95ms 71.14%- Req/Sec 59.95 7.17 80.00 52.34%+ Latency 15.81ms 2.73ms 30.86ms 71.49%+ Req/Sec 63.08 7.41 90.00 84.66%Latency Distribution- 50% 16.30ms- 75% 18.19ms- 90% 20.30ms- 99% 24.95ms- 1535 requests in 801.08ms, 7.93MB read-Requests/sec: 1916.18-Transfer/sec: 9.90MB+ 50% 15.65ms+ 75% 17.46ms+ 90% 19.18ms+ 99% 23.26ms+ 4441 requests in 2.20s, 23.04MB read+Requests/sec: 2015.82+Transfer/sec: 10.46MB```The results for llama.cpp in CPU mode were:diff --git a/llamafile/tinyblas_cpu_mixmul.inc b/llamafile/tinyblas_cpu_mixmul.incindex 732a3677..20ede965 100644--- a/llamafile/tinyblas_cpu_mixmul.inc+++ b/llamafile/tinyblas_cpu_mixmul.inc@@ -282,7 +282,7 @@ class MixMul {if (thought->type != ggml_type_trait<TB>::id)quantize_thought(ggml_type_trait<TB>::id);build_row_pointers(ggml_type_trait<TB>::id);- ggml_barrier(params->shared);+ ggml_barrier(params);assert(!(cols % BS));assert(!(weights->nb[1] % sizeof(TA)));// TODO(jart): parallelize this loop