Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Fix --threads-batch flag
Justine Tunney64a60526db2c
diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex 2a84c5d7..80be6440 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -11,6 +11,7 @@#include "json-schema-to-grammar.h"#include "llama.h"#include "llamafile/debug.h"+#include "llamafile/macros.h"#include "string.h"#include <cosmo.h>@@ -272,7 +273,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_paCHECK_ARGparams.n_threads = std::stoi(argv[i]);if (params.n_threads <= 0) {- params.n_threads = std::thread::hardware_concurrency();+ params.n_threads = MIN(cpu_get_num_math(), 20); // [jart]}FLAG_threads = params.n_threads; // [jart]return true;@@ -281,7 +282,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_paCHECK_ARGparams.n_threads_batch = std::stoi(argv[i]);if (params.n_threads_batch <= 0) {- params.n_threads_batch = std::thread::hardware_concurrency();+ params.n_threads_batch = cpu_get_num_math(); // [jart]}return true;}@@ -289,7 +290,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_paCHECK_ARGparams.n_threads_draft = std::stoi(argv[i]);if (params.n_threads_draft <= 0) {- params.n_threads_draft = std::thread::hardware_concurrency();+ params.n_threads_draft = cpu_get_num_math(); // [jart]}return true;}@@ -297,7 +298,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_paCHECK_ARGparams.n_threads_batch_draft = std::stoi(argv[i]);if (params.n_threads_batch_draft <= 0) {- params.n_threads_batch_draft = std::thread::hardware_concurrency();+ params.n_threads_batch_draft = cpu_get_num_math(); // [jart]}return true;}@@ -1701,7 +1702,7 @@ std::string gpt_params_get_system_info(const gpt_params & params) {if (params.n_threads_batch != -1) {os << " (n_threads_batch = " << params.n_threads_batch << ")";}- os << " / " << std::thread::hardware_concurrency() << " | " << llama_print_system_info();+ os << " / " << cpu_get_num_math() << " | " << llama_print_system_info(); // [jart]return os.str();}@@ -3247,7 +3248,7 @@ void yaml_dump_non_result_info(FILE * stream, const gpt_params & params, const lyaml_dump_vector_float(stream, "tensor_split", tensor_split_vector);fprintf(stream, "tfs: %f # default: 1.0\n", sparams.tfs_z);- fprintf(stream, "threads: %d # default: %u\n", params.n_threads, std::thread::hardware_concurrency());+ fprintf(stream, "threads: %d # default: %u\n", params.n_threads, cpu_get_num_math()); // [jart]fprintf(stream, "top_k: %d # default: 40\n", sparams.top_k);fprintf(stream, "top_p: %f # default: 0.95\n", sparams.top_p);fprintf(stream, "min_p: %f # default: 0.0\n", sparams.min_p);diff --git a/llama.cpp/common.h b/llama.cpp/common.hindex 10012492..118bd301 100644--- a/llama.cpp/common.h+++ b/llama.cpp/common.h@@ -7,6 +7,7 @@#include "llamafile/log.h"#include "llama.cpp/cores.h"+#include "llamafile/macros.h"#include "llama.h"#include "sampling.h"@@ -76,9 +77,9 @@ enum dimre_method {struct gpt_params {uint32_t seed = LLAMA_DEFAULT_SEED; // RNG seed- int32_t n_threads = cpu_get_num_math();+ int32_t n_threads = MIN(cpu_get_num_math(), 20);int32_t n_threads_draft = -1;- int32_t n_threads_batch = -1; // number of threads to use for batch processing (-1 = use n_threads)+ int32_t n_threads_batch = cpu_get_num_math(); // number of threads to use for batch processing (-1 = use n_threads)int32_t n_threads_batch_draft = -1;int32_t n_predict = -1; // new tokens to predictint32_t n_ctx = 8192; // context size [jart]diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex 7b870c7c..18da7731 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -14775,7 +14775,7 @@ static int llama_decode_internal(lctx.n_outputs = n_outputs_new;}- int n_threads = n_tokens == 1 ? cparams.n_threads : cparams.n_threads_batch;+ int n_threads = n_tokens <= 2 ? cparams.n_threads : cparams.n_threads_batch; // [jart]GGML_ASSERT(n_threads > 0);// helpers for smoother batch API transition@@ -15026,7 +15026,7 @@ static int llama_encode_internal(lctx.inp_embd_enc = NULL;lctx.n_outputs = n_tokens;- int n_threads = n_tokens == 1 ? cparams.n_threads : cparams.n_threads_batch;+ int n_threads = n_tokens <= 2 ? cparams.n_threads : cparams.n_threads_batch; // [jart]GGML_ASSERT(n_threads > 0);// helpers for smoother batch API transition@@ -15084,14 +15084,6 @@ static int llama_encode_internal(llama_set_inputs(lctx, batch);- // [jart] On CPUs with many cores (e.g. EPYC, Threadripper)- // using more than twenty threads for token prediction- // never helps. This number appears to be optimal for all- // models ranging from TinyLLaMA 1.1B to mighty Mixtral 8x22B.- if (n_tokens <= 2) {- n_threads = std::min(20, n_threads);- }-llama_graph_compute(lctx, gf, n_threads);// extract embeddingsdiff --git a/llama.cpp/main/main.1 b/llama.cpp/main/main.1index 2727fbbc..391b64f3 100644--- a/llama.cpp/main/main.1+++ b/llama.cpp/main/main.1@@ -112,6 +112,10 @@ Default: -1.It Fl t Ar N , Fl Fl threads Ar NNumber of threads to use during generation..Pp+Default: $(nproc)/2 max 20+.It Fl tb Ar N , Fl Fl threads-batch Ar N+Number of threads to use during prompt processing.+.PpDefault: $(nproc)/2.It Fl Fl in-prefix-bosPrefix BOS to user inputs, preceding thediff --git a/llama.cpp/main/main.1.asc b/llama.cpp/main/main.1.ascindex fc1648c4..55467b6d 100644--- a/llama.cpp/main/main.1.asc+++ b/llama.cpp/main/main.1.asc@@ -72,6 +72,11 @@[1m-t [4m[22mN[24m, [1m--threads [4m[22mN[0mNumber of threads to use during generation.+ Default: $(nproc)/2 max 20++ [1m-tb [4m[22mN[24m, [1m--threads-batch [4m[22mN[0m+ Number of threads to use during prompt processing.+Default: $(nproc)/2[1m--in-prefix-bos[0mdiff --git a/llamafile/server/completion.cpp b/llamafile/server/completion.cppindex 1ecc33b8..5a98f1f8 100644--- a/llamafile/server/completion.cpp+++ b/llamafile/server/completion.cpp@@ -24,6 +24,7 @@#include "llama.cpp/llama.h"#include "llama.cpp/sampling.h"+#include "llamafile/macros.h"#include "cleanup.h"#include "fastjson.h"@@ -212,7 +213,7 @@ Client::completion()cparams.n_ubatch = count;cparams.n_seq_max = 1;cparams.n_threads = FLAG_threads;- cparams.n_threads_batch = FLAG_threads;+ cparams.n_threads_batch = MIN(FLAG_threads, 20);cparams.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_NONE;cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;cparams.rope_freq_base = 0;