Skip to content

Distribute and run LLMs with a single file.

Fix --threads-batch flag

Justine Tunney64a60526db2c
diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cpp
index 2a84c5d7..80be6440 100644
--- a/llama.cpp/common.cpp
+++ b/llama.cpp/common.cpp
@@ -11,6 +11,7 @@
#include "json-schema-to-grammar.h"
#include "llama.h"
#include "llamafile/debug.h"
+#include "llamafile/macros.h"
#include "string.h"
#include <cosmo.h>
@@ -272,7 +273,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
CHECK_ARG
params.n_threads = std::stoi(argv[i]);
if (params.n_threads <= 0) {
- params.n_threads = std::thread::hardware_concurrency();
+ params.n_threads = MIN(cpu_get_num_math(), 20); // [jart]
}
FLAG_threads = params.n_threads; // [jart]
return true;
@@ -281,7 +282,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
CHECK_ARG
params.n_threads_batch = std::stoi(argv[i]);
if (params.n_threads_batch <= 0) {
- params.n_threads_batch = std::thread::hardware_concurrency();
+ params.n_threads_batch = cpu_get_num_math(); // [jart]
}
return true;
}
@@ -289,7 +290,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
CHECK_ARG
params.n_threads_draft = std::stoi(argv[i]);
if (params.n_threads_draft <= 0) {
- params.n_threads_draft = std::thread::hardware_concurrency();
+ params.n_threads_draft = cpu_get_num_math(); // [jart]
}
return true;
}
@@ -297,7 +298,7 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
CHECK_ARG
params.n_threads_batch_draft = std::stoi(argv[i]);
if (params.n_threads_batch_draft <= 0) {
- params.n_threads_batch_draft = std::thread::hardware_concurrency();
+ params.n_threads_batch_draft = cpu_get_num_math(); // [jart]
}
return true;
}
@@ -1701,7 +1702,7 @@ std::string gpt_params_get_system_info(const gpt_params & params) {
if (params.n_threads_batch != -1) {
os << " (n_threads_batch = " << params.n_threads_batch << ")";
}
- os << " / " << std::thread::hardware_concurrency() << " | " << llama_print_system_info();
+ os << " / " << cpu_get_num_math() << " | " << llama_print_system_info(); // [jart]
return os.str();
}
@@ -3247,7 +3248,7 @@ void yaml_dump_non_result_info(FILE * stream, const gpt_params & params, const l
yaml_dump_vector_float(stream, "tensor_split", tensor_split_vector);
fprintf(stream, "tfs: %f # default: 1.0\n", sparams.tfs_z);
- fprintf(stream, "threads: %d # default: %u\n", params.n_threads, std::thread::hardware_concurrency());
+ fprintf(stream, "threads: %d # default: %u\n", params.n_threads, cpu_get_num_math()); // [jart]
fprintf(stream, "top_k: %d # default: 40\n", sparams.top_k);
fprintf(stream, "top_p: %f # default: 0.95\n", sparams.top_p);
fprintf(stream, "min_p: %f # default: 0.0\n", sparams.min_p);
diff --git a/llama.cpp/common.h b/llama.cpp/common.h
index 10012492..118bd301 100644
--- a/llama.cpp/common.h
+++ b/llama.cpp/common.h
@@ -7,6 +7,7 @@
#include "llamafile/log.h"
#include "llama.cpp/cores.h"
+#include "llamafile/macros.h"
#include "llama.h"
#include "sampling.h"
@@ -76,9 +77,9 @@ enum dimre_method {
struct gpt_params {
uint32_t seed = LLAMA_DEFAULT_SEED; // RNG seed
- int32_t n_threads = cpu_get_num_math();
+ int32_t n_threads = MIN(cpu_get_num_math(), 20);
int32_t n_threads_draft = -1;
- int32_t n_threads_batch = -1; // number of threads to use for batch processing (-1 = use n_threads)
+ int32_t n_threads_batch = cpu_get_num_math(); // number of threads to use for batch processing (-1 = use n_threads)
int32_t n_threads_batch_draft = -1;
int32_t n_predict = -1; // new tokens to predict
int32_t n_ctx = 8192; // context size [jart]
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cpp
index 7b870c7c..18da7731 100644
--- a/llama.cpp/llama.cpp
+++ b/llama.cpp/llama.cpp
@@ -14775,7 +14775,7 @@ static int llama_decode_internal(
lctx.n_outputs = n_outputs_new;
}
- int n_threads = n_tokens == 1 ? cparams.n_threads : cparams.n_threads_batch;
+ int n_threads = n_tokens <= 2 ? cparams.n_threads : cparams.n_threads_batch; // [jart]
GGML_ASSERT(n_threads > 0);
// helpers for smoother batch API transition
@@ -15026,7 +15026,7 @@ static int llama_encode_internal(
lctx.inp_embd_enc = NULL;
lctx.n_outputs = n_tokens;
- int n_threads = n_tokens == 1 ? cparams.n_threads : cparams.n_threads_batch;
+ int n_threads = n_tokens <= 2 ? cparams.n_threads : cparams.n_threads_batch; // [jart]
GGML_ASSERT(n_threads > 0);
// helpers for smoother batch API transition
@@ -15084,14 +15084,6 @@ static int llama_encode_internal(
llama_set_inputs(lctx, batch);
- // [jart] On CPUs with many cores (e.g. EPYC, Threadripper)
- // using more than twenty threads for token prediction
- // never helps. This number appears to be optimal for all
- // models ranging from TinyLLaMA 1.1B to mighty Mixtral 8x22B.
- if (n_tokens <= 2) {
- n_threads = std::min(20, n_threads);
- }
-
llama_graph_compute(lctx, gf, n_threads);
// extract embeddings
diff --git a/llama.cpp/main/main.1 b/llama.cpp/main/main.1
index 2727fbbc..391b64f3 100644
--- a/llama.cpp/main/main.1
+++ b/llama.cpp/main/main.1
@@ -112,6 +112,10 @@ Default: -1
.It Fl t Ar N , Fl Fl threads Ar N
Number of threads to use during generation.
.Pp
+Default: $(nproc)/2 max 20
+.It Fl tb Ar N , Fl Fl threads-batch Ar N
+Number of threads to use during prompt processing.
+.Pp
Default: $(nproc)/2
.It Fl Fl in-prefix-bos
Prefix BOS to user inputs, preceding the
diff --git a/llama.cpp/main/main.1.asc b/llama.cpp/main/main.1.asc
index fc1648c4..55467b6d 100644
--- a/llama.cpp/main/main.1.asc
+++ b/llama.cpp/main/main.1.asc
@@ -72,6 +72,11 @@
-t N, --threads N
Number of threads to use during generation.
+ Default: $(nproc)/2 max 20
+
+ -tb N, --threads-batch N
+ Number of threads to use during prompt processing.
+
Default: $(nproc)/2
--in-prefix-bos
diff --git a/llamafile/server/completion.cpp b/llamafile/server/completion.cpp
index 1ecc33b8..5a98f1f8 100644
--- a/llamafile/server/completion.cpp
+++ b/llamafile/server/completion.cpp
@@ -24,6 +24,7 @@
#include "llama.cpp/llama.h"
#include "llama.cpp/sampling.h"
+#include "llamafile/macros.h"
#include "cleanup.h"
#include "fastjson.h"
@@ -212,7 +213,7 @@ Client::completion()
cparams.n_ubatch = count;
cparams.n_seq_max = 1;
cparams.n_threads = FLAG_threads;
- cparams.n_threads_batch = FLAG_threads;
+ cparams.n_threads_batch = MIN(FLAG_threads, 20);
cparams.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_NONE;
cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;
cparams.rope_freq_base = 0;