Skip to content

Distribute and run LLMs with a single file.

Add --precise and --fast flags

Justine Tunneybbae0f6aaf35
diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cpp
index e57f083f..89db03a7 100644
--- a/llama.cpp/common.cpp
+++ b/llama.cpp/common.cpp
@@ -279,6 +279,16 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
if (arg == "--cli") {
return true;
}
+ if (arg == "--fast") {
+ FLAG_precise = false;
+ FLAG_precision_specified = true;
+ return true;
+ }
+ if (arg == "--precise") {
+ FLAG_precise = true;
+ FLAG_precision_specified = true;
+ return true;
+ }
if (arg == "--trap") {
FLAG_trap = true;
FLAG_unsecure = true; // for better backtraces
diff --git a/llama.cpp/ggml-vector.inc b/llama.cpp/ggml-vector.inc
index 62614a77..ead2de86 100644
--- a/llama.cpp/ggml-vector.inc
+++ b/llama.cpp/ggml-vector.inc
@@ -1189,7 +1189,7 @@ void ggml_vec_silu_f32(const int n, float * y, const float * x) {
int i = 0;
#if defined(__AVX512F__)
- if (!FLAG_trap) {
+ if (!FLAG_trap && !FLAG_precise) {
for (; i + 15 < n; i += 16) {
_mm512_storeu_ps(y + i, llamafile_silu_avx512(_mm512_loadu_ps(x + i)));
}
@@ -1197,7 +1197,7 @@ void ggml_vec_silu_f32(const int n, float * y, const float * x) {
#endif
#if defined(__AVX2__) && defined(__FMA__)
- if (!FLAG_trap) {
+ if (!FLAG_trap && !FLAG_precise) {
for (; i + 7 < n; i += 8) {
_mm256_storeu_ps(y + i, llamafile_silu_avx2fma(_mm256_loadu_ps(x + i)));
}
@@ -1288,7 +1288,7 @@ float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max)
ggml_float sum = 0;
#if defined(__AVX512F__)
- if (!FLAG_trap) {
+ if (!FLAG_trap && !FLAG_precise) {
for (; i + 15 < n; i += 16) {
__m512 val = llamafile_expf_avx512(_mm512_sub_ps(_mm512_loadu_ps(x + i),
_mm512_set1_ps(max)));
@@ -1299,7 +1299,7 @@ float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max)
#endif
#if defined(__AVX2__) && defined(__FMA__)
- if (!FLAG_trap) {
+ if (!FLAG_trap && !FLAG_precise) {
for (; i + 7 < n; i += 8) {
__m256 val = llamafile_expf_avx2fma(_mm256_sub_ps(_mm256_loadu_ps(x + i),
_mm256_set1_ps(max)));
@@ -1314,10 +1314,12 @@ float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max)
#endif
#ifdef __ARM_NEON
- for (; i + 3 < n; i += 4) {
- float32x4_t val = llamafile_expf_neon(vsubq_f32(vld1q_f32(x + i), vdupq_n_f32(max)));
- vst1q_f32(y + i, val);
- sum += vaddvq_f32(val);
+ if (!FLAG_trap) {
+ for (; i + 3 < n; i += 4) {
+ float32x4_t val = llamafile_expf_neon(vsubq_f32(vld1q_f32(x + i), vdupq_n_f32(max)));
+ vst1q_f32(y + i, val);
+ sum += vaddvq_f32(val);
+ }
}
#endif
diff --git a/llama.cpp/main/main.1 b/llama.cpp/main/main.1
index 06acc11a..49a3edfd 100644
--- a/llama.cpp/main/main.1
+++ b/llama.cpp/main/main.1
@@ -600,10 +600,36 @@ useful for shell scripts when the
flag is also supplied.
.It Fl Fl grammar-file Ar FNAME
File to read grammar from.
+.It Fl Fl fast
+Put llamafile into fast math mode. This disables algorithms that reduce
+floating point rounding, e.g. Kahan summation, and certain functions
+like expf() will be vectorized but handle underflows less gracefully.
+It's unspecified whether llamafile runs in fast or precise math mode
+when neither flag is specified.
+.It Fl Fl precise
+Put llamafile into precise math mode. This enables algorithms that
+reduce floating point rounding, e.g. Kahan summation, and certain
+functions like expf() will always handle subnormals correctly. It's
+unspecified whether llamafile runs in fast or precise math mode when
+neither flag is specified.
+.It Fl Fl trap
+Put llamafile into math trapping mode. When floating point exceptions
+occur, such as NaNs, overflow, and divide by zero, llamafile will print
+a warning to the console. This warning will include a C++ backtrace the
+first time an exception is trapped. The op graph will also be dumped to
+a file, and llamafile will report the specific op where the exception
+occurred. This is useful for troubleshooting when reporting issues.
+USing this feature will disable sandboxing. Math trapping is only
+possible if your CPU supports it. That is generally the case on AMD64,
+however it's less common on ARM64.
.It Fl Fl prompt-cache Ar FNAME
File to cache prompt state for faster startup.
.Pp
Default: none
+.It Fl fa Ar FNAME , Fl Fl flash-attn
+Enable Flash Attention. This is a mathematical shortcut that can speed
+up inference for certain models. This feature is still under active
+development.
.It Fl Fl prompt-cache-all
If specified, saves user input and generations to cache as well. Not supported with
.Fl Fl interactive
@@ -653,8 +679,14 @@ Enable colorized output to differentiate visually distinguishing between
prompts, user input, and generated text.
.It Fl Fl no-display-prompt , Fl Fl silent-prompt
Don't echo the prompt itself to standard output.
+.It Fl Fl keep Ar N
+Specifies number of tokens to keep from the initial prompt. The default
+is -1 which means all tokens.
.It Fl Fl multiline-input
Allows you to write or paste multiple lines without ending each in '\[rs]'.
+.It Fl Fl cont-batching
+Enables continuous batching, a.k.a. dynamic batching.
+is -1 which means all tokens.
.El
.Sh SERVER OPTIONS
The following options may be specified when
diff --git a/llama.cpp/main/main.1.asc b/llama.cpp/main/main.1.asc
index 49eefb57..ed2b50e6 100644
--- a/llama.cpp/main/main.1.asc
+++ b/llama.cpp/main/main.1.asc
@@ -1,836 +1,883 @@
-LLAMAFILE(1) BSD General Commands Manual LLAMAFILE(1)
+LLAMAFILE(1) General Commands Manual LLAMAFILE(1)
-NNAAMMEE
- llllaammaaffiillee — large language model runner
+NAME
+ llamafile — large language model runner
-SSYYNNOOPPSSIISS
- llllaammaaffiillee [----sseerrvveerr] [flags...] --mm _m_o_d_e_l_._g_g_u_f [----mmmmpprroojj _v_i_s_i_o_n_._g_g_u_f]
- llllaammaaffiillee [----ccllii] [flags...] --mm _m_o_d_e_l_._g_g_u_f --pp _p_r_o_m_p_t
- llllaammaaffiillee [----ccllii] [flags...] --mm _m_o_d_e_l_._g_g_u_f ----mmmmpprroojj _v_i_s_i_o_n_._g_g_u_f ----iimmaaggee
- _g_r_a_p_h_i_c_._p_n_g --pp _p_r_o_m_p_t
+SYNOPSIS
+ llamafile [--server] [flags...] -m model.gguf [--mmproj vision.gguf]
+ llamafile [--cli] [flags...] -m model.gguf -p prompt
+ llamafile [--cli] [flags...] -m model.gguf --mmproj vision.gguf --image
+ graphic.png -p prompt
-DDEESSCCRRIIPPTTIIOONN
- llllaammaaffiillee is a large language model tool. It has use cases such as:
+DESCRIPTION
+ llamafile is a large language model tool. It has use cases such as:
- -- Code completion
- -- Prose composition
- -- Chatbot that passes the Turing test
- -- Text/image summarization and analysis
+ - Code completion
+ - Prose composition
+ - Chatbot that passes the Turing test
+ - Text/image summarization and analysis
-OOPPTTIIOONNSS
- The following options are available:
+OPTIONS
+ The following options are available:
- ----vveerrssiioonn
- Print version and exit.
+ --version
+ Print version and exit.
- --hh, ----hheellpp
- Show help message and exit.
+ -h, --help
+ Show help message and exit.
- ----ccllii Puts program in command line interface mode. This flag is implied
- when a prompt is supplied using either the --pp or --ff flags.
+ --cli Puts program in command line interface mode. This flag is im‐
+ plied when a prompt is supplied using either the -p or -f
+ flags.
- ----sseerrvveerr
- Puts program in server mode. This will launch an HTTP server on a
- local port. This server has both a web UI and an OpenAI API com‐
- patible completions endpoint. When the server is run on a desk
- system, a tab browser tab will be launched automatically that
- displays the web UI. This ----sseerrvveerr flag is implied if no prompt
- is specified, i.e. neither the --pp or --ff flags are passed.
+ --server
+ Puts program in server mode. This will launch an HTTP server on
+ a local port. This server has both a web UI and an OpenAI API
+ compatible completions endpoint. When the server is run on a
+ desk system, a tab browser tab will be launched automatically
+ that displays the web UI. This --server flag is implied if no
+ prompt is specified, i.e. neither the -p or -f flags are
+ passed.
- --mm _F_N_A_M_E, ----mmooddeell _F_N_A_M_E
- Model path in the GGUF file format.
+ -m FNAME, --model FNAME
+ Model path in the GGUF file format.
- Default: _m_o_d_e_l_s_/_7_B_/_g_g_m_l_-_m_o_d_e_l_-_f_1_6_._g_g_u_f
+ Default: models/7B/ggml‐model‐f16.gguf
- ----mmmmpprroojj _F_N_A_M_E
- Specifies path of the LLaVA vision model in the GGUF file format.
- If this flag is supplied, then the ----mmooddeell and ----iimmaaggee flags
- should also be supplied.
+ --mmproj FNAME
+ Specifies path of the LLaVA vision model in the GGUF file for‐
+ mat. If this flag is supplied, then the --model and --image
+ flags should also be supplied.
- --ss _S_E_E_D, ----sseeeedd _S_E_E_D
- Random Number Generator (RNG) seed. A random seed is used if this
- is less than zero.
+ -s SEED, --seed SEED
+ Random Number Generator (RNG) seed. A random seed is used if
+ this is less than zero.
- Default: -1
+ Default: ‐1
- --tt _N, ----tthhrreeaaddss _N
- Number of threads to use during generation.
+ -t N, --threads N
+ Number of threads to use during generation.
- Default: $(nproc)/2
+ Default: $(nproc)/2
- --ttbb _N, ----tthhrreeaaddss--bbaattcchh _N
- Set the number of threads to use during batch and prompt process‐
- ing. In some systems, it is beneficial to use a higher number of
- threads during batch processing than during generation. If not
- specified, the number of threads used for batch processing will
- be the same as the number of threads used for generation.
+ -tb N, --threads‐batch N
+ Set the number of threads to use during batch and prompt pro‐
+ cessing. In some systems, it is beneficial to use a higher num‐
+ ber of threads during batch processing than during generation.
+ If not specified, the number of threads used for batch process‐
+ ing will be the same as the number of threads used for genera‐
+ tion.
- Default: Same as ----tthhrreeaaddss
+ Default: Same as --threads
- --ttdd _N, ----tthhrreeaaddss--ddrraafftt _N
- Number of threads to use during generation.
+ -td N, --threads‐draft N
+ Number of threads to use during generation.
- Default: Same as ----tthhrreeaaddss
+ Default: Same as --threads
- --ttbbdd _N, ----tthhrreeaaddss--bbaattcchh--ddrraafftt _N
- Number of threads to use during batch and prompt processing.
+ -tbd N, --threads‐batch‐draft N
+ Number of threads to use during batch and prompt processing.
- Default: Same as ----tthhrreeaaddss--ddrraafftt
+ Default: Same as --threads‐draft
- ----iinn--pprreeffiixx--bbooss
- Prefix BOS to user inputs, preceding the ----iinn--pprreeffiixx string.
+ --in‐prefix‐bos
+ Prefix BOS to user inputs, preceding the --in‐prefix string.
- ----iinn--pprreeffiixx _S_T_R_I_N_G
- This flag is used to add a prefix to your input, primarily, this
- is used to insert a space after the reverse prompt. Here's an ex‐
- ample of how to use the ----iinn--pprreeffiixx flag in conjunction with the
- ----rreevveerrssee--pprroommpptt flag:
+ --in‐prefix STRING
+ This flag is used to add a prefix to your input, primarily,
+ this is used to insert a space after the reverse prompt. Here’s
+ an example of how to use the --in‐prefix flag in conjunction
+ with the --reverse‐prompt flag:
- ./main -r "User:" --in-prefix " "
+ ./main ‐r "User:" ‐‐in‐prefix " "
- Default: empty
+ Default: empty
- ----iinn--ssuuffffiixx _S_T_R_I_N_G
- This flag is used to add a suffix after your input. This is use‐
- ful for adding an "Assistant:" prompt after the user's input.
- It's added after the new-line character (\n) that's automatically
- added to the end of the user's input. Here's an example of how to
- use the ----iinn--ssuuffffiixx flag in conjunction with the ----rreevveerrssee--pprroommpptt
- flag:
+ --in‐suffix STRING
+ This flag is used to add a suffix after your input. This is
+ useful for adding an "Assistant:" prompt after the user’s in‐
+ put. It’s added after the new‐line character (\n) that’s auto‐
+ matically added to the end of the user’s input. Here’s an exam‐
+ ple of how to use the --in‐suffix flag in conjunction with the
+ --reverse‐prompt flag:
- ./main -r "User:" --in-prefix " " --in-suffix "Assistant:"
+ ./main ‐r "User:" ‐‐in‐prefix " " ‐‐in‐suffix
+ "Assistant:"
- Default: empty
+ Default: empty
- --nn _N, ----nn--pprreeddiicctt _N
- Number of tokens to predict.
+ -n N, --n‐predict N
+ Number of tokens to predict.
- -- -1 = infinity
- -- -2 = until context filled
+ - ‐1 = infinity
+ - ‐2 = until context filled
- Default: -1
+ Default: ‐1
- --cc _N, ----ccttxx--ssiizzee _N
- Set the size of the prompt context. A larger context size helps
- the model to better comprehend and generate responses for longer
- input or conversations. The LLaMA models were built with a con‐
- text of 2048, which yields the best results on longer input / in‐
- ference.
+ -c N, --ctx‐size N
+ Set the size of the prompt context. A larger context size helps
+ the model to better comprehend and generate responses for
+ longer input or conversations. The LLaMA models were built with
+ a context of 2048, which yields the best results on longer in‐
+ put / inference.
- -- 0 = loaded automatically from model
+ - 0 = loaded automatically from model
- Default: 512
+ Default: 512
- --bb _N, ----bbaattcchh--ssiizzee _N
- Batch size for prompt processing.
+ -b N, --batch‐size N
+ Batch size for prompt processing.
- Default: 512
+ Default: 512
- ----ttoopp--kk _N
- Top-k sampling.
+ --top‐k N
+ Top‐k sampling.
- -- 0 = disabled
+ - 0 = disabled
- Default: 40
+ Default: 40
- ----ttoopp--pp _N
- Top-p sampling.
+ --top‐p N
+ Top‐p sampling.
- -- 1.0 = disabled
+ - 1.0 = disabled
- Default: 0.9
+ Default: 0.9
- ----mmiinn--pp _N
- Min-p sampling.
+ --min‐p N
+ Min‐p sampling.
- -- 0.0 = disabled
+ - 0.0 = disabled
- Default: 0.1
+ Default: 0.1
- ----ttffss _N
- Tail free sampling, parameter z.
+ --tfs N
+ Tail free sampling, parameter z.
- -- 1.0 = disabled
+ - 1.0 = disabled
- Default: 1.0
+ Default: 1.0
- ----ttyyppiiccaall _N
- Locally typical sampling, parameter p.
+ --typical N
+ Locally typical sampling, parameter p.
- -- 1.0 = disabled
+ - 1.0 = disabled
- Default: 1.0
+ Default: 1.0
- ----rreeppeeaatt--llaasstt--nn _N
- Last n tokens to consider for penalize.
+ --repeat‐last‐n N
+ Last n tokens to consider for penalize.
- -- 0 = disabled
- -- -1 = ctx_size
+ - 0 = disabled
+ - ‐1 = ctx_size
- Default: 64
+ Default: 64
- ----rreeppeeaatt--ppeennaallttyy _N
- Penalize repeat sequence of tokens.
+ --repeat‐penalty N
+ Penalize repeat sequence of tokens.
- -- 1.0 = disabled
+ - 1.0 = disabled
- Default: 1.1
+ Default: 1.1
- ----pprreesseennccee--ppeennaallttyy _N
- Repeat alpha presence penalty.
+ --presence‐penalty N
+ Repeat alpha presence penalty.
- -- 0.0 = disabled
+ - 0.0 = disabled
- Default: 0.0
+ Default: 0.0
- ----ffrreeqquueennccyy--ppeennaallttyy _N
- Repeat alpha frequency penalty.
+ --frequency‐penalty N
+ Repeat alpha frequency penalty.
- -- 0.0 = disabled
+ - 0.0 = disabled
- Default: 0.0
+ Default: 0.0
- ----mmiirroossttaatt _N
- Use Mirostat sampling. Top K, Nucleus, Tail Free and Locally Typ‐
- ical samplers are ignored if used..
+ --mirostat N
+ Use Mirostat sampling. Top K, Nucleus, Tail Free and Locally
+ Typical samplers are ignored if used..
- -- 0 = disabled
- -- 1 = Mirostat
- -- 2 = Mirostat 2.0
+ - 0 = disabled
+ - 1 = Mirostat
+ - 2 = Mirostat 2.0
- Default: 0
+ Default: 0
- ----mmiirroossttaatt--llrr _N
- Mirostat learning rate, parameter eta.
+ --mirostat‐lr N
+ Mirostat learning rate, parameter eta.
- Default: 0.1
+ Default: 0.1
- ----mmiirroossttaatt--eenntt _N
- Mirostat target entropy, parameter tau.
+ --mirostat‐ent N
+ Mirostat target entropy, parameter tau.
- Default: 5.0
+ Default: 5.0
- --ll _T_O_K_E_N___I_D_(_+_/_-_)_B_I_A_S, ----llooggiitt--bbiiaass _T_O_K_E_N___I_D_(_+_/_-_)_B_I_A_S
- Modifies the likelihood of token appearing in the completion,
- i.e. ----llooggiitt--bbiiaass _1_5_0_4_3_+_1 to increase likelihood of token
- _'_ _H_e_l_l_o_', or ----llooggiitt--bbiiaass _1_5_0_4_3_-_1 to decrease likelihood of token
- _'_ _H_e_l_l_o_'.
+ -l TOKEN_ID(+/‐)BIAS, --logit‐bias TOKEN_ID(+/‐)BIAS
+ Modifies the likelihood of token appearing in the completion,
+ i.e. --logit‐bias 15043+1 to increase likelihood of token
+ ’ Hello’, or --logit‐bias 15043‐1 to decrease likelihood of to‐
+ ken ’ Hello’.
- --mmdd _F_N_A_M_E, ----mmooddeell--ddrraafftt _F_N_A_M_E
- Draft model for speculative decoding.
+ -md FNAME, --model‐draft FNAME
+ Draft model for speculative decoding.
- Default: _m_o_d_e_l_s_/_7_B_/_g_g_m_l_-_m_o_d_e_l_-_f_1_6_._g_g_u_f
+ Default: models/7B/ggml‐model‐f16.gguf
- ----ccffgg--nneeggaattiivvee--pprroommpptt _P_R_O_M_P_T
- Negative prompt to use for guidance..
+ --cfg‐negative‐prompt PROMPT
+ Negative prompt to use for guidance..
- Default: empty
+ Default: empty
- ----ccffgg--nneeggaattiivvee--pprroommpptt--ffiillee _F_N_A_M_E
- Negative prompt file to use for guidance.
+ --cfg‐negative‐prompt‐file FNAME
+ Negative prompt file to use for guidance.
- Default: empty
+ Default: empty
- ----ccffgg--ssccaallee _N
- Strength of guidance.
+ --cfg‐scale N
+ Strength of guidance.
- -- 1.0 = disable
+ - 1.0 = disable
- Default: 1.0
+ Default: 1.0
- ----rrooppee--ssccaalliinngg _{_n_o_n_e_,_l_i_n_e_a_r_,_y_a_r_n_}
- RoPE frequency scaling method, defaults to linear unless speci‐
- fied by the model
+ --rope‐scaling {none,linear,yarn}
+ RoPE frequency scaling method, defaults to linear unless speci‐
+ fied by the model
- ----rrooppee--ssccaallee _N
- RoPE context scaling factor, expands context by a factor of _N
- where _N is the linear scaling factor used by the fine-tuned
- model. Some fine-tuned models have extended the context length by
- scaling RoPE. For example, if the original pre-trained model have
- a context length (max sequence length) of 4096 (4k) and the fine-
- tuned model have 32k. That is a scaling factor of 8, and should
- work by setting the above ----ccttxx--ssiizzee to 32768 (32k) and
- ----rrooppee--ssccaallee to 8.
+ --rope‐scale N
+ RoPE context scaling factor, expands context by a factor of N
+ where N is the linear scaling factor used by the fine‐tuned
+ model. Some fine‐tuned models have extended the context length
+ by scaling RoPE. For example, if the original pre‐trained model
+ have a context length (max sequence length) of 4096 (4k) and
+ the fine‐tuned model have 32k. That is a scaling factor of 8,
+ and should work by setting the above --ctx‐size to 32768 (32k)
+ and --rope‐scale to 8.
- ----rrooppee--ffrreeqq--bbaassee _N
- RoPE base frequency, used by NTK-aware scaling.
+ --rope‐freq‐base N
+ RoPE base frequency, used by NTK‐aware scaling.
- Default: loaded from model
+ Default: loaded from model
- ----rrooppee--ffrreeqq--ssccaallee _N
- RoPE frequency scaling factor, expands context by a factor of 1/N
+ --rope‐freq‐scale N
+ RoPE frequency scaling factor, expands context by a factor of
+ 1/N
- ----yyaarrnn--oorriigg--ccttxx _N
- YaRN: original context size of model.
+ --yarn‐orig‐ctx N
+ YaRN: original context size of model.
- Default: 0 = model training context size
+ Default: 0 = model training context size
- ----yyaarrnn--eexxtt--ffaaccttoorr _N
- YaRN: extrapolation mix factor.
+ --yarn‐ext‐factor N
+ YaRN: extrapolation mix factor.
- -- 0.0 = full interpolation
+ - 0.0 = full interpolation
- Default: 1.0
+ Default: 1.0
- ----yyaarrnn--aattttnn--ffaaccttoorr _N
- YaRN: scale sqrt(t) or attention magnitude.
+ --yarn‐attn‐factor N
+ YaRN: scale sqrt(t) or attention magnitude.
- Default: 1.0
+ Default: 1.0
- ----yyaarrnn--bbeettaa--ssllooww _N
- YaRN: high correction dim or alpha.
+ --yarn‐beta‐slow N
+ YaRN: high correction dim or alpha.
- Default: 1.0
+ Default: 1.0
- ----yyaarrnn--bbeettaa--ffaasstt _N
- YaRN: low correction dim or beta.
+ --yarn‐beta‐fast N
+ YaRN: low correction dim or beta.
- Default: 32.0
+ Default: 32.0
- ----iiggnnoorree--eeooss
- Ignore end of stream token and continue generating (implies
- ----llooggiitt--bbiiaass _2_-_i_n_f)
+ --ignore‐eos
+ Ignore end of stream token and continue generating (implies
+ --logit‐bias 2‐inf)
- ----nnoo--ppeennaalliizzee--nnll
- Do not penalize newline token.
+ --no‐penalize‐nl
+ Do not penalize newline token.
- ----tteemmpp _N
- Temperature.
+ --temp N
+ Temperature.
- Default: 0.8
+ Default: 0.8
- ----llooggiittss--aallll
- Return logits for all tokens in the batch.
+ --logits‐all
+ Return logits for all tokens in the batch.
- Default: disabled
+ Default: disabled
- ----hheellllaasswwaagg
- Compute HellaSwag score over random tasks from datafile supplied
- with -f
+ --hellaswag
+ Compute HellaSwag score over random tasks from datafile sup‐
+ plied with ‐f
- ----hheellllaasswwaagg--ttaasskkss _N
- Number of tasks to use when computing the HellaSwag score.
+ --hellaswag‐tasks N
+ Number of tasks to use when computing the HellaSwag score.
- Default: 400
+ Default: 400
- ----kkeeeepp _N
- This flag allows users to retain the original prompt when the
- model runs out of context, ensuring a connection to the initial
- instruction or conversation topic is maintained, where _N is the
- number of tokens from the initial prompt to retain when the model
- resets its internal context.
+ --keep N
+ This flag allows users to retain the original prompt when the
+ model runs out of context, ensuring a connection to the initial
+ instruction or conversation topic is maintained, where N is the
+ number of tokens from the initial prompt to retain when the
+ model resets its internal context.
- -- 0 = no tokens are kept from initial prompt
- -- -1 = retain all tokens from initial prompt
+ - 0 = no tokens are kept from initial prompt
+ - ‐1 = retain all tokens from initial prompt
- Default: 0
+ Default: 0
- ----ddrraafftt _N
- Number of tokens to draft for speculative decoding.
+ --draft N
+ Number of tokens to draft for speculative decoding.
- Default: 16
+ Default: 16
- ----cchhuunnkkss _N
- Max number of chunks to process.
+ --chunks N
+ Max number of chunks to process.
- -- -1 = all
+ - ‐1 = all
- Default: -1
+ Default: ‐1
- --nnss _N, ----sseeqquueenncceess _N
- Number of sequences to decode.
+ -ns N, --sequences N
+ Number of sequences to decode.
- Default: 1
+ Default: 1
- --ppaa _N, ----pp--aacccceepptt _N
- speculative decoding accept probability.
+ -pa N, --p‐accept N
+ speculative decoding accept probability.
- Default: 0.5
+ Default: 0.5
- --ppss _N, ----pp--sspplliitt _N
- Speculative decoding split probability.
+ -ps N, --p‐split N
+ Speculative decoding split probability.
- Default: 0.1
+ Default: 0.1
- ----mmlloocckk
- Force system to keep model in RAM rather than swapping or com‐
- pressing.
+ --mlock
+ Force system to keep model in RAM rather than swapping or com‐
+ pressing.
- ----nnoo--mmmmaapp
- Do not memory-map model (slower load but may reduce pageouts if
- not using mlock).
+ --no‐mmap
+ Do not memory‐map model (slower load but may reduce pageouts if
+ not using mlock).
- ----nnuummaa Attempt optimizations that help on some NUMA systems if run with‐
- out this previously, it is recommended to drop the system page
- cache before using this. See
- https://github.com/ggerganov/llama.cpp/issues/1437.
+ --numa Attempt optimizations that help on some NUMA systems if run
+ without this previously, it is recommended to drop the system
+ page cache before using this. See
+ https://github.com/ggerganov/llama.cpp/issues/1437.
- ----rreeccoommppiillee
- Force GPU support to be recompiled at runtime if possible.
+ --recompile
+ Force GPU support to be recompiled at runtime if possible.
- ----nnooccoommppiillee
- Never compile GPU support at runtime.
+ --nocompile
+ Never compile GPU support at runtime.
- If the appropriate DSO file already exists under _~_/_._l_l_a_m_a_f_i_l_e_/
- then it'll be linked as-is without question. If a prebuilt DSO is
- present in the PKZIP content of the executable, then it'll be ex‐
- tracted and linked if possible. Otherwise, llllaammaaffiillee will skip
- any attempt to compile GPU support and simply fall back to using
- CPU inference.
+ If the appropriate DSO file already exists under ~/.llamafile/
+ then it’ll be linked as‐is without question. If a prebuilt DSO
+ is present in the PKZIP content of the executable, then it’ll
+ be extracted and linked if possible. Otherwise, llamafile will
+ skip any attempt to compile GPU support and simply fall back to
+ using CPU inference.
- ----ggppuu _G_P_U
- Specifies which brand of GPU should be used. Valid choices are:
+ --gpu GPU
+ Specifies which brand of GPU should be used. Valid choices are:
- -- _A_U_T_O: Use any GPU if possible, otherwise fall back to CPU in‐
- ference (default)
+ - AUTO: Use any GPU if possible, otherwise fall back to CPU
+ inference (default)
- -- _A_P_P_L_E: Use Apple Metal GPU. This is only available on MacOS
- ARM64. If Metal could not be used for any reason, then a fa‐
- tal error will be raised.
+ - APPLE: Use Apple Metal GPU. This is only available on MacOS
+ ARM64. If Metal could not be used for any reason, then a
+ fatal error will be raised.
- -- _A_M_D: Use AMD GPUs. The AMD HIP ROCm SDK should be installed
- in which case we assume the HIP_PATH environment variable has
- been defined. The set of gfx microarchitectures needed to run
- on the host machine is determined automatically based on the
- output of the hipInfo command. On Windows, llllaammaaffiillee release
- binaries are distributed with a tinyBLAS DLL so it'll work
- out of the box without requiring the HIP SDK to be installed.
- However, tinyBLAS is slower than rocBLAS for batch and image
- processing, so it's recommended that the SDK be installed
- anyway. If an AMD GPU could not be used for any reason, then
- a fatal error will be raised.
+ - AMD: Use AMD GPUs. The AMD HIP ROCm SDK should be installed
+ in which case we assume the HIP_PATH environment variable
+ has been defined. The set of gfx microarchitectures needed
+ to run on the host machine is determined automatically
+ based on the output of the hipInfo command. On Windows,
+ llamafile release binaries are distributed with a tinyBLAS
+ DLL so it’ll work out of the box without requiring the HIP
+ SDK to be installed. However, tinyBLAS is slower than
+ rocBLAS for batch and image processing, so it’s recommended
+ that the SDK be installed anyway. If an AMD GPU could not
+ be used for any reason, then a fatal error will be raised.
- -- _N_V_I_D_I_A: Use NVIDIA GPUs. If an NVIDIA GPU could not be used
- for any reason, a fatal error will be raised. On Windows,
- NVIDIA GPU support will use our tinyBLAS library, since it
- works on stock Windows installs. However, tinyBLAS goes
- slower for batch and image processing. It's possible to use
- NVIDIA's closed-source cuBLAS library instead. To do that,
- both MSVC and CUDA need to be installed and the llllaammaaffiillee
- command should be run once from the x64 MSVC command prompt
- with the ----rreeccoommppiillee flag passed. The GGML library will then
- be compiled and saved to _~_/_._l_l_a_m_a_f_i_l_e_/ so the special process
- only needs to happen a single time.
+ - NVIDIA: Use NVIDIA GPUs. If an NVIDIA GPU could not be used
+ for any reason, a fatal error will be raised. On Windows,
+ NVIDIA GPU support will use our tinyBLAS library, since it
+ works on stock Windows installs. However, tinyBLAS goes
+ slower for batch and image processing. It’s possible to use
+ NVIDIA’s closed‐source cuBLAS library instead. To do that,
+ both MSVC and CUDA need to be installed and the llamafile
+ command should be run once from the x64 MSVC command prompt
+ with the --recompile flag passed. The GGML library will
+ then be compiled and saved to ~/.llamafile/ so the special
+ process only needs to happen a single time.
- -- _D_I_S_A_B_L_E: Never use GPU and instead use CPU inference. This
- setting is implied by --nnggll _0.
+ - DISABLE: Never use GPU and instead use CPU inference. This
+ setting is implied by -ngl 0.
- --nnggll _N, ----nn--ggppuu--llaayyeerrss _N
- Number of layers to store in VRAM.
+ -ngl N, --n‐gpu‐layers N
+ Number of layers to store in VRAM.
- --nngglldd _N, ----nn--ggppuu--llaayyeerrss--ddrraafftt _N
- Number of layers to store in VRAM for the draft model.
+ -ngld N, --n‐gpu‐layers‐draft N
+ Number of layers to store in VRAM for the draft model.
- --ssmm _S_P_L_I_T___M_O_D_E, ----sspplliitt--mmooddee _S_P_L_I_T___M_O_D_E
- How to split the model across multiple GPUs, one of:
- -- none: use one GPU only
- -- layer (default): split layers and KV across GPUs
- -- row: split rows across GPUs
+ -sm SPLIT_MODE, --split‐mode SPLIT_MODE
+ How to split the model across multiple GPUs, one of:
+ - none: use one GPU only
+ - layer (default): split layers and KV across GPUs
+ - row: split rows across GPUs
- --ttss _S_P_L_I_T, ----tteennssoorr--sspplliitt _S_P_L_I_T
- When using multiple GPUs this option controls how large tensors
- should be split across all GPUs. _S_P_L_I_T is a comma-separated list
- of non-negative values that assigns the proportion of data that
- each GPU should get in order. For example, "3,2" will assign 60%
- of the data to GPU 0 and 40% to GPU 1. By default the data is
- split in proportion to VRAM but this may not be optimal for per‐
- formance. Requires cuBLAS. How to split tensors across multiple
- GPUs, comma-separated list of proportions, e.g. 3,1
+ -ts SPLIT, --tensor‐split SPLIT
+ When using multiple GPUs this option controls how large tensors
+ should be split across all GPUs. SPLIT is a comma‐separated
+ list of non‐negative values that assigns the proportion of data
+ that each GPU should get in order. For example, "3,2" will as‐
+ sign 60% of the data to GPU 0 and 40% to GPU 1. By default the
+ data is split in proportion to VRAM but this may not be optimal
+ for performance. Requires cuBLAS. How to split tensors across
+ multiple GPUs, comma‐separated list of proportions, e.g. 3,1
- --mmgg _i, ----mmaaiinn--ggppuu _i
- The GPU to use for scratch and small tensors.
+ -mg i, --main‐gpu i
+ The GPU to use for scratch and small tensors.
- --nnoommmmqq, ----nnoo--mmuull--mmaatt--qq
- Use cuBLAS instead of custom mul_mat_q CUDA kernels. Not recom‐
- mended since this is both slower and uses more VRAM.
+ -nommq, --no‐mul‐mat‐q
+ Use cuBLAS instead of custom mul_mat_q CUDA kernels. Not recom‐
+ mended since this is both slower and uses more VRAM.
- ----vveerrbboossee--pprroommpptt
- Print prompt before generation.
+ --verbose‐prompt
+ Print prompt before generation.
- ----ssiimmppllee--iioo
- Use basic IO for better compatibility in subprocesses and limited
- consoles.
+ --simple‐io
+ Use basic IO for better compatibility in subprocesses and lim‐
+ ited consoles.
- ----lloorraa _F_N_A_M_E
- Apply LoRA adapter (implies ----nnoo--mmmmaapp)
+ --lora FNAME
+ Apply LoRA adapter (implies --no‐mmap)
- ----lloorraa--ssccaalleedd _F_N_A_M_E _S
- Apply LoRA adapter with user defined scaling S (implies
- ----nnoo--mmmmaapp)
+ --lora‐scaled FNAME S
+ Apply LoRA adapter with user defined scaling S (implies
+ --no‐mmap)
- ----lloorraa--bbaassee _F_N_A_M_E
- Optional model to use as a base for the layers modified by the
- LoRA adapter
+ --lora‐base FNAME
+ Optional model to use as a base for the layers modified by the
+ LoRA adapter
- ----uunnsseeccuurree
- Disables pledge() sandboxing on Linux and OpenBSD.
+ --unsecure
+ Disables pledge() sandboxing on Linux and OpenBSD.
- ----ssaammpplleerrss
- Samplers that will be used for generation in the order, separated
- by semicolon, for example: top_k;tfs;typical;top_p;min_p;temp
+ --samplers
+ Samplers that will be used for generation in the order, sepa‐
+ rated by semicolon, for example: top_k;tfs;typi‐
+ cal;top_p;min_p;temp
- ----ssaammpplleerrss--sseeqq
- Simplified sequence for samplers that will be used.
+ --samplers‐seq
+ Simplified sequence for samplers that will be used.
- --ccmmll, ----cchhaattmmll
- Run in chatml mode (use with ChatML-compatible models)
+ -cml, --chatml
+ Run in chatml mode (use with ChatML‐compatible models)
- --ddkkvvcc, ----dduummpp--kkvv--ccaacchhee
- Verbose print of the KV cache.
+ -dkvc, --dump‐kv‐cache
+ Verbose print of the KV cache.
- --nnkkvvoo, ----nnoo--kkvv--ooffffllooaadd
- Disable KV offload.
+ -nkvo, --no‐kv‐offload
+ Disable KV offload.
- --ccttkk _T_Y_P_E, ----ccaacchhee--ttyyppee--kk _T_Y_P_E
- KV cache data type for K.
+ -ctk TYPE, --cache‐type‐k TYPE
+ KV cache data type for K.
- --ccttvv _T_Y_P_E, ----ccaacchhee--ttyyppee--vv _T_Y_P_E
- KV cache data type for V.
+ -ctv TYPE, --cache‐type‐v TYPE
+ KV cache data type for V.
- --ggaann _N, ----ggrrpp--aattttnn--nn _N
- Group-attention factor.
+ -gan N, --grp‐attn‐n N
+ Group‐attention factor.
- Default: 1
+ Default: 1
- --ggaaww _N, ----ggrrpp--aattttnn--ww _N
- Group-attention width.
+ -gaw N, --grp‐attn‐w N
+ Group‐attention width.
- Default: 512
+ Default: 512
- --bbff _F_N_A_M_E, ----bbiinnaarryy--ffiillee _F_N_A_M_E
- Binary file containing multiple choice tasks.
+ -bf FNAME, --binary‐file FNAME
+ Binary file containing multiple choice tasks.
- ----wwiinnooggrraannddee
- Compute Winogrande score over random tasks from datafile supplied
- by the --ff flag.
+ --winogrande
+ Compute Winogrande score over random tasks from datafile sup‐
+ plied by the -f flag.
- ----wwiinnooggrraannddee--ttaasskkss _N
- Number of tasks to use when computing the Winogrande score.
+ --winogrande‐tasks N
+ Number of tasks to use when computing the Winogrande score.
- Default: 0
+ Default: 0
- ----mmuullttiippllee--cchhooiiccee
- Compute multiple choice score over random tasks from datafile
- supplied by the --ff flag.
+ --multiple‐choice
+ Compute multiple choice score over random tasks from datafile
+ supplied by the -f flag.
- ----mmuullttiippllee--cchhooiiccee--ttaasskkss _N
- Number of tasks to use when computing the multiple choice score.
+ --multiple‐choice‐tasks N
+ Number of tasks to use when computing the multiple choice
+ score.
- Default: 0
+ Default: 0
- ----kkll--ddiivveerrggeennccee
- Computes KL-divergence to logits provided via the
- ----kkll--ddiivveerrggeennccee--bbaassee flag.
+ --kl‐divergence
+ Computes KL‐divergence to logits provided via the
+ --kl‐divergence‐base flag.
- ----ssaavvee--aallll--llooggiittss _F_N_A_M_E, ----kkll--ddiivveerrggeennccee--bbaassee _F_N_A_M_E
- Save logits to filename.
+ --save‐all‐logits FNAME, --kl‐divergence‐base FNAME
+ Save logits to filename.
- --ppttcc _N, ----pprriinntt--ttookkeenn--ccoouunntt _N
- Print token count every _N tokens.
+ -ptc N, --print‐token‐count N
+ Print token count every N tokens.
- Default: -1
+ Default: ‐1
-CCLLII OOPPTTIIOONNSS
- The following options may be specified when llllaammaaffiillee is running in ----ccllii
- mode.
+CLI OPTIONS
+ The following options may be specified when llamafile is running in
+ --cli mode.
- --ee, ----eessccaappee
- Process prompt escapes sequences (\n, \r, \t, \´, \", \\)
+ -e, --escape
+ Process prompt escapes sequences (\n, \r, \t, \´, \", \\)
- --pp _S_T_R_I_N_G, ----pprroommpptt _S_T_R_I_N_G
- Prompt to start text generation. Your LLM works by auto-complet‐
- ing this text. For example:
+ -p STRING, --prompt STRING
+ Prompt to start text generation. Your LLM works by auto‐com‐
+ pleting this text. For example:
- llamafile -m model.gguf -p "four score and"
+ llamafile ‐m model.gguf ‐p "four score and"
- Stands a pretty good chance of printing Lincoln's Gettysburg Ad‐
- dress. Prompts can take on a structured format too. Depending on
- how your model was trained, it may specify in its docs an in‐
- struction notation. With some models that might be:
+ Stands a pretty good chance of printing Lincoln’s Gettysburg
+ Address. Prompts can take on a structured format too. Depend‐
+ ing on how your model was trained, it may specify in its docs
+ an instruction notation. With some models that might be:
- llamafile -p "[INST]Summarize this: $(cat file)[/INST]"
+ llamafile ‐p "[INST]Summarize this: $(cat file)[/INST]"
- In most cases, simply colons and newlines will work too:
+ In most cases, simply colons and newlines will work too:
- llamafile -e -p "User: What is best in life?\nAssistant:"
+ llamafile ‐e ‐p "User: What is best in life?\nAssistant:"
- --ff _F_N_A_M_E, ----ffiillee _F_N_A_M_E
- Prompt file to start generation.
+ -f FNAME, --file FNAME
+ Prompt file to start generation.
- ----ggrraammmmaarr _G_R_A_M_M_A_R
- BNF-like grammar to constrain which tokens may be selected when
- generating text. For example, the grammar:
+ --grammar GRAMMAR
+ BNF‐like grammar to constrain which tokens may be selected when
+ generating text. For example, the grammar:
- root ::= "yes" | "no"
+ root ::= "yes" | "no"
- will force the LLM to only output yes or no before exiting. This
- is useful for shell scripts when the ----nnoo--ddiissppllaayy--pprroommpptt flag is
- also supplied.
+ will force the LLM to only output yes or no before exiting.
+ This is useful for shell scripts when the --no‐display‐prompt
+ flag is also supplied.
- ----ggrraammmmaarr--ffiillee _F_N_A_M_E
- File to read grammar from.
+ --grammar‐file FNAME
+ File to read grammar from.
- ----pprroommpptt--ccaacchhee _F_N_A_M_E
- File to cache prompt state for faster startup.
+ --fast Put llamafile into fast math mode. This disables algorithms
+ that reduce floating point rounding, e.g. Kahan summation, and
+ certain functions like expf() will be vectorized but handle un‐
+ derflows less gracefully. It’s unspecified whether llamafile
+ runs in fast or precise math mode when neither flag is speci‐
+ fied.
- Default: none
+ --precise
+ Put llamafile into precise math mode. This enables algorithms
+ that reduce floating point rounding, e.g. Kahan summation, and
+ certain functions like expf() will always handle subnormals
+ correctly. It’s unspecified whether llamafile runs in fast or
+ precise math mode when neither flag is specified.
- ----pprroommpptt--ccaacchhee--aallll
- If specified, saves user input and generations to cache as well.
- Not supported with ----iinntteerraaccttiivvee or other interactive options.
+ --trap Put llamafile into math trapping mode. When floating point ex‐
+ ceptions occur, such as NaNs, overflow, and divide by zero,
+ llamafile will print a warning to the console. This warning
+ will include a C++ backtrace the first time an exception is
+ trapped. The op graph will also be dumped to a file, and lla‐
+ mafile will report the specific op where the exception oc‐
+ curred. This is useful for troubleshooting when reporting is‐
+ sues. USing this feature will disable sandboxing. Math trap‐
+ ping is only possible if your CPU supports it. That is gener‐
+ ally the case on AMD64, however it’s less common on ARM64.
- ----pprroommpptt--ccaacchhee--rroo
- If specified, uses the prompt cache but does not update it.
+ --prompt‐cache FNAME
+ File to cache prompt state for faster startup.
- ----rraannddoomm--pprroommpptt
- Start with a randomized prompt.
+ Default: none
- ----iimmaaggee _I_M_A_G_E___F_I_L_E
- Path to an image file. This should be used with multimodal mod‐
- els. Alternatively, it's possible to embed an image directly
- into the prompt instead; in which case, it must be base64 encoded
- into an HTML img tag URL with the image/jpeg MIME type. See also
- the ----mmmmpprroojj flag for supplying the vision model.
+ -fa FNAME, --flash‐attn
+ Enable Flash Attention. This is a mathematical shortcut that
+ can speed up inference for certain models. This feature is
+ still under active development.
- --ii, ----iinntteerraaccttiivvee
- Run the program in interactive mode, allowing users to engage in
- real-time conversations or provide specific instructions to the
- model.
+ --prompt‐cache‐all
+ If specified, saves user input and generations to cache as
+ well. Not supported with --interactive or other interactive op‐
+ tions.
- ----iinntteerraaccttiivvee--ffiirrsstt
- Run the program in interactive mode and immediately wait for user
- input before starting the text generation.
+ --prompt‐cache‐ro
+ If specified, uses the prompt cache but does not update it.
- --iinnss, ----iinnssttrruucctt
- Run the program in instruction mode, which is specifically de‐
- signed to work with Alpaca models that excel in completing tasks
- based on user instructions.
+ --random‐prompt
+ Start with a randomized prompt.
- Technical details: The user's input is internally prefixed with
- the reverse prompt (or "### Instruction:" as the default), and
- followed by "### Response:" (except if you just press Return
- without any input, to keep generating a longer response).
+ --image IMAGE_FILE
+ Path to an image file. This should be used with multimodal mod‐
+ els. Alternatively, it’s possible to embed an image directly
+ into the prompt instead; in which case, it must be base64 en‐
+ coded into an HTML img tag URL with the image/jpeg MIME type.
+ See also the --mmproj flag for supplying the vision model.
- By understanding and utilizing these interaction options, you can
- create engaging and dynamic experiences with the LLaMA models,
- tailoring the text generation process to your specific needs.
+ -i, --interactive
+ Run the program in interactive mode, allowing users to engage
+ in real‐time conversations or provide specific instructions to
+ the model.
- --rr _P_R_O_M_P_T, ----rreevveerrssee--pprroommpptt _P_R_O_M_P_T
- Specify one or multiple reverse prompts to pause text generation
- and switch to interactive mode. For example, --rr _"_U_s_e_r_:_" can be
- used to jump back into the conversation whenever it's the user's
- turn to speak. This helps create a more interactive and conversa‐
- tional experience. However, the reverse prompt doesn't work when
- it ends with a space. To overcome this limitation, you can use
- the ----iinn--pprreeffiixx flag to add a space or any other characters after
- the reverse prompt.
+ --interactive‐first
+ Run the program in interactive mode and immediately wait for
+ user input before starting the text generation.
- ----ccoolloorr
- Enable colorized output to differentiate visually distinguishing
- between prompts, user input, and generated text.
+ -ins, --instruct
+ Run the program in instruction mode, which is specifically de‐
+ signed to work with Alpaca models that excel in completing
+ tasks based on user instructions.
- ----nnoo--ddiissppllaayy--pprroommpptt, ----ssiilleenntt--pprroommpptt
- Don't echo the prompt itself to standard output.
+ Technical details: The user’s input is internally prefixed with
+ the reverse prompt (or "### Instruction:" as the default), and
+ followed by "### Response:" (except if you just press Return
+ without any input, to keep generating a longer response).
- ----mmuullttiilliinnee--iinnppuutt
- Allows you to write or paste multiple lines without ending each
- in '\'.
+ By understanding and utilizing these interaction options, you
+ can create engaging and dynamic experiences with the LLaMA mod‐
+ els, tailoring the text generation process to your specific
+ needs.
-SSEERRVVEERR OOPPTTIIOONNSS
- The following options may be specified when llllaammaaffiillee is running in
- ----sseerrvveerr mode.
+ -r PROMPT, --reverse‐prompt PROMPT
+ Specify one or multiple reverse prompts to pause text genera‐
+ tion and switch to interactive mode. For example, -r "User:"
+ can be used to jump back into the conversation whenever it’s
+ the user’s turn to speak. This helps create a more interactive
+ and conversational experience. However, the reverse prompt
+ doesn’t work when it ends with a space. To overcome this limi‐
+ tation, you can use the --in‐prefix flag to add a space or any
+ other characters after the reverse prompt.
- ----ppoorrtt _P_O_R_T
- Port to listen
+ --color
+ Enable colorized output to differentiate visually distinguish‐
+ ing between prompts, user input, and generated text.
- Default: 8080
+ --no‐display‐prompt, --silent‐prompt
+ Don’t echo the prompt itself to standard output.
- ----hhoosstt _I_P_A_D_D_R
- IP address to listen.
+ --keep N
+ Specifies number of tokens to keep from the initial prompt. The
+ default is ‐1 which means all tokens.
- Default: 127.0.0.1
+ --multiline‐input
+ Allows you to write or paste multiple lines without ending each
+ in ’\’.
- --ttoo _N, ----ttiimmeeoouutt _N
- Server read/write timeout in seconds.
+ --cont‐batching
+ Enables continuous batching, a.k.a. dynamic batching. is ‐1
+ which means all tokens.
- Default: 600
+SERVER OPTIONS
+ The following options may be specified when llamafile is running in
+ --server mode.
- --nnpp _N, ----ppaarraalllleell _N
- Number of slots for process requests.
+ --port PORT
+ Port to listen
- Default: 1
+ Default: 8080
- --ccbb, ----ccoonntt--bbaattcchhiinngg
- Enable continuous batching (a.k.a dynamic batching).
+ --host IPADDR
+ IP address to listen.
- Default: disabled
+ Default: 127.0.0.1
- --ssppff _F_N_A_M_E, ----ssyysstteemm--pprroommpptt--ffiillee _F_N_A_M_E
- Set a file to load a system prompt (initial prompt of all slots),
- this is useful for chat applications.
+ -to N, --timeout N
+ Server read/write timeout in seconds.
- --aa _A_L_I_A_S, ----aalliiaass _A_L_I_A_S
- Set an alias for the model. This will be added as the _m_o_d_e_l field
- in completion responses.
+ Default: 600
- ----ppaatthh _P_U_B_L_I_C___P_A_T_H
- Path from which to serve static files.
+ -np N, --parallel N
+ Number of slots for process requests.
- Default: _/_z_i_p_/_l_l_a_m_a_._c_p_p_/_s_e_r_v_e_r_/_p_u_b_l_i_c
+ Default: 1
- ----eemmbbeeddddiinngg
- Enable embedding vector output.
+ -cb, --cont‐batching
+ Enable continuous batching (a.k.a dynamic batching).
- Default: disabled
+ Default: disabled
- ----nnoobbrroowwsseerr
- Do not attempt to open a web browser tab at startup.
+ -spf FNAME, --system‐prompt‐file FNAME
+ Set a file to load a system prompt (initial prompt of all
+ slots), this is useful for chat applications.
- --ggaann _N, ----ggrrpp--aattttnn--nn _N
- Set the group attention factor to extend context size through
- self-extend. The default value is _1 which means disabled. This
- flag is used together with ----ggrrpp--aattttnn--ww.
+ -a ALIAS, --alias ALIAS
+ Set an alias for the model. This will be added as the model
+ field in completion responses.
- --ggaaww _N, ----ggrrpp--aattttnn--ww _N
- Set the group attention width to extend context size through
- self-extend. The default value is _5_1_2. This flag is used to‐
- gether with ----ggrrpp--aattttnn--nn.
+ --path PUBLIC_PATH
+ Path from which to serve static files.
-LLOOGG OOPPTTIIOONNSS
- The following log options are available:
+ Default: /zip/llama.cpp/server/public
- --lldd _L_O_G_D_I_R, ----llooggddiirr _L_O_G_D_I_R
- Path under which to save YAML logs (no logging if unset)
-
- ----lloogg--tteesstt
- Run simple logging test
+ --embedding
+ Enable embedding vector output.
- ----lloogg--ddiissaabbllee
- Disable trace logs
-
- ----lloogg--eennaabbllee
- Enable trace logs
-
- ----lloogg--ffiillee
- Specify a log filename (without extension)
-
- ----lloogg--nneeww
- Create a separate new log file on start. Each log file will have
- unique name: _<_n_a_m_e_>_._<_I_D_>_._l_o_g
-
- ----lloogg--aappppeenndd
- Don't truncate the old log file.
-
-EEXXAAMMPPLLEESS
- Here's an example of how to run llama.cpp's built-in HTTP server. This
- example uses LLaVA v1.5-7B, a multimodal LLM that works with llama.cpp's
- recently-added support for image inputs.
-
- llamafile \
- -m llava-v1.5-7b-Q8_0.gguf \
- --mmproj llava-v1.5-7b-mmproj-Q8_0.gguf \
- --host 0.0.0.0
-
- Here's an example of how to generate code for a libc function using the
- llama.cpp command line interface, utilizing WizardCoder-Python-13B
- weights:
-
- llamafile \
- -m wizardcoder-python-13b-v1.0.Q8_0.gguf --temp 0 -r '}\n' -r '```\n' \
- -e -p '```c\nvoid *memcpy(void *dst, const void *src, size_t size) {\n'
-
- Here's a similar example that instead utilizes Mistral-7B-Instruct
- weights for prose composition:
-
- llamafile \
- -m mistral-7b-instruct-v0.2.Q5_K_M.gguf \
- -p '[INST]Write a story about llamas[/INST]'
-
- Here's an example of how llamafile can be used as an interactive chatbot
- that lets you query knowledge contained in training data:
-
- llamafile -m llama-65b-Q5_K.gguf -p '
- The following is a conversation between a Researcher and their helpful AI
- assistant Digital Athena which is a large language model trained on the
- sum of human knowledge.
- Researcher: Good morning.
- Digital Athena: How can I help you today?
- Researcher:' --interactive --color --batch_size 1024 --ctx_size 4096 \
- --keep -1 --temp 0 --mirostat 2 --in-prefix ' ' --interactive-first \
- --in-suffix 'Digital Athena:' --reverse-prompt 'Researcher:'
-
- Here's an example of how you can use llamafile to summarize HTML URLs:
-
- (
- echo '[INST]Summarize the following text:'
- links -codepage utf-8 \
- -force-html \
- -width 500 \
- -dump https://www.poetryfoundation.org/poems/48860/the-raven |
- sed 's/ */ /g'
- echo '[/INST]'
- ) | llamafile \
- -m mistral-7b-instruct-v0.2.Q5_K_M.gguf \
- -f /dev/stdin \
- -c 0 \
- --temp 0 \
- -n 500 \
- --no-display-prompt 2>/dev/null
-
- Here's how you can use llamafile to describe a jpg/png/gif/bmp image:
-
- llamafile --temp 0 \
- --image lemurs.jpg \
- -m llava-v1.5-7b-Q4_K.gguf \
- --mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \
- -e -p '### User: What do you see?\n### Assistant: ' \
- --no-display-prompt 2>/dev/null
-
- If you wanted to write a script to rename all your image files, you could
- use the following command to generate a safe filename:
-
- llamafile --temp 0 \
- --image ~/Pictures/lemurs.jpg \
- -m llava-v1.5-7b-Q4_K.gguf \
- --mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \
- --grammar 'root ::= [a-z]+ (" " [a-z]+)+' \
- -e -p '### User: The image has...\n### Assistant: ' \
- --no-display-prompt 2>/dev/null |
- sed -e's/ /_/g' -e's/$/.jpg/'
- three_baby_lemurs_on_the_back_of_an_adult_lemur.jpg
-
- Here's an example of how to make an API request to the OpenAI API compat‐
- ible completions endpoint when your llllaammaaffiillee is running in the back‐
- ground in ----sseerrvveerr mode.
-
- curl -s http://localhost:8080/v1/chat/completions \
- -H "Content-Type: application/json" -d '{
- "model": "gpt-3.5-turbo",
- "stream": true,
- "messages": [
- {
- "role": "system",
- "content": "You are a poetic assistant."
- },
- {
- "role": "user",
- "content": "Compose a poem that explains FORTRAN."
- }
- ]
- }' | python3 -c '
- import json
- import sys
- json.dump(json.load(sys.stdin), sys.stdout, indent=2)
- print()
-
-PPRROOTTIIPP
- The --nnggll _3_5 flag needs to be passed in order to use GPUs made by NVIDIA
- and AMD. It's not enabled by default since it sometimes needs to be
- tuned based on the system hardware and model architecture, in order to
- achieve optimal performance, and avoid compromising a shared display.
-
-SSEEEE AALLSSOO
- llamafile-quantize(1), llamafile-perplexity(1), llava-quantize(1),
- zipalign(1), unzip(1)
-
-Mozilla Ocho January 1, 2024 Mozilla Ocho
+ Default: disabled
+
+ --nobrowser
+ Do not attempt to open a web browser tab at startup.
+
+ -gan N, --grp‐attn‐n N
+ Set the group attention factor to extend context size through
+ self‐extend. The default value is 1 which means disabled. This
+ flag is used together with --grp‐attn‐w.
+
+ -gaw N, --grp‐attn‐w N
+ Set the group attention width to extend context size through
+ self‐extend. The default value is 512. This flag is used to‐
+ gether with --grp‐attn‐n.
+
+LOG OPTIONS
+ The following log options are available:
+
+ -ld LOGDIR, --logdir LOGDIR
+ Path under which to save YAML logs (no logging if unset)
+
+ --log‐test
+ Run simple logging test
+
+ --log‐disable
+ Disable trace logs
+
+ --log‐enable
+ Enable trace logs
+
+ --log‐file
+ Specify a log filename (without extension)
+
+ --log‐new
+ Create a separate new log file on start. Each log file will
+ have unique name: <name>.<ID>.log
+
+ --log‐append
+ Don’t truncate the old log file.
+
+EXAMPLES
+ Here’s an example of how to run llama.cpp’s built‐in HTTP server. This
+ example uses LLaVA v1.5‐7B, a multimodal LLM that works with
+ llama.cpp’s recently‐added support for image inputs.
+
+ llamafile \
+ ‐m llava‐v1.5‐7b‐Q8_0.gguf \
+ ‐‐mmproj llava‐v1.5‐7b‐mmproj‐Q8_0.gguf \
+ ‐‐host 0.0.0.0
+
+ Here’s an example of how to generate code for a libc function using the
+ llama.cpp command line interface, utilizing WizardCoder‐Python‐13B
+ weights:
+
+ llamafile \
+ ‐m wizardcoder‐python‐13b‐v1.0.Q8_0.gguf ‐‐temp 0 ‐r ’}\n’ ‐r ’```\n’ \
+ ‐e ‐p ’```c\nvoid *memcpy(void *dst, const void *src, size_t size) {\n’
+
+ Here’s a similar example that instead utilizes Mistral‐7B‐Instruct
+ weights for prose composition:
+
+ llamafile \
+ ‐m mistral‐7b‐instruct‐v0.2.Q5_K_M.gguf \
+ ‐p ’[INST]Write a story about llamas[/INST]’
+
+ Here’s an example of how llamafile can be used as an interactive chat‐
+ bot that lets you query knowledge contained in training data:
+
+ llamafile ‐m llama‐65b‐Q5_K.gguf ‐p ’
+ The following is a conversation between a Researcher and their helpful AI
+ assistant Digital Athena which is a large language model trained on the
+ sum of human knowledge.
+ Researcher: Good morning.
+ Digital Athena: How can I help you today?
+ Researcher:’ ‐‐interactive ‐‐color ‐‐batch_size 1024 ‐‐ctx_size 4096 \
+ ‐‐keep ‐1 ‐‐temp 0 ‐‐mirostat 2 ‐‐in‐prefix ’ ’ ‐‐interactive‐first \
+ ‐‐in‐suffix ’Digital Athena:’ ‐‐reverse‐prompt ’Researcher:’
+
+ Here’s an example of how you can use llamafile to summarize HTML URLs:
+
+ (
+ echo ’[INST]Summarize the following text:’
+ links ‐codepage utf‐8 \
+ ‐force‐html \
+ ‐width 500 \
+ ‐dump https://www.poetryfoundation.org/poems/48860/the‐raven |
+ sed ’s/ */ /g’
+ echo ’[/INST]’
+ ) | llamafile \
+ ‐m mistral‐7b‐instruct‐v0.2.Q5_K_M.gguf \
+ ‐f /dev/stdin \
+ ‐c 0 \
+ ‐‐temp 0 \
+ ‐n 500 \
+ ‐‐no‐display‐prompt 2>/dev/null
+
+ Here’s how you can use llamafile to describe a jpg/png/gif/bmp image:
+
+ llamafile ‐‐temp 0 \
+ ‐‐image lemurs.jpg \
+ ‐m llava‐v1.5‐7b‐Q4_K.gguf \
+ ‐‐mmproj llava‐v1.5‐7b‐mmproj‐Q4_0.gguf \
+ ‐e ‐p ’### User: What do you see?\n### Assistant: ’ \
+ ‐‐no‐display‐prompt 2>/dev/null
+
+ If you wanted to write a script to rename all your image files, you
+ could use the following command to generate a safe filename:
+
+ llamafile ‐‐temp 0 \
+ ‐‐image ~/Pictures/lemurs.jpg \
+ ‐m llava‐v1.5‐7b‐Q4_K.gguf \
+ ‐‐mmproj llava‐v1.5‐7b‐mmproj‐Q4_0.gguf \
+ ‐‐grammar ’root ::= [a‐z]+ (" " [a‐z]+)+’ \
+ ‐e ‐p ’### User: The image has...\n### Assistant: ’ \
+ ‐‐no‐display‐prompt 2>/dev/null |
+ sed ‐e’s/ /_/g’ ‐e’s/$/.jpg/’
+ three_baby_lemurs_on_the_back_of_an_adult_lemur.jpg
+
+ Here’s an example of how to make an API request to the OpenAI API com‐
+ patible completions endpoint when your llamafile is running in the
+ background in --server mode.
+
+ curl ‐s http://localhost:8080/v1/chat/completions \
+ ‐H "Content‐Type: application/json" ‐d ’{
+ "model": "gpt‐3.5‐turbo",
+ "stream": true,
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are a poetic assistant."
+ },
+ {
+ "role": "user",
+ "content": "Compose a poem that explains FORTRAN."
+ }
+ ]
+ }’ | python3 ‐c ’
+ import json
+ import sys
+ json.dump(json.load(sys.stdin), sys.stdout, indent=2)
+ print()
+
+PROTIP
+ The -ngl 35 flag needs to be passed in order to use GPUs made by NVIDIA
+ and AMD. It’s not enabled by default since it sometimes needs to be
+ tuned based on the system hardware and model architecture, in order to
+ achieve optimal performance, and avoid compromising a shared display.
+
+SEE ALSO
+ llamafile‐quantize(1), llamafile‐perplexity(1), llava‐quantize(1),
+ zipalign(1), unzip(1)
+
+Mozilla Ocho January 1, 2024 LLAMAFILE(1)
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index 989295d8..a6114ef4 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -107,10 +107,19 @@ o/$(MODE)/llamafile/tinyblas_cpu_mixmul_arm82.o: private TARGET_ARCH += -Xaarch6
################################################################################
# testing
+o/$(MODE)/llamafile/sgemm_sss_test: private LDFLAGS += -fopenmp
+o/$(MODE)/llamafile/sgemm_sss_test.o: private CCFLAGS += -fopenmp
+o/$(MODE)/llamafile/sgemm_matmul_test: private LDFLAGS += -fopenmp
+o/$(MODE)/llamafile/sgemm_matmul_test.o: private CCFLAGS += -fopenmp
+
o/$(MODE)/llamafile/sgemm_sss_test: \
o/$(MODE)/llamafile/sgemm_sss_test.o \
o/$(MODE)/llama.cpp/llama.cpp.a
+o/$(MODE)/llamafile/sgemm_matmul_test: \
+ o/$(MODE)/llamafile/sgemm_matmul_test.o \
+ o/$(MODE)/llama.cpp/llama.cpp.a
+
o/$(MODE)/llamafile/sgemm_vecdot_test: \
o/$(MODE)/llamafile/sgemm_vecdot_test.o \
o/$(MODE)/llama.cpp/llama.cpp.a
diff --git a/llamafile/ansiblas.h b/llamafile/ansiblas.h
new file mode 100644
index 00000000..c3a20358
--- /dev/null
+++ b/llamafile/ansiblas.h
@@ -0,0 +1,145 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <cmath>
+#include <unistd.h>
+
+namespace {
+namespace ansiBLAS {
+
+static constexpr int KN = 8;
+
+alignas(64) union Vector {
+ double v[KN];
+};
+
+inline Vector load(const float *p) {
+ Vector x;
+ for (int i = 0; i < KN; ++i) {
+ x.v[i] = p[i];
+ }
+ return x;
+}
+
+inline Vector madd(Vector x, Vector y, Vector s) {
+ for (int i = 0; i < KN; ++i) {
+ s.v[i] = fma(x.v[i], y.v[i], s.v[i]);
+ }
+ return s;
+}
+
+inline float hsum(Vector x) {
+ double s = 0;
+ for (int i = 0; i < KN; ++i) {
+ s += x.v[i];
+ }
+ return s;
+}
+
+struct ansiBLAS {
+ public:
+ ansiBLAS(int k, const float *A, int lda, const float *B, int ldb, float *C, int ldc, int ith,
+ int nth)
+ : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith), nth(nth) {
+ }
+
+ void matmul(int m, int n) {
+ mnpack(0, m, 0, n);
+ }
+
+ private:
+ void mnpack(int m0, int m, int n0, int n) {
+ int mc, nc, mp, np;
+ if (m - m0 <= 0 || n - n0 <= 0)
+ return;
+ if (m - m0 >= 4 && n - n0 >= 3) {
+ mc = 4;
+ nc = 3;
+ gemm<4, 3>(m0, m, n0, n);
+ } else if (n - n0 >= 4) {
+ mc = 1;
+ nc = 4;
+ gemm<1, 4>(m0, m, n0, n);
+ } else if (m - m0 >= 4) {
+ mc = 4;
+ nc = 1;
+ gemm<4, 1>(m0, m, n0, n);
+ } else {
+ mc = 1;
+ nc = 1;
+ gemm<1, 1>(m0, m, n0, n);
+ }
+ mp = m0 + (m - m0) / mc * mc;
+ np = n0 + (n - n0) / nc * nc;
+ mnpack(mp, m, n0, np);
+ mnpack(m0, m, np, n);
+ }
+
+ template <int RM, int RN>
+ __target_clones("avx512f,fma") void gemm(int m0, int m, int n0, int n) {
+ int ytiles = (m - m0) / RM;
+ int xtiles = (n - n0) / RN;
+ int tiles = xtiles * ytiles;
+ int duty = (tiles + nth - 1) / nth;
+ int start = duty * ith;
+ int end = start + duty;
+ if (end > tiles)
+ end = tiles;
+ for (int job = start; job < end; ++job) {
+ int ii = m0 + job / xtiles * RM;
+ int jj = n0 + job % xtiles * RN;
+ Vector Cv[RN][RM] = {0};
+ Vector Ce[RN][RM] = {0};
+ for (int l = 0; l < k; l += KN)
+ for (int j = 0; j < RN; ++j)
+ for (int i = 0; i < RM; ++i)
+ Cv[j][i] = madd(load(A + lda * (ii + i) + l), //
+ load(B + ldb * (jj + j) + l), //
+ Cv[j][i]);
+ for (int j = 0; j < RN; ++j)
+ for (int i = 0; i < RM; ++i)
+ C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);
+ }
+ }
+
+ const int k;
+ const float *const A;
+ const int lda;
+ const float *const B;
+ const int ldb;
+ float *const C;
+ const int ldc;
+ const int ith;
+ const int nth;
+};
+
+void sgemm(int m, int n, int k, //
+ const float *A, int lda, //
+ const float *B, int ldb, //
+ float *C, int ldc) {
+ int nth = sysconf(_SC_NPROCESSORS_ONLN);
+#pragma omp parallel for
+ for (int ith = 0; ith < nth; ++ith) {
+ ansiBLAS tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ }
+}
+
+} // namespace ansiBLAS
+} // namespace
diff --git a/llamafile/llamafile.h b/llamafile/llamafile.h
index 316953f9..e592d193 100644
--- a/llamafile/llamafile.h
+++ b/llamafile/llamafile.h
@@ -34,7 +34,9 @@ void llamafile_get_app_dir(char *, size_t);
void llamafile_launch_browser(const char *);
extern bool FLAG_trap;
+extern bool FLAG_precise;
extern bool FLAG_unsecure;
+extern bool FLAG_precision_specified;
#define LLAMAFILE_GPU_ERROR -2
#define LLAMAFILE_GPU_DISABLE -1
diff --git a/llamafile/precise.cpp b/llamafile/precise.cpp
new file mode 100644
index 00000000..49b9d96e
--- /dev/null
+++ b/llamafile/precise.cpp
@@ -0,0 +1,21 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "llamafile.h"
+
+bool FLAG_precise = true;
+bool FLAG_precision_specified;
diff --git a/llamafile/sgemm_matmul_test.cpp b/llamafile/sgemm_matmul_test.cpp
new file mode 100644
index 00000000..bd595734
--- /dev/null
+++ b/llamafile/sgemm_matmul_test.cpp
@@ -0,0 +1,138 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "ansiblas.h"
+#include "bench.h"
+#include "float.h"
+#include "llama.cpp/ggml.h"
+#include "llamafile.h"
+#include "macros.h"
+#include "numba.h"
+#include <cassert>
+#include <cstdlib>
+
+#define ITERATIONS 30
+#define ALLOC(n) (float *)memalign(4096, sizeof(float) * (n))
+
+void llamafile_sgemm_openmp(long m, long n, long k, const void *A, long lda, const void *B,
+ long ldb, void *C, long ldc, int task, int Atype, int Btype, int Ctype,
+ int precision) {
+ int nth = sysconf(_SC_NPROCESSORS_ONLN);
+#pragma omp parallel for
+ for (int ith = 0; ith < nth; ++ith) {
+ bool res = llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,
+ Ctype, precision);
+ assert(res);
+ }
+}
+
+int test(void) {
+ int m = 256;
+ int n = 500;
+ int k = 32768 * 2;
+ int lda = ROUNDUP(k, 16);
+ int ldb = ROUNDUP(k, 16);
+ int ldc = ROUNDUP(m, 16);
+ float *A = ALLOC(lda * m);
+ float *B = ALLOC(ldb * n);
+ float *C = ALLOC(ldc * n);
+ float *G = ALLOC(ldc * n);
+ broadcast(A, lda * m, NAN);
+ broadcast(B, ldb * n, NAN);
+ broadcast(C, ldc * n, NAN);
+ broadcast(G, ldc * n, NAN);
+ randomize(k, m, A, lda);
+ randomize(k, n, B, ldb);
+
+ BENCH(ansiBLAS::sgemm(m, n, k, A, lda, B, ldb, G, ldc));
+ BENCH(llamafile_sgemm_openmp(m, n, k, A, lda, B, ldb, C, ldc, GGML_TASK_TYPE_COMPUTE,
+ GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));
+
+ double err_sum = 0;
+ long long err_worst = 0;
+ for (int i = 0; i < m; ++i)
+ for (int j = 0; j < n; ++j) {
+ float g = G[ldc * j + i];
+ float c = C[ldc * j + i];
+ if (flt::isnan(g)) {
+ fprintf(stderr, "%s:%err: found nan in reference matrix: i=%err j=%err\n", __FILE__,
+ __LINE__, i, j);
+ return 3;
+ }
+ if (flt::isnan(c)) {
+ fprintf(stderr, "%s:%err: found nan in output matrix: i=%err j=%err\n", __FILE__,
+ __LINE__, i, j);
+ return 4;
+ }
+ long long gi = flt::toint(g);
+ long long ci = flt::toint(c);
+ long long err = gi - ci;
+ if (err < 0)
+ err = -err;
+ err_sum += err;
+ if (err > err_worst)
+ err_worst = err;
+ }
+
+ double err_avg = err_sum / (m * n);
+ fprintf(stderr, "%9g ulp average\n", err_avg);
+ fprintf(stderr, "%9lld ulp worst\n", err_worst);
+
+ // using one accumulator
+ // 87015 us gemm
+ // 1859 us llamafile_sgemm
+ // 94.7872 ulp average
+ // 27610661 ulp worst
+
+ // using three accumulators
+ // 22.291 average ulp
+ // 1566 worst ulp
+
+ // using kahan summation
+ // 40190 us gemm
+ // 3028 us llamafile_sgemm
+ // 2.14244 ulp average
+ // 134 ulp worst
+
+ // if (err_avg > 2.15)
+ // return 5;
+ // if (err_worst > 134)
+ // return 6;
+
+ free(G);
+ free(C);
+ free(B);
+ free(A);
+
+ return 0;
+}
+
+int main(int argc, char *argv[]) {
+ int rc;
+
+ llamafile_trapping_enabled(+1);
+
+ printf("\nFLAG_precise = false;\n");
+ FLAG_precise = false;
+ if ((rc = test()))
+ return rc;
+
+ printf("\nFLAG_precise = true;\n");
+ FLAG_precise = true;
+ if ((rc = test()))
+ return rc;
+}
diff --git a/llamafile/sgemm_sss_test.cpp b/llamafile/sgemm_sss_test.cpp
index b7e49462..1d86dca9 100644
--- a/llamafile/sgemm_sss_test.cpp
+++ b/llamafile/sgemm_sss_test.cpp
@@ -15,22 +15,35 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include "ansiblas.h"
#include "bench.h"
#include "float.h"
-#include "gemm.h"
#include "llama.cpp/ggml.h"
#include "llamafile.h"
#include "macros.h"
#include "numba.h"
+#include <cassert>
#define ITERATIONS 5
#define ALLOC(n) (float *)memalign(4096, sizeof(float) * (n))
-int main(int argc, char *argv[]) {
+void llamafile_sgemm_openmp(long m, long n, long k, const void *A, long lda, const void *B,
+ long ldb, void *C, long ldc, int task, int Atype, int Btype, int Ctype,
+ int precision) {
+ int nth = sysconf(_SC_NPROCESSORS_ONLN);
+#pragma omp parallel for
+ for (int ith = 0; ith < nth; ++ith) {
+ bool res = llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,
+ Ctype, precision);
+ assert(res);
+ }
+}
+
+int test(void) {
float tolerance = 2e-5;
int m = 510;
int n = 513;
- int k = 512;
+ int k = 512 * 8;
int lda = ROUNDUP(k, 16);
int ldb = ROUNDUP(k, 16);
int ldc = ROUNDUP(m, 16);
@@ -45,36 +58,60 @@ int main(int argc, char *argv[]) {
randomize(k, m, A, lda);
randomize(k, n, B, ldb);
- BENCH(gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc));
+ BENCH(ansiBLAS::sgemm(m, n, k, A, lda, B, ldb, G, ldc));
BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,
GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));
+ double err_sum = 0;
+ long long err_worst = 0;
for (int i = 0; i < m; ++i)
for (int j = 0; j < n; ++j) {
- auto g = G[ldc * j + i];
- auto c = C[ldc * j + i];
+ float g = G[ldc * j + i];
+ float c = C[ldc * j + i];
if (flt::isnan(g)) {
- fprintf(stderr, "%s:%d: found nan in reference matrix: i=%d j=%d\n", __FILE__,
+ fprintf(stderr, "%s:%err: found nan in reference matrix: i=%err j=%err\n", __FILE__,
__LINE__, i, j);
return 3;
}
if (flt::isnan(c)) {
- fprintf(stderr, "%s:%d: found nan in output matrix: i=%d j=%d\n", __FILE__,
+ fprintf(stderr, "%s:%err: found nan in output matrix: i=%err j=%err\n", __FILE__,
__LINE__, i, j);
return 4;
}
- auto d = g - c;
- if (d < 0)
- d = -d;
- if (d > tolerance) {
- fprintf(stderr, "%s:%d: difference exceeds tolerance: %g vs. %g (%g) i=%d j=%d\n",
- __FILE__, __LINE__, g, c, d, i, j);
- return 5;
- }
+ long long gi = flt::toint(g);
+ long long ci = flt::toint(c);
+ long long err = gi - ci;
+ if (err < 0)
+ err = -err;
+ err_sum += err;
+ if (err > err_worst)
+ err_worst = err;
}
+ double err_avg = err_sum / (m * n);
+ fprintf(stderr, "%12g ulp average\n", err_avg);
+ fprintf(stderr, "%12lld ulp worst\n", err_worst);
+
free(G);
free(C);
free(B);
free(A);
+
+ return 0;
+}
+
+int main(int argc, char *argv[]) {
+ int rc;
+
+ llamafile_trapping_enabled(+1);
+
+ printf("\nFLAG_precise = false;\n");
+ FLAG_precise = false;
+ if ((rc = test()))
+ return rc;
+
+ printf("\nFLAG_precise = true;\n");
+ FLAG_precise = true;
+ if ((rc = test()))
+ return rc;
}
diff --git a/llamafile/sgemm_vecdot_test.cpp b/llamafile/sgemm_vecdot_test.cpp
index 651e230c..f8374f88 100644
--- a/llamafile/sgemm_vecdot_test.cpp
+++ b/llamafile/sgemm_vecdot_test.cpp
@@ -15,9 +15,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include "ansiblas.h"
#include "bench.h"
#include "float.h"
-#include "gemm.h"
#include "llama.cpp/ggml.h"
#include "llamafile.h"
#include "macros.h"
@@ -26,7 +26,7 @@
#define ITERATIONS 30
#define ALLOC(n) (float *)memalign(4096, sizeof(float) * (n))
-int main(int argc, char *argv[]) {
+int test(void) {
int m = 1025;
int n = 1;
int k = 32768;
@@ -44,7 +44,7 @@ int main(int argc, char *argv[]) {
randomize(k, m, A, lda);
randomize(k, n, B, ldb);
- BENCH(gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc));
+ BENCH(ansiBLAS::sgemm(m, n, k, A, lda, B, ldb, G, ldc));
BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,
GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));
@@ -94,13 +94,29 @@ int main(int argc, char *argv[]) {
// 2.14244 ulp average
// 134 ulp worst
- if (err_avg > 2.15)
- return 5;
- if (err_worst > 134)
- return 6;
+ // if (err_avg > 2.15)
+ // return 5;
+ // if (err_worst > 134)
+ // return 6;
free(G);
free(C);
free(B);
free(A);
+
+ return 0;
+}
+
+int main(int argc, char *argv[]) {
+ int rc;
+
+ printf("\nFLAG_precise = false;\n");
+ FLAG_precise = false;
+ if ((rc = test()))
+ return rc;
+
+ printf("\nFLAG_precise = true;\n");
+ FLAG_precise = true;
+ if ((rc = test()))
+ return rc;
}
diff --git a/llamafile/tinyblas_cpu.h b/llamafile/tinyblas_cpu.h
index e4b4ec8c..be85892e 100644
--- a/llamafile/tinyblas_cpu.h
+++ b/llamafile/tinyblas_cpu.h
@@ -431,152 +431,182 @@ class tinyBLAS {
private:
NOINLINE void mnpack(long m0, long m, long n0, long n) {
long mc, nc, mp, np;
- switch ((MIN(m - m0, 5) << 4) | MIN(n - n0, 5)) {
+
#if VECTOR_REGISTERS == 32
- case 0x55:
- case 0x54:
- mc = 5;
- nc = 4;
- gemm<5, 4, false>(m0, m, n0, n);
- break;
- case 0x45:
- mc = 4;
- nc = 5;
- gemm<4, 5, false>(m0, m, n0, n);
- break;
- case 0x44:
- mc = 4;
- nc = 4;
- gemm<4, 4, false>(m0, m, n0, n);
- break;
- case 0x53:
- mc = 5;
- nc = 3;
- gemm<5, 3, false>(m0, m, n0, n);
- break;
- case 0x35:
- mc = 3;
- nc = 5;
- gemm<3, 5, false>(m0, m, n0, n);
- break;
- case 0x43:
- mc = 4;
- nc = 3;
- gemm<4, 3, false>(m0, m, n0, n);
- break;
-#else
- case 0x55:
- case 0x54:
- case 0x53:
- case 0x45:
- case 0x44:
- case 0x43:
- mc = 4;
- nc = 3;
- gemm<4, 3, false>(m0, m, n0, n);
- break;
- case 0x35:
+ if (!FLAG_precise) {
+ switch ((MIN(m - m0, 5) << 4) | MIN(n - n0, 5)) {
+ case 0x55:
+ mc = 5;
+ nc = 5;
+ gemm<5, 5, false>(m0, m, n0, n);
+ break;
+ case 0x54:
+ case 0x53:
+ case 0x52:
+ case 0x45:
+ case 0x44:
+ case 0x43:
+ case 0x42:
+ case 0x35:
+ case 0x34:
+ case 0x33:
+ case 0x32:
+ case 0x25:
+ case 0x24:
+ case 0x23:
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2, true>(m0, m, n0, n);
+ break;
+ case 0x51:
+ case 0x41:
+ case 0x31:
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1, true>(m0, m, n0, n);
+ break;
+ case 0x15:
+ case 0x14:
+ case 0x13:
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2, true>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1, true>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
+ } else {
+ switch ((MIN(m - m0, 4) << 4) | MIN(n - n0, 3)) {
+ case 0x43:
+ mc = 4;
+ nc = 3;
+ gemm<4, 3, true>(m0, m, n0, n);
+ break;
+ case 0x42:
+ case 0x33:
+ case 0x32:
+ case 0x23:
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2, true>(m0, m, n0, n);
+ break;
+ case 0x41:
+ case 0x31:
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1, true>(m0, m, n0, n);
+ break;
+ case 0x13:
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2, true>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1, true>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
+ }
#endif
- case 0x34:
- mc = 3;
- nc = 4;
- gemm<3, 4, false>(m0, m, n0, n);
- break;
- case 0x52:
- mc = 5;
- nc = 2;
- gemm<5, 2, false>(m0, m, n0, n);
- break;
- case 0x33:
- mc = 3;
- nc = 3;
- gemm<3, 3, false>(m0, m, n0, n);
- break;
- case 0x25:
- mc = 2;
- nc = 5;
- gemm<2, 5, false>(m0, m, n0, n);
- break;
- case 0x42:
- mc = 4;
- nc = 2;
- gemm<4, 2, false>(m0, m, n0, n);
- break;
- case 0x24:
- mc = 2;
- nc = 4;
- gemm<2, 4, false>(m0, m, n0, n);
- break;
- case 0x32:
- mc = 3;
- nc = 2;
- gemm<3, 2, true>(m0, m, n0, n);
- break;
- case 0x23:
- mc = 2;
- nc = 3;
- gemm<2, 3, true>(m0, m, n0, n);
- break;
- case 0x51:
- mc = 5;
- nc = 1;
- gemm<5, 1, true>(m0, m, n0, n);
- break;
- case 0x41:
- mc = 4;
- nc = 1;
- gemm<4, 1, true>(m0, m, n0, n);
- break;
- case 0x22:
- mc = 2;
- nc = 2;
- gemm<2, 2, true>(m0, m, n0, n);
- break;
- case 0x15:
- mc = 1;
- nc = 5;
- gemm<1, 5, true>(m0, m, n0, n);
- break;
- case 0x14:
- mc = 1;
- nc = 4;
- gemm<1, 4, true>(m0, m, n0, n);
- break;
- case 0x31:
- mc = 3;
- nc = 1;
- gemm<3, 1, true>(m0, m, n0, n);
- break;
- case 0x13:
- mc = 1;
- nc = 3;
- gemm<1, 3, true>(m0, m, n0, n);
- break;
- case 0x21:
- mc = 2;
- nc = 1;
- gemm<2, 1, true>(m0, m, n0, n);
- break;
- case 0x12:
- mc = 1;
- nc = 2;
- gemm<1, 2, true>(m0, m, n0, n);
- break;
- case 0x11:
- mc = 1;
- nc = 1;
- gemm<1, 1, true>(m0, m, n0, n);
- break;
- default:
- return;
+
+#if VECTOR_REGISTERS == 16
+ if (!FLAG_precise) {
+ switch ((MIN(m - m0, 4) << 4) | MIN(n - n0, 3)) {
+ case 0x43:
+ mc = 4;
+ nc = 3;
+ gemm<4, 3, false>(m0, m, n0, n);
+ break;
+ case 0x42:
+ case 0x33:
+ case 0x32:
+ case 0x23:
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2, true>(m0, m, n0, n);
+ break;
+ case 0x41:
+ case 0x31:
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1, true>(m0, m, n0, n);
+ break;
+ case 0x13:
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2, true>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1, true>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
+ } else {
+ switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 2)) {
+ case 0x32:
+ mc = 3;
+ nc = 2;
+ gemm<3, 2, true>(m0, m, n0, n);
+ break;
+ case 0x23:
+ mc = 2;
+ nc = 3;
+ gemm<2, 3, true>(m0, m, n0, n);
+ break;
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2, true>(m0, m, n0, n);
+ break;
+ case 0x31:
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1, true>(m0, m, n0, n);
+ break;
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2, true>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1, true>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
}
+#endif
+
mp = m0 + (m - m0) / mc * mc;
np = n0 + (n - n0) / nc * nc;
mnpack(mp, m, n0, np);
mnpack(m0, m, np, n);
}
- template <int RM, int RN, int KAHAN>
+ template <int RM, int RN, int PRECISE>
NOINLINE void gemm(long m0, long m, long n0, long n) {
long ytiles = RM > 1 ? (m - m0) / RM : 1;
long xtiles = RN > 1 ? (n - n0) / RN : 1;
@@ -594,7 +624,7 @@ class tinyBLAS {
for (long l = 0; l < k; l += KN)
for (int j = 0; j < RN; ++j)
for (int i = 0; i < RM; ++i)
- if (KAHAN)
+ if (PRECISE)
Cv[j][i] = madder(load<V>(INDEX(A, lda, ii + i, l)), //
load<V>(INDEX(B, ldb, jj + j, l)), //
Cv[j][i], &Ce[j][i]);
@@ -639,62 +669,84 @@ class tinyBLAS_Q0_ARM {
private:
NOINLINE void mnpack(long m0, long m, long n0, long n) {
long mc, nc, mp, np;
- switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 3)) {
- case 0x33:
- mc = 3;
- nc = 3;
- gemm<3, 3, false>(m0, m, n0, n);
- break;
- case 0x32:
- mc = 3;
- nc = 2;
- gemm<3, 2, false>(m0, m, n0, n);
- break;
- case 0x23:
- mc = 2;
- nc = 3;
- gemm<2, 3, false>(m0, m, n0, n);
- break;
- case 0x22:
- mc = 2;
- nc = 2;
- gemm<2, 2, true>(m0, m, n0, n);
- break;
- case 0x31:
- mc = 3;
- nc = 1;
- gemm<3, 1, true>(m0, m, n0, n);
- break;
- case 0x13:
- mc = 1;
- nc = 3;
- gemm<1, 3, true>(m0, m, n0, n);
- break;
- case 0x21:
- mc = 2;
- nc = 1;
- gemm<2, 1, true>(m0, m, n0, n);
- break;
- case 0x12:
- mc = 1;
- nc = 2;
- gemm<1, 2, true>(m0, m, n0, n);
- break;
- case 0x11:
- mc = 1;
- nc = 1;
- gemm<1, 1, true>(m0, m, n0, n);
- break;
- default:
- return;
+
+ if (!FLAG_precise || (!FLAG_precision_specified && sizeof(TB) == sizeof(block_q4_0))) {
+ switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 3)) {
+ case 0x33:
+ mc = 3;
+ nc = 3;
+ gemm<3, 3, false>(m0, m, n0, n);
+ break;
+ case 0x32:
+ case 0x23:
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2, false>(m0, m, n0, n);
+ break;
+ case 0x31:
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1, true>(m0, m, n0, n);
+ break;
+ case 0x13:
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2, true>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1, true>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
+ } else {
+ switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 3)) {
+ case 0x33:
+ mc = 3;
+ nc = 3;
+ gemm<3, 3, true>(m0, m, n0, n);
+ break;
+ case 0x32:
+ case 0x23:
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2, true>(m0, m, n0, n);
+ break;
+ case 0x31:
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1, true>(m0, m, n0, n);
+ break;
+ case 0x13:
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2, true>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1, true>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
}
+
mp = m0 + (m - m0) / mc * mc;
np = n0 + (n - n0) / nc * nc;
mnpack(mp, m, n0, np);
mnpack(m0, m, np, n);
}
- template <int RM, int RN, int KAHAN>
+ template <int RM, int RN, int PRECISE>
NOINLINE void gemm(long m0, long m, long n0, long n) {
long ytiles = RM > 1 ? (m - m0) / RM : 1;
long xtiles = RN > 1 ? (n - n0) / RN : 1;
@@ -718,7 +770,7 @@ class tinyBLAS_Q0_ARM {
load_hi(INDEX(A, lda, ii + i, l)), load_hi(INDEX(B, ldb, jj + j, l))));
float b = unhalf(INDEX(A, lda, ii + i, l)->d) *
unhalf(INDEX(B, ldb, jj + j, l)->d);
- if (KAHAN)
+ if (PRECISE)
Cv[j][i] = badder(a, b, Cv[j][i], &Ce[j][i]);
else
Cv[j][i] = vmlaq_n_f32(Cv[j][i], a, b);
@@ -775,114 +827,146 @@ class tinyBLAS_Q0_AVX2 {
private:
void mnpack(long m0, long m, long n0, long n) {
long mc, nc, mp, np;
- switch ((MIN(m - m0, 4) << 4) | MIN(n - n0, 4)) {
+
#if VECTOR_REGISTERS == 32
- case 0x44:
- mc = 4;
- nc = 4;
- gemm<4, 4, false>(m0, m, n0, n);
- break;
- case 0x43:
- mc = 4;
- nc = 3;
- gemm<4, 3, false>(m0, m, n0, n);
- break;
- case 0x34:
- mc = 3;
- nc = 4;
- gemm<3, 4, false>(m0, m, n0, n);
- break;
- case 0x33:
- mc = 3;
- nc = 3;
- gemm<3, 3, false>(m0, m, n0, n);
- break;
- case 0x42:
- mc = 4;
- nc = 2;
- gemm<4, 2, false>(m0, m, n0, n);
- break;
- case 0x24:
- mc = 2;
- nc = 4;
- gemm<2, 4, false>(m0, m, n0, n);
- break;
-#else
- case 0x44:
- case 0x43:
- case 0x42:
- mc = 4;
- nc = 2;
- gemm<4, 2, false>(m0, m, n0, n);
- break;
- case 0x34:
- case 0x24:
- mc = 2;
- nc = 4;
- gemm<2, 4, false>(m0, m, n0, n);
- break;
- case 0x33:
+ if (!FLAG_precise || (!FLAG_precision_specified && sizeof(TB) == sizeof(block_q4_0))) {
+ switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 3)) {
+ case 0x33:
+ mc = 3;
+ nc = 3;
+ gemm<3, 3, false>(m0, m, n0, n);
+ break;
+ case 0x32:
+ case 0x23:
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2, false>(m0, m, n0, n);
+ break;
+ case 0x31:
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1, true>(m0, m, n0, n);
+ break;
+ case 0x13:
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2, true>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1, true>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
+ } else {
+ switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 3)) {
+ case 0x33:
+ mc = 3;
+ nc = 3;
+ gemm<3, 3, true>(m0, m, n0, n);
+ break;
+ case 0x32:
+ case 0x23:
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2, true>(m0, m, n0, n);
+ break;
+ case 0x31:
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1, true>(m0, m, n0, n);
+ break;
+ case 0x13:
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2, true>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1, true>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
+ }
#endif
- case 0x32:
- mc = 3;
- nc = 2;
- gemm<3, 2, false>(m0, m, n0, n);
- break;
- case 0x23:
- mc = 2;
- nc = 3;
- gemm<2, 3, false>(m0, m, n0, n);
- break;
- case 0x41:
- mc = 4;
- nc = 1;
- gemm<4, 1, true>(m0, m, n0, n);
- break;
- case 0x22:
- mc = 2;
- nc = 2;
- gemm<2, 2, true>(m0, m, n0, n);
- break;
- case 0x14:
- mc = 1;
- nc = 4;
- gemm<1, 4, true>(m0, m, n0, n);
- break;
- case 0x31:
- mc = 3;
- nc = 1;
- gemm<3, 1, true>(m0, m, n0, n);
- break;
- case 0x13:
- mc = 1;
- nc = 3;
- gemm<1, 3, true>(m0, m, n0, n);
- break;
- case 0x21:
- mc = 2;
- nc = 1;
- gemm<2, 1, true>(m0, m, n0, n);
- break;
- case 0x12:
- mc = 1;
- nc = 2;
- gemm<1, 2, true>(m0, m, n0, n);
- break;
- case 0x11:
- mc = 1;
- nc = 1;
- gemm<1, 1, true>(m0, m, n0, n);
- break;
- default:
- return;
+
+#if VECTOR_REGISTERS == 16
+ if (!FLAG_precise || (!FLAG_precision_specified && sizeof(TB) == sizeof(block_q4_0))) {
+ switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 2)) {
+ case 0x32:
+ mc = 3;
+ nc = 2;
+ gemm<3, 2, false>(m0, m, n0, n);
+ break;
+ case 0x23:
+ mc = 2;
+ nc = 3;
+ gemm<2, 3, false>(m0, m, n0, n);
+ break;
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2, false>(m0, m, n0, n);
+ break;
+ case 0x31:
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1, true>(m0, m, n0, n);
+ break;
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2, true>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1, true>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
+ } else {
+ switch ((MIN(m - m0, 2) << 4) | MIN(n - n0, 1)) {
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1, true>(m0, m, n0, n);
+ break;
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2, true>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1, true>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
}
+#endif
+
mp = m0 + (m - m0) / mc * mc;
np = n0 + (n - n0) / nc * nc;
mnpack(mp, m, n0, np);
mnpack(m0, m, np, n);
}
- template <int RM, int RN, int KAHAN>
+ template <int RM, int RN, int PRECISE>
NOINLINE void gemm(long m0, long m, long n0, long n) {
long ytiles = RM > 1 ? (m - m0) / RM : 1;
long xtiles = RN > 1 ? (n - n0) / RN : 1;
@@ -906,7 +990,7 @@ class tinyBLAS_Q0_AVX2 {
load(INDEX(A, lda, ii + i, l))),
_mm256_sign_epi8(load(INDEX(B, ldb, jj + j, l)),
load(INDEX(A, lda, ii + i, l))));
- if (KAHAN)
+ if (PRECISE)
Cv[j][i] = madder(a, b, Cv[j][i], &Ce[j][i]);
else
Cv[j][i] = madd(a, b, Cv[j][i]);
diff --git a/llamafile/tinyblas_cpu_sgemm.inc b/llamafile/tinyblas_cpu_sgemm.inc
index 84520fd1..108ced2d 100644
--- a/llamafile/tinyblas_cpu_sgemm.inc
+++ b/llamafile/tinyblas_cpu_sgemm.inc
@@ -45,7 +45,7 @@ namespace {
template <typename TC>
bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const void *B, long ldb,
TC *C, long ldc, int ith, int nth, int task, int Atype, int Btype,
- int precision) {
+ int Ctype, int precision) {
switch (Atype) {
@@ -316,13 +316,13 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const void
switch (Ctype) {
case GGML_TYPE_F32:
return llamafile_sgemm_impl(m, n, k, A, lda, B, ldb, (float *)C, ldc, ith, nth, task, Atype,
- Btype, precision);
+ Btype, Ctype, precision);
case GGML_TYPE_BF16:
return llamafile_sgemm_impl(m, n, k, A, lda, B, ldb, (ggml_bf16_t *)C, ldc, ith, nth, task,
- Atype, Btype, precision);
+ Atype, Btype, Ctype, precision);
case GGML_TYPE_F16:
return llamafile_sgemm_impl(m, n, k, A, lda, B, ldb, (ggml_fp16_t *)C, ldc, ith, nth, task,
- Atype, Btype, precision);
+ Atype, Btype, Ctype, precision);
default:
return NOT_SUPPORTED;
}
diff --git a/llamafile/tinyblas_mnpack.py b/llamafile/tinyblas_mnpack.py
new file mode 100644
index 00000000..191c065c
--- /dev/null
+++ b/llamafile/tinyblas_mnpack.py
@@ -0,0 +1,128 @@
+# Copyright 2024 Mozilla Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# # tinyBLAS
+# MAX_M = 5
+# MAX_N = 5
+# EDGE_M = 2
+# EDGE_N = 2
+# OVERHEAD = 1
+
+# tinyBLAS_Q0
+MAX_M = 3
+MAX_N = 3
+EDGE_M = 2
+EDGE_N = 2
+OVERHEAD = 8
+
+def doit(VECTOR_REGISTERS, PRECISE):
+ # choose tile size that exploits all vector registers
+
+ specified = {}
+ precise = set()
+
+ for mc in range(1, MAX_M + 1):
+ for nc in range(1, MAX_N + 1):
+ accumulators = mc * nc
+ memory_loads = min(mc, nc)
+
+ if (mc > EDGE_M or nc > EDGE_N) and (mc == 1 or nc == 1):
+ continue
+
+ # always use precise if there's enough registers
+ v = accumulators * 2 + memory_loads + OVERHEAD
+ if v <= VECTOR_REGISTERS:
+ if mc % 8 == 0:
+ v += 2
+ if mc % 4 == 0:
+ v += 1
+ specified[mc, nc] = v
+ if PRECISE or (mc == 1 or nc == 1):
+ precise.add((mc, nc))
+ elif not PRECISE:
+ v = accumulators + memory_loads + OVERHEAD
+ if v <= VECTOR_REGISTERS:
+ if mc % 8 == 0:
+ v += 2
+ if mc % 4 == 0:
+ v += 1
+ specified[mc, nc] = v
+
+ # if PRECISE:
+ # v = accumulators * 2 + memory_loads + OVERHEAD
+ # if v <= VECTOR_REGISTERS:
+ # if mc % 8 == 0:
+ # v += 2
+ # if mc % 4 == 0:
+ # v += 1
+ # specified[mc, nc] = v
+ # precise.add((mc, nc))
+ # else:
+ # v = accumulators + memory_loads + OVERHEAD
+ # if v <= VECTOR_REGISTERS:
+ # if mc % 8 == 0:
+ # v += 2
+ # if mc % 4 == 0:
+ # v += 1
+ # specified[mc, nc] = v
+
+ # generate code for handling biggest tile (e.g. 5x5)
+ # generate code for handling edge tiles (i.e. <=2x2)
+ # avoid long compile times to generate tiles between
+ (best_mc, best_nc), best_v = list(sorted(specified.items(), key=lambda s: s[1]))[-1]
+ for (mc, nc), v in list(specified.items()):
+ if v < best_v and (mc > EDGE_M or nc > EDGE_N):
+ del specified[mc, nc]
+
+ print("switch ((MIN(m - m0, %d) << 4) | MIN(n - n0, %d)) {" % (best_mc, best_nc))
+
+ a = []
+ for (mc, nc), v in specified.items():
+ s = ""
+ s += "case 0x%x%x:\n" % (mc, nc)
+ s += " mc = %d;\n" % (mc)
+ s += " nc = %d;\n" % (nc)
+ s += " gemm<%d, %d, %s>(m0, m, n0, n);\n" % (mc, nc, "true" if (mc, nc) in precise else "false")
+ s += " break;"
+ a.append((v, mc, nc, s, []))
+
+ a = list(reversed(sorted(a)))
+
+ for mc in range(1, best_mc + 1):
+ for nc in range(1, best_nc + 1):
+ if (mc, nc) in specified:
+ continue
+ for v_, mc_, nc_, s_, extra in a:
+ if mc_ <= mc and nc_ <= nc:
+ extra.append("case 0x%x%x:" % (mc, nc))
+ break
+
+ for v, mc, nc, s, extra in a:
+ for e in list(reversed(sorted(extra))):
+ print(e)
+ print(s)
+
+ print("default:")
+ print(" return;")
+ print("}")
+
+for VECTOR_REGISTERS in (32, 16):
+ print()
+ print("#if VECTOR_REGISTERS == %d" % (VECTOR_REGISTERS))
+ print("if (!FLAG_precise) {")
+ doit(VECTOR_REGISTERS, False)
+ print("} else {")
+ doit(VECTOR_REGISTERS, True)
+ print("}")
+ print("#endif")