Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add --precise and --fast flags
Justine Tunneybbae0f6aaf35
diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex e57f083f..89db03a7 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -279,6 +279,16 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_paif (arg == "--cli") {return true;}+ if (arg == "--fast") {+ FLAG_precise = false;+ FLAG_precision_specified = true;+ return true;+ }+ if (arg == "--precise") {+ FLAG_precise = true;+ FLAG_precision_specified = true;+ return true;+ }if (arg == "--trap") {FLAG_trap = true;FLAG_unsecure = true; // for better backtracesdiff --git a/llama.cpp/ggml-vector.inc b/llama.cpp/ggml-vector.incindex 62614a77..ead2de86 100644--- a/llama.cpp/ggml-vector.inc+++ b/llama.cpp/ggml-vector.inc@@ -1189,7 +1189,7 @@ void ggml_vec_silu_f32(const int n, float * y, const float * x) {int i = 0;#if defined(__AVX512F__)- if (!FLAG_trap) {+ if (!FLAG_trap && !FLAG_precise) {for (; i + 15 < n; i += 16) {_mm512_storeu_ps(y + i, llamafile_silu_avx512(_mm512_loadu_ps(x + i)));}@@ -1197,7 +1197,7 @@ void ggml_vec_silu_f32(const int n, float * y, const float * x) {#endif#if defined(__AVX2__) && defined(__FMA__)- if (!FLAG_trap) {+ if (!FLAG_trap && !FLAG_precise) {for (; i + 7 < n; i += 8) {_mm256_storeu_ps(y + i, llamafile_silu_avx2fma(_mm256_loadu_ps(x + i)));}@@ -1288,7 +1288,7 @@ float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max)ggml_float sum = 0;#if defined(__AVX512F__)- if (!FLAG_trap) {+ if (!FLAG_trap && !FLAG_precise) {for (; i + 15 < n; i += 16) {__m512 val = llamafile_expf_avx512(_mm512_sub_ps(_mm512_loadu_ps(x + i),_mm512_set1_ps(max)));@@ -1299,7 +1299,7 @@ float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max)#endif#if defined(__AVX2__) && defined(__FMA__)- if (!FLAG_trap) {+ if (!FLAG_trap && !FLAG_precise) {for (; i + 7 < n; i += 8) {__m256 val = llamafile_expf_avx2fma(_mm256_sub_ps(_mm256_loadu_ps(x + i),_mm256_set1_ps(max)));@@ -1314,10 +1314,12 @@ float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float max)#endif#ifdef __ARM_NEON- for (; i + 3 < n; i += 4) {- float32x4_t val = llamafile_expf_neon(vsubq_f32(vld1q_f32(x + i), vdupq_n_f32(max)));- vst1q_f32(y + i, val);- sum += vaddvq_f32(val);+ if (!FLAG_trap) {+ for (; i + 3 < n; i += 4) {+ float32x4_t val = llamafile_expf_neon(vsubq_f32(vld1q_f32(x + i), vdupq_n_f32(max)));+ vst1q_f32(y + i, val);+ sum += vaddvq_f32(val);+ }}#endifdiff --git a/llama.cpp/main/main.1 b/llama.cpp/main/main.1index 06acc11a..49a3edfd 100644--- a/llama.cpp/main/main.1+++ b/llama.cpp/main/main.1@@ -600,10 +600,36 @@ useful for shell scripts when theflag is also supplied..It Fl Fl grammar-file Ar FNAMEFile to read grammar from.+.It Fl Fl fast+Put llamafile into fast math mode. This disables algorithms that reduce+floating point rounding, e.g. Kahan summation, and certain functions+like expf() will be vectorized but handle underflows less gracefully.+It's unspecified whether llamafile runs in fast or precise math mode+when neither flag is specified.+.It Fl Fl precise+Put llamafile into precise math mode. This enables algorithms that+reduce floating point rounding, e.g. Kahan summation, and certain+functions like expf() will always handle subnormals correctly. It's+unspecified whether llamafile runs in fast or precise math mode when+neither flag is specified.+.It Fl Fl trap+Put llamafile into math trapping mode. When floating point exceptions+occur, such as NaNs, overflow, and divide by zero, llamafile will print+a warning to the console. This warning will include a C++ backtrace the+first time an exception is trapped. The op graph will also be dumped to+a file, and llamafile will report the specific op where the exception+occurred. This is useful for troubleshooting when reporting issues.+USing this feature will disable sandboxing. Math trapping is only+possible if your CPU supports it. That is generally the case on AMD64,+however it's less common on ARM64..It Fl Fl prompt-cache Ar FNAMEFile to cache prompt state for faster startup..PpDefault: none+.It Fl fa Ar FNAME , Fl Fl flash-attn+Enable Flash Attention. This is a mathematical shortcut that can speed+up inference for certain models. This feature is still under active+development..It Fl Fl prompt-cache-allIf specified, saves user input and generations to cache as well. Not supported with.Fl Fl interactive@@ -653,8 +679,14 @@ Enable colorized output to differentiate visually distinguishing betweenprompts, user input, and generated text..It Fl Fl no-display-prompt , Fl Fl silent-promptDon't echo the prompt itself to standard output.+.It Fl Fl keep Ar N+Specifies number of tokens to keep from the initial prompt. The default+is -1 which means all tokens..It Fl Fl multiline-inputAllows you to write or paste multiple lines without ending each in '\[rs]'.+.It Fl Fl cont-batching+Enables continuous batching, a.k.a. dynamic batching.+is -1 which means all tokens..El.Sh SERVER OPTIONSThe following options may be specified whendiff --git a/llama.cpp/main/main.1.asc b/llama.cpp/main/main.1.ascindex 49eefb57..ed2b50e6 100644--- a/llama.cpp/main/main.1.asc+++ b/llama.cpp/main/main.1.asc@@ -1,836 +1,883 @@-LLAMAFILE(1) BSD General Commands Manual LLAMAFILE(1)+[4mLLAMAFILE[24m(1) General Commands Manual [4mLLAMAFILE[24m(1)-NNAAMMEE- llllaammaaffiillee — large language model runner+[1mNAME[0m+ llamafile — large language model runner-SSYYNNOOPPSSIISS- llllaammaaffiillee [----sseerrvveerr] [flags...] --mm _m_o_d_e_l_._g_g_u_f [----mmmmpprroojj _v_i_s_i_o_n_._g_g_u_f]- llllaammaaffiillee [----ccllii] [flags...] --mm _m_o_d_e_l_._g_g_u_f --pp _p_r_o_m_p_t- llllaammaaffiillee [----ccllii] [flags...] --mm _m_o_d_e_l_._g_g_u_f ----mmmmpprroojj _v_i_s_i_o_n_._g_g_u_f ----iimmaaggee- _g_r_a_p_h_i_c_._p_n_g --pp _p_r_o_m_p_t+[1mSYNOPSIS[0m+ [1mllamafile [22m[[1m--server[22m] [flags...] [1m-m [4m[22mmodel.gguf[24m [[1m--mmproj [4m[22mvision.gguf[24m]+ [1mllamafile [22m[[1m--cli[22m] [flags...] [1m-m [4m[22mmodel.gguf[24m [1m-p [4m[22mprompt[0m+ [1mllamafile [22m[[1m--cli[22m] [flags...] [1m-m [4m[22mmodel.gguf[24m [1m--mmproj [4m[22mvision.gguf[24m [1m--image[0m+ [4mgraphic.png[24m [1m-p [4m[22mprompt[0m-DDEESSCCRRIIPPTTIIOONN- llllaammaaffiillee is a large language model tool. It has use cases such as:+[1mDESCRIPTION[0m+ [1mllamafile [22mis a large language model tool. It has use cases such as:- -- Code completion- -- Prose composition- -- Chatbot that passes the Turing test- -- Text/image summarization and analysis+ [1m- [22mCode completion+ [1m- [22mProse composition+ [1m- [22mChatbot that passes the Turing test+ [1m- [22mText/image summarization and analysis-OOPPTTIIOONNSS- The following options are available:+[1mOPTIONS[0m+ The following options are available:- ----vveerrssiioonn- Print version and exit.+ [1m--version[0m+ Print version and exit.- --hh, ----hheellpp- Show help message and exit.+ [1m-h[22m, [1m--help[0m+ Show help message and exit.- ----ccllii Puts program in command line interface mode. This flag is implied- when a prompt is supplied using either the --pp or --ff flags.+ [1m--cli [22mPuts program in command line interface mode. This flag is im‐+ plied when a prompt is supplied using either the [1m-p [22mor [1m-f[0m+ flags.- ----sseerrvveerr- Puts program in server mode. This will launch an HTTP server on a- local port. This server has both a web UI and an OpenAI API com‐- patible completions endpoint. When the server is run on a desk- system, a tab browser tab will be launched automatically that- displays the web UI. This ----sseerrvveerr flag is implied if no prompt- is specified, i.e. neither the --pp or --ff flags are passed.+ [1m--server[0m+ Puts program in server mode. This will launch an HTTP server on+ a local port. This server has both a web UI and an OpenAI API+ compatible completions endpoint. When the server is run on a+ desk system, a tab browser tab will be launched automatically+ that displays the web UI. This [1m--server [22mflag is implied if no+ prompt is specified, i.e. neither the [1m-p [22mor [1m-f [22mflags are+ passed.- --mm _F_N_A_M_E, ----mmooddeell _F_N_A_M_E- Model path in the GGUF file format.+ [1m-m [4m[22mFNAME[24m, [1m--model [4m[22mFNAME[0m+ Model path in the GGUF file format.- Default: _m_o_d_e_l_s_/_7_B_/_g_g_m_l_-_m_o_d_e_l_-_f_1_6_._g_g_u_f+ Default: [4mmodels/7B/ggml‐model‐f16.gguf[0m- ----mmmmpprroojj _F_N_A_M_E- Specifies path of the LLaVA vision model in the GGUF file format.- If this flag is supplied, then the ----mmooddeell and ----iimmaaggee flags- should also be supplied.+ [1m--mmproj [4m[22mFNAME[0m+ Specifies path of the LLaVA vision model in the GGUF file for‐+ mat. If this flag is supplied, then the [1m--model [22mand [1m--image[0m+ flags should also be supplied.- --ss _S_E_E_D, ----sseeeedd _S_E_E_D- Random Number Generator (RNG) seed. A random seed is used if this- is less than zero.+ [1m-s [4m[22mSEED[24m, [1m--seed [4m[22mSEED[0m+ Random Number Generator (RNG) seed. A random seed is used if+ this is less than zero.- Default: -1+ Default: ‐1- --tt _N, ----tthhrreeaaddss _N- Number of threads to use during generation.+ [1m-t [4m[22mN[24m, [1m--threads [4m[22mN[0m+ Number of threads to use during generation.- Default: $(nproc)/2+ Default: $(nproc)/2- --ttbb _N, ----tthhrreeaaddss--bbaattcchh _N- Set the number of threads to use during batch and prompt process‐- ing. In some systems, it is beneficial to use a higher number of- threads during batch processing than during generation. If not- specified, the number of threads used for batch processing will- be the same as the number of threads used for generation.+ [1m-tb [4m[22mN[24m, [1m--threads‐batch [4m[22mN[0m+ Set the number of threads to use during batch and prompt pro‐+ cessing. In some systems, it is beneficial to use a higher num‐+ ber of threads during batch processing than during generation.+ If not specified, the number of threads used for batch process‐+ ing will be the same as the number of threads used for genera‐+ tion.- Default: Same as ----tthhrreeaaddss+ Default: Same as [1m--threads[0m- --ttdd _N, ----tthhrreeaaddss--ddrraafftt _N- Number of threads to use during generation.+ [1m-td [4m[22mN[24m, [1m--threads‐draft [4m[22mN[0m+ Number of threads to use during generation.- Default: Same as ----tthhrreeaaddss+ Default: Same as [1m--threads[0m- --ttbbdd _N, ----tthhrreeaaddss--bbaattcchh--ddrraafftt _N- Number of threads to use during batch and prompt processing.+ [1m-tbd [4m[22mN[24m, [1m--threads‐batch‐draft [4m[22mN[0m+ Number of threads to use during batch and prompt processing.- Default: Same as ----tthhrreeaaddss--ddrraafftt+ Default: Same as [1m--threads‐draft[0m- ----iinn--pprreeffiixx--bbooss- Prefix BOS to user inputs, preceding the ----iinn--pprreeffiixx string.+ [1m--in‐prefix‐bos[0m+ Prefix BOS to user inputs, preceding the [1m--in‐prefix [22mstring.- ----iinn--pprreeffiixx _S_T_R_I_N_G- This flag is used to add a prefix to your input, primarily, this- is used to insert a space after the reverse prompt. Here's an ex‐- ample of how to use the ----iinn--pprreeffiixx flag in conjunction with the- ----rreevveerrssee--pprroommpptt flag:+ [1m--in‐prefix [4m[22mSTRING[0m+ This flag is used to add a prefix to your input, primarily,+ this is used to insert a space after the reverse prompt. Here’s+ an example of how to use the [1m--in‐prefix [22mflag in conjunction+ with the [1m--reverse‐prompt [22mflag:- ./main -r "User:" --in-prefix " "+ [1m./main ‐r "User:" ‐‐in‐prefix " "[0m- Default: empty+ Default: empty- ----iinn--ssuuffffiixx _S_T_R_I_N_G- This flag is used to add a suffix after your input. This is use‐- ful for adding an "Assistant:" prompt after the user's input.- It's added after the new-line character (\n) that's automatically- added to the end of the user's input. Here's an example of how to- use the ----iinn--ssuuffffiixx flag in conjunction with the ----rreevveerrssee--pprroommpptt- flag:+ [1m--in‐suffix [4m[22mSTRING[0m+ This flag is used to add a suffix after your input. This is+ useful for adding an "Assistant:" prompt after the user’s in‐+ put. It’s added after the new‐line character (\n) that’s auto‐+ matically added to the end of the user’s input. Here’s an exam‐+ ple of how to use the [1m--in‐suffix [22mflag in conjunction with the+ [1m--reverse‐prompt [22mflag:- ./main -r "User:" --in-prefix " " --in-suffix "Assistant:"+ [1m./main ‐r "User:" ‐‐in‐prefix " " ‐‐in‐suffix[0m+ [1m"Assistant:"[0m- Default: empty+ Default: empty- --nn _N, ----nn--pprreeddiicctt _N- Number of tokens to predict.+ [1m-n [4m[22mN[24m, [1m--n‐predict [4m[22mN[0m+ Number of tokens to predict.- -- -1 = infinity- -- -2 = until context filled+ [1m- [22m‐1 = infinity+ [1m- [22m‐2 = until context filled- Default: -1+ Default: ‐1- --cc _N, ----ccttxx--ssiizzee _N- Set the size of the prompt context. A larger context size helps- the model to better comprehend and generate responses for longer- input or conversations. The LLaMA models were built with a con‐- text of 2048, which yields the best results on longer input / in‐- ference.+ [1m-c [4m[22mN[24m, [1m--ctx‐size [4m[22mN[0m+ Set the size of the prompt context. A larger context size helps+ the model to better comprehend and generate responses for+ longer input or conversations. The LLaMA models were built with+ a context of 2048, which yields the best results on longer in‐+ put / inference.- -- 0 = loaded automatically from model+ [1m- [22m0 = loaded automatically from model- Default: 512+ Default: 512- --bb _N, ----bbaattcchh--ssiizzee _N- Batch size for prompt processing.+ [1m-b [4m[22mN[24m, [1m--batch‐size [4m[22mN[0m+ Batch size for prompt processing.- Default: 512+ Default: 512- ----ttoopp--kk _N- Top-k sampling.+ [1m--top‐k [4m[22mN[0m+ Top‐k sampling.- -- 0 = disabled+ [1m- [22m0 = disabled- Default: 40+ Default: 40- ----ttoopp--pp _N- Top-p sampling.+ [1m--top‐p [4m[22mN[0m+ Top‐p sampling.- -- 1.0 = disabled+ [1m- [22m1.0 = disabled- Default: 0.9+ Default: 0.9- ----mmiinn--pp _N- Min-p sampling.+ [1m--min‐p [4m[22mN[0m+ Min‐p sampling.- -- 0.0 = disabled+ [1m- [22m0.0 = disabled- Default: 0.1+ Default: 0.1- ----ttffss _N- Tail free sampling, parameter z.+ [1m--tfs [4m[22mN[0m+ Tail free sampling, parameter z.- -- 1.0 = disabled+ [1m- [22m1.0 = disabled- Default: 1.0+ Default: 1.0- ----ttyyppiiccaall _N- Locally typical sampling, parameter p.+ [1m--typical [4m[22mN[0m+ Locally typical sampling, parameter p.- -- 1.0 = disabled+ [1m- [22m1.0 = disabled- Default: 1.0+ Default: 1.0- ----rreeppeeaatt--llaasstt--nn _N- Last n tokens to consider for penalize.+ [1m--repeat‐last‐n [4m[22mN[0m+ Last n tokens to consider for penalize.- -- 0 = disabled- -- -1 = ctx_size+ [1m- [22m0 = disabled+ [1m- [22m‐1 = ctx_size- Default: 64+ Default: 64- ----rreeppeeaatt--ppeennaallttyy _N- Penalize repeat sequence of tokens.+ [1m--repeat‐penalty [4m[22mN[0m+ Penalize repeat sequence of tokens.- -- 1.0 = disabled+ [1m- [22m1.0 = disabled- Default: 1.1+ Default: 1.1- ----pprreesseennccee--ppeennaallttyy _N- Repeat alpha presence penalty.+ [1m--presence‐penalty [4m[22mN[0m+ Repeat alpha presence penalty.- -- 0.0 = disabled+ [1m- [22m0.0 = disabled- Default: 0.0+ Default: 0.0- ----ffrreeqquueennccyy--ppeennaallttyy _N- Repeat alpha frequency penalty.+ [1m--frequency‐penalty [4m[22mN[0m+ Repeat alpha frequency penalty.- -- 0.0 = disabled+ [1m- [22m0.0 = disabled- Default: 0.0+ Default: 0.0- ----mmiirroossttaatt _N- Use Mirostat sampling. Top K, Nucleus, Tail Free and Locally Typ‐- ical samplers are ignored if used..+ [1m--mirostat [4m[22mN[0m+ Use Mirostat sampling. Top K, Nucleus, Tail Free and Locally+ Typical samplers are ignored if used..- -- 0 = disabled- -- 1 = Mirostat- -- 2 = Mirostat 2.0+ [1m- [22m0 = disabled+ [1m- [22m1 = Mirostat+ [1m- [22m2 = Mirostat 2.0- Default: 0+ Default: 0- ----mmiirroossttaatt--llrr _N- Mirostat learning rate, parameter eta.+ [1m--mirostat‐lr [4m[22mN[0m+ Mirostat learning rate, parameter eta.- Default: 0.1+ Default: 0.1- ----mmiirroossttaatt--eenntt _N- Mirostat target entropy, parameter tau.+ [1m--mirostat‐ent [4m[22mN[0m+ Mirostat target entropy, parameter tau.- Default: 5.0+ Default: 5.0- --ll _T_O_K_E_N___I_D_(_+_/_-_)_B_I_A_S, ----llooggiitt--bbiiaass _T_O_K_E_N___I_D_(_+_/_-_)_B_I_A_S- Modifies the likelihood of token appearing in the completion,- i.e. ----llooggiitt--bbiiaass _1_5_0_4_3_+_1 to increase likelihood of token- _'_ _H_e_l_l_o_', or ----llooggiitt--bbiiaass _1_5_0_4_3_-_1 to decrease likelihood of token- _'_ _H_e_l_l_o_'.+ [1m-l [4m[22mTOKEN_ID(+/‐)BIAS[24m, [1m--logit‐bias [4m[22mTOKEN_ID(+/‐)BIAS[0m+ Modifies the likelihood of token appearing in the completion,+ i.e. [1m--logit‐bias [4m[22m15043+1[24m to increase likelihood of token+ [4m’[24m [4mHello’[24m, or [1m--logit‐bias [4m[22m15043‐1[24m to decrease likelihood of to‐+ ken [4m’[24m [4mHello’[24m.- --mmdd _F_N_A_M_E, ----mmooddeell--ddrraafftt _F_N_A_M_E- Draft model for speculative decoding.+ [1m-md [4m[22mFNAME[24m, [1m--model‐draft [4m[22mFNAME[0m+ Draft model for speculative decoding.- Default: _m_o_d_e_l_s_/_7_B_/_g_g_m_l_-_m_o_d_e_l_-_f_1_6_._g_g_u_f+ Default: [4mmodels/7B/ggml‐model‐f16.gguf[0m- ----ccffgg--nneeggaattiivvee--pprroommpptt _P_R_O_M_P_T- Negative prompt to use for guidance..+ [1m--cfg‐negative‐prompt [4m[22mPROMPT[0m+ Negative prompt to use for guidance..- Default: empty+ Default: empty- ----ccffgg--nneeggaattiivvee--pprroommpptt--ffiillee _F_N_A_M_E- Negative prompt file to use for guidance.+ [1m--cfg‐negative‐prompt‐file [4m[22mFNAME[0m+ Negative prompt file to use for guidance.- Default: empty+ Default: empty- ----ccffgg--ssccaallee _N- Strength of guidance.+ [1m--cfg‐scale [4m[22mN[0m+ Strength of guidance.- -- 1.0 = disable+ [1m- [22m1.0 = disable- Default: 1.0+ Default: 1.0- ----rrooppee--ssccaalliinngg _{_n_o_n_e_,_l_i_n_e_a_r_,_y_a_r_n_}- RoPE frequency scaling method, defaults to linear unless speci‐- fied by the model+ [1m--rope‐scaling [4m[22m{none,linear,yarn}[0m+ RoPE frequency scaling method, defaults to linear unless speci‐+ fied by the model- ----rrooppee--ssccaallee _N- RoPE context scaling factor, expands context by a factor of _N- where _N is the linear scaling factor used by the fine-tuned- model. Some fine-tuned models have extended the context length by- scaling RoPE. For example, if the original pre-trained model have- a context length (max sequence length) of 4096 (4k) and the fine-- tuned model have 32k. That is a scaling factor of 8, and should- work by setting the above ----ccttxx--ssiizzee to 32768 (32k) and- ----rrooppee--ssccaallee to 8.+ [1m--rope‐scale [4m[22mN[0m+ RoPE context scaling factor, expands context by a factor of [4mN[0m+ where [4mN[24m is the linear scaling factor used by the fine‐tuned+ model. Some fine‐tuned models have extended the context length+ by scaling RoPE. For example, if the original pre‐trained model+ have a context length (max sequence length) of 4096 (4k) and+ the fine‐tuned model have 32k. That is a scaling factor of 8,+ and should work by setting the above [1m--ctx‐size [22mto 32768 (32k)+ and [1m--rope‐scale [22mto 8.- ----rrooppee--ffrreeqq--bbaassee _N- RoPE base frequency, used by NTK-aware scaling.+ [1m--rope‐freq‐base [4m[22mN[0m+ RoPE base frequency, used by NTK‐aware scaling.- Default: loaded from model+ Default: loaded from model- ----rrooppee--ffrreeqq--ssccaallee _N- RoPE frequency scaling factor, expands context by a factor of 1/N+ [1m--rope‐freq‐scale [4m[22mN[0m+ RoPE frequency scaling factor, expands context by a factor of+ 1/N- ----yyaarrnn--oorriigg--ccttxx _N- YaRN: original context size of model.+ [1m--yarn‐orig‐ctx [4m[22mN[0m+ YaRN: original context size of model.- Default: 0 = model training context size+ Default: 0 = model training context size- ----yyaarrnn--eexxtt--ffaaccttoorr _N- YaRN: extrapolation mix factor.+ [1m--yarn‐ext‐factor [4m[22mN[0m+ YaRN: extrapolation mix factor.- -- 0.0 = full interpolation+ [1m- [22m0.0 = full interpolation- Default: 1.0+ Default: 1.0- ----yyaarrnn--aattttnn--ffaaccttoorr _N- YaRN: scale sqrt(t) or attention magnitude.+ [1m--yarn‐attn‐factor [4m[22mN[0m+ YaRN: scale sqrt(t) or attention magnitude.- Default: 1.0+ Default: 1.0- ----yyaarrnn--bbeettaa--ssllooww _N- YaRN: high correction dim or alpha.+ [1m--yarn‐beta‐slow [4m[22mN[0m+ YaRN: high correction dim or alpha.- Default: 1.0+ Default: 1.0- ----yyaarrnn--bbeettaa--ffaasstt _N- YaRN: low correction dim or beta.+ [1m--yarn‐beta‐fast [4m[22mN[0m+ YaRN: low correction dim or beta.- Default: 32.0+ Default: 32.0- ----iiggnnoorree--eeooss- Ignore end of stream token and continue generating (implies- ----llooggiitt--bbiiaass _2_-_i_n_f)+ [1m--ignore‐eos[0m+ Ignore end of stream token and continue generating (implies+ [1m--logit‐bias [4m[22m2‐inf[24m)- ----nnoo--ppeennaalliizzee--nnll- Do not penalize newline token.+ [1m--no‐penalize‐nl[0m+ Do not penalize newline token.- ----tteemmpp _N- Temperature.+ [1m--temp [4m[22mN[0m+ Temperature.- Default: 0.8+ Default: 0.8- ----llooggiittss--aallll- Return logits for all tokens in the batch.+ [1m--logits‐all[0m+ Return logits for all tokens in the batch.- Default: disabled+ Default: disabled- ----hheellllaasswwaagg- Compute HellaSwag score over random tasks from datafile supplied- with -f+ [1m--hellaswag[0m+ Compute HellaSwag score over random tasks from datafile sup‐+ plied with ‐f- ----hheellllaasswwaagg--ttaasskkss _N- Number of tasks to use when computing the HellaSwag score.+ [1m--hellaswag‐tasks [4m[22mN[0m+ Number of tasks to use when computing the HellaSwag score.- Default: 400+ Default: 400- ----kkeeeepp _N- This flag allows users to retain the original prompt when the- model runs out of context, ensuring a connection to the initial- instruction or conversation topic is maintained, where _N is the- number of tokens from the initial prompt to retain when the model- resets its internal context.+ [1m--keep [4m[22mN[0m+ This flag allows users to retain the original prompt when the+ model runs out of context, ensuring a connection to the initial+ instruction or conversation topic is maintained, where [4mN[24m is the+ number of tokens from the initial prompt to retain when the+ model resets its internal context.- -- 0 = no tokens are kept from initial prompt- -- -1 = retain all tokens from initial prompt+ [1m- [22m0 = no tokens are kept from initial prompt+ [1m- [22m‐1 = retain all tokens from initial prompt- Default: 0+ Default: 0- ----ddrraafftt _N- Number of tokens to draft for speculative decoding.+ [1m--draft [4m[22mN[0m+ Number of tokens to draft for speculative decoding.- Default: 16+ Default: 16- ----cchhuunnkkss _N- Max number of chunks to process.+ [1m--chunks [4m[22mN[0m+ Max number of chunks to process.- -- -1 = all+ [1m- [22m‐1 = all- Default: -1+ Default: ‐1- --nnss _N, ----sseeqquueenncceess _N- Number of sequences to decode.+ [1m-ns [4m[22mN[24m, [1m--sequences [4m[22mN[0m+ Number of sequences to decode.- Default: 1+ Default: 1- --ppaa _N, ----pp--aacccceepptt _N- speculative decoding accept probability.+ [1m-pa [4m[22mN[24m, [1m--p‐accept [4m[22mN[0m+ speculative decoding accept probability.- Default: 0.5+ Default: 0.5- --ppss _N, ----pp--sspplliitt _N- Speculative decoding split probability.+ [1m-ps [4m[22mN[24m, [1m--p‐split [4m[22mN[0m+ Speculative decoding split probability.- Default: 0.1+ Default: 0.1- ----mmlloocckk- Force system to keep model in RAM rather than swapping or com‐- pressing.+ [1m--mlock[0m+ Force system to keep model in RAM rather than swapping or com‐+ pressing.- ----nnoo--mmmmaapp- Do not memory-map model (slower load but may reduce pageouts if- not using mlock).+ [1m--no‐mmap[0m+ Do not memory‐map model (slower load but may reduce pageouts if+ not using mlock).- ----nnuummaa Attempt optimizations that help on some NUMA systems if run with‐- out this previously, it is recommended to drop the system page- cache before using this. See- https://github.com/ggerganov/llama.cpp/issues/1437.+ [1m--numa [22mAttempt optimizations that help on some NUMA systems if run+ without this previously, it is recommended to drop the system+ page cache before using this. See+ https://github.com/ggerganov/llama.cpp/issues/1437.- ----rreeccoommppiillee- Force GPU support to be recompiled at runtime if possible.+ [1m--recompile[0m+ Force GPU support to be recompiled at runtime if possible.- ----nnooccoommppiillee- Never compile GPU support at runtime.+ [1m--nocompile[0m+ Never compile GPU support at runtime.- If the appropriate DSO file already exists under _~_/_._l_l_a_m_a_f_i_l_e_/- then it'll be linked as-is without question. If a prebuilt DSO is- present in the PKZIP content of the executable, then it'll be ex‐- tracted and linked if possible. Otherwise, llllaammaaffiillee will skip- any attempt to compile GPU support and simply fall back to using- CPU inference.+ If the appropriate DSO file already exists under [4m~/.llamafile/[0m+ then it’ll be linked as‐is without question. If a prebuilt DSO+ is present in the PKZIP content of the executable, then it’ll+ be extracted and linked if possible. Otherwise, [1mllamafile [22mwill+ skip any attempt to compile GPU support and simply fall back to+ using CPU inference.- ----ggppuu _G_P_U- Specifies which brand of GPU should be used. Valid choices are:+ [1m--gpu [4m[22mGPU[0m+ Specifies which brand of GPU should be used. Valid choices are:- -- _A_U_T_O: Use any GPU if possible, otherwise fall back to CPU in‐- ference (default)+ [1m- [4m[22mAUTO[24m: Use any GPU if possible, otherwise fall back to CPU+ inference (default)- -- _A_P_P_L_E: Use Apple Metal GPU. This is only available on MacOS- ARM64. If Metal could not be used for any reason, then a fa‐- tal error will be raised.+ [1m- [4m[22mAPPLE[24m: Use Apple Metal GPU. This is only available on MacOS+ ARM64. If Metal could not be used for any reason, then a+ fatal error will be raised.- -- _A_M_D: Use AMD GPUs. The AMD HIP ROCm SDK should be installed- in which case we assume the HIP_PATH environment variable has- been defined. The set of gfx microarchitectures needed to run- on the host machine is determined automatically based on the- output of the hipInfo command. On Windows, llllaammaaffiillee release- binaries are distributed with a tinyBLAS DLL so it'll work- out of the box without requiring the HIP SDK to be installed.- However, tinyBLAS is slower than rocBLAS for batch and image- processing, so it's recommended that the SDK be installed- anyway. If an AMD GPU could not be used for any reason, then- a fatal error will be raised.+ [1m- [4m[22mAMD[24m: Use AMD GPUs. The AMD HIP ROCm SDK should be installed+ in which case we assume the HIP_PATH environment variable+ has been defined. The set of gfx microarchitectures needed+ to run on the host machine is determined automatically+ based on the output of the hipInfo command. On Windows,+ [1mllamafile [22mrelease binaries are distributed with a tinyBLAS+ DLL so it’ll work out of the box without requiring the HIP+ SDK to be installed. However, tinyBLAS is slower than+ rocBLAS for batch and image processing, so it’s recommended+ that the SDK be installed anyway. If an AMD GPU could not+ be used for any reason, then a fatal error will be raised.- -- _N_V_I_D_I_A: Use NVIDIA GPUs. If an NVIDIA GPU could not be used- for any reason, a fatal error will be raised. On Windows,- NVIDIA GPU support will use our tinyBLAS library, since it- works on stock Windows installs. However, tinyBLAS goes- slower for batch and image processing. It's possible to use- NVIDIA's closed-source cuBLAS library instead. To do that,- both MSVC and CUDA need to be installed and the llllaammaaffiillee- command should be run once from the x64 MSVC command prompt- with the ----rreeccoommppiillee flag passed. The GGML library will then- be compiled and saved to _~_/_._l_l_a_m_a_f_i_l_e_/ so the special process- only needs to happen a single time.+ [1m- [4m[22mNVIDIA[24m: Use NVIDIA GPUs. If an NVIDIA GPU could not be used+ for any reason, a fatal error will be raised. On Windows,+ NVIDIA GPU support will use our tinyBLAS library, since it+ works on stock Windows installs. However, tinyBLAS goes+ slower for batch and image processing. It’s possible to use+ NVIDIA’s closed‐source cuBLAS library instead. To do that,+ both MSVC and CUDA need to be installed and the [1mllamafile[0m+ command should be run once from the x64 MSVC command prompt+ with the [1m--recompile [22mflag passed. The GGML library will+ then be compiled and saved to [4m~/.llamafile/[24m so the special+ process only needs to happen a single time.- -- _D_I_S_A_B_L_E: Never use GPU and instead use CPU inference. This- setting is implied by --nnggll _0.+ [1m- [4m[22mDISABLE[24m: Never use GPU and instead use CPU inference. This+ setting is implied by [1m-ngl [4m[22m0[24m.- --nnggll _N, ----nn--ggppuu--llaayyeerrss _N- Number of layers to store in VRAM.+ [1m-ngl [4m[22mN[24m, [1m--n‐gpu‐layers [4m[22mN[0m+ Number of layers to store in VRAM.- --nngglldd _N, ----nn--ggppuu--llaayyeerrss--ddrraafftt _N- Number of layers to store in VRAM for the draft model.+ [1m-ngld [4m[22mN[24m, [1m--n‐gpu‐layers‐draft [4m[22mN[0m+ Number of layers to store in VRAM for the draft model.- --ssmm _S_P_L_I_T___M_O_D_E, ----sspplliitt--mmooddee _S_P_L_I_T___M_O_D_E- How to split the model across multiple GPUs, one of:- -- none: use one GPU only- -- layer (default): split layers and KV across GPUs- -- row: split rows across GPUs+ [1m-sm [4m[22mSPLIT_MODE[24m, [1m--split‐mode [4m[22mSPLIT_MODE[0m+ How to split the model across multiple GPUs, one of:+ [1m- [22mnone: use one GPU only+ [1m- [22mlayer (default): split layers and KV across GPUs+ [1m- [22mrow: split rows across GPUs- --ttss _S_P_L_I_T, ----tteennssoorr--sspplliitt _S_P_L_I_T- When using multiple GPUs this option controls how large tensors- should be split across all GPUs. _S_P_L_I_T is a comma-separated list- of non-negative values that assigns the proportion of data that- each GPU should get in order. For example, "3,2" will assign 60%- of the data to GPU 0 and 40% to GPU 1. By default the data is- split in proportion to VRAM but this may not be optimal for per‐- formance. Requires cuBLAS. How to split tensors across multiple- GPUs, comma-separated list of proportions, e.g. 3,1+ [1m-ts [4m[22mSPLIT[24m, [1m--tensor‐split [4m[22mSPLIT[0m+ When using multiple GPUs this option controls how large tensors+ should be split across all GPUs. [4mSPLIT[24m is a comma‐separated+ list of non‐negative values that assigns the proportion of data+ that each GPU should get in order. For example, "3,2" will as‐+ sign 60% of the data to GPU 0 and 40% to GPU 1. By default the+ data is split in proportion to VRAM but this may not be optimal+ for performance. Requires cuBLAS. How to split tensors across+ multiple GPUs, comma‐separated list of proportions, e.g. 3,1- --mmgg _i, ----mmaaiinn--ggppuu _i- The GPU to use for scratch and small tensors.+ [1m-mg [4m[22mi[24m, [1m--main‐gpu [4m[22mi[0m+ The GPU to use for scratch and small tensors.- --nnoommmmqq, ----nnoo--mmuull--mmaatt--qq- Use cuBLAS instead of custom mul_mat_q CUDA kernels. Not recom‐- mended since this is both slower and uses more VRAM.+ [1m-nommq[22m, [1m--no‐mul‐mat‐q[0m+ Use cuBLAS instead of custom mul_mat_q CUDA kernels. Not recom‐+ mended since this is both slower and uses more VRAM.- ----vveerrbboossee--pprroommpptt- Print prompt before generation.+ [1m--verbose‐prompt[0m+ Print prompt before generation.- ----ssiimmppllee--iioo- Use basic IO for better compatibility in subprocesses and limited- consoles.+ [1m--simple‐io[0m+ Use basic IO for better compatibility in subprocesses and lim‐+ ited consoles.- ----lloorraa _F_N_A_M_E- Apply LoRA adapter (implies ----nnoo--mmmmaapp)+ [1m--lora [4m[22mFNAME[0m+ Apply LoRA adapter (implies [1m--no‐mmap[22m)- ----lloorraa--ssccaalleedd _F_N_A_M_E _S- Apply LoRA adapter with user defined scaling S (implies- ----nnoo--mmmmaapp)+ [1m--lora‐scaled [4m[22mFNAME[24m [4mS[0m+ Apply LoRA adapter with user defined scaling S (implies+ [1m--no‐mmap[22m)- ----lloorraa--bbaassee _F_N_A_M_E- Optional model to use as a base for the layers modified by the- LoRA adapter+ [1m--lora‐base [4m[22mFNAME[0m+ Optional model to use as a base for the layers modified by the+ LoRA adapter- ----uunnsseeccuurree- Disables pledge() sandboxing on Linux and OpenBSD.+ [1m--unsecure[0m+ Disables pledge() sandboxing on Linux and OpenBSD.- ----ssaammpplleerrss- Samplers that will be used for generation in the order, separated- by semicolon, for example: top_k;tfs;typical;top_p;min_p;temp+ [1m--samplers[0m+ Samplers that will be used for generation in the order, sepa‐+ rated by semicolon, for example: top_k;tfs;typi‐+ cal;top_p;min_p;temp- ----ssaammpplleerrss--sseeqq- Simplified sequence for samplers that will be used.+ [1m--samplers‐seq[0m+ Simplified sequence for samplers that will be used.- --ccmmll, ----cchhaattmmll- Run in chatml mode (use with ChatML-compatible models)+ [1m-cml[22m, [1m--chatml[0m+ Run in chatml mode (use with ChatML‐compatible models)- --ddkkvvcc, ----dduummpp--kkvv--ccaacchhee- Verbose print of the KV cache.+ [1m-dkvc[22m, [1m--dump‐kv‐cache[0m+ Verbose print of the KV cache.- --nnkkvvoo, ----nnoo--kkvv--ooffffllooaadd- Disable KV offload.+ [1m-nkvo[22m, [1m--no‐kv‐offload[0m+ Disable KV offload.- --ccttkk _T_Y_P_E, ----ccaacchhee--ttyyppee--kk _T_Y_P_E- KV cache data type for K.+ [1m-ctk [4m[22mTYPE[24m, [1m--cache‐type‐k [4m[22mTYPE[0m+ KV cache data type for K.- --ccttvv _T_Y_P_E, ----ccaacchhee--ttyyppee--vv _T_Y_P_E- KV cache data type for V.+ [1m-ctv [4m[22mTYPE[24m, [1m--cache‐type‐v [4m[22mTYPE[0m+ KV cache data type for V.- --ggaann _N, ----ggrrpp--aattttnn--nn _N- Group-attention factor.+ [1m-gan [4m[22mN[24m, [1m--grp‐attn‐n [4m[22mN[0m+ Group‐attention factor.- Default: 1+ Default: 1- --ggaaww _N, ----ggrrpp--aattttnn--ww _N- Group-attention width.+ [1m-gaw [4m[22mN[24m, [1m--grp‐attn‐w [4m[22mN[0m+ Group‐attention width.- Default: 512+ Default: 512- --bbff _F_N_A_M_E, ----bbiinnaarryy--ffiillee _F_N_A_M_E- Binary file containing multiple choice tasks.+ [1m-bf [4m[22mFNAME[24m, [1m--binary‐file [4m[22mFNAME[0m+ Binary file containing multiple choice tasks.- ----wwiinnooggrraannddee- Compute Winogrande score over random tasks from datafile supplied- by the --ff flag.+ [1m--winogrande[0m+ Compute Winogrande score over random tasks from datafile sup‐+ plied by the [1m-f [22mflag.- ----wwiinnooggrraannddee--ttaasskkss _N- Number of tasks to use when computing the Winogrande score.+ [1m--winogrande‐tasks [4m[22mN[0m+ Number of tasks to use when computing the Winogrande score.- Default: 0+ Default: 0- ----mmuullttiippllee--cchhooiiccee- Compute multiple choice score over random tasks from datafile- supplied by the --ff flag.+ [1m--multiple‐choice[0m+ Compute multiple choice score over random tasks from datafile+ supplied by the [1m-f [22mflag.- ----mmuullttiippllee--cchhooiiccee--ttaasskkss _N- Number of tasks to use when computing the multiple choice score.+ [1m--multiple‐choice‐tasks [4m[22mN[0m+ Number of tasks to use when computing the multiple choice+ score.- Default: 0+ Default: 0- ----kkll--ddiivveerrggeennccee- Computes KL-divergence to logits provided via the- ----kkll--ddiivveerrggeennccee--bbaassee flag.+ [1m--kl‐divergence[0m+ Computes KL‐divergence to logits provided via the+ [1m--kl‐divergence‐base [22mflag.- ----ssaavvee--aallll--llooggiittss _F_N_A_M_E, ----kkll--ddiivveerrggeennccee--bbaassee _F_N_A_M_E- Save logits to filename.+ [1m--save‐all‐logits [4m[22mFNAME[24m, [1m--kl‐divergence‐base [4m[22mFNAME[0m+ Save logits to filename.- --ppttcc _N, ----pprriinntt--ttookkeenn--ccoouunntt _N- Print token count every _N tokens.+ [1m-ptc [4m[22mN[24m, [1m--print‐token‐count [4m[22mN[0m+ Print token count every [4mN[24m tokens.- Default: -1+ Default: ‐1-CCLLII OOPPTTIIOONNSS- The following options may be specified when llllaammaaffiillee is running in ----ccllii- mode.+[1mCLI OPTIONS[0m+ The following options may be specified when [1mllamafile [22mis running in+ [1m--cli [22mmode.- --ee, ----eessccaappee- Process prompt escapes sequences (\n, \r, \t, \´, \", \\)+ [1m-e[22m, [1m--escape[0m+ Process prompt escapes sequences (\n, \r, \t, \´, \", \\)- --pp _S_T_R_I_N_G, ----pprroommpptt _S_T_R_I_N_G- Prompt to start text generation. Your LLM works by auto-complet‐- ing this text. For example:+ [1m-p [4m[22mSTRING[24m, [1m--prompt [4m[22mSTRING[0m+ Prompt to start text generation. Your LLM works by auto‐com‐+ pleting this text. For example:- llamafile -m model.gguf -p "four score and"+ [1mllamafile ‐m model.gguf ‐p "four score and"[0m- Stands a pretty good chance of printing Lincoln's Gettysburg Ad‐- dress. Prompts can take on a structured format too. Depending on- how your model was trained, it may specify in its docs an in‐- struction notation. With some models that might be:+ Stands a pretty good chance of printing Lincoln’s Gettysburg+ Address. Prompts can take on a structured format too. Depend‐+ ing on how your model was trained, it may specify in its docs+ an instruction notation. With some models that might be:- llamafile -p "[INST]Summarize this: $(cat file)[/INST]"+ [1mllamafile ‐p "[INST]Summarize this: $(cat file)[/INST]"[0m- In most cases, simply colons and newlines will work too:+ In most cases, simply colons and newlines will work too:- llamafile -e -p "User: What is best in life?\nAssistant:"+ [1mllamafile ‐e ‐p "User: What is best in life?\nAssistant:"[0m- --ff _F_N_A_M_E, ----ffiillee _F_N_A_M_E- Prompt file to start generation.+ [1m-f [4m[22mFNAME[24m, [1m--file [4m[22mFNAME[0m+ Prompt file to start generation.- ----ggrraammmmaarr _G_R_A_M_M_A_R- BNF-like grammar to constrain which tokens may be selected when- generating text. For example, the grammar:+ [1m--grammar [4m[22mGRAMMAR[0m+ BNF‐like grammar to constrain which tokens may be selected when+ generating text. For example, the grammar:- root ::= "yes" | "no"+ [1mroot ::= "yes" | "no"[0m- will force the LLM to only output yes or no before exiting. This- is useful for shell scripts when the ----nnoo--ddiissppllaayy--pprroommpptt flag is- also supplied.+ will force the LLM to only output yes or no before exiting.+ This is useful for shell scripts when the [1m--no‐display‐prompt[0m+ flag is also supplied.- ----ggrraammmmaarr--ffiillee _F_N_A_M_E- File to read grammar from.+ [1m--grammar‐file [4m[22mFNAME[0m+ File to read grammar from.- ----pprroommpptt--ccaacchhee _F_N_A_M_E- File to cache prompt state for faster startup.+ [1m--fast [22mPut llamafile into fast math mode. This disables algorithms+ that reduce floating point rounding, e.g. Kahan summation, and+ certain functions like expf() will be vectorized but handle un‐+ derflows less gracefully. It’s unspecified whether llamafile+ runs in fast or precise math mode when neither flag is speci‐+ fied.- Default: none+ [1m--precise[0m+ Put llamafile into precise math mode. This enables algorithms+ that reduce floating point rounding, e.g. Kahan summation, and+ certain functions like expf() will always handle subnormals+ correctly. It’s unspecified whether llamafile runs in fast or+ precise math mode when neither flag is specified.- ----pprroommpptt--ccaacchhee--aallll- If specified, saves user input and generations to cache as well.- Not supported with ----iinntteerraaccttiivvee or other interactive options.+ [1m--trap [22mPut llamafile into math trapping mode. When floating point ex‐+ ceptions occur, such as NaNs, overflow, and divide by zero,+ llamafile will print a warning to the console. This warning+ will include a C++ backtrace the first time an exception is+ trapped. The op graph will also be dumped to a file, and lla‐+ mafile will report the specific op where the exception oc‐+ curred. This is useful for troubleshooting when reporting is‐+ sues. USing this feature will disable sandboxing. Math trap‐+ ping is only possible if your CPU supports it. That is gener‐+ ally the case on AMD64, however it’s less common on ARM64.- ----pprroommpptt--ccaacchhee--rroo- If specified, uses the prompt cache but does not update it.+ [1m--prompt‐cache [4m[22mFNAME[0m+ File to cache prompt state for faster startup.- ----rraannddoomm--pprroommpptt- Start with a randomized prompt.+ Default: none- ----iimmaaggee _I_M_A_G_E___F_I_L_E- Path to an image file. This should be used with multimodal mod‐- els. Alternatively, it's possible to embed an image directly- into the prompt instead; in which case, it must be base64 encoded- into an HTML img tag URL with the image/jpeg MIME type. See also- the ----mmmmpprroojj flag for supplying the vision model.+ [1m-fa [4m[22mFNAME[24m, [1m--flash‐attn[0m+ Enable Flash Attention. This is a mathematical shortcut that+ can speed up inference for certain models. This feature is+ still under active development.- --ii, ----iinntteerraaccttiivvee- Run the program in interactive mode, allowing users to engage in- real-time conversations or provide specific instructions to the- model.+ [1m--prompt‐cache‐all[0m+ If specified, saves user input and generations to cache as+ well. Not supported with [1m--interactive [22mor other interactive op‐+ tions.- ----iinntteerraaccttiivvee--ffiirrsstt- Run the program in interactive mode and immediately wait for user- input before starting the text generation.+ [1m--prompt‐cache‐ro[0m+ If specified, uses the prompt cache but does not update it.- --iinnss, ----iinnssttrruucctt- Run the program in instruction mode, which is specifically de‐- signed to work with Alpaca models that excel in completing tasks- based on user instructions.+ [1m--random‐prompt[0m+ Start with a randomized prompt.- Technical details: The user's input is internally prefixed with- the reverse prompt (or "### Instruction:" as the default), and- followed by "### Response:" (except if you just press Return- without any input, to keep generating a longer response).+ [1m--image [4m[22mIMAGE_FILE[0m+ Path to an image file. This should be used with multimodal mod‐+ els. Alternatively, it’s possible to embed an image directly+ into the prompt instead; in which case, it must be base64 en‐+ coded into an HTML img tag URL with the image/jpeg MIME type.+ See also the [1m--mmproj [22mflag for supplying the vision model.- By understanding and utilizing these interaction options, you can- create engaging and dynamic experiences with the LLaMA models,- tailoring the text generation process to your specific needs.+ [1m-i[22m, [1m--interactive[0m+ Run the program in interactive mode, allowing users to engage+ in real‐time conversations or provide specific instructions to+ the model.- --rr _P_R_O_M_P_T, ----rreevveerrssee--pprroommpptt _P_R_O_M_P_T- Specify one or multiple reverse prompts to pause text generation- and switch to interactive mode. For example, --rr _"_U_s_e_r_:_" can be- used to jump back into the conversation whenever it's the user's- turn to speak. This helps create a more interactive and conversa‐- tional experience. However, the reverse prompt doesn't work when- it ends with a space. To overcome this limitation, you can use- the ----iinn--pprreeffiixx flag to add a space or any other characters after- the reverse prompt.+ [1m--interactive‐first[0m+ Run the program in interactive mode and immediately wait for+ user input before starting the text generation.- ----ccoolloorr- Enable colorized output to differentiate visually distinguishing- between prompts, user input, and generated text.+ [1m-ins[22m, [1m--instruct[0m+ Run the program in instruction mode, which is specifically de‐+ signed to work with Alpaca models that excel in completing+ tasks based on user instructions.- ----nnoo--ddiissppllaayy--pprroommpptt, ----ssiilleenntt--pprroommpptt- Don't echo the prompt itself to standard output.+ Technical details: The user’s input is internally prefixed with+ the reverse prompt (or "### Instruction:" as the default), and+ followed by "### Response:" (except if you just press Return+ without any input, to keep generating a longer response).- ----mmuullttiilliinnee--iinnppuutt- Allows you to write or paste multiple lines without ending each- in '\'.+ By understanding and utilizing these interaction options, you+ can create engaging and dynamic experiences with the LLaMA mod‐+ els, tailoring the text generation process to your specific+ needs.-SSEERRVVEERR OOPPTTIIOONNSS- The following options may be specified when llllaammaaffiillee is running in- ----sseerrvveerr mode.+ [1m-r [4m[22mPROMPT[24m, [1m--reverse‐prompt [4m[22mPROMPT[0m+ Specify one or multiple reverse prompts to pause text genera‐+ tion and switch to interactive mode. For example, [1m-r [4m[22m"User:"[0m+ can be used to jump back into the conversation whenever it’s+ the user’s turn to speak. This helps create a more interactive+ and conversational experience. However, the reverse prompt+ doesn’t work when it ends with a space. To overcome this limi‐+ tation, you can use the [1m--in‐prefix [22mflag to add a space or any+ other characters after the reverse prompt.- ----ppoorrtt _P_O_R_T- Port to listen+ [1m--color[0m+ Enable colorized output to differentiate visually distinguish‐+ ing between prompts, user input, and generated text.- Default: 8080+ [1m--no‐display‐prompt[22m, [1m--silent‐prompt[0m+ Don’t echo the prompt itself to standard output.- ----hhoosstt _I_P_A_D_D_R- IP address to listen.+ [1m--keep [4m[22mN[0m+ Specifies number of tokens to keep from the initial prompt. The+ default is ‐1 which means all tokens.- Default: 127.0.0.1+ [1m--multiline‐input[0m+ Allows you to write or paste multiple lines without ending each+ in ’\’.- --ttoo _N, ----ttiimmeeoouutt _N- Server read/write timeout in seconds.+ [1m--cont‐batching[0m+ Enables continuous batching, a.k.a. dynamic batching. is ‐1+ which means all tokens.- Default: 600+[1mSERVER OPTIONS[0m+ The following options may be specified when [1mllamafile [22mis running in+ [1m--server [22mmode.- --nnpp _N, ----ppaarraalllleell _N- Number of slots for process requests.+ [1m--port [4m[22mPORT[0m+ Port to listen- Default: 1+ Default: 8080- --ccbb, ----ccoonntt--bbaattcchhiinngg- Enable continuous batching (a.k.a dynamic batching).+ [1m--host [4m[22mIPADDR[0m+ IP address to listen.- Default: disabled+ Default: 127.0.0.1- --ssppff _F_N_A_M_E, ----ssyysstteemm--pprroommpptt--ffiillee _F_N_A_M_E- Set a file to load a system prompt (initial prompt of all slots),- this is useful for chat applications.+ [1m-to [4m[22mN[24m, [1m--timeout [4m[22mN[0m+ Server read/write timeout in seconds.- --aa _A_L_I_A_S, ----aalliiaass _A_L_I_A_S- Set an alias for the model. This will be added as the _m_o_d_e_l field- in completion responses.+ Default: 600- ----ppaatthh _P_U_B_L_I_C___P_A_T_H- Path from which to serve static files.+ [1m-np [4m[22mN[24m, [1m--parallel [4m[22mN[0m+ Number of slots for process requests.- Default: _/_z_i_p_/_l_l_a_m_a_._c_p_p_/_s_e_r_v_e_r_/_p_u_b_l_i_c+ Default: 1- ----eemmbbeeddddiinngg- Enable embedding vector output.+ [1m-cb[22m, [1m--cont‐batching[0m+ Enable continuous batching (a.k.a dynamic batching).- Default: disabled+ Default: disabled- ----nnoobbrroowwsseerr- Do not attempt to open a web browser tab at startup.+ [1m-spf [4m[22mFNAME[24m, [1m--system‐prompt‐file [4m[22mFNAME[0m+ Set a file to load a system prompt (initial prompt of all+ slots), this is useful for chat applications.- --ggaann _N, ----ggrrpp--aattttnn--nn _N- Set the group attention factor to extend context size through- self-extend. The default value is _1 which means disabled. This- flag is used together with ----ggrrpp--aattttnn--ww.+ [1m-a [4m[22mALIAS[24m, [1m--alias [4m[22mALIAS[0m+ Set an alias for the model. This will be added as the [4mmodel[0m+ field in completion responses.- --ggaaww _N, ----ggrrpp--aattttnn--ww _N- Set the group attention width to extend context size through- self-extend. The default value is _5_1_2. This flag is used to‐- gether with ----ggrrpp--aattttnn--nn.+ [1m--path [4m[22mPUBLIC_PATH[0m+ Path from which to serve static files.-LLOOGG OOPPTTIIOONNSS- The following log options are available:+ Default: [4m/zip/llama.cpp/server/public[0m- --lldd _L_O_G_D_I_R, ----llooggddiirr _L_O_G_D_I_R- Path under which to save YAML logs (no logging if unset)-- ----lloogg--tteesstt- Run simple logging test+ [1m--embedding[0m+ Enable embedding vector output.- ----lloogg--ddiissaabbllee- Disable trace logs-- ----lloogg--eennaabbllee- Enable trace logs-- ----lloogg--ffiillee- Specify a log filename (without extension)-- ----lloogg--nneeww- Create a separate new log file on start. Each log file will have- unique name: _<_n_a_m_e_>_._<_I_D_>_._l_o_g-- ----lloogg--aappppeenndd- Don't truncate the old log file.--EEXXAAMMPPLLEESS- Here's an example of how to run llama.cpp's built-in HTTP server. This- example uses LLaVA v1.5-7B, a multimodal LLM that works with llama.cpp's- recently-added support for image inputs.-- llamafile \- -m llava-v1.5-7b-Q8_0.gguf \- --mmproj llava-v1.5-7b-mmproj-Q8_0.gguf \- --host 0.0.0.0-- Here's an example of how to generate code for a libc function using the- llama.cpp command line interface, utilizing WizardCoder-Python-13B- weights:-- llamafile \- -m wizardcoder-python-13b-v1.0.Q8_0.gguf --temp 0 -r '}\n' -r '```\n' \- -e -p '```c\nvoid *memcpy(void *dst, const void *src, size_t size) {\n'-- Here's a similar example that instead utilizes Mistral-7B-Instruct- weights for prose composition:-- llamafile \- -m mistral-7b-instruct-v0.2.Q5_K_M.gguf \- -p '[INST]Write a story about llamas[/INST]'-- Here's an example of how llamafile can be used as an interactive chatbot- that lets you query knowledge contained in training data:-- llamafile -m llama-65b-Q5_K.gguf -p '- The following is a conversation between a Researcher and their helpful AI- assistant Digital Athena which is a large language model trained on the- sum of human knowledge.- Researcher: Good morning.- Digital Athena: How can I help you today?- Researcher:' --interactive --color --batch_size 1024 --ctx_size 4096 \- --keep -1 --temp 0 --mirostat 2 --in-prefix ' ' --interactive-first \- --in-suffix 'Digital Athena:' --reverse-prompt 'Researcher:'-- Here's an example of how you can use llamafile to summarize HTML URLs:-- (- echo '[INST]Summarize the following text:'- links -codepage utf-8 \- -force-html \- -width 500 \- -dump https://www.poetryfoundation.org/poems/48860/the-raven |- sed 's/ */ /g'- echo '[/INST]'- ) | llamafile \- -m mistral-7b-instruct-v0.2.Q5_K_M.gguf \- -f /dev/stdin \- -c 0 \- --temp 0 \- -n 500 \- --no-display-prompt 2>/dev/null-- Here's how you can use llamafile to describe a jpg/png/gif/bmp image:-- llamafile --temp 0 \- --image lemurs.jpg \- -m llava-v1.5-7b-Q4_K.gguf \- --mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \- -e -p '### User: What do you see?\n### Assistant: ' \- --no-display-prompt 2>/dev/null-- If you wanted to write a script to rename all your image files, you could- use the following command to generate a safe filename:-- llamafile --temp 0 \- --image ~/Pictures/lemurs.jpg \- -m llava-v1.5-7b-Q4_K.gguf \- --mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \- --grammar 'root ::= [a-z]+ (" " [a-z]+)+' \- -e -p '### User: The image has...\n### Assistant: ' \- --no-display-prompt 2>/dev/null |- sed -e's/ /_/g' -e's/$/.jpg/'- three_baby_lemurs_on_the_back_of_an_adult_lemur.jpg-- Here's an example of how to make an API request to the OpenAI API compat‐- ible completions endpoint when your llllaammaaffiillee is running in the back‐- ground in ----sseerrvveerr mode.-- curl -s http://localhost:8080/v1/chat/completions \- -H "Content-Type: application/json" -d '{- "model": "gpt-3.5-turbo",- "stream": true,- "messages": [- {- "role": "system",- "content": "You are a poetic assistant."- },- {- "role": "user",- "content": "Compose a poem that explains FORTRAN."- }- ]- }' | python3 -c '- import json- import sys- json.dump(json.load(sys.stdin), sys.stdout, indent=2)- print()--PPRROOTTIIPP- The --nnggll _3_5 flag needs to be passed in order to use GPUs made by NVIDIA- and AMD. It's not enabled by default since it sometimes needs to be- tuned based on the system hardware and model architecture, in order to- achieve optimal performance, and avoid compromising a shared display.--SSEEEE AALLSSOO- llamafile-quantize(1), llamafile-perplexity(1), llava-quantize(1),- zipalign(1), unzip(1)--Mozilla Ocho January 1, 2024 Mozilla Ocho+ Default: disabled++ [1m--nobrowser[0m+ Do not attempt to open a web browser tab at startup.++ [1m-gan [4m[22mN[24m, [1m--grp‐attn‐n [4m[22mN[0m+ Set the group attention factor to extend context size through+ self‐extend. The default value is [4m1[24m which means disabled. This+ flag is used together with [1m--grp‐attn‐w[22m.++ [1m-gaw [4m[22mN[24m, [1m--grp‐attn‐w [4m[22mN[0m+ Set the group attention width to extend context size through+ self‐extend. The default value is [4m512[24m. This flag is used to‐+ gether with [1m--grp‐attn‐n[22m.++[1mLOG OPTIONS[0m+ The following log options are available:++ [1m-ld [4m[22mLOGDIR[24m, [1m--logdir [4m[22mLOGDIR[0m+ Path under which to save YAML logs (no logging if unset)++ [1m--log‐test[0m+ Run simple logging test++ [1m--log‐disable[0m+ Disable trace logs++ [1m--log‐enable[0m+ Enable trace logs++ [1m--log‐file[0m+ Specify a log filename (without extension)++ [1m--log‐new[0m+ Create a separate new log file on start. Each log file will+ have unique name: [4m<name>.<ID>.log[0m++ [1m--log‐append[0m+ Don’t truncate the old log file.++[1mEXAMPLES[0m+ Here’s an example of how to run llama.cpp’s built‐in HTTP server. This+ example uses LLaVA v1.5‐7B, a multimodal LLM that works with+ llama.cpp’s recently‐added support for image inputs.++ llamafile \+ ‐m llava‐v1.5‐7b‐Q8_0.gguf \+ ‐‐mmproj llava‐v1.5‐7b‐mmproj‐Q8_0.gguf \+ ‐‐host 0.0.0.0++ Here’s an example of how to generate code for a libc function using the+ llama.cpp command line interface, utilizing WizardCoder‐Python‐13B+ weights:++ llamafile \+ ‐m wizardcoder‐python‐13b‐v1.0.Q8_0.gguf ‐‐temp 0 ‐r ’}\n’ ‐r ’```\n’ \+ ‐e ‐p ’```c\nvoid *memcpy(void *dst, const void *src, size_t size) {\n’++ Here’s a similar example that instead utilizes Mistral‐7B‐Instruct+ weights for prose composition:++ llamafile \+ ‐m mistral‐7b‐instruct‐v0.2.Q5_K_M.gguf \+ ‐p ’[INST]Write a story about llamas[/INST]’++ Here’s an example of how llamafile can be used as an interactive chat‐+ bot that lets you query knowledge contained in training data:++ llamafile ‐m llama‐65b‐Q5_K.gguf ‐p ’+ The following is a conversation between a Researcher and their helpful AI+ assistant Digital Athena which is a large language model trained on the+ sum of human knowledge.+ Researcher: Good morning.+ Digital Athena: How can I help you today?+ Researcher:’ ‐‐interactive ‐‐color ‐‐batch_size 1024 ‐‐ctx_size 4096 \+ ‐‐keep ‐1 ‐‐temp 0 ‐‐mirostat 2 ‐‐in‐prefix ’ ’ ‐‐interactive‐first \+ ‐‐in‐suffix ’Digital Athena:’ ‐‐reverse‐prompt ’Researcher:’++ Here’s an example of how you can use llamafile to summarize HTML URLs:++ (+ echo ’[INST]Summarize the following text:’+ links ‐codepage utf‐8 \+ ‐force‐html \+ ‐width 500 \+ ‐dump https://www.poetryfoundation.org/poems/48860/the‐raven |+ sed ’s/ */ /g’+ echo ’[/INST]’+ ) | llamafile \+ ‐m mistral‐7b‐instruct‐v0.2.Q5_K_M.gguf \+ ‐f /dev/stdin \+ ‐c 0 \+ ‐‐temp 0 \+ ‐n 500 \+ ‐‐no‐display‐prompt 2>/dev/null++ Here’s how you can use llamafile to describe a jpg/png/gif/bmp image:++ llamafile ‐‐temp 0 \+ ‐‐image lemurs.jpg \+ ‐m llava‐v1.5‐7b‐Q4_K.gguf \+ ‐‐mmproj llava‐v1.5‐7b‐mmproj‐Q4_0.gguf \+ ‐e ‐p ’### User: What do you see?\n### Assistant: ’ \+ ‐‐no‐display‐prompt 2>/dev/null++ If you wanted to write a script to rename all your image files, you+ could use the following command to generate a safe filename:++ llamafile ‐‐temp 0 \+ ‐‐image ~/Pictures/lemurs.jpg \+ ‐m llava‐v1.5‐7b‐Q4_K.gguf \+ ‐‐mmproj llava‐v1.5‐7b‐mmproj‐Q4_0.gguf \+ ‐‐grammar ’root ::= [a‐z]+ (" " [a‐z]+)+’ \+ ‐e ‐p ’### User: The image has...\n### Assistant: ’ \+ ‐‐no‐display‐prompt 2>/dev/null |+ sed ‐e’s/ /_/g’ ‐e’s/$/.jpg/’+ three_baby_lemurs_on_the_back_of_an_adult_lemur.jpg++ Here’s an example of how to make an API request to the OpenAI API com‐+ patible completions endpoint when your [1mllamafile [22mis running in the+ background in [1m--server [22mmode.++ curl ‐s http://localhost:8080/v1/chat/completions \+ ‐H "Content‐Type: application/json" ‐d ’{+ "model": "gpt‐3.5‐turbo",+ "stream": true,+ "messages": [+ {+ "role": "system",+ "content": "You are a poetic assistant."+ },+ {+ "role": "user",+ "content": "Compose a poem that explains FORTRAN."+ }+ ]+ }’ | python3 ‐c ’+ import json+ import sys+ json.dump(json.load(sys.stdin), sys.stdout, indent=2)+ print()++[1mPROTIP[0m+ The [1m-ngl [4m[22m35[24m flag needs to be passed in order to use GPUs made by NVIDIA+ and AMD. It’s not enabled by default since it sometimes needs to be+ tuned based on the system hardware and model architecture, in order to+ achieve optimal performance, and avoid compromising a shared display.++[1mSEE ALSO[0m+ [4mllamafile‐quantize[24m(1), [4mllamafile‐perplexity[24m(1), [4mllava‐quantize[24m(1),+ [4mzipalign[24m(1), [4munzip[24m(1)++Mozilla Ocho January 1, 2024 [4mLLAMAFILE[24m(1)diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex 989295d8..a6114ef4 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -107,10 +107,19 @@ o/$(MODE)/llamafile/tinyblas_cpu_mixmul_arm82.o: private TARGET_ARCH += -Xaarch6################################################################################# testing+o/$(MODE)/llamafile/sgemm_sss_test: private LDFLAGS += -fopenmp+o/$(MODE)/llamafile/sgemm_sss_test.o: private CCFLAGS += -fopenmp+o/$(MODE)/llamafile/sgemm_matmul_test: private LDFLAGS += -fopenmp+o/$(MODE)/llamafile/sgemm_matmul_test.o: private CCFLAGS += -fopenmp+o/$(MODE)/llamafile/sgemm_sss_test: \o/$(MODE)/llamafile/sgemm_sss_test.o \o/$(MODE)/llama.cpp/llama.cpp.a+o/$(MODE)/llamafile/sgemm_matmul_test: \+ o/$(MODE)/llamafile/sgemm_matmul_test.o \+ o/$(MODE)/llama.cpp/llama.cpp.a+o/$(MODE)/llamafile/sgemm_vecdot_test: \o/$(MODE)/llamafile/sgemm_vecdot_test.o \o/$(MODE)/llama.cpp/llama.cpp.adiff --git a/llamafile/ansiblas.h b/llamafile/ansiblas.hnew file mode 100644index 00000000..c3a20358--- /dev/null+++ b/llamafile/ansiblas.h@@ -0,0 +1,145 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once++#include <cmath>+#include <unistd.h>++namespace {+namespace ansiBLAS {++static constexpr int KN = 8;++alignas(64) union Vector {+ double v[KN];+};++inline Vector load(const float *p) {+ Vector x;+ for (int i = 0; i < KN; ++i) {+ x.v[i] = p[i];+ }+ return x;+}++inline Vector madd(Vector x, Vector y, Vector s) {+ for (int i = 0; i < KN; ++i) {+ s.v[i] = fma(x.v[i], y.v[i], s.v[i]);+ }+ return s;+}++inline float hsum(Vector x) {+ double s = 0;+ for (int i = 0; i < KN; ++i) {+ s += x.v[i];+ }+ return s;+}++struct ansiBLAS {+ public:+ ansiBLAS(int k, const float *A, int lda, const float *B, int ldb, float *C, int ldc, int ith,+ int nth)+ : k(k), A(A), lda(lda), B(B), ldb(ldb), C(C), ldc(ldc), ith(ith), nth(nth) {+ }++ void matmul(int m, int n) {+ mnpack(0, m, 0, n);+ }++ private:+ void mnpack(int m0, int m, int n0, int n) {+ int mc, nc, mp, np;+ if (m - m0 <= 0 || n - n0 <= 0)+ return;+ if (m - m0 >= 4 && n - n0 >= 3) {+ mc = 4;+ nc = 3;+ gemm<4, 3>(m0, m, n0, n);+ } else if (n - n0 >= 4) {+ mc = 1;+ nc = 4;+ gemm<1, 4>(m0, m, n0, n);+ } else if (m - m0 >= 4) {+ mc = 4;+ nc = 1;+ gemm<4, 1>(m0, m, n0, n);+ } else {+ mc = 1;+ nc = 1;+ gemm<1, 1>(m0, m, n0, n);+ }+ mp = m0 + (m - m0) / mc * mc;+ np = n0 + (n - n0) / nc * nc;+ mnpack(mp, m, n0, np);+ mnpack(m0, m, np, n);+ }++ template <int RM, int RN>+ __target_clones("avx512f,fma") void gemm(int m0, int m, int n0, int n) {+ int ytiles = (m - m0) / RM;+ int xtiles = (n - n0) / RN;+ int tiles = xtiles * ytiles;+ int duty = (tiles + nth - 1) / nth;+ int start = duty * ith;+ int end = start + duty;+ if (end > tiles)+ end = tiles;+ for (int job = start; job < end; ++job) {+ int ii = m0 + job / xtiles * RM;+ int jj = n0 + job % xtiles * RN;+ Vector Cv[RN][RM] = {0};+ Vector Ce[RN][RM] = {0};+ for (int l = 0; l < k; l += KN)+ for (int j = 0; j < RN; ++j)+ for (int i = 0; i < RM; ++i)+ Cv[j][i] = madd(load(A + lda * (ii + i) + l), //+ load(B + ldb * (jj + j) + l), //+ Cv[j][i]);+ for (int j = 0; j < RN; ++j)+ for (int i = 0; i < RM; ++i)+ C[ldc * (jj + j) + (ii + i)] = hsum(Cv[j][i]);+ }+ }++ const int k;+ const float *const A;+ const int lda;+ const float *const B;+ const int ldb;+ float *const C;+ const int ldc;+ const int ith;+ const int nth;+};++void sgemm(int m, int n, int k, //+ const float *A, int lda, //+ const float *B, int ldb, //+ float *C, int ldc) {+ int nth = sysconf(_SC_NPROCESSORS_ONLN);+#pragma omp parallel for+ for (int ith = 0; ith < nth; ++ith) {+ ansiBLAS tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ }+}++} // namespace ansiBLAS+} // namespacediff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex 316953f9..e592d193 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -34,7 +34,9 @@ void llamafile_get_app_dir(char *, size_t);void llamafile_launch_browser(const char *);extern bool FLAG_trap;+extern bool FLAG_precise;extern bool FLAG_unsecure;+extern bool FLAG_precision_specified;#define LLAMAFILE_GPU_ERROR -2#define LLAMAFILE_GPU_DISABLE -1diff --git a/llamafile/precise.cpp b/llamafile/precise.cppnew file mode 100644index 00000000..49b9d96e--- /dev/null+++ b/llamafile/precise.cpp@@ -0,0 +1,21 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "llamafile.h"++bool FLAG_precise = true;+bool FLAG_precision_specified;diff --git a/llamafile/sgemm_matmul_test.cpp b/llamafile/sgemm_matmul_test.cppnew file mode 100644index 00000000..bd595734--- /dev/null+++ b/llamafile/sgemm_matmul_test.cpp@@ -0,0 +1,138 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "ansiblas.h"+#include "bench.h"+#include "float.h"+#include "llama.cpp/ggml.h"+#include "llamafile.h"+#include "macros.h"+#include "numba.h"+#include <cassert>+#include <cstdlib>++#define ITERATIONS 30+#define ALLOC(n) (float *)memalign(4096, sizeof(float) * (n))++void llamafile_sgemm_openmp(long m, long n, long k, const void *A, long lda, const void *B,+ long ldb, void *C, long ldc, int task, int Atype, int Btype, int Ctype,+ int precision) {+ int nth = sysconf(_SC_NPROCESSORS_ONLN);+#pragma omp parallel for+ for (int ith = 0; ith < nth; ++ith) {+ bool res = llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,+ Ctype, precision);+ assert(res);+ }+}++int test(void) {+ int m = 256;+ int n = 500;+ int k = 32768 * 2;+ int lda = ROUNDUP(k, 16);+ int ldb = ROUNDUP(k, 16);+ int ldc = ROUNDUP(m, 16);+ float *A = ALLOC(lda * m);+ float *B = ALLOC(ldb * n);+ float *C = ALLOC(ldc * n);+ float *G = ALLOC(ldc * n);+ broadcast(A, lda * m, NAN);+ broadcast(B, ldb * n, NAN);+ broadcast(C, ldc * n, NAN);+ broadcast(G, ldc * n, NAN);+ randomize(k, m, A, lda);+ randomize(k, n, B, ldb);++ BENCH(ansiBLAS::sgemm(m, n, k, A, lda, B, ldb, G, ldc));+ BENCH(llamafile_sgemm_openmp(m, n, k, A, lda, B, ldb, C, ldc, GGML_TASK_TYPE_COMPUTE,+ GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));++ double err_sum = 0;+ long long err_worst = 0;+ for (int i = 0; i < m; ++i)+ for (int j = 0; j < n; ++j) {+ float g = G[ldc * j + i];+ float c = C[ldc * j + i];+ if (flt::isnan(g)) {+ fprintf(stderr, "%s:%err: found nan in reference matrix: i=%err j=%err\n", __FILE__,+ __LINE__, i, j);+ return 3;+ }+ if (flt::isnan(c)) {+ fprintf(stderr, "%s:%err: found nan in output matrix: i=%err j=%err\n", __FILE__,+ __LINE__, i, j);+ return 4;+ }+ long long gi = flt::toint(g);+ long long ci = flt::toint(c);+ long long err = gi - ci;+ if (err < 0)+ err = -err;+ err_sum += err;+ if (err > err_worst)+ err_worst = err;+ }++ double err_avg = err_sum / (m * n);+ fprintf(stderr, "%9g ulp average\n", err_avg);+ fprintf(stderr, "%9lld ulp worst\n", err_worst);++ // using one accumulator+ // 87015 us gemm+ // 1859 us llamafile_sgemm+ // 94.7872 ulp average+ // 27610661 ulp worst++ // using three accumulators+ // 22.291 average ulp+ // 1566 worst ulp++ // using kahan summation+ // 40190 us gemm+ // 3028 us llamafile_sgemm+ // 2.14244 ulp average+ // 134 ulp worst++ // if (err_avg > 2.15)+ // return 5;+ // if (err_worst > 134)+ // return 6;++ free(G);+ free(C);+ free(B);+ free(A);++ return 0;+}++int main(int argc, char *argv[]) {+ int rc;++ llamafile_trapping_enabled(+1);++ printf("\nFLAG_precise = false;\n");+ FLAG_precise = false;+ if ((rc = test()))+ return rc;++ printf("\nFLAG_precise = true;\n");+ FLAG_precise = true;+ if ((rc = test()))+ return rc;+}diff --git a/llamafile/sgemm_sss_test.cpp b/llamafile/sgemm_sss_test.cppindex b7e49462..1d86dca9 100644--- a/llamafile/sgemm_sss_test.cpp+++ b/llamafile/sgemm_sss_test.cpp@@ -15,22 +15,35 @@// See the License for the specific language governing permissions and// limitations under the License.+#include "ansiblas.h"#include "bench.h"#include "float.h"-#include "gemm.h"#include "llama.cpp/ggml.h"#include "llamafile.h"#include "macros.h"#include "numba.h"+#include <cassert>#define ITERATIONS 5#define ALLOC(n) (float *)memalign(4096, sizeof(float) * (n))-int main(int argc, char *argv[]) {+void llamafile_sgemm_openmp(long m, long n, long k, const void *A, long lda, const void *B,+ long ldb, void *C, long ldc, int task, int Atype, int Btype, int Ctype,+ int precision) {+ int nth = sysconf(_SC_NPROCESSORS_ONLN);+#pragma omp parallel for+ for (int ith = 0; ith < nth; ++ith) {+ bool res = llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, ith, nth, task, Atype, Btype,+ Ctype, precision);+ assert(res);+ }+}++int test(void) {float tolerance = 2e-5;int m = 510;int n = 513;- int k = 512;+ int k = 512 * 8;int lda = ROUNDUP(k, 16);int ldb = ROUNDUP(k, 16);int ldc = ROUNDUP(m, 16);@@ -45,36 +58,60 @@ int main(int argc, char *argv[]) {randomize(k, m, A, lda);randomize(k, n, B, ldb);- BENCH(gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc));+ BENCH(ansiBLAS::sgemm(m, n, k, A, lda, B, ldb, G, ldc));BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));+ double err_sum = 0;+ long long err_worst = 0;for (int i = 0; i < m; ++i)for (int j = 0; j < n; ++j) {- auto g = G[ldc * j + i];- auto c = C[ldc * j + i];+ float g = G[ldc * j + i];+ float c = C[ldc * j + i];if (flt::isnan(g)) {- fprintf(stderr, "%s:%d: found nan in reference matrix: i=%d j=%d\n", __FILE__,+ fprintf(stderr, "%s:%err: found nan in reference matrix: i=%err j=%err\n", __FILE__,__LINE__, i, j);return 3;}if (flt::isnan(c)) {- fprintf(stderr, "%s:%d: found nan in output matrix: i=%d j=%d\n", __FILE__,+ fprintf(stderr, "%s:%err: found nan in output matrix: i=%err j=%err\n", __FILE__,__LINE__, i, j);return 4;}- auto d = g - c;- if (d < 0)- d = -d;- if (d > tolerance) {- fprintf(stderr, "%s:%d: difference exceeds tolerance: %g vs. %g (%g) i=%d j=%d\n",- __FILE__, __LINE__, g, c, d, i, j);- return 5;- }+ long long gi = flt::toint(g);+ long long ci = flt::toint(c);+ long long err = gi - ci;+ if (err < 0)+ err = -err;+ err_sum += err;+ if (err > err_worst)+ err_worst = err;}+ double err_avg = err_sum / (m * n);+ fprintf(stderr, "%12g ulp average\n", err_avg);+ fprintf(stderr, "%12lld ulp worst\n", err_worst);+free(G);free(C);free(B);free(A);++ return 0;+}++int main(int argc, char *argv[]) {+ int rc;++ llamafile_trapping_enabled(+1);++ printf("\nFLAG_precise = false;\n");+ FLAG_precise = false;+ if ((rc = test()))+ return rc;++ printf("\nFLAG_precise = true;\n");+ FLAG_precise = true;+ if ((rc = test()))+ return rc;}diff --git a/llamafile/sgemm_vecdot_test.cpp b/llamafile/sgemm_vecdot_test.cppindex 651e230c..f8374f88 100644--- a/llamafile/sgemm_vecdot_test.cpp+++ b/llamafile/sgemm_vecdot_test.cpp@@ -15,9 +15,9 @@// See the License for the specific language governing permissions and// limitations under the License.+#include "ansiblas.h"#include "bench.h"#include "float.h"-#include "gemm.h"#include "llama.cpp/ggml.h"#include "llamafile.h"#include "macros.h"@@ -26,7 +26,7 @@#define ITERATIONS 30#define ALLOC(n) (float *)memalign(4096, sizeof(float) * (n))-int main(int argc, char *argv[]) {+int test(void) {int m = 1025;int n = 1;int k = 32768;@@ -44,7 +44,7 @@ int main(int argc, char *argv[]) {randomize(k, m, A, lda);randomize(k, n, B, ldb);- BENCH(gemm(true, false, m, n, k, 1., A, lda, B, ldb, 0., G, ldc));+ BENCH(ansiBLAS::sgemm(m, n, k, A, lda, B, ldb, G, ldc));BENCH(llamafile_sgemm(m, n, k, A, lda, B, ldb, C, ldc, 0, 1, GGML_TASK_TYPE_COMPUTE,GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, GGML_PREC_DEFAULT));@@ -94,13 +94,29 @@ int main(int argc, char *argv[]) {// 2.14244 ulp average// 134 ulp worst- if (err_avg > 2.15)- return 5;- if (err_worst > 134)- return 6;+ // if (err_avg > 2.15)+ // return 5;+ // if (err_worst > 134)+ // return 6;free(G);free(C);free(B);free(A);++ return 0;+}++int main(int argc, char *argv[]) {+ int rc;++ printf("\nFLAG_precise = false;\n");+ FLAG_precise = false;+ if ((rc = test()))+ return rc;++ printf("\nFLAG_precise = true;\n");+ FLAG_precise = true;+ if ((rc = test()))+ return rc;}diff --git a/llamafile/tinyblas_cpu.h b/llamafile/tinyblas_cpu.hindex e4b4ec8c..be85892e 100644--- a/llamafile/tinyblas_cpu.h+++ b/llamafile/tinyblas_cpu.h@@ -431,152 +431,182 @@ class tinyBLAS {private:NOINLINE void mnpack(long m0, long m, long n0, long n) {long mc, nc, mp, np;- switch ((MIN(m - m0, 5) << 4) | MIN(n - n0, 5)) {+#if VECTOR_REGISTERS == 32- case 0x55:- case 0x54:- mc = 5;- nc = 4;- gemm<5, 4, false>(m0, m, n0, n);- break;- case 0x45:- mc = 4;- nc = 5;- gemm<4, 5, false>(m0, m, n0, n);- break;- case 0x44:- mc = 4;- nc = 4;- gemm<4, 4, false>(m0, m, n0, n);- break;- case 0x53:- mc = 5;- nc = 3;- gemm<5, 3, false>(m0, m, n0, n);- break;- case 0x35:- mc = 3;- nc = 5;- gemm<3, 5, false>(m0, m, n0, n);- break;- case 0x43:- mc = 4;- nc = 3;- gemm<4, 3, false>(m0, m, n0, n);- break;-#else- case 0x55:- case 0x54:- case 0x53:- case 0x45:- case 0x44:- case 0x43:- mc = 4;- nc = 3;- gemm<4, 3, false>(m0, m, n0, n);- break;- case 0x35:+ if (!FLAG_precise) {+ switch ((MIN(m - m0, 5) << 4) | MIN(n - n0, 5)) {+ case 0x55:+ mc = 5;+ nc = 5;+ gemm<5, 5, false>(m0, m, n0, n);+ break;+ case 0x54:+ case 0x53:+ case 0x52:+ case 0x45:+ case 0x44:+ case 0x43:+ case 0x42:+ case 0x35:+ case 0x34:+ case 0x33:+ case 0x32:+ case 0x25:+ case 0x24:+ case 0x23:+ case 0x22:+ mc = 2;+ nc = 2;+ gemm<2, 2, true>(m0, m, n0, n);+ break;+ case 0x51:+ case 0x41:+ case 0x31:+ case 0x21:+ mc = 2;+ nc = 1;+ gemm<2, 1, true>(m0, m, n0, n);+ break;+ case 0x15:+ case 0x14:+ case 0x13:+ case 0x12:+ mc = 1;+ nc = 2;+ gemm<1, 2, true>(m0, m, n0, n);+ break;+ case 0x11:+ mc = 1;+ nc = 1;+ gemm<1, 1, true>(m0, m, n0, n);+ break;+ default:+ return;+ }+ } else {+ switch ((MIN(m - m0, 4) << 4) | MIN(n - n0, 3)) {+ case 0x43:+ mc = 4;+ nc = 3;+ gemm<4, 3, true>(m0, m, n0, n);+ break;+ case 0x42:+ case 0x33:+ case 0x32:+ case 0x23:+ case 0x22:+ mc = 2;+ nc = 2;+ gemm<2, 2, true>(m0, m, n0, n);+ break;+ case 0x41:+ case 0x31:+ case 0x21:+ mc = 2;+ nc = 1;+ gemm<2, 1, true>(m0, m, n0, n);+ break;+ case 0x13:+ case 0x12:+ mc = 1;+ nc = 2;+ gemm<1, 2, true>(m0, m, n0, n);+ break;+ case 0x11:+ mc = 1;+ nc = 1;+ gemm<1, 1, true>(m0, m, n0, n);+ break;+ default:+ return;+ }+ }#endif- case 0x34:- mc = 3;- nc = 4;- gemm<3, 4, false>(m0, m, n0, n);- break;- case 0x52:- mc = 5;- nc = 2;- gemm<5, 2, false>(m0, m, n0, n);- break;- case 0x33:- mc = 3;- nc = 3;- gemm<3, 3, false>(m0, m, n0, n);- break;- case 0x25:- mc = 2;- nc = 5;- gemm<2, 5, false>(m0, m, n0, n);- break;- case 0x42:- mc = 4;- nc = 2;- gemm<4, 2, false>(m0, m, n0, n);- break;- case 0x24:- mc = 2;- nc = 4;- gemm<2, 4, false>(m0, m, n0, n);- break;- case 0x32:- mc = 3;- nc = 2;- gemm<3, 2, true>(m0, m, n0, n);- break;- case 0x23:- mc = 2;- nc = 3;- gemm<2, 3, true>(m0, m, n0, n);- break;- case 0x51:- mc = 5;- nc = 1;- gemm<5, 1, true>(m0, m, n0, n);- break;- case 0x41:- mc = 4;- nc = 1;- gemm<4, 1, true>(m0, m, n0, n);- break;- case 0x22:- mc = 2;- nc = 2;- gemm<2, 2, true>(m0, m, n0, n);- break;- case 0x15:- mc = 1;- nc = 5;- gemm<1, 5, true>(m0, m, n0, n);- break;- case 0x14:- mc = 1;- nc = 4;- gemm<1, 4, true>(m0, m, n0, n);- break;- case 0x31:- mc = 3;- nc = 1;- gemm<3, 1, true>(m0, m, n0, n);- break;- case 0x13:- mc = 1;- nc = 3;- gemm<1, 3, true>(m0, m, n0, n);- break;- case 0x21:- mc = 2;- nc = 1;- gemm<2, 1, true>(m0, m, n0, n);- break;- case 0x12:- mc = 1;- nc = 2;- gemm<1, 2, true>(m0, m, n0, n);- break;- case 0x11:- mc = 1;- nc = 1;- gemm<1, 1, true>(m0, m, n0, n);- break;- default:- return;++#if VECTOR_REGISTERS == 16+ if (!FLAG_precise) {+ switch ((MIN(m - m0, 4) << 4) | MIN(n - n0, 3)) {+ case 0x43:+ mc = 4;+ nc = 3;+ gemm<4, 3, false>(m0, m, n0, n);+ break;+ case 0x42:+ case 0x33:+ case 0x32:+ case 0x23:+ case 0x22:+ mc = 2;+ nc = 2;+ gemm<2, 2, true>(m0, m, n0, n);+ break;+ case 0x41:+ case 0x31:+ case 0x21:+ mc = 2;+ nc = 1;+ gemm<2, 1, true>(m0, m, n0, n);+ break;+ case 0x13:+ case 0x12:+ mc = 1;+ nc = 2;+ gemm<1, 2, true>(m0, m, n0, n);+ break;+ case 0x11:+ mc = 1;+ nc = 1;+ gemm<1, 1, true>(m0, m, n0, n);+ break;+ default:+ return;+ }+ } else {+ switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 2)) {+ case 0x32:+ mc = 3;+ nc = 2;+ gemm<3, 2, true>(m0, m, n0, n);+ break;+ case 0x23:+ mc = 2;+ nc = 3;+ gemm<2, 3, true>(m0, m, n0, n);+ break;+ case 0x22:+ mc = 2;+ nc = 2;+ gemm<2, 2, true>(m0, m, n0, n);+ break;+ case 0x31:+ case 0x21:+ mc = 2;+ nc = 1;+ gemm<2, 1, true>(m0, m, n0, n);+ break;+ case 0x12:+ mc = 1;+ nc = 2;+ gemm<1, 2, true>(m0, m, n0, n);+ break;+ case 0x11:+ mc = 1;+ nc = 1;+ gemm<1, 1, true>(m0, m, n0, n);+ break;+ default:+ return;+ }}+#endif+mp = m0 + (m - m0) / mc * mc;np = n0 + (n - n0) / nc * nc;mnpack(mp, m, n0, np);mnpack(m0, m, np, n);}- template <int RM, int RN, int KAHAN>+ template <int RM, int RN, int PRECISE>NOINLINE void gemm(long m0, long m, long n0, long n) {long ytiles = RM > 1 ? (m - m0) / RM : 1;long xtiles = RN > 1 ? (n - n0) / RN : 1;@@ -594,7 +624,7 @@ class tinyBLAS {for (long l = 0; l < k; l += KN)for (int j = 0; j < RN; ++j)for (int i = 0; i < RM; ++i)- if (KAHAN)+ if (PRECISE)Cv[j][i] = madder(load<V>(INDEX(A, lda, ii + i, l)), //load<V>(INDEX(B, ldb, jj + j, l)), //Cv[j][i], &Ce[j][i]);@@ -639,62 +669,84 @@ class tinyBLAS_Q0_ARM {private:NOINLINE void mnpack(long m0, long m, long n0, long n) {long mc, nc, mp, np;- switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 3)) {- case 0x33:- mc = 3;- nc = 3;- gemm<3, 3, false>(m0, m, n0, n);- break;- case 0x32:- mc = 3;- nc = 2;- gemm<3, 2, false>(m0, m, n0, n);- break;- case 0x23:- mc = 2;- nc = 3;- gemm<2, 3, false>(m0, m, n0, n);- break;- case 0x22:- mc = 2;- nc = 2;- gemm<2, 2, true>(m0, m, n0, n);- break;- case 0x31:- mc = 3;- nc = 1;- gemm<3, 1, true>(m0, m, n0, n);- break;- case 0x13:- mc = 1;- nc = 3;- gemm<1, 3, true>(m0, m, n0, n);- break;- case 0x21:- mc = 2;- nc = 1;- gemm<2, 1, true>(m0, m, n0, n);- break;- case 0x12:- mc = 1;- nc = 2;- gemm<1, 2, true>(m0, m, n0, n);- break;- case 0x11:- mc = 1;- nc = 1;- gemm<1, 1, true>(m0, m, n0, n);- break;- default:- return;++ if (!FLAG_precise || (!FLAG_precision_specified && sizeof(TB) == sizeof(block_q4_0))) {+ switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 3)) {+ case 0x33:+ mc = 3;+ nc = 3;+ gemm<3, 3, false>(m0, m, n0, n);+ break;+ case 0x32:+ case 0x23:+ case 0x22:+ mc = 2;+ nc = 2;+ gemm<2, 2, false>(m0, m, n0, n);+ break;+ case 0x31:+ case 0x21:+ mc = 2;+ nc = 1;+ gemm<2, 1, true>(m0, m, n0, n);+ break;+ case 0x13:+ case 0x12:+ mc = 1;+ nc = 2;+ gemm<1, 2, true>(m0, m, n0, n);+ break;+ case 0x11:+ mc = 1;+ nc = 1;+ gemm<1, 1, true>(m0, m, n0, n);+ break;+ default:+ return;+ }+ } else {+ switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 3)) {+ case 0x33:+ mc = 3;+ nc = 3;+ gemm<3, 3, true>(m0, m, n0, n);+ break;+ case 0x32:+ case 0x23:+ case 0x22:+ mc = 2;+ nc = 2;+ gemm<2, 2, true>(m0, m, n0, n);+ break;+ case 0x31:+ case 0x21:+ mc = 2;+ nc = 1;+ gemm<2, 1, true>(m0, m, n0, n);+ break;+ case 0x13:+ case 0x12:+ mc = 1;+ nc = 2;+ gemm<1, 2, true>(m0, m, n0, n);+ break;+ case 0x11:+ mc = 1;+ nc = 1;+ gemm<1, 1, true>(m0, m, n0, n);+ break;+ default:+ return;+ }}+mp = m0 + (m - m0) / mc * mc;np = n0 + (n - n0) / nc * nc;mnpack(mp, m, n0, np);mnpack(m0, m, np, n);}- template <int RM, int RN, int KAHAN>+ template <int RM, int RN, int PRECISE>NOINLINE void gemm(long m0, long m, long n0, long n) {long ytiles = RM > 1 ? (m - m0) / RM : 1;long xtiles = RN > 1 ? (n - n0) / RN : 1;@@ -718,7 +770,7 @@ class tinyBLAS_Q0_ARM {load_hi(INDEX(A, lda, ii + i, l)), load_hi(INDEX(B, ldb, jj + j, l))));float b = unhalf(INDEX(A, lda, ii + i, l)->d) *unhalf(INDEX(B, ldb, jj + j, l)->d);- if (KAHAN)+ if (PRECISE)Cv[j][i] = badder(a, b, Cv[j][i], &Ce[j][i]);elseCv[j][i] = vmlaq_n_f32(Cv[j][i], a, b);@@ -775,114 +827,146 @@ class tinyBLAS_Q0_AVX2 {private:void mnpack(long m0, long m, long n0, long n) {long mc, nc, mp, np;- switch ((MIN(m - m0, 4) << 4) | MIN(n - n0, 4)) {+#if VECTOR_REGISTERS == 32- case 0x44:- mc = 4;- nc = 4;- gemm<4, 4, false>(m0, m, n0, n);- break;- case 0x43:- mc = 4;- nc = 3;- gemm<4, 3, false>(m0, m, n0, n);- break;- case 0x34:- mc = 3;- nc = 4;- gemm<3, 4, false>(m0, m, n0, n);- break;- case 0x33:- mc = 3;- nc = 3;- gemm<3, 3, false>(m0, m, n0, n);- break;- case 0x42:- mc = 4;- nc = 2;- gemm<4, 2, false>(m0, m, n0, n);- break;- case 0x24:- mc = 2;- nc = 4;- gemm<2, 4, false>(m0, m, n0, n);- break;-#else- case 0x44:- case 0x43:- case 0x42:- mc = 4;- nc = 2;- gemm<4, 2, false>(m0, m, n0, n);- break;- case 0x34:- case 0x24:- mc = 2;- nc = 4;- gemm<2, 4, false>(m0, m, n0, n);- break;- case 0x33:+ if (!FLAG_precise || (!FLAG_precision_specified && sizeof(TB) == sizeof(block_q4_0))) {+ switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 3)) {+ case 0x33:+ mc = 3;+ nc = 3;+ gemm<3, 3, false>(m0, m, n0, n);+ break;+ case 0x32:+ case 0x23:+ case 0x22:+ mc = 2;+ nc = 2;+ gemm<2, 2, false>(m0, m, n0, n);+ break;+ case 0x31:+ case 0x21:+ mc = 2;+ nc = 1;+ gemm<2, 1, true>(m0, m, n0, n);+ break;+ case 0x13:+ case 0x12:+ mc = 1;+ nc = 2;+ gemm<1, 2, true>(m0, m, n0, n);+ break;+ case 0x11:+ mc = 1;+ nc = 1;+ gemm<1, 1, true>(m0, m, n0, n);+ break;+ default:+ return;+ }+ } else {+ switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 3)) {+ case 0x33:+ mc = 3;+ nc = 3;+ gemm<3, 3, true>(m0, m, n0, n);+ break;+ case 0x32:+ case 0x23:+ case 0x22:+ mc = 2;+ nc = 2;+ gemm<2, 2, true>(m0, m, n0, n);+ break;+ case 0x31:+ case 0x21:+ mc = 2;+ nc = 1;+ gemm<2, 1, true>(m0, m, n0, n);+ break;+ case 0x13:+ case 0x12:+ mc = 1;+ nc = 2;+ gemm<1, 2, true>(m0, m, n0, n);+ break;+ case 0x11:+ mc = 1;+ nc = 1;+ gemm<1, 1, true>(m0, m, n0, n);+ break;+ default:+ return;+ }+ }#endif- case 0x32:- mc = 3;- nc = 2;- gemm<3, 2, false>(m0, m, n0, n);- break;- case 0x23:- mc = 2;- nc = 3;- gemm<2, 3, false>(m0, m, n0, n);- break;- case 0x41:- mc = 4;- nc = 1;- gemm<4, 1, true>(m0, m, n0, n);- break;- case 0x22:- mc = 2;- nc = 2;- gemm<2, 2, true>(m0, m, n0, n);- break;- case 0x14:- mc = 1;- nc = 4;- gemm<1, 4, true>(m0, m, n0, n);- break;- case 0x31:- mc = 3;- nc = 1;- gemm<3, 1, true>(m0, m, n0, n);- break;- case 0x13:- mc = 1;- nc = 3;- gemm<1, 3, true>(m0, m, n0, n);- break;- case 0x21:- mc = 2;- nc = 1;- gemm<2, 1, true>(m0, m, n0, n);- break;- case 0x12:- mc = 1;- nc = 2;- gemm<1, 2, true>(m0, m, n0, n);- break;- case 0x11:- mc = 1;- nc = 1;- gemm<1, 1, true>(m0, m, n0, n);- break;- default:- return;++#if VECTOR_REGISTERS == 16+ if (!FLAG_precise || (!FLAG_precision_specified && sizeof(TB) == sizeof(block_q4_0))) {+ switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 2)) {+ case 0x32:+ mc = 3;+ nc = 2;+ gemm<3, 2, false>(m0, m, n0, n);+ break;+ case 0x23:+ mc = 2;+ nc = 3;+ gemm<2, 3, false>(m0, m, n0, n);+ break;+ case 0x22:+ mc = 2;+ nc = 2;+ gemm<2, 2, false>(m0, m, n0, n);+ break;+ case 0x31:+ case 0x21:+ mc = 2;+ nc = 1;+ gemm<2, 1, true>(m0, m, n0, n);+ break;+ case 0x12:+ mc = 1;+ nc = 2;+ gemm<1, 2, true>(m0, m, n0, n);+ break;+ case 0x11:+ mc = 1;+ nc = 1;+ gemm<1, 1, true>(m0, m, n0, n);+ break;+ default:+ return;+ }+ } else {+ switch ((MIN(m - m0, 2) << 4) | MIN(n - n0, 1)) {+ case 0x21:+ mc = 2;+ nc = 1;+ gemm<2, 1, true>(m0, m, n0, n);+ break;+ case 0x12:+ mc = 1;+ nc = 2;+ gemm<1, 2, true>(m0, m, n0, n);+ break;+ case 0x11:+ mc = 1;+ nc = 1;+ gemm<1, 1, true>(m0, m, n0, n);+ break;+ default:+ return;+ }}+#endif+mp = m0 + (m - m0) / mc * mc;np = n0 + (n - n0) / nc * nc;mnpack(mp, m, n0, np);mnpack(m0, m, np, n);}- template <int RM, int RN, int KAHAN>+ template <int RM, int RN, int PRECISE>NOINLINE void gemm(long m0, long m, long n0, long n) {long ytiles = RM > 1 ? (m - m0) / RM : 1;long xtiles = RN > 1 ? (n - n0) / RN : 1;@@ -906,7 +990,7 @@ class tinyBLAS_Q0_AVX2 {load(INDEX(A, lda, ii + i, l))),_mm256_sign_epi8(load(INDEX(B, ldb, jj + j, l)),load(INDEX(A, lda, ii + i, l))));- if (KAHAN)+ if (PRECISE)Cv[j][i] = madder(a, b, Cv[j][i], &Ce[j][i]);elseCv[j][i] = madd(a, b, Cv[j][i]);diff --git a/llamafile/tinyblas_cpu_sgemm.inc b/llamafile/tinyblas_cpu_sgemm.incindex 84520fd1..108ced2d 100644--- a/llamafile/tinyblas_cpu_sgemm.inc+++ b/llamafile/tinyblas_cpu_sgemm.inc@@ -45,7 +45,7 @@ namespace {template <typename TC>bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const void *B, long ldb,TC *C, long ldc, int ith, int nth, int task, int Atype, int Btype,- int precision) {+ int Ctype, int precision) {switch (Atype) {@@ -316,13 +316,13 @@ bool llamafile_sgemm(long m, long n, long k, const void *A, long lda, const voidswitch (Ctype) {case GGML_TYPE_F32:return llamafile_sgemm_impl(m, n, k, A, lda, B, ldb, (float *)C, ldc, ith, nth, task, Atype,- Btype, precision);+ Btype, Ctype, precision);case GGML_TYPE_BF16:return llamafile_sgemm_impl(m, n, k, A, lda, B, ldb, (ggml_bf16_t *)C, ldc, ith, nth, task,- Atype, Btype, precision);+ Atype, Btype, Ctype, precision);case GGML_TYPE_F16:return llamafile_sgemm_impl(m, n, k, A, lda, B, ldb, (ggml_fp16_t *)C, ldc, ith, nth, task,- Atype, Btype, precision);+ Atype, Btype, Ctype, precision);default:return NOT_SUPPORTED;}diff --git a/llamafile/tinyblas_mnpack.py b/llamafile/tinyblas_mnpack.pynew file mode 100644index 00000000..191c065c--- /dev/null+++ b/llamafile/tinyblas_mnpack.py@@ -0,0 +1,128 @@+# Copyright 2024 Mozilla Foundation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# # tinyBLAS+# MAX_M = 5+# MAX_N = 5+# EDGE_M = 2+# EDGE_N = 2+# OVERHEAD = 1++# tinyBLAS_Q0+MAX_M = 3+MAX_N = 3+EDGE_M = 2+EDGE_N = 2+OVERHEAD = 8++def doit(VECTOR_REGISTERS, PRECISE):+ # choose tile size that exploits all vector registers++ specified = {}+ precise = set()++ for mc in range(1, MAX_M + 1):+ for nc in range(1, MAX_N + 1):+ accumulators = mc * nc+ memory_loads = min(mc, nc)++ if (mc > EDGE_M or nc > EDGE_N) and (mc == 1 or nc == 1):+ continue++ # always use precise if there's enough registers+ v = accumulators * 2 + memory_loads + OVERHEAD+ if v <= VECTOR_REGISTERS:+ if mc % 8 == 0:+ v += 2+ if mc % 4 == 0:+ v += 1+ specified[mc, nc] = v+ if PRECISE or (mc == 1 or nc == 1):+ precise.add((mc, nc))+ elif not PRECISE:+ v = accumulators + memory_loads + OVERHEAD+ if v <= VECTOR_REGISTERS:+ if mc % 8 == 0:+ v += 2+ if mc % 4 == 0:+ v += 1+ specified[mc, nc] = v++ # if PRECISE:+ # v = accumulators * 2 + memory_loads + OVERHEAD+ # if v <= VECTOR_REGISTERS:+ # if mc % 8 == 0:+ # v += 2+ # if mc % 4 == 0:+ # v += 1+ # specified[mc, nc] = v+ # precise.add((mc, nc))+ # else:+ # v = accumulators + memory_loads + OVERHEAD+ # if v <= VECTOR_REGISTERS:+ # if mc % 8 == 0:+ # v += 2+ # if mc % 4 == 0:+ # v += 1+ # specified[mc, nc] = v++ # generate code for handling biggest tile (e.g. 5x5)+ # generate code for handling edge tiles (i.e. <=2x2)+ # avoid long compile times to generate tiles between+ (best_mc, best_nc), best_v = list(sorted(specified.items(), key=lambda s: s[1]))[-1]+ for (mc, nc), v in list(specified.items()):+ if v < best_v and (mc > EDGE_M or nc > EDGE_N):+ del specified[mc, nc]++ print("switch ((MIN(m - m0, %d) << 4) | MIN(n - n0, %d)) {" % (best_mc, best_nc))++ a = []+ for (mc, nc), v in specified.items():+ s = ""+ s += "case 0x%x%x:\n" % (mc, nc)+ s += " mc = %d;\n" % (mc)+ s += " nc = %d;\n" % (nc)+ s += " gemm<%d, %d, %s>(m0, m, n0, n);\n" % (mc, nc, "true" if (mc, nc) in precise else "false")+ s += " break;"+ a.append((v, mc, nc, s, []))++ a = list(reversed(sorted(a)))++ for mc in range(1, best_mc + 1):+ for nc in range(1, best_nc + 1):+ if (mc, nc) in specified:+ continue+ for v_, mc_, nc_, s_, extra in a:+ if mc_ <= mc and nc_ <= nc:+ extra.append("case 0x%x%x:" % (mc, nc))+ break++ for v, mc, nc, s, extra in a:+ for e in list(reversed(sorted(extra))):+ print(e)+ print(s)++ print("default:")+ print(" return;")+ print("}")++for VECTOR_REGISTERS in (32, 16):+ print()+ print("#if VECTOR_REGISTERS == %d" % (VECTOR_REGISTERS))+ print("if (!FLAG_precise) {")+ doit(VECTOR_REGISTERS, False)+ print("} else {")+ doit(VECTOR_REGISTERS, True)+ print("}")+ print("#endif")