Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add --cli flag and show unified man page on --help
Justine Tunney5cf6e76647d0
diff --git a/Makefile b/Makefileindex 1b71047c..69543a1a 100644--- a/Makefile+++ b/Makefile@@ -23,7 +23,6 @@ o/$(MODE)/: o/$(MODE)/llama.cpp o/$(MODE)/llamafile.PHONY: installinstall: llamafile/zipalign.1 \llama.cpp/main/main.1 \- llama.cpp/server/server.1 \llama.cpp/quantize/quantize.1 \llama.cpp/perplexity/perplexity.1 \llama.cpp/llava/llava-quantize.1 \@@ -42,7 +41,6 @@ install: llamafile/zipalign.1 \mkdir -p $(PREFIX)/share/man/man1$(INSTALL) -m 0644 llamafile/zipalign.1 $(PREFIX)/share/man/man1/zipalign.1$(INSTALL) -m 0644 llama.cpp/main/main.1 $(PREFIX)/share/man/man1/llamafile.1- $(INSTALL) -m 0644 llama.cpp/server/server.1 $(PREFIX)/share/man/man1/llamafile-server.1$(INSTALL) -m 0644 llama.cpp/quantize/quantize.1 $(PREFIX)/share/man/man1/llamafile-quantize.1$(INSTALL) -m 0644 llama.cpp/perplexity/perplexity.1 $(PREFIX)/share/man/man1/llamafile-perplexity.1$(INSTALL) -m 0644 llama.cpp/llava/llava-quantize.1 $(PREFIX)/share/man/man1/llava-quantize.1diff --git a/README.md b/README.mdindex 486fb65b..38f176a2 100644--- a/README.md+++ b/README.md@@ -275,9 +275,8 @@ weights:```shllamafile \-m wizardcoder-python-13b-v1.0.Q8_0.gguf \- --temp 0 -e \- -r '```\n' \- -p '```c\nvoid *memcpy(char *dst, const char *src, size_t size) {\n'+ --temp 0 -r '}\n' -r '```\n' \+ -e -p '```c\nvoid *memcpy(void *dst, const void *src, size_t size) {\n'```Here's a similar example that instead utilizes Mistral-7B-Instruct@@ -314,7 +313,7 @@ Here's an example of how you can use llamafile to summarize HTML URLs:sed 's/ */ /g'echo '[/INST]') | llamafile \- -m mistral-7b-instruct-v0.1.Q4_K_M.gguf \+ -m mistral-7b-instruct-v0.2.Q5_K_M.gguf \-f /dev/stdin \-c 0 \--temp 0 \@@ -329,7 +328,7 @@ llamafile --temp 0 \--image ~/Pictures/lemurs.jpg \-m llava-v1.5-7b-Q4_K.gguf \--mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \- -p $'### User: What do you see?\n### Assistant: ' \+ -e -p '### User: What do you see?\n### Assistant: ' \--silent-prompt 2>/dev/null```@@ -341,11 +340,11 @@ wanted to write a script to rename all your image files, you could say:```shllamafile --temp 0 \- --image ~/Pictures/lemurs.jpg \+ --image lemurs.jpg \-m llava-v1.5-7b-Q4_K.gguf \--mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \--grammar 'root ::= [a-z]+ (" " [a-z]+)+' \- -p $'### User: What do you see?\n### Assistant: ' \+ -e -p '### User: What do you see?\n### Assistant: ' \--silent-prompt 2>/dev/null |sed -e's/ /_/g' -e's/$/.jpg/'a_baby_monkey_on_the_back_of_a_mother.jpgdiff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex 42b1ed1c..7381efa2 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -154,7 +154,9 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {std::replace(arg.begin(), arg.end(), '_', '-');}- if (arg == "-s" || arg == "--seed") {+ if (arg == "--cli") {+ // do nothing+ } else if (arg == "-s" || arg == "--seed") {if (++i >= argc) {invalid_param = true;break;diff --git a/llama.cpp/main/BUILD.mk b/llama.cpp/main/BUILD.mkindex 64a53dde..6e255e06 100644--- a/llama.cpp/main/BUILD.mk+++ b/llama.cpp/main/BUILD.mk@@ -13,8 +13,13 @@ o/$(MODE)/llama.cpp/main/main: \o/$(MODE)/llama.cpp/server/server.a \o/$(MODE)/llama.cpp/llava/llava.a \o/$(MODE)/llama.cpp/llama.cpp.a \+ o/$(MODE)/llama.cpp/main/main.1.asc.zip.o \$(LLAMA_CPP_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)+llama.cpp/main/main.1.asc: llama.cpp/main/main.1+ -man $< >[email protected] && mv -f [email protected] $@+ @rm -f [email protected]+.PHONY: o/$(MODE)/llama.cpp/maino/$(MODE)/llama.cpp/main: \o/$(MODE)/llama.cpp/main/maindiff --git a/llama.cpp/main/main.1 b/llama.cpp/main/main.1index ef371543..b08e3b6b 100644--- a/llama.cpp/main/main.1+++ b/llama.cpp/main/main.1@@ -1,20 +1,22 @@-.Dd December 5, 2023-.Dt llamafile 1-.Os+.Dd January 1, 2024+.Dt LLAMAFILE 1+.Os Mozilla Ocho.Sh NAME.Nm llamafile.Nd large language model runner.Sh SYNOPSIS.Nm+.Op Fl Fl server.Op flags....Fl m Ar model.gguf-.Fl p Ar prompt+.Op Fl Fl mmproj Ar vision.gguf.Nm+.Op Fl Fl cli.Op flags...-.Op Fl Fl server.Fl m Ar model.gguf-.Op Fl Fl mmproj Ar vision.gguf+.Fl p Ar prompt.Nm+.Op Fl Fl cli.Op flags....Fl m Ar model.gguf.Fl Fl mmproj Ar vision.gguf@@ -37,84 +39,41 @@ Text/image summarization and analysis.Sh OPTIONSThe following options are available:.Bl -tag -width indent+.It Fl Fl version+Print version and exit..It Fl h , Fl Fl helpShow help message and exit.-.It Fl Fl version-Print llamafile version and exit.+.It Fl Fl cli+Puts program in command line interface mode. This flag is implied when a+prompt is supplied using either the+.Fl p+or+.Fl f+flags..It Fl Fl server-.Xr llamafile-server 1-is launched via the-.Nm llamafile-command when the+Puts program in server mode. This will launch an HTTP server on a local+port. This server has both a web UI and an OpenAI API compatible+completions endpoint. When the server is run on a desk system, a tab+browser tab will be launched automatically that displays the web UI.+This.Fl Fl server-flag is passed. The server flag is implied by default if no prompt is-specified, i.e. neither the+flag is implied if no prompt is specified, i.e. neither the.Fl por.Fl fflags are passed..It Fl m Ar FNAME , Fl Fl model Ar FNAME-Model path in GGUF file format.-.Pp-Default: models/7B/ggml-model-f16.gguf-.It Fl p Ar STRING , Fl Fl prompt Ar STRING-Prompt to start text generation. Your LLM works by auto-completing this-text. For example:-.Bd -literal -offset indent-.Nm Fl m Li model.gguf Fl p Li \[dq]four score and\[dq]-.Ed-.Pp-Stands a pretty good chance of printing Lincoln's Gettysburg Address.-Prompts can take on a structured format too. Depending on how your model-was trained, it may specify in its docs an instruction notation. With-some models that might be:-.Bd -literal -offset indent-.Nm Fl p Li \[dq][INST]Summarize this: $(cat file)[/INST]\[dq]-.Ed-.Pp-In most cases, simply colons and newlines will work too:-.Bd -literal -offset indent-.Nm Fl e Fl p Li \[dq]User: What is best in life?\[rs]nAssistant:\[dq]-.Ed+Model path in the GGUF file format..Pp+Default:+.Pa models/7B/ggml-model-f16.gguf.It Fl Fl mmproj Ar FNAME-Specifies path of vision model in the GGUF file format. If this flag is supplied, then the+Specifies path of the LLaVA vision model in the GGUF file format. If+this flag is supplied, then the.Fl Fl modeland.Fl Fl imageflags should also be supplied.-.It Fl Fl image Ar IMAGE_FILE-Path to an image file. This should be used with multimodal models.-Alternatively, it's possible to embed an image directly into the prompt-instead; in which case, it must be base64 encoded into an HTML img tag-URL with the image/jpeg MIME type. See also the-.Fl Fl mmproj-flag for supplying the vision model.-.It Fl Fl grammar Ar GRAMMAR-BNF-like grammar to constrain which tokens may be selected when-generating text. For example, the grammar:-.Bd -literal -offset indent-root ::= "yes" | "no"-.Ed-.Pp-will force the LLM to only output yes or no before exiting. This is-useful for shell scripts when the-.Fl Fl silent-prompt-flag is also supplied.-.It Fl i , Fl Fl interactive-Run in interactive mode.-.It Fl Fl interactive-first-Run in interactive mode and wait for input right away.-.It Fl ins , Fl Fl instruct-Run in instruction mode (use with Alpaca models).-.It Fl Fl multiline-input-Allows you to write or paste multiple lines without ending each in '\[rs]'.-.It Fl r Ar PROMPT , Fl Fl reverse-prompt Ar PROMPT-Halt generation at-.Ar PROMPT-and return control in interactive mode (can be specified more than once for multiple prompts).-.It Fl Fl color-colorise output to distinguish prompt and user input from generations.It Fl s Ar SEED , Fl Fl seed Ar SEEDRandom Number Generator (RNG) seed. A random seed is used if this isless than zero.@@ -129,22 +88,6 @@ Number of threads to use during batch and prompt processing..PpDefault: Same as.Fl Fl threads-.It Fl f Ar FNAME , Fl Fl file Ar FNAME-Prompt file to start generation.-.It Fl e , Fl Fl escape-Process prompt escapes sequences (\[rs]n, \[rs]r, \[rs]t, \[rs]\[aa], \[rs]\[dq], \[rs]\[rs])-.It Fl Fl prompt-cache Ar FNAME-File to cache prompt state for faster startup.-.Pp-Default: none-.It Fl Fl prompt-cache-all-If specified, saves user input and generations to cache as well. Not supported with-.Fl Fl interactive-or other interactive options.-.It Fl Fl prompt-cache-ro-If specified, uses the prompt cache but does not update it.-.It Fl Fl random-prompt-Start with a randomized prompt..It Fl Fl in-prefix-bosPrefix BOS to user inputs, preceding the.Fl Fl in-prefix@@ -297,9 +240,8 @@ to decrease likelihood of token.It Fl md Ar FNAME , Fl Fl model-draft Ar FNAMEDraft model for speculative decoding..Pp-Default: models/7B/ggml-model-f16.gguf-.It Fl Fl grammar-file Ar FNAME-File to read grammar from.+Default:+.Pa models/7B/ggml-model-f16.gguf.It Fl Fl cfg-negative-prompt Ar PROMPTNegative prompt to use for guidance...Pp@@ -393,10 +335,6 @@ Max number of chunks to process..El.PpDefault: -1-.It Fl np Ar N , Fl Fl parallel Ar N-Number of parallel sequences to decode.-.Pp-Default: 1.It Fl ns Ar N , Fl Fl sequences Ar NNumber of sequences to decode..Pp@@ -409,10 +347,6 @@ Default: 0.5Speculative decoding split probability..PpDefault: 0.1-.It Fl cb , Fl Fl cont-batching-Enable continuous batching (a.k.a dynamic batching).-.Pp-Default: disabled.It Fl Fl mlockForce system to keep model in RAM rather than swapping or compressing..It Fl Fl no-mmap@@ -431,8 +365,6 @@ The GPU to use for scratch and small tensors.Use cuBLAS instead of custom mul_mat_q CUDA kernels. Not recommended since this is both slower and uses more VRAM..It Fl Fl verbose-promptPrint prompt before generation.-.It Fl Fl silent-prompt-Don't echo the prompt itself to standard output..It Fl Fl simple-ioUse basic IO for better compatibility in subprocesses and limited consoles..It Fl Fl lora Ar FNAME@@ -461,6 +393,128 @@ KV cache data type for K..It Fl ctv Ar TYPE , Fl Fl cache-type-v Ar TYPEKV cache data type for V..El+.Sh CLI OPTIONS+The following options may be specified when+.Nm+is running in+.Fl Fl cli+mode.+.Bl -tag -width indent+.It Fl e , Fl Fl escape+Process prompt escapes sequences (\[rs]n, \[rs]r, \[rs]t, \[rs]\[aa], \[rs]\[dq], \[rs]\[rs])+.It Fl p Ar STRING , Fl Fl prompt Ar STRING+Prompt to start text generation. Your LLM works by auto-completing this+text. For example:+.Pp+.Dl "llamafile -m model.gguf -p \[dq]four score and\[dq]"+.Pp+Stands a pretty good chance of printing Lincoln's Gettysburg Address.+Prompts can take on a structured format too. Depending on how your model+was trained, it may specify in its docs an instruction notation. With+some models that might be:+.Pp+.Dl "llamafile -p \[dq][INST]Summarize this: $(cat file)[/INST]\[dq]"+.Pp+In most cases, simply colons and newlines will work too:+.Pp+.Dl "llamafile -e -p \[dq]User: What is best in life?\[rs]nAssistant:\[dq]"+.Pp+.It Fl f Ar FNAME , Fl Fl file Ar FNAME+Prompt file to start generation.+.It Fl Fl grammar Ar GRAMMAR+BNF-like grammar to constrain which tokens may be selected when+generating text. For example, the grammar:+.Pp+.Dl "root ::= \[dq]yes\[dq] | \[dq]no\[dq]"+.Pp+will force the LLM to only output yes or no before exiting. This is+useful for shell scripts when the+.It Fl Fl grammar-file Ar FNAME+File to read grammar from.+.It Fl Fl prompt-cache Ar FNAME+File to cache prompt state for faster startup.+.Pp+Default: none+.It Fl Fl prompt-cache-all+If specified, saves user input and generations to cache as well. Not supported with+.Fl Fl interactive+or other interactive options.+.It Fl Fl prompt-cache-ro+If specified, uses the prompt cache but does not update it.+.It Fl Fl random-prompt+Start with a randomized prompt.+.It Fl Fl image Ar IMAGE_FILE+Path to an image file. This should be used with multimodal models.+Alternatively, it's possible to embed an image directly into the prompt+instead; in which case, it must be base64 encoded into an HTML img tag+URL with the image/jpeg MIME type. See also the+.Fl Fl mmproj+flag for supplying the vision model.+.It Fl i , Fl Fl interactive+Run in interactive mode.+.It Fl Fl interactive-first+Run in interactive mode and wait for input right away.+.It Fl ins , Fl Fl instruct+Run in instruction mode (use with Alpaca models).+.Fl Fl silent-prompt+flag is also supplied.+.It Fl r Ar PROMPT , Fl Fl reverse-prompt Ar PROMPT+Halt generation at+.Ar PROMPT+and return control in interactive mode (can be specified more than once for multiple prompts).+.It Fl Fl color+Colorise output to distinguish prompt and user input from generations.+.It Fl Fl silent-prompt+Don't echo the prompt itself to standard output.+.It Fl Fl multiline-input+Allows you to write or paste multiple lines without ending each in '\[rs]'.+.El+.Sh SERVER OPTIONS+The following options may be specified when+.Nm+is running in+.Fl Fl server+mode.+.Bl -tag -width indent+.It Fl Fl port Ar PORT+Port to listen+.Pp+Default: 8080+.It Fl Fl host Ar IPADDR+IP address to listen.+.Pp+Default: 127.0.0.1+.It Fl to Ar N , Fl Fl timeout Ar N+Server read/write timeout in seconds.+.Pp+Default: 600+.It Fl np Ar N , Fl Fl parallel Ar N+Number of slots for process requests.+.Pp+Default: 1+.It Fl cb , Fl Fl cont-batching+Enable continuous batching (a.k.a dynamic batching).+.Pp+Default: disabled+.It Fl spf Ar FNAME , Fl Fl system-prompt-file Ar FNAME+Set a file to load a system prompt (initial prompt of all slots), this+is useful for chat applications.+.It Fl a Ar ALIAS , Fl Fl alias Ar ALIAS+Set an alias for the model. This will be added as the+.Ar model+field in completion responses.+.It Fl Fl path Ar PUBLIC_PATH+Path from which to serve static files.+.Pp+Default:+.Pa /zip/llama.cpp/server/public+.It Fl Fl embedding+Enable embedding vector output.+.Pp+Default: disabled+.It Fl Fl nobrowser+Do not attempt to open a web browser tab at startup.+.El.Sh LOG OPTIONSThe following log options are available:.Bl -tag -width indent@@ -475,12 +529,136 @@ Enable trace logs.It Fl Fl log-fileSpecify a log filename (without extension).It Fl Fl log-new-Create a separate new log file on start. Each log file will have unique name: "<name>.<ID>.log"+Create a separate new log file on start. Each log file will have unique name:+.Fa <name>.<ID>.log.It Fl Fl log-appendDon't truncate the old log file..El+.Sh EXAMPLES+Here's an example of how to run llama.cpp's built-in HTTP server. This+example uses LLaVA v1.5-7B, a multimodal LLM that works with llama.cpp's+recently-added support for image inputs.+.Bd -literal -offset indent+llamafile \[rs]+ -m llava-v1.5-7b-Q8_0.gguf \[rs]+ --mmproj llava-v1.5-7b-mmproj-Q8_0.gguf \[rs]+ --host 0.0.0.0+.Ed+.Pp+Here's an example of how to generate code for a libc function using the+llama.cpp command line interface, utilizing WizardCoder-Python-13B+weights:+.Bd -literal -offset indent+llamafile \[rs]+ -m wizardcoder-python-13b-v1.0.Q8_0.gguf \[rs]+ --temp 0 -r '}\[rs]n' -r '\`\`\`\[rs]n' \[rs]+ -e -p '\`\`\`c\[rs]nvoid *memcpy(void *dst, const void *src, size_t size) {\[rs]n'+.Ed+.Pp+Here's a similar example that instead utilizes Mistral-7B-Instruct+weights for prose composition:+.Bd -literal -offset indent+llamafile \[rs]+ -m mistral-7b-instruct-v0.2.Q5_K_M.gguf \[rs]+ -p '[INST]Write a story about llamas[/INST]'+.Ed+.Pp+Here's an example of how llamafile can be used as an interactive chatbot+that lets you query knowledge contained in training data:+.Bd -literal -offset indent+llamafile -m llama-65b-Q5_K.gguf -p '+The following is a conversation between a Researcher and their helpful AI+assistant Digital Athena which is a large language model trained on the+sum of human knowledge.+Researcher: Good morning.+Digital Athena: How can I help you today?+Researcher:' --interactive --color --batch_size 1024 --ctx_size 4096 \[rs]+--keep -1 --temp 0 --mirostat 2 --in-prefix ' ' --interactive-first \[rs]+--in-suffix 'Digital Athena:' --reverse-prompt 'Researcher:'+.Ed+.Pp+Here's an example of how you can use llamafile to summarize HTML URLs:+.Bd -literal -offset indent+(+ echo '[INST]Summarize the following text:'+ links -codepage utf-8 \[rs]+ -force-html \[rs]+ -width 500 \[rs]+ -dump https://www.poetryfoundation.org/poems/48860/the-raven |+ sed 's/ */ /g'+ echo '[/INST]'+) | llamafile \[rs]+ -m mistral-7b-instruct-v0.2.Q5_K_M.gguf \[rs]+ -f /dev/stdin \[rs]+ -c 0 \[rs]+ --temp 0 \[rs]+ -n 500 \[rs]+ --silent-prompt 2>/dev/null+.Ed+.Pp+Here's how you can use llamafile to describe a jpg/png/gif/bmp image:+.Bd -literal -offset indent+llamafile --temp 0 \[rs]+ --image lemurs.jpg \[rs]+ -m llava-v1.5-7b-Q4_K.gguf \[rs]+ --mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \[rs]+ -e -p '### User: What do you see?\[rs]n### Assistant: ' \[rs]+ --silent-prompt 2>/dev/null+.Ed+.Pp+If you wanted to write a script to rename all your image files, you+could use the following command to generate a safe filename:+.Bd -literal -offset indent+llamafile --temp 0 \[rs]+ --image ~/Pictures/lemurs.jpg \[rs]+ -m llava-v1.5-7b-Q4_K.gguf \[rs]+ --mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \[rs]+ --grammar 'root ::= [a-z]+ (" " [a-z]+)+' \[rs]+ -e -p '### User: What do you see?\[rs]n### Assistant: ' \[rs]+ --silent-prompt 2>/dev/null |+ sed -e's/ /_/g' -e's/$/.jpg/'+a_baby_monkey_on_the_back_of_a_mother.jpg+.Ed+.Pp+Here's an example of how to make an API request to the OpenAI API+compatible completions endpoint when your+.Nm+is running in the background in+.Fl Fl server+mode.+.Bd -literal -offset indent+curl -s http://localhost:8080/v1/chat/completions \[rs]+ -H "Content-Type: application/json" -d '{+ "model": "gpt-3.5-turbo",+ "stream": true,+ "messages": [+ {+ "role": "system",+ "content": "You are a poetic assistant."+ },+ {+ "role": "user",+ "content": "Compose a poem that explains FORTRAN."+ }+ ]+}' | python3 -c '+import json+import sys+json.dump(json.load(sys.stdin), sys.stdout, indent=2)+print()+'+.Ed+.Sh BUGS+Your HTTP server is great for local hosting, but it would need much more+work to be a production worthy component of a public-facing service. For+example, C++ exceptions caused by JSON parsing errors will make it abort+and print a backtrace.+.Sh PROTIP+NVIDIA users need to pass the+.Fl ngl Ar 35+flag to enable GPU acceleration. It's not enabled by default since it+sometimes needs to be tuned for system hardware and model architecture..Sh SEE ALSO-.Xr llamafile-server 1 ,.Xr llamafile-quantize 1 ,.Xr llamafile-perplexity 1 ,.Xr llava-quantize 1 ,diff --git a/llama.cpp/main/main.1.asc b/llama.cpp/main/main.1.ascnew file mode 100644index 00000000..682ac9ae--- /dev/null+++ b/llama.cpp/main/main.1.asc@@ -0,0 +1,666 @@+LLAMAFILE(1) General Commands Manual LLAMAFILE(1)++NNAAMMEE+ llllaammaaffiillee - large language model runner++SSYYNNOOPPSSIISS+ llllaammaaffiillee [----sseerrvveerr] [flags...] --mm _m_o_d_e_l_._g_g_u_f [----mmmmpprroojj _v_i_s_i_o_n_._g_g_u_f]+ llllaammaaffiillee [----ccllii] [flags...] --mm _m_o_d_e_l_._g_g_u_f --pp _p_r_o_m_p_t+ llllaammaaffiillee [----ccllii] [flags...]+ --mm _m_o_d_e_l_._g_g_u_f ----mmmmpprroojj _v_i_s_i_o_n_._g_g_u_f ----iimmaaggee _g_r_a_p_h_i_c_._p_n_g+ --pp _p_r_o_m_p_t++DDEESSCCRRIIPPTTIIOONN+ llllaammaaffiillee is a large language model tool. It has use cases such as:++ -- Code completion+ -- Prose composition+ -- Chatbot that passes the Turing test+ -- Text/image summarization and analysis++OOPPTTIIOONNSS+ The following options are available:++ ----vveerrssiioonn+ Print version and exit.++ --hh, ----hheellpp+ Show help message and exit.++ ----ccllii Puts program in command line interface mode. This flag is implied+ when a prompt is supplied using either the --pp or --ff flags.++ ----sseerrvveerr+ Puts program in server mode. This will launch an HTTP server on a+ local port. This server has both a web UI and an OpenAI API+ compatible completions endpoint. When the server is run on a desk+ system, a tab browser tab will be launched automatically that+ displays the web UI. This ----sseerrvveerr flag is implied if no prompt+ is specified, i.e. neither the --pp or --ff flags are passed.++ --mm _F_N_A_M_E, ----mmooddeell _F_N_A_M_E+ Model path in the GGUF file format.++ Default: _m_o_d_e_l_s_/_7_B_/_g_g_m_l_-_m_o_d_e_l_-_f_1_6_._g_g_u_f++ ----mmmmpprroojj _F_N_A_M_E+ Specifies path of the LLaVA vision model in the GGUF file format.+ If this flag is supplied, then the ----mmooddeell and ----iimmaaggee flags+ should also be supplied.++ --ss _S_E_E_D, ----sseeeedd _S_E_E_D+ Random Number Generator (RNG) seed. A random seed is used if this+ is less than zero.++ Default: -1++ --tt _N, ----tthhrreeaaddss _N+ Number of threads to use during generation.++ Default: $(nproc)/2++ --ttbb _N, ----tthhrreeaaddss--bbaattcchh _N+ Number of threads to use during batch and prompt processing.++ Default: Same as ----tthhrreeaaddss++ ----iinn--pprreeffiixx--bbooss+ Prefix BOS to user inputs, preceding the ----iinn--pprreeffiixx string.++ ----iinn--pprreeffiixx _S_T_R_I_N_G+ String to prefix user inputs with.++ Default: empty++ ----iinn--ssuuffffiixx _S_T_R_I_N_G+ String to suffix after user inputs with.++ Default: empty++ --nn _N, ----nn--pprreeddiicctt _N+ Number of tokens to predict.++ -- -1 = infinity+ -- -2 = until context filled++ Default: -1++ --cc _N, ----ccttxx--ssiizzee _N+ Size of the prompt context.++ -- 0 = loaded from model++ Default: 512++ --bb _N, ----bbaattcchh--ssiizzee _N+ Batch size for prompt processing.++ Default: 512++ ----ttoopp--kk _N+ Top-k sampling.++ -- 0 = disabled++ Default: 40++ ----ttoopp--pp _N+ Top-p sampling.++ -- 1.0 = disabled++ Default: 0.9++ ----mmiinn--pp _N+ Min-p sampling.++ -- 0.0 = disabled++ Default: 0.1++ ----ttffss _N+ Tail free sampling, parameter z.++ -- 1.0 = disabled++ Default: 1.0++ ----ttyyppiiccaall _N+ Locally typical sampling, parameter p.++ -- 1.0 = disabled++ Default: 1.0++ ----rreeppeeaatt--llaasstt--nn _N+ Last n tokens to consider for penalize.++ -- 0 = disabled+ -- -1 = ctx_size++ Default: 64++ ----rreeppeeaatt--ppeennaallttyy _N+ Penalize repeat sequence of tokens.++ -- 1.0 = disabled++ Default: 1.1++ ----pprreesseennccee--ppeennaallttyy _N+ Repeat alpha presence penalty.++ -- 0.0 = disabled++ Default: 0.0++ ----ffrreeqquueennccyy--ppeennaallttyy _N+ Repeat alpha frequency penalty.++ -- 0.0 = disabled++ Default: 0.0++ ----mmiirroossttaatt _N+ Use Mirostat sampling. Top K, Nucleus, Tail Free and Locally+ Typical samplers are ignored if used..++ -- 0 = disabled+ -- 1 = Mirostat+ -- 2 = Mirostat 2.0++ Default: 0++ ----mmiirroossttaatt--llrr _N+ Mirostat learning rate, parameter eta.++ Default: 0.1++ ----mmiirroossttaatt--eenntt _N+ Mirostat target entropy, parameter tau.++ Default: 5.0++ --ll _T_O_K_E_N___I_D_(_+_/_-_)_B_I_A_S, ----llooggiitt--bbiiaass _T_O_K_E_N___I_D_(_+_/_-_)_B_I_A_S+ Modifies the likelihood of token appearing in the completion,+ i.e. ----llooggiitt--bbiiaass _1_5_0_4_3_+_1 to increase likelihood of token+ _' _H_e_l_l_o_', or ----llooggiitt--bbiiaass _1_5_0_4_3_-_1 to decrease likelihood of token+ _' _H_e_l_l_o_'.++ --mmdd _F_N_A_M_E, ----mmooddeell--ddrraafftt _F_N_A_M_E+ Draft model for speculative decoding.++ Default: _m_o_d_e_l_s_/_7_B_/_g_g_m_l_-_m_o_d_e_l_-_f_1_6_._g_g_u_f++ ----ccffgg--nneeggaattiivvee--pprroommpptt _P_R_O_M_P_T+ Negative prompt to use for guidance..++ Default: empty++ ----ccffgg--nneeggaattiivvee--pprroommpptt--ffiillee _F_N_A_M_E+ Negative prompt file to use for guidance.++ Default: empty++ ----ccffgg--ssccaallee _N+ Strength of guidance.++ -- 1.0 = disable++ Default: 1.0++ ----rrooppee--ssccaalliinngg _{_n_o_n_e_,_l_i_n_e_a_r_,_y_a_r_n_}+ RoPE frequency scaling method, defaults to linear unless+ specified by the model++ ----rrooppee--ssccaallee _N+ RoPE context scaling factor, expands context by a factor of N++ ----rrooppee--ffrreeqq--bbaassee _N+ RoPE base frequency, used by NTK-aware scaling.++ Default: loaded from model++ ----rrooppee--ffrreeqq--ssccaallee _N+ RoPE frequency scaling factor, expands context by a factor of 1/N++ ----yyaarrnn--oorriigg--ccttxx _N+ YaRN: original context size of model.++ Default: 0 = model training context size++ ----yyaarrnn--eexxtt--ffaaccttoorr _N+ YaRN: extrapolation mix factor.++ -- 0.0 = full interpolation++ Default: 1.0++ ----yyaarrnn--aattttnn--ffaaccttoorr _N+ YaRN: scale sqrt(t) or attention magnitude.++ Default: 1.0++ ----yyaarrnn--bbeettaa--ssllooww _N+ YaRN: high correction dim or alpha.++ Default: 1.0++ ----yyaarrnn--bbeettaa--ffaasstt _N+ YaRN: low correction dim or beta.++ Default: 32.0++ ----iiggnnoorree--eeooss+ Ignore end of stream token and continue generating (implies+ ----llooggiitt--bbiiaass _2_-_i_n_f)++ ----nnoo--ppeennaalliizzee--nnll+ Do not penalize newline token.++ ----tteemmpp _N+ Temperature.++ Default: 0.8++ ----llooggiittss--aallll+ Return logits for all tokens in the batch.++ Default: disabled++ ----hheellllaasswwaagg+ Compute HellaSwag score over random tasks from datafile supplied+ with -f++ ----hheellllaasswwaagg--ttaasskkss _N+ Number of tasks to use when computing the HellaSwag score.++ Default: 400++ ----kkeeeepp _N+ Number of tokens to keep from the initial prompt.++ -- -1 = all++ Default: 0++ ----ddrraafftt _N+ Number of tokens to draft for speculative decoding.++ Default: 16++ ----cchhuunnkkss _N+ Max number of chunks to process.++ -- -1 = all++ Default: -1++ --nnss _N, ----sseeqquueenncceess _N+ Number of sequences to decode.++ Default: 1++ --ppaa _N, ----pp--aacccceepptt _N+ speculative decoding accept probability.++ Default: 0.5++ --ppss _N, ----pp--sspplliitt _N+ Speculative decoding split probability.++ Default: 0.1++ ----mmlloocckk+ Force system to keep model in RAM rather than swapping or+ compressing.++ ----nnoo--mmmmaapp+ Do not memory-map model (slower load but may reduce pageouts if+ not using mlock).++ ----nnuummaa Attempt optimizations that help on some NUMA systems if run+ without this previously, it is recommended to drop the system+ page cache before using this. See+ https://github.com/ggerganov/llama.cpp/issues/1437.++ --nnggll _N, ----nn--ggppuu--llaayyeerrss _N+ Number of layers to store in VRAM.++ --nngglldd _N, ----nn--ggppuu--llaayyeerrss--ddrraafftt _N+ Number of layers to store in VRAM for the draft model.++ --ttss _S_P_L_I_T, ----tteennssoorr--sspplliitt _S_P_L_I_T+ How to split tensors across multiple GPUs, comma-separated list+ of proportions, e.g. 3,1++ --mmgg _i, ----mmaaiinn--ggppuu _i+ The GPU to use for scratch and small tensors.++ --nnoommmmqq, ----nnoo--mmuull--mmaatt--qq+ Use cuBLAS instead of custom mul_mat_q CUDA kernels. Not+ recommended since this is both slower and uses more VRAM.++ ----vveerrbboossee--pprroommpptt+ Print prompt before generation.++ ----ssiimmppllee--iioo+ Use basic IO for better compatibility in subprocesses and limited+ consoles.++ ----lloorraa _F_N_A_M_E+ Apply LoRA adapter (implies ----nnoo--mmmmaapp)++ ----lloorraa--ssccaalleedd _F_N_A_M_E _S+ Apply LoRA adapter with user defined scaling S (implies+ ----nnoo--mmmmaapp)++ ----lloorraa--bbaassee _F_N_A_M_E+ Optional model to use as a base for the layers modified by the+ LoRA adapter++ ----uunnsseeccuurree+ Disables pledge() sandboxing on Linux and OpenBSD.++ ----ssaammpplleerrss+ Samplers that will be used for generation in the order, separated+ by semicolon, for example: top_k;tfs;typical;top_p;min_p;temp++ ----ssaammpplleerrss--sseeqq+ Simplified sequence for samplers that will be used.++ --ccmmll, ----cchhaattmmll+ Run in chatml mode (use with ChatML-compatible models)++ --ddkkvvcc, ----dduummpp--kkvv--ccaacchhee+ Verbose print of the KV cache.++ --nnkkvvoo, ----nnoo--kkvv--ooffffllooaadd+ Disable KV offload.++ --ccttkk _T_Y_P_E, ----ccaacchhee--ttyyppee--kk _T_Y_P_E+ KV cache data type for K.++ --ccttvv _T_Y_P_E, ----ccaacchhee--ttyyppee--vv _T_Y_P_E+ KV cache data type for V.++CCLLII OOPPTTIIOONNSS+ The following options may be specified when llllaammaaffiillee is running in ----ccllii+ mode.++ --ee, ----eessccaappee+ Process prompt escapes sequences (\n, \r, \t, \', \", \\)++ --pp _S_T_R_I_N_G, ----pprroommpptt _S_T_R_I_N_G+ Prompt to start text generation. Your LLM works by auto-+ completing this text. For example:++ llamafile -m model.gguf -p "four score and"++ Stands a pretty good chance of printing Lincoln's Gettysburg+ Address. Prompts can take on a structured format too. Depending+ on how your model was trained, it may specify in its docs an+ instruction notation. With some models that might be:++ llamafile -p "[INST]Summarize this: $(cat file)[/INST]"++ In most cases, simply colons and newlines will work too:++ llamafile -e -p "User: What is best in life?\nAssistant:"++ --ff _F_N_A_M_E, ----ffiillee _F_N_A_M_E+ Prompt file to start generation.++ ----ggrraammmmaarr _G_R_A_M_M_A_R+ BNF-like grammar to constrain which tokens may be selected when+ generating text. For example, the grammar:++ root ::= "yes" | "no"++ will force the LLM to only output yes or no before exiting. This+ is useful for shell scripts when the++ ----ggrraammmmaarr--ffiillee _F_N_A_M_E+ File to read grammar from.++ ----pprroommpptt--ccaacchhee _F_N_A_M_E+ File to cache prompt state for faster startup.++ Default: none++ ----pprroommpptt--ccaacchhee--aallll+ If specified, saves user input and generations to cache as well.+ Not supported with ----iinntteerraaccttiivvee or other interactive options.++ ----pprroommpptt--ccaacchhee--rroo+ If specified, uses the prompt cache but does not update it.++ ----rraannddoomm--pprroommpptt+ Start with a randomized prompt.++ ----iimmaaggee _I_M_A_G_E___F_I_L_E+ Path to an image file. This should be used with multimodal+ models. Alternatively, it's possible to embed an image directly+ into the prompt instead; in which case, it must be base64 encoded+ into an HTML img tag URL with the image/jpeg MIME type. See also+ the ----mmmmpprroojj flag for supplying the vision model.++ --ii, ----iinntteerraaccttiivvee+ Run in interactive mode.++ ----iinntteerraaccttiivvee--ffiirrsstt+ Run in interactive mode and wait for input right away.++ --iinnss, ----iinnssttrruucctt+ Run in instruction mode (use with Alpaca models).+ ----ssiilleenntt--pprroommpptt flag is also supplied.++ --rr _P_R_O_M_P_T, ----rreevveerrssee--pprroommpptt _P_R_O_M_P_T+ Halt generation at _P_R_O_M_P_T and return control in interactive mode+ (can be specified more than once for multiple prompts).++ ----ccoolloorr+ Colorise output to distinguish prompt and user input from+ generations.++ ----ssiilleenntt--pprroommpptt+ Don't echo the prompt itself to standard output.++ ----mmuullttiilliinnee--iinnppuutt+ Allows you to write or paste multiple lines without ending each+ in '\'.++SSEERRVVEERR OOPPTTIIOONNSS+ The following options may be specified when llllaammaaffiillee is running in+ ----sseerrvveerr mode.++ ----ppoorrtt _P_O_R_T+ Port to listen++ Default: 8080++ ----hhoosstt _I_P_A_D_D_R+ IP address to listen.++ Default: 127.0.0.1++ --ttoo _N, ----ttiimmeeoouutt _N+ Server read/write timeout in seconds.++ Default: 600++ --nnpp _N, ----ppaarraalllleell _N+ Number of slots for process requests.++ Default: 1++ --ccbb, ----ccoonntt--bbaattcchhiinngg+ Enable continuous batching (a.k.a dynamic batching).++ Default: disabled++ --ssppff _F_N_A_M_E, ----ssyysstteemm--pprroommpptt--ffiillee _F_N_A_M_E+ Set a file to load a system prompt (initial prompt of all slots),+ this is useful for chat applications.++ --aa _A_L_I_A_S, ----aalliiaass _A_L_I_A_S+ Set an alias for the model. This will be added as the _m_o_d_e_l field+ in completion responses.++ ----ppaatthh _P_U_B_L_I_C___P_A_T_H+ Path from which to serve static files.++ Default: _/_z_i_p_/_l_l_a_m_a_._c_p_p_/_s_e_r_v_e_r_/_p_u_b_l_i_c++ ----eemmbbeeddddiinngg+ Enable embedding vector output.++ Default: disabled++ ----nnoobbrroowwsseerr+ Do not attempt to open a web browser tab at startup.++LLOOGG OOPPTTIIOONNSS+ The following log options are available:++ --lldd _L_O_G_D_I_R, ----llooggddiirr _L_O_G_D_I_R+ Path under which to save YAML logs (no logging if unset)++ ----lloogg--tteesstt+ Run simple logging test++ ----lloogg--ddiissaabbllee+ Disable trace logs++ ----lloogg--eennaabbllee+ Enable trace logs++ ----lloogg--ffiillee+ Specify a log filename (without extension)++ ----lloogg--nneeww+ Create a separate new log file on start. Each log file will have+ unique name: _<_n_a_m_e_>_._<_I_D_>_._l_o_g++ ----lloogg--aappppeenndd+ Don't truncate the old log file.++EEXXAAMMPPLLEESS+ Here's an example of how to run llama.cpp's built-in HTTP server. This+ example uses LLaVA v1.5-7B, a multimodal LLM that works with llama.cpp's+ recently-added support for image inputs.++ llamafile \+ -m llava-v1.5-7b-Q8_0.gguf \+ --mmproj llava-v1.5-7b-mmproj-Q8_0.gguf \+ --host 0.0.0.0++ Here's an example of how to generate code for a libc function using the+ llama.cpp command line interface, utilizing WizardCoder-Python-13B+ weights:++ llamafile \+ -m wizardcoder-python-13b-v1.0.Q8_0.gguf \+ --temp 0 -r '}\n' -r '```\n' \+ -e -p '```c\nvoid *memcpy(void *dst, const void *src, size_t size) {\n'++ Here's a similar example that instead utilizes Mistral-7B-Instruct+ weights for prose composition:++ llamafile \+ -m mistral-7b-instruct-v0.2.Q5_K_M.gguf \+ -p '[INST]Write a story about llamas[/INST]'++ Here's an example of how llamafile can be used as an interactive chatbot+ that lets you query knowledge contained in training data:++ llamafile -m llama-65b-Q5_K.gguf -p '+ The following is a conversation between a Researcher and their helpful AI+ assistant Digital Athena which is a large language model trained on the+ sum of human knowledge.+ Researcher: Good morning.+ Digital Athena: How can I help you today?+ Researcher:' --interactive --color --batch_size 1024 --ctx_size 4096 \+ --keep -1 --temp 0 --mirostat 2 --in-prefix ' ' --interactive-first \+ --in-suffix 'Digital Athena:' --reverse-prompt 'Researcher:'++ Here's an example of how you can use llamafile to summarize HTML URLs:++ (+ echo '[INST]Summarize the following text:'+ links -codepage utf-8 \+ -force-html \+ -width 500 \+ -dump https://www.poetryfoundation.org/poems/48860/the-raven |+ sed 's/ */ /g'+ echo '[/INST]'+ ) | llamafile \+ -m mistral-7b-instruct-v0.2.Q5_K_M.gguf \+ -f /dev/stdin \+ -c 0 \+ --temp 0 \+ -n 500 \+ --silent-prompt 2>/dev/null++ Here's how you can use llamafile to describe a jpg/png/gif/bmp image:++ llamafile --temp 0 \+ --image lemurs.jpg \+ -m llava-v1.5-7b-Q4_K.gguf \+ --mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \+ -e -p '### User: What do you see?\n### Assistant: ' \+ --silent-prompt 2>/dev/null++ If you wanted to write a script to rename all your image files, you could+ use the following command to generate a safe filename:++ llamafile --temp 0 \+ --image ~/Pictures/lemurs.jpg \+ -m llava-v1.5-7b-Q4_K.gguf \+ --mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \+ --grammar 'root ::= [a-z]+ (" " [a-z]+)+' \+ -e -p '### User: What do you see?\n### Assistant: ' \+ --silent-prompt 2>/dev/null |+ sed -e's/ /_/g' -e's/$/.jpg/'+ a_baby_monkey_on_the_back_of_a_mother.jpg++ Here's an example of how to make an API request to the OpenAI API+ compatible completions endpoint when your llllaammaaffiillee is running in the+ background in ----sseerrvveerr mode.++ curl -s http://localhost:8080/v1/chat/completions \+ -H "Content-Type: application/json" -d '{+ "model": "gpt-3.5-turbo",+ "stream": true,+ "messages": [+ {+ "role": "system",+ "content": "You are a poetic assistant."+ },+ {+ "role": "user",+ "content": "Compose a poem that explains FORTRAN."+ }+ ]+ }' | python3 -c '+ import json+ import sys+ json.dump(json.load(sys.stdin), sys.stdout, indent=2)+ print()++BBUUGGSS+ Your HTTP server is great for local hosting, but it would need much more+ work to be a production worthy component of a public-facing service. For+ example, C++ exceptions caused by JSON parsing errors will make it abort+ and print a backtrace.++PPRROOTTIIPP+ NVIDIA users need to pass the --nnggll _3_5 flag to enable GPU acceleration.+ It's not enabled by default since it sometimes needs to be tuned for+ system hardware and model architecture.++SSEEEE AALLSSOO+ llamafile-quantize(1), llamafile-perplexity(1), llava-quantize(1),+ zipalign(1), unzip(1)++Mozilla Ocho January 1, 2024 Mozilla Ochodiff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cppindex 641b318b..09f03904 100644--- a/llama.cpp/main/main.cpp+++ b/llama.cpp/main/main.cpp@@ -109,22 +109,27 @@ static bool has_argument(int argc, char ** argv, const char * arg) {}int main(int argc, char ** argv) {+ llamafile_init();+ llamafile_check_cpu();+ ShowCrashReports();+ LoadZipArgs(&argc, &argv);if (has_argument(argc, argv, "--version")) {printf("llamafile v" LLAMAFILE_VERSION_STRING "\n");- exit(0);+ return 0;}- if (has_argument(argc, argv, "--server") ||- (!has_argument(argc, argv, "-p") &&- !has_argument(argc, argv, "-f"))) {- return server_cli(argc, argv);+ if (has_argument(argc, argv, "--help")) {+ llamafile_help("/zip/llama.cpp/main/main.1.asc");}- llamafile_init();- llamafile_check_cpu();- ShowCrashReports();- LoadZipArgs(&argc, &argv);+ if (!has_argument(argc, argv, "--cli") &&+ (has_argument(argc, argv, "--server") ||+ (!has_argument(argc, argv, "-p") &&+ !has_argument(argc, argv, "-f") &&+ !has_argument(argc, argv, "--random-prompt")))) {+ return server_cli(argc, argv);+ }if (has_argument(argc, argv, "--mmproj")) {return llava_cli(argc, argv);diff --git a/llama.cpp/quantize/quantize.1 b/llama.cpp/quantize/quantize.1index 103ec208..50068d0e 100644--- a/llama.cpp/quantize/quantize.1+++ b/llama.cpp/quantize/quantize.1@@ -84,7 +84,6 @@ COPY Only copy tensors, no quantizing..El.Sh SEE ALSO.Xr llamafile 1 ,-.Xr llamafile-server 1 ,.Xr llamafile-perplexity 1 ,.Xr llava-quantize 1 ,.Xr zipalign 1 ,diff --git a/llama.cpp/server/server.1 b/llama.cpp/server/server.1deleted file mode 100644index 69d18ca7..00000000--- a/llama.cpp/server/server.1+++ /dev/null@@ -1,121 +0,0 @@-.Dd December 5, 2023-.Dt llamafile-server 1-.Os-.Sh NAME-.Nm llamafile-server-.Nd large language model server-.Sh SYNOPSIS-.Nm llamafile-.Fl Fl server-.Op flags...-.Op Fl m Ar weights.gguf-.Sh DESCRIPTION-.Nm-launches an HTTP server and browser tab with a web user interface for-the llama.cpp software. The UI lets you chat with your LLM and upload-images it can analyze. Furthermore, this server provides an OpenAI API-compatible completions endpoint.-.Pp-.Nm llamafile-server-is launched via the-.Nm llamafile-command when the-.Fl Fl server-flag is passed. The server flag is implied by default if no prompt is-specified, i.e. neither the-.Fl p-or-.Fl f-flags are passed.-.Sh OPTIONS-The following options are available:-.Bl -tag -width indent-.It Fl h , Fl Fl help-Show this help message and exit.-.It Fl v , Fl Fl verbose-Verbose output (default: disabled)-.It Fl t Ar N , Fl Fl threads Ar N-Number of threads to use during computation (default: nproc/2)-.It Fl tb Ar N , Fl Fl threads-batch Ar N-Number of threads to use during batch and prompt processing (default:-same as-.Fl Fl threads )-.It Fl c Ar N , Fl Fl ctx-size Ar N-Size of the prompt context (default: 512, 0 = loaded from model)-.It Fl Fl rope-scaling Ar {none,linear,yarn}-RoPE frequency scaling method, defaults to linear unless specified by the model-.It Fl Fl rope-scale Ar N-RoPE context scaling factor, expands context by a factor of N-.It Fl Fl rope-freq-base Ar N-RoPE base frequency, used by NTK-aware scaling (default: loaded from model)-.It Fl Fl rope-freq-scale Ar N-RoPE frequency scaling factor, expands context by a factor of 1/N-.It Fl Fl yarn-orig-ctx Ar N-YaRN: original context size of model (default: 0 = model training context size)-.It Fl Fl yarn-ext-factor Ar N-YaRN: extrapolation mix factor (default: 1.0, 0.0 = full interpolation)-.It Fl Fl yarn-attn-factor Ar N-YaRN: scale sqrt(t) or attention magnitude (default: 1.0)-.It Fl Fl yarn-beta-slow Ar N-YaRN: high correction dim or alpha (default: 1.0)-.It Fl Fl yarn-beta-fast Ar N-YaRN: low correction dim or beta (default: 32.0)-.It Fl b Ar N , Fl Fl batch-size Ar N-Batch size for prompt processing (default: 512)-.It Fl Fl memory-f32-Use f32 instead of f16 for memory key+value (default: disabled) Not recommended: doubles context memory required and no measurable increase in quality.-.It Fl Fl mlock-Force system to keep model in RAM rather than swapping or compressing.-.It Fl Fl no-mmap-Do not memory-map model (slower load but may reduce pageouts if not using mlock).-.It Fl Fl numa-Attempt optimizations that help on some NUMA systems if run without this previously, it is recommended to drop the system page cache before using this. See https://github.com/ggerganov/llama.cpp/issues/1437.-.It Fl ngl Ar N , Fl Fl n-gpu-layers Ar N-Number of layers to store in VRAM.-.It Fl ts Ar SPLIT , Fl Fl tensor-split Ar SPLIT-How to split tensors across multiple GPUs, comma-separated list of proportions, e.g. 3,1-.It Fl mg Ar i , Fl Fl main-gpu Ar i-The GPU to use for scratch and small tensors.-.It Fl nommq , Fl Fl no-mul-mat-q-Use cuBLAS instead of custom mul_mat_q CUDA kernels. Not recommended since this is both slower and uses more VRAM.-.It Fl m Ar FNAME , Fl Fl model Ar FNAME-Model path (default: models/7B/ggml-model-f16.gguf)-.It Fl a Ar ALIAS , Fl Fl alias Ar ALIAS-Set an alias for the model, will be added as `model` field in completion response.-.It Fl Fl lora Ar FNAME-Apply LoRA adapter (implies-.Fl Fl no-mmap )-.It Fl Fl lora-base Ar FNAME-Optional model to use as a base for the layers modified by the LoRA adapter-.It Fl Fl host Ar IPADDR-IP address to listen (default: 127.0.0.1)-.It Fl Fl port Ar PORT-Port to listen (default (default: 8080)-.It Fl Fl path Ar PUBLIC_PATH-Path from which to serve static files (default /zip/llama.cpp/server/public)-.It Fl to Ar N , Fl Fl timeout Ar N-Server read/write timeout in seconds (default: 600)-.It Fl Fl embedding-Enable embedding vector output (default: disabled)-.It Fl np Ar N , Fl Fl parallel Ar N-Number of slots for process requests (default: 1)-.It Fl cb , Fl Fl cont-batching-Enable continuous batching (a.k.a dynamic batching) (default: disabled)-.It Fl spf Ar FNAME , Fl Fl system-prompt-file Ar FNAME-Set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications.-.It Fl Fl mmproj Ar MMPROJ_FILE-Path to a multimodal projector file for LLaVA.-.It Fl Fl log-disable-Disables logging to a file.-.It Fl Fl nobrowser-Do not attempt to open a web browser tab at startup.-.It Fl Fl unsecure-Disables pledge() sandboxing on Linux and OpenBSD-.El-.Sh SEE ALSO-.Xr llamafile 1 ,-.Xr llamafile-quantize 1 ,-.Xr llamafile-perplexity 1 ,-.Xr llava-quantize 1 ,-.Xr zipalign 1 ,-.Xr unzip 1diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex 87ea917d..4da15d94 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -2651,17 +2651,7 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_con}}-int server_cli(int argc, char ** argv)-{- if (argc == 2 && !strcmp(argv[1], "--version")) {- printf("llamafile v" LLAMAFILE_VERSION_STRING " server\n");- exit(0);- }-- llamafile_init();- llamafile_check_cpu();- LoadZipArgs(&argc, &argv);- ShowCrashReports();+int server_cli(int argc, char ** argv) {// own arguments required by this examplegpt_params params;diff --git a/llamafile/help.c b/llamafile/help.cnew file mode 100644index 00000000..54c73010--- /dev/null+++ b/llamafile/help.c@@ -0,0 +1,78 @@+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "llamafile.h"+#include <fcntl.h>+#include <stdio.h>+#include <unistd.h>+#include <stdlib.h>++/**+ * Displays man page.+ *+ * This function uses the system pagination program (less or more) when+ * it's available. That'll be the case for all OSes, including Windows.+ * When the standard output file descriptor isn't a teletypewriter, the+ * man page will be formatted as plain text utf8 without any paginator.+ *+ * @param path should be a .txt or .asc file, which is allowed to use+ * ascii combining characters (for bold and underscore text)+ * @noreturn+ */+wontreturn void llamafile_help(const char *path) {++ // slurp rendered `man llamafile` page+ int fd;+ fd = open(path, O_RDONLY);+ if (fd == -1) {+ perror(path);+ exit(1);+ }+ int size;+ size = lseek(fd, 0, SEEK_END);+ if (size == -1) {+ perror(path);+ exit(1);+ }+ char *text = calloc(1, size + 1);+ if (pread(fd, text, size, 0) != size) {+ perror(path);+ exit(1);+ }+ close(fd);++ // strip ascii combining sequences for bold and underscore text+ if (!isatty(1)) {+ int j = 0;+ for (int i = 0; i < size; ++i) {+ switch (text[i]) {+ case '\b':+ --j;+ break;+ default:+ text[j++] = text[i];+ break;+ }+ }+ text[j] = 0;+ }++ // diplay manual page in less or more if available+ __paginate(1, text);+ free(text);+ exit(0);+}diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex d51c63dd..c380ed52 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -2,9 +2,7 @@#define LLAMAFILE_H_#include <stdio.h>#include <stdbool.h>-#ifdef __cplusplus-extern "C" {-#endif+COSMOPOLITAN_C_START_struct llamafile;struct llamafile *llamafile_open(const char *, const char *);@@ -19,6 +17,7 @@ FILE *llamafile_fp(struct llamafile *);void llamafile_init(void);void llamafile_check_cpu(void);+void llamafile_help(const char *) wontreturn;const char *llamafile_get_tmp_dir(void);bool llamafile_extract(const char *, const char *);int llamafile_is_file_newer_than(const char *, const char *);@@ -26,7 +25,5 @@ void llamafile_schlep(const void *, size_t);void llamafile_get_app_dir(char *, size_t);bool llamafile_launch_browser(const char *);-#ifdef __cplusplus-}-#endif+COSMOPOLITAN_C_END_#endif /* LLAMAFILE_H_ */