Skip to content

Distribute and run LLMs with a single file.

Add --cli flag and show unified man page on --help

Justine Tunney5cf6e76647d0
diff --git a/Makefile b/Makefile
index 1b71047c..69543a1a 100644
--- a/Makefile
+++ b/Makefile
@@ -23,7 +23,6 @@ o/$(MODE)/: o/$(MODE)/llama.cpp o/$(MODE)/llamafile
.PHONY: install
install: llamafile/zipalign.1 \
llama.cpp/main/main.1 \
- llama.cpp/server/server.1 \
llama.cpp/quantize/quantize.1 \
llama.cpp/perplexity/perplexity.1 \
llama.cpp/llava/llava-quantize.1 \
@@ -42,7 +41,6 @@ install: llamafile/zipalign.1 \
mkdir -p $(PREFIX)/share/man/man1
$(INSTALL) -m 0644 llamafile/zipalign.1 $(PREFIX)/share/man/man1/zipalign.1
$(INSTALL) -m 0644 llama.cpp/main/main.1 $(PREFIX)/share/man/man1/llamafile.1
- $(INSTALL) -m 0644 llama.cpp/server/server.1 $(PREFIX)/share/man/man1/llamafile-server.1
$(INSTALL) -m 0644 llama.cpp/quantize/quantize.1 $(PREFIX)/share/man/man1/llamafile-quantize.1
$(INSTALL) -m 0644 llama.cpp/perplexity/perplexity.1 $(PREFIX)/share/man/man1/llamafile-perplexity.1
$(INSTALL) -m 0644 llama.cpp/llava/llava-quantize.1 $(PREFIX)/share/man/man1/llava-quantize.1
diff --git a/README.md b/README.md
index 486fb65b..38f176a2 100644
--- a/README.md
+++ b/README.md
@@ -275,9 +275,8 @@ weights:
```sh
llamafile \
-m wizardcoder-python-13b-v1.0.Q8_0.gguf \
- --temp 0 -e \
- -r '```\n' \
- -p '```c\nvoid *memcpy(char *dst, const char *src, size_t size) {\n'
+ --temp 0 -r '}\n' -r '```\n' \
+ -e -p '```c\nvoid *memcpy(void *dst, const void *src, size_t size) {\n'
```
Here's a similar example that instead utilizes Mistral-7B-Instruct
@@ -314,7 +313,7 @@ Here's an example of how you can use llamafile to summarize HTML URLs:
sed 's/ */ /g'
echo '[/INST]'
) | llamafile \
- -m mistral-7b-instruct-v0.1.Q4_K_M.gguf \
+ -m mistral-7b-instruct-v0.2.Q5_K_M.gguf \
-f /dev/stdin \
-c 0 \
--temp 0 \
@@ -329,7 +328,7 @@ llamafile --temp 0 \
--image ~/Pictures/lemurs.jpg \
-m llava-v1.5-7b-Q4_K.gguf \
--mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \
- -p $'### User: What do you see?\n### Assistant: ' \
+ -e -p '### User: What do you see?\n### Assistant: ' \
--silent-prompt 2>/dev/null
```
@@ -341,11 +340,11 @@ wanted to write a script to rename all your image files, you could say:
```sh
llamafile --temp 0 \
- --image ~/Pictures/lemurs.jpg \
+ --image lemurs.jpg \
-m llava-v1.5-7b-Q4_K.gguf \
--mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \
--grammar 'root ::= [a-z]+ (" " [a-z]+)+' \
- -p $'### User: What do you see?\n### Assistant: ' \
+ -e -p '### User: What do you see?\n### Assistant: ' \
--silent-prompt 2>/dev/null |
sed -e's/ /_/g' -e's/$/.jpg/'
a_baby_monkey_on_the_back_of_a_mother.jpg
diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cpp
index 42b1ed1c..7381efa2 100644
--- a/llama.cpp/common.cpp
+++ b/llama.cpp/common.cpp
@@ -154,7 +154,9 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {
std::replace(arg.begin(), arg.end(), '_', '-');
}
- if (arg == "-s" || arg == "--seed") {
+ if (arg == "--cli") {
+ // do nothing
+ } else if (arg == "-s" || arg == "--seed") {
if (++i >= argc) {
invalid_param = true;
break;
diff --git a/llama.cpp/main/BUILD.mk b/llama.cpp/main/BUILD.mk
index 64a53dde..6e255e06 100644
--- a/llama.cpp/main/BUILD.mk
+++ b/llama.cpp/main/BUILD.mk
@@ -13,8 +13,13 @@ o/$(MODE)/llama.cpp/main/main: \
o/$(MODE)/llama.cpp/server/server.a \
o/$(MODE)/llama.cpp/llava/llava.a \
o/$(MODE)/llama.cpp/llama.cpp.a \
+ o/$(MODE)/llama.cpp/main/main.1.asc.zip.o \
$(LLAMA_CPP_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)
+llama.cpp/main/main.1.asc: llama.cpp/main/main.1
+ -man $< >[email protected] && mv -f [email protected] $@
+
.PHONY: o/$(MODE)/llama.cpp/main
o/$(MODE)/llama.cpp/main: \
o/$(MODE)/llama.cpp/main/main
diff --git a/llama.cpp/main/main.1 b/llama.cpp/main/main.1
index ef371543..b08e3b6b 100644
--- a/llama.cpp/main/main.1
+++ b/llama.cpp/main/main.1
@@ -1,20 +1,22 @@
-.Dd December 5, 2023
-.Dt llamafile 1
-.Os
+.Dd January 1, 2024
+.Dt LLAMAFILE 1
+.Os Mozilla Ocho
.Sh NAME
.Nm llamafile
.Nd large language model runner
.Sh SYNOPSIS
.Nm
+.Op Fl Fl server
.Op flags...
.Fl m Ar model.gguf
-.Fl p Ar prompt
+.Op Fl Fl mmproj Ar vision.gguf
.Nm
+.Op Fl Fl cli
.Op flags...
-.Op Fl Fl server
.Fl m Ar model.gguf
-.Op Fl Fl mmproj Ar vision.gguf
+.Fl p Ar prompt
.Nm
+.Op Fl Fl cli
.Op flags...
.Fl m Ar model.gguf
.Fl Fl mmproj Ar vision.gguf
@@ -37,84 +39,41 @@ Text/image summarization and analysis
.Sh OPTIONS
The following options are available:
.Bl -tag -width indent
+.It Fl Fl version
+Print version and exit.
.It Fl h , Fl Fl help
Show help message and exit.
-.It Fl Fl version
-Print llamafile version and exit.
+.It Fl Fl cli
+Puts program in command line interface mode. This flag is implied when a
+prompt is supplied using either the
+.Fl p
+or
+.Fl f
+flags.
.It Fl Fl server
-.Xr llamafile-server 1
-is launched via the
-.Nm llamafile
-command when the
+Puts program in server mode. This will launch an HTTP server on a local
+port. This server has both a web UI and an OpenAI API compatible
+completions endpoint. When the server is run on a desk system, a tab
+browser tab will be launched automatically that displays the web UI.
+This
.Fl Fl server
-flag is passed. The server flag is implied by default if no prompt is
-specified, i.e. neither the
+flag is implied if no prompt is specified, i.e. neither the
.Fl p
or
.Fl f
flags are passed.
.It Fl m Ar FNAME , Fl Fl model Ar FNAME
-Model path in GGUF file format.
-.Pp
-Default: models/7B/ggml-model-f16.gguf
-.It Fl p Ar STRING , Fl Fl prompt Ar STRING
-Prompt to start text generation. Your LLM works by auto-completing this
-text. For example:
-.Bd -literal -offset indent
-.Nm Fl m Li model.gguf Fl p Li \[dq]four score and\[dq]
-.Ed
-.Pp
-Stands a pretty good chance of printing Lincoln's Gettysburg Address.
-Prompts can take on a structured format too. Depending on how your model
-was trained, it may specify in its docs an instruction notation. With
-some models that might be:
-.Bd -literal -offset indent
-.Nm Fl p Li \[dq][INST]Summarize this: $(cat file)[/INST]\[dq]
-.Ed
-.Pp
-In most cases, simply colons and newlines will work too:
-.Bd -literal -offset indent
-.Nm Fl e Fl p Li \[dq]User: What is best in life?\[rs]nAssistant:\[dq]
-.Ed
+Model path in the GGUF file format.
.Pp
+Default:
+.Pa models/7B/ggml-model-f16.gguf
.It Fl Fl mmproj Ar FNAME
-Specifies path of vision model in the GGUF file format. If this flag is supplied, then the
+Specifies path of the LLaVA vision model in the GGUF file format. If
+this flag is supplied, then the
.Fl Fl model
and
.Fl Fl image
flags should also be supplied.
-.It Fl Fl image Ar IMAGE_FILE
-Path to an image file. This should be used with multimodal models.
-Alternatively, it's possible to embed an image directly into the prompt
-instead; in which case, it must be base64 encoded into an HTML img tag
-URL with the image/jpeg MIME type. See also the
-.Fl Fl mmproj
-flag for supplying the vision model.
-.It Fl Fl grammar Ar GRAMMAR
-BNF-like grammar to constrain which tokens may be selected when
-generating text. For example, the grammar:
-.Bd -literal -offset indent
-root ::= "yes" | "no"
-.Ed
-.Pp
-will force the LLM to only output yes or no before exiting. This is
-useful for shell scripts when the
-.Fl Fl silent-prompt
-flag is also supplied.
-.It Fl i , Fl Fl interactive
-Run in interactive mode.
-.It Fl Fl interactive-first
-Run in interactive mode and wait for input right away.
-.It Fl ins , Fl Fl instruct
-Run in instruction mode (use with Alpaca models).
-.It Fl Fl multiline-input
-Allows you to write or paste multiple lines without ending each in '\[rs]'.
-.It Fl r Ar PROMPT , Fl Fl reverse-prompt Ar PROMPT
-Halt generation at
-.Ar PROMPT
-and return control in interactive mode (can be specified more than once for multiple prompts).
-.It Fl Fl color
-colorise output to distinguish prompt and user input from generations
.It Fl s Ar SEED , Fl Fl seed Ar SEED
Random Number Generator (RNG) seed. A random seed is used if this is
less than zero.
@@ -129,22 +88,6 @@ Number of threads to use during batch and prompt processing.
.Pp
Default: Same as
.Fl Fl threads
-.It Fl f Ar FNAME , Fl Fl file Ar FNAME
-Prompt file to start generation.
-.It Fl e , Fl Fl escape
-Process prompt escapes sequences (\[rs]n, \[rs]r, \[rs]t, \[rs]\[aa], \[rs]\[dq], \[rs]\[rs])
-.It Fl Fl prompt-cache Ar FNAME
-File to cache prompt state for faster startup.
-.Pp
-Default: none
-.It Fl Fl prompt-cache-all
-If specified, saves user input and generations to cache as well. Not supported with
-.Fl Fl interactive
-or other interactive options.
-.It Fl Fl prompt-cache-ro
-If specified, uses the prompt cache but does not update it.
-.It Fl Fl random-prompt
-Start with a randomized prompt.
.It Fl Fl in-prefix-bos
Prefix BOS to user inputs, preceding the
.Fl Fl in-prefix
@@ -297,9 +240,8 @@ to decrease likelihood of token
.It Fl md Ar FNAME , Fl Fl model-draft Ar FNAME
Draft model for speculative decoding.
.Pp
-Default: models/7B/ggml-model-f16.gguf
-.It Fl Fl grammar-file Ar FNAME
-File to read grammar from.
+Default:
+.Pa models/7B/ggml-model-f16.gguf
.It Fl Fl cfg-negative-prompt Ar PROMPT
Negative prompt to use for guidance..
.Pp
@@ -393,10 +335,6 @@ Max number of chunks to process.
.El
.Pp
Default: -1
-.It Fl np Ar N , Fl Fl parallel Ar N
-Number of parallel sequences to decode.
-.Pp
-Default: 1
.It Fl ns Ar N , Fl Fl sequences Ar N
Number of sequences to decode.
.Pp
@@ -409,10 +347,6 @@ Default: 0.5
Speculative decoding split probability.
.Pp
Default: 0.1
-.It Fl cb , Fl Fl cont-batching
-Enable continuous batching (a.k.a dynamic batching).
-.Pp
-Default: disabled
.It Fl Fl mlock
Force system to keep model in RAM rather than swapping or compressing.
.It Fl Fl no-mmap
@@ -431,8 +365,6 @@ The GPU to use for scratch and small tensors.
Use cuBLAS instead of custom mul_mat_q CUDA kernels. Not recommended since this is both slower and uses more VRAM.
.It Fl Fl verbose-prompt
Print prompt before generation.
-.It Fl Fl silent-prompt
-Don't echo the prompt itself to standard output.
.It Fl Fl simple-io
Use basic IO for better compatibility in subprocesses and limited consoles.
.It Fl Fl lora Ar FNAME
@@ -461,6 +393,128 @@ KV cache data type for K.
.It Fl ctv Ar TYPE , Fl Fl cache-type-v Ar TYPE
KV cache data type for V.
.El
+.Sh CLI OPTIONS
+The following options may be specified when
+.Nm
+is running in
+.Fl Fl cli
+mode.
+.Bl -tag -width indent
+.It Fl e , Fl Fl escape
+Process prompt escapes sequences (\[rs]n, \[rs]r, \[rs]t, \[rs]\[aa], \[rs]\[dq], \[rs]\[rs])
+.It Fl p Ar STRING , Fl Fl prompt Ar STRING
+Prompt to start text generation. Your LLM works by auto-completing this
+text. For example:
+.Pp
+.Dl "llamafile -m model.gguf -p \[dq]four score and\[dq]"
+.Pp
+Stands a pretty good chance of printing Lincoln's Gettysburg Address.
+Prompts can take on a structured format too. Depending on how your model
+was trained, it may specify in its docs an instruction notation. With
+some models that might be:
+.Pp
+.Dl "llamafile -p \[dq][INST]Summarize this: $(cat file)[/INST]\[dq]"
+.Pp
+In most cases, simply colons and newlines will work too:
+.Pp
+.Dl "llamafile -e -p \[dq]User: What is best in life?\[rs]nAssistant:\[dq]"
+.Pp
+.It Fl f Ar FNAME , Fl Fl file Ar FNAME
+Prompt file to start generation.
+.It Fl Fl grammar Ar GRAMMAR
+BNF-like grammar to constrain which tokens may be selected when
+generating text. For example, the grammar:
+.Pp
+.Dl "root ::= \[dq]yes\[dq] | \[dq]no\[dq]"
+.Pp
+will force the LLM to only output yes or no before exiting. This is
+useful for shell scripts when the
+.It Fl Fl grammar-file Ar FNAME
+File to read grammar from.
+.It Fl Fl prompt-cache Ar FNAME
+File to cache prompt state for faster startup.
+.Pp
+Default: none
+.It Fl Fl prompt-cache-all
+If specified, saves user input and generations to cache as well. Not supported with
+.Fl Fl interactive
+or other interactive options.
+.It Fl Fl prompt-cache-ro
+If specified, uses the prompt cache but does not update it.
+.It Fl Fl random-prompt
+Start with a randomized prompt.
+.It Fl Fl image Ar IMAGE_FILE
+Path to an image file. This should be used with multimodal models.
+Alternatively, it's possible to embed an image directly into the prompt
+instead; in which case, it must be base64 encoded into an HTML img tag
+URL with the image/jpeg MIME type. See also the
+.Fl Fl mmproj
+flag for supplying the vision model.
+.It Fl i , Fl Fl interactive
+Run in interactive mode.
+.It Fl Fl interactive-first
+Run in interactive mode and wait for input right away.
+.It Fl ins , Fl Fl instruct
+Run in instruction mode (use with Alpaca models).
+.Fl Fl silent-prompt
+flag is also supplied.
+.It Fl r Ar PROMPT , Fl Fl reverse-prompt Ar PROMPT
+Halt generation at
+.Ar PROMPT
+and return control in interactive mode (can be specified more than once for multiple prompts).
+.It Fl Fl color
+Colorise output to distinguish prompt and user input from generations.
+.It Fl Fl silent-prompt
+Don't echo the prompt itself to standard output.
+.It Fl Fl multiline-input
+Allows you to write or paste multiple lines without ending each in '\[rs]'.
+.El
+.Sh SERVER OPTIONS
+The following options may be specified when
+.Nm
+is running in
+.Fl Fl server
+mode.
+.Bl -tag -width indent
+.It Fl Fl port Ar PORT
+Port to listen
+.Pp
+Default: 8080
+.It Fl Fl host Ar IPADDR
+IP address to listen.
+.Pp
+Default: 127.0.0.1
+.It Fl to Ar N , Fl Fl timeout Ar N
+Server read/write timeout in seconds.
+.Pp
+Default: 600
+.It Fl np Ar N , Fl Fl parallel Ar N
+Number of slots for process requests.
+.Pp
+Default: 1
+.It Fl cb , Fl Fl cont-batching
+Enable continuous batching (a.k.a dynamic batching).
+.Pp
+Default: disabled
+.It Fl spf Ar FNAME , Fl Fl system-prompt-file Ar FNAME
+Set a file to load a system prompt (initial prompt of all slots), this
+is useful for chat applications.
+.It Fl a Ar ALIAS , Fl Fl alias Ar ALIAS
+Set an alias for the model. This will be added as the
+.Ar model
+field in completion responses.
+.It Fl Fl path Ar PUBLIC_PATH
+Path from which to serve static files.
+.Pp
+Default:
+.Pa /zip/llama.cpp/server/public
+.It Fl Fl embedding
+Enable embedding vector output.
+.Pp
+Default: disabled
+.It Fl Fl nobrowser
+Do not attempt to open a web browser tab at startup.
+.El
.Sh LOG OPTIONS
The following log options are available:
.Bl -tag -width indent
@@ -475,12 +529,136 @@ Enable trace logs
.It Fl Fl log-file
Specify a log filename (without extension)
.It Fl Fl log-new
-Create a separate new log file on start. Each log file will have unique name: "<name>.<ID>.log"
+Create a separate new log file on start. Each log file will have unique name:
+.Fa <name>.<ID>.log
.It Fl Fl log-append
Don't truncate the old log file.
.El
+.Sh EXAMPLES
+Here's an example of how to run llama.cpp's built-in HTTP server. This
+example uses LLaVA v1.5-7B, a multimodal LLM that works with llama.cpp's
+recently-added support for image inputs.
+.Bd -literal -offset indent
+llamafile \[rs]
+ -m llava-v1.5-7b-Q8_0.gguf \[rs]
+ --mmproj llava-v1.5-7b-mmproj-Q8_0.gguf \[rs]
+ --host 0.0.0.0
+.Ed
+.Pp
+Here's an example of how to generate code for a libc function using the
+llama.cpp command line interface, utilizing WizardCoder-Python-13B
+weights:
+.Bd -literal -offset indent
+llamafile \[rs]
+ -m wizardcoder-python-13b-v1.0.Q8_0.gguf \[rs]
+ --temp 0 -r '}\[rs]n' -r '\`\`\`\[rs]n' \[rs]
+ -e -p '\`\`\`c\[rs]nvoid *memcpy(void *dst, const void *src, size_t size) {\[rs]n'
+.Ed
+.Pp
+Here's a similar example that instead utilizes Mistral-7B-Instruct
+weights for prose composition:
+.Bd -literal -offset indent
+llamafile \[rs]
+ -m mistral-7b-instruct-v0.2.Q5_K_M.gguf \[rs]
+ -p '[INST]Write a story about llamas[/INST]'
+.Ed
+.Pp
+Here's an example of how llamafile can be used as an interactive chatbot
+that lets you query knowledge contained in training data:
+.Bd -literal -offset indent
+llamafile -m llama-65b-Q5_K.gguf -p '
+The following is a conversation between a Researcher and their helpful AI
+assistant Digital Athena which is a large language model trained on the
+sum of human knowledge.
+Researcher: Good morning.
+Digital Athena: How can I help you today?
+Researcher:' --interactive --color --batch_size 1024 --ctx_size 4096 \[rs]
+--keep -1 --temp 0 --mirostat 2 --in-prefix ' ' --interactive-first \[rs]
+--in-suffix 'Digital Athena:' --reverse-prompt 'Researcher:'
+.Ed
+.Pp
+Here's an example of how you can use llamafile to summarize HTML URLs:
+.Bd -literal -offset indent
+(
+ echo '[INST]Summarize the following text:'
+ links -codepage utf-8 \[rs]
+ -force-html \[rs]
+ -width 500 \[rs]
+ -dump https://www.poetryfoundation.org/poems/48860/the-raven |
+ sed 's/ */ /g'
+ echo '[/INST]'
+) | llamafile \[rs]
+ -m mistral-7b-instruct-v0.2.Q5_K_M.gguf \[rs]
+ -f /dev/stdin \[rs]
+ -c 0 \[rs]
+ --temp 0 \[rs]
+ -n 500 \[rs]
+ --silent-prompt 2>/dev/null
+.Ed
+.Pp
+Here's how you can use llamafile to describe a jpg/png/gif/bmp image:
+.Bd -literal -offset indent
+llamafile --temp 0 \[rs]
+ --image lemurs.jpg \[rs]
+ -m llava-v1.5-7b-Q4_K.gguf \[rs]
+ --mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \[rs]
+ -e -p '### User: What do you see?\[rs]n### Assistant: ' \[rs]
+ --silent-prompt 2>/dev/null
+.Ed
+.Pp
+If you wanted to write a script to rename all your image files, you
+could use the following command to generate a safe filename:
+.Bd -literal -offset indent
+llamafile --temp 0 \[rs]
+ --image ~/Pictures/lemurs.jpg \[rs]
+ -m llava-v1.5-7b-Q4_K.gguf \[rs]
+ --mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \[rs]
+ --grammar 'root ::= [a-z]+ (" " [a-z]+)+' \[rs]
+ -e -p '### User: What do you see?\[rs]n### Assistant: ' \[rs]
+ --silent-prompt 2>/dev/null |
+ sed -e's/ /_/g' -e's/$/.jpg/'
+a_baby_monkey_on_the_back_of_a_mother.jpg
+.Ed
+.Pp
+Here's an example of how to make an API request to the OpenAI API
+compatible completions endpoint when your
+.Nm
+is running in the background in
+.Fl Fl server
+mode.
+.Bd -literal -offset indent
+curl -s http://localhost:8080/v1/chat/completions \[rs]
+ -H "Content-Type: application/json" -d '{
+ "model": "gpt-3.5-turbo",
+ "stream": true,
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are a poetic assistant."
+ },
+ {
+ "role": "user",
+ "content": "Compose a poem that explains FORTRAN."
+ }
+ ]
+}' | python3 -c '
+import json
+import sys
+json.dump(json.load(sys.stdin), sys.stdout, indent=2)
+print()
+'
+.Ed
+.Sh BUGS
+Your HTTP server is great for local hosting, but it would need much more
+work to be a production worthy component of a public-facing service. For
+example, C++ exceptions caused by JSON parsing errors will make it abort
+and print a backtrace.
+.Sh PROTIP
+NVIDIA users need to pass the
+.Fl ngl Ar 35
+flag to enable GPU acceleration. It's not enabled by default since it
+sometimes needs to be tuned for system hardware and model architecture.
.Sh SEE ALSO
-.Xr llamafile-server 1 ,
.Xr llamafile-quantize 1 ,
.Xr llamafile-perplexity 1 ,
.Xr llava-quantize 1 ,
diff --git a/llama.cpp/main/main.1.asc b/llama.cpp/main/main.1.asc
new file mode 100644
index 00000000..682ac9ae
--- /dev/null
+++ b/llama.cpp/main/main.1.asc
@@ -0,0 +1,666 @@
+LLAMAFILE(1) General Commands Manual LLAMAFILE(1)
+
+NNAAMMEE
+ llllaammaaffiillee - large language model runner
+
+SSYYNNOOPPSSIISS
+ llllaammaaffiillee [----sseerrvveerr] [flags...] --mm _m_o_d_e_l_._g_g_u_f [----mmmmpprroojj _v_i_s_i_o_n_._g_g_u_f]
+ llllaammaaffiillee [----ccllii] [flags...] --mm _m_o_d_e_l_._g_g_u_f --pp _p_r_o_m_p_t
+ llllaammaaffiillee [----ccllii] [flags...]
+ --mm _m_o_d_e_l_._g_g_u_f ----mmmmpprroojj _v_i_s_i_o_n_._g_g_u_f ----iimmaaggee _g_r_a_p_h_i_c_._p_n_g
+ --pp _p_r_o_m_p_t
+
+DDEESSCCRRIIPPTTIIOONN
+ llllaammaaffiillee is a large language model tool. It has use cases such as:
+
+ -- Code completion
+ -- Prose composition
+ -- Chatbot that passes the Turing test
+ -- Text/image summarization and analysis
+
+OOPPTTIIOONNSS
+ The following options are available:
+
+ ----vveerrssiioonn
+ Print version and exit.
+
+ --hh, ----hheellpp
+ Show help message and exit.
+
+ ----ccllii Puts program in command line interface mode. This flag is implied
+ when a prompt is supplied using either the --pp or --ff flags.
+
+ ----sseerrvveerr
+ Puts program in server mode. This will launch an HTTP server on a
+ local port. This server has both a web UI and an OpenAI API
+ compatible completions endpoint. When the server is run on a desk
+ system, a tab browser tab will be launched automatically that
+ displays the web UI. This ----sseerrvveerr flag is implied if no prompt
+ is specified, i.e. neither the --pp or --ff flags are passed.
+
+ --mm _F_N_A_M_E, ----mmooddeell _F_N_A_M_E
+ Model path in the GGUF file format.
+
+ Default: _m_o_d_e_l_s_/_7_B_/_g_g_m_l_-_m_o_d_e_l_-_f_1_6_._g_g_u_f
+
+ ----mmmmpprroojj _F_N_A_M_E
+ Specifies path of the LLaVA vision model in the GGUF file format.
+ If this flag is supplied, then the ----mmooddeell and ----iimmaaggee flags
+ should also be supplied.
+
+ --ss _S_E_E_D, ----sseeeedd _S_E_E_D
+ Random Number Generator (RNG) seed. A random seed is used if this
+ is less than zero.
+
+ Default: -1
+
+ --tt _N, ----tthhrreeaaddss _N
+ Number of threads to use during generation.
+
+ Default: $(nproc)/2
+
+ --ttbb _N, ----tthhrreeaaddss--bbaattcchh _N
+ Number of threads to use during batch and prompt processing.
+
+ Default: Same as ----tthhrreeaaddss
+
+ ----iinn--pprreeffiixx--bbooss
+ Prefix BOS to user inputs, preceding the ----iinn--pprreeffiixx string.
+
+ ----iinn--pprreeffiixx _S_T_R_I_N_G
+ String to prefix user inputs with.
+
+ Default: empty
+
+ ----iinn--ssuuffffiixx _S_T_R_I_N_G
+ String to suffix after user inputs with.
+
+ Default: empty
+
+ --nn _N, ----nn--pprreeddiicctt _N
+ Number of tokens to predict.
+
+ -- -1 = infinity
+ -- -2 = until context filled
+
+ Default: -1
+
+ --cc _N, ----ccttxx--ssiizzee _N
+ Size of the prompt context.
+
+ -- 0 = loaded from model
+
+ Default: 512
+
+ --bb _N, ----bbaattcchh--ssiizzee _N
+ Batch size for prompt processing.
+
+ Default: 512
+
+ ----ttoopp--kk _N
+ Top-k sampling.
+
+ -- 0 = disabled
+
+ Default: 40
+
+ ----ttoopp--pp _N
+ Top-p sampling.
+
+ -- 1.0 = disabled
+
+ Default: 0.9
+
+ ----mmiinn--pp _N
+ Min-p sampling.
+
+ -- 0.0 = disabled
+
+ Default: 0.1
+
+ ----ttffss _N
+ Tail free sampling, parameter z.
+
+ -- 1.0 = disabled
+
+ Default: 1.0
+
+ ----ttyyppiiccaall _N
+ Locally typical sampling, parameter p.
+
+ -- 1.0 = disabled
+
+ Default: 1.0
+
+ ----rreeppeeaatt--llaasstt--nn _N
+ Last n tokens to consider for penalize.
+
+ -- 0 = disabled
+ -- -1 = ctx_size
+
+ Default: 64
+
+ ----rreeppeeaatt--ppeennaallttyy _N
+ Penalize repeat sequence of tokens.
+
+ -- 1.0 = disabled
+
+ Default: 1.1
+
+ ----pprreesseennccee--ppeennaallttyy _N
+ Repeat alpha presence penalty.
+
+ -- 0.0 = disabled
+
+ Default: 0.0
+
+ ----ffrreeqquueennccyy--ppeennaallttyy _N
+ Repeat alpha frequency penalty.
+
+ -- 0.0 = disabled
+
+ Default: 0.0
+
+ ----mmiirroossttaatt _N
+ Use Mirostat sampling. Top K, Nucleus, Tail Free and Locally
+ Typical samplers are ignored if used..
+
+ -- 0 = disabled
+ -- 1 = Mirostat
+ -- 2 = Mirostat 2.0
+
+ Default: 0
+
+ ----mmiirroossttaatt--llrr _N
+ Mirostat learning rate, parameter eta.
+
+ Default: 0.1
+
+ ----mmiirroossttaatt--eenntt _N
+ Mirostat target entropy, parameter tau.
+
+ Default: 5.0
+
+ --ll _T_O_K_E_N___I_D_(_+_/_-_)_B_I_A_S, ----llooggiitt--bbiiaass _T_O_K_E_N___I_D_(_+_/_-_)_B_I_A_S
+ Modifies the likelihood of token appearing in the completion,
+ i.e. ----llooggiitt--bbiiaass _1_5_0_4_3_+_1 to increase likelihood of token
+ _' _H_e_l_l_o_', or ----llooggiitt--bbiiaass _1_5_0_4_3_-_1 to decrease likelihood of token
+ _' _H_e_l_l_o_'.
+
+ --mmdd _F_N_A_M_E, ----mmooddeell--ddrraafftt _F_N_A_M_E
+ Draft model for speculative decoding.
+
+ Default: _m_o_d_e_l_s_/_7_B_/_g_g_m_l_-_m_o_d_e_l_-_f_1_6_._g_g_u_f
+
+ ----ccffgg--nneeggaattiivvee--pprroommpptt _P_R_O_M_P_T
+ Negative prompt to use for guidance..
+
+ Default: empty
+
+ ----ccffgg--nneeggaattiivvee--pprroommpptt--ffiillee _F_N_A_M_E
+ Negative prompt file to use for guidance.
+
+ Default: empty
+
+ ----ccffgg--ssccaallee _N
+ Strength of guidance.
+
+ -- 1.0 = disable
+
+ Default: 1.0
+
+ ----rrooppee--ssccaalliinngg _{_n_o_n_e_,_l_i_n_e_a_r_,_y_a_r_n_}
+ RoPE frequency scaling method, defaults to linear unless
+ specified by the model
+
+ ----rrooppee--ssccaallee _N
+ RoPE context scaling factor, expands context by a factor of N
+
+ ----rrooppee--ffrreeqq--bbaassee _N
+ RoPE base frequency, used by NTK-aware scaling.
+
+ Default: loaded from model
+
+ ----rrooppee--ffrreeqq--ssccaallee _N
+ RoPE frequency scaling factor, expands context by a factor of 1/N
+
+ ----yyaarrnn--oorriigg--ccttxx _N
+ YaRN: original context size of model.
+
+ Default: 0 = model training context size
+
+ ----yyaarrnn--eexxtt--ffaaccttoorr _N
+ YaRN: extrapolation mix factor.
+
+ -- 0.0 = full interpolation
+
+ Default: 1.0
+
+ ----yyaarrnn--aattttnn--ffaaccttoorr _N
+ YaRN: scale sqrt(t) or attention magnitude.
+
+ Default: 1.0
+
+ ----yyaarrnn--bbeettaa--ssllooww _N
+ YaRN: high correction dim or alpha.
+
+ Default: 1.0
+
+ ----yyaarrnn--bbeettaa--ffaasstt _N
+ YaRN: low correction dim or beta.
+
+ Default: 32.0
+
+ ----iiggnnoorree--eeooss
+ Ignore end of stream token and continue generating (implies
+ ----llooggiitt--bbiiaass _2_-_i_n_f)
+
+ ----nnoo--ppeennaalliizzee--nnll
+ Do not penalize newline token.
+
+ ----tteemmpp _N
+ Temperature.
+
+ Default: 0.8
+
+ ----llooggiittss--aallll
+ Return logits for all tokens in the batch.
+
+ Default: disabled
+
+ ----hheellllaasswwaagg
+ Compute HellaSwag score over random tasks from datafile supplied
+ with -f
+
+ ----hheellllaasswwaagg--ttaasskkss _N
+ Number of tasks to use when computing the HellaSwag score.
+
+ Default: 400
+
+ ----kkeeeepp _N
+ Number of tokens to keep from the initial prompt.
+
+ -- -1 = all
+
+ Default: 0
+
+ ----ddrraafftt _N
+ Number of tokens to draft for speculative decoding.
+
+ Default: 16
+
+ ----cchhuunnkkss _N
+ Max number of chunks to process.
+
+ -- -1 = all
+
+ Default: -1
+
+ --nnss _N, ----sseeqquueenncceess _N
+ Number of sequences to decode.
+
+ Default: 1
+
+ --ppaa _N, ----pp--aacccceepptt _N
+ speculative decoding accept probability.
+
+ Default: 0.5
+
+ --ppss _N, ----pp--sspplliitt _N
+ Speculative decoding split probability.
+
+ Default: 0.1
+
+ ----mmlloocckk
+ Force system to keep model in RAM rather than swapping or
+ compressing.
+
+ ----nnoo--mmmmaapp
+ Do not memory-map model (slower load but may reduce pageouts if
+ not using mlock).
+
+ ----nnuummaa Attempt optimizations that help on some NUMA systems if run
+ without this previously, it is recommended to drop the system
+ page cache before using this. See
+ https://github.com/ggerganov/llama.cpp/issues/1437.
+
+ --nnggll _N, ----nn--ggppuu--llaayyeerrss _N
+ Number of layers to store in VRAM.
+
+ --nngglldd _N, ----nn--ggppuu--llaayyeerrss--ddrraafftt _N
+ Number of layers to store in VRAM for the draft model.
+
+ --ttss _S_P_L_I_T, ----tteennssoorr--sspplliitt _S_P_L_I_T
+ How to split tensors across multiple GPUs, comma-separated list
+ of proportions, e.g. 3,1
+
+ --mmgg _i, ----mmaaiinn--ggppuu _i
+ The GPU to use for scratch and small tensors.
+
+ --nnoommmmqq, ----nnoo--mmuull--mmaatt--qq
+ Use cuBLAS instead of custom mul_mat_q CUDA kernels. Not
+ recommended since this is both slower and uses more VRAM.
+
+ ----vveerrbboossee--pprroommpptt
+ Print prompt before generation.
+
+ ----ssiimmppllee--iioo
+ Use basic IO for better compatibility in subprocesses and limited
+ consoles.
+
+ ----lloorraa _F_N_A_M_E
+ Apply LoRA adapter (implies ----nnoo--mmmmaapp)
+
+ ----lloorraa--ssccaalleedd _F_N_A_M_E _S
+ Apply LoRA adapter with user defined scaling S (implies
+ ----nnoo--mmmmaapp)
+
+ ----lloorraa--bbaassee _F_N_A_M_E
+ Optional model to use as a base for the layers modified by the
+ LoRA adapter
+
+ ----uunnsseeccuurree
+ Disables pledge() sandboxing on Linux and OpenBSD.
+
+ ----ssaammpplleerrss
+ Samplers that will be used for generation in the order, separated
+ by semicolon, for example: top_k;tfs;typical;top_p;min_p;temp
+
+ ----ssaammpplleerrss--sseeqq
+ Simplified sequence for samplers that will be used.
+
+ --ccmmll, ----cchhaattmmll
+ Run in chatml mode (use with ChatML-compatible models)
+
+ --ddkkvvcc, ----dduummpp--kkvv--ccaacchhee
+ Verbose print of the KV cache.
+
+ --nnkkvvoo, ----nnoo--kkvv--ooffffllooaadd
+ Disable KV offload.
+
+ --ccttkk _T_Y_P_E, ----ccaacchhee--ttyyppee--kk _T_Y_P_E
+ KV cache data type for K.
+
+ --ccttvv _T_Y_P_E, ----ccaacchhee--ttyyppee--vv _T_Y_P_E
+ KV cache data type for V.
+
+CCLLII OOPPTTIIOONNSS
+ The following options may be specified when llllaammaaffiillee is running in ----ccllii
+ mode.
+
+ --ee, ----eessccaappee
+ Process prompt escapes sequences (\n, \r, \t, \', \", \\)
+
+ --pp _S_T_R_I_N_G, ----pprroommpptt _S_T_R_I_N_G
+ Prompt to start text generation. Your LLM works by auto-
+ completing this text. For example:
+
+ llamafile -m model.gguf -p "four score and"
+
+ Stands a pretty good chance of printing Lincoln's Gettysburg
+ Address. Prompts can take on a structured format too. Depending
+ on how your model was trained, it may specify in its docs an
+ instruction notation. With some models that might be:
+
+ llamafile -p "[INST]Summarize this: $(cat file)[/INST]"
+
+ In most cases, simply colons and newlines will work too:
+
+ llamafile -e -p "User: What is best in life?\nAssistant:"
+
+ --ff _F_N_A_M_E, ----ffiillee _F_N_A_M_E
+ Prompt file to start generation.
+
+ ----ggrraammmmaarr _G_R_A_M_M_A_R
+ BNF-like grammar to constrain which tokens may be selected when
+ generating text. For example, the grammar:
+
+ root ::= "yes" | "no"
+
+ will force the LLM to only output yes or no before exiting. This
+ is useful for shell scripts when the
+
+ ----ggrraammmmaarr--ffiillee _F_N_A_M_E
+ File to read grammar from.
+
+ ----pprroommpptt--ccaacchhee _F_N_A_M_E
+ File to cache prompt state for faster startup.
+
+ Default: none
+
+ ----pprroommpptt--ccaacchhee--aallll
+ If specified, saves user input and generations to cache as well.
+ Not supported with ----iinntteerraaccttiivvee or other interactive options.
+
+ ----pprroommpptt--ccaacchhee--rroo
+ If specified, uses the prompt cache but does not update it.
+
+ ----rraannddoomm--pprroommpptt
+ Start with a randomized prompt.
+
+ ----iimmaaggee _I_M_A_G_E___F_I_L_E
+ Path to an image file. This should be used with multimodal
+ models. Alternatively, it's possible to embed an image directly
+ into the prompt instead; in which case, it must be base64 encoded
+ into an HTML img tag URL with the image/jpeg MIME type. See also
+ the ----mmmmpprroojj flag for supplying the vision model.
+
+ --ii, ----iinntteerraaccttiivvee
+ Run in interactive mode.
+
+ ----iinntteerraaccttiivvee--ffiirrsstt
+ Run in interactive mode and wait for input right away.
+
+ --iinnss, ----iinnssttrruucctt
+ Run in instruction mode (use with Alpaca models).
+ ----ssiilleenntt--pprroommpptt flag is also supplied.
+
+ --rr _P_R_O_M_P_T, ----rreevveerrssee--pprroommpptt _P_R_O_M_P_T
+ Halt generation at _P_R_O_M_P_T and return control in interactive mode
+ (can be specified more than once for multiple prompts).
+
+ ----ccoolloorr
+ Colorise output to distinguish prompt and user input from
+ generations.
+
+ ----ssiilleenntt--pprroommpptt
+ Don't echo the prompt itself to standard output.
+
+ ----mmuullttiilliinnee--iinnppuutt
+ Allows you to write or paste multiple lines without ending each
+ in '\'.
+
+SSEERRVVEERR OOPPTTIIOONNSS
+ The following options may be specified when llllaammaaffiillee is running in
+ ----sseerrvveerr mode.
+
+ ----ppoorrtt _P_O_R_T
+ Port to listen
+
+ Default: 8080
+
+ ----hhoosstt _I_P_A_D_D_R
+ IP address to listen.
+
+ Default: 127.0.0.1
+
+ --ttoo _N, ----ttiimmeeoouutt _N
+ Server read/write timeout in seconds.
+
+ Default: 600
+
+ --nnpp _N, ----ppaarraalllleell _N
+ Number of slots for process requests.
+
+ Default: 1
+
+ --ccbb, ----ccoonntt--bbaattcchhiinngg
+ Enable continuous batching (a.k.a dynamic batching).
+
+ Default: disabled
+
+ --ssppff _F_N_A_M_E, ----ssyysstteemm--pprroommpptt--ffiillee _F_N_A_M_E
+ Set a file to load a system prompt (initial prompt of all slots),
+ this is useful for chat applications.
+
+ --aa _A_L_I_A_S, ----aalliiaass _A_L_I_A_S
+ Set an alias for the model. This will be added as the _m_o_d_e_l field
+ in completion responses.
+
+ ----ppaatthh _P_U_B_L_I_C___P_A_T_H
+ Path from which to serve static files.
+
+ Default: _/_z_i_p_/_l_l_a_m_a_._c_p_p_/_s_e_r_v_e_r_/_p_u_b_l_i_c
+
+ ----eemmbbeeddddiinngg
+ Enable embedding vector output.
+
+ Default: disabled
+
+ ----nnoobbrroowwsseerr
+ Do not attempt to open a web browser tab at startup.
+
+LLOOGG OOPPTTIIOONNSS
+ The following log options are available:
+
+ --lldd _L_O_G_D_I_R, ----llooggddiirr _L_O_G_D_I_R
+ Path under which to save YAML logs (no logging if unset)
+
+ ----lloogg--tteesstt
+ Run simple logging test
+
+ ----lloogg--ddiissaabbllee
+ Disable trace logs
+
+ ----lloogg--eennaabbllee
+ Enable trace logs
+
+ ----lloogg--ffiillee
+ Specify a log filename (without extension)
+
+ ----lloogg--nneeww
+ Create a separate new log file on start. Each log file will have
+ unique name: _<_n_a_m_e_>_._<_I_D_>_._l_o_g
+
+ ----lloogg--aappppeenndd
+ Don't truncate the old log file.
+
+EEXXAAMMPPLLEESS
+ Here's an example of how to run llama.cpp's built-in HTTP server. This
+ example uses LLaVA v1.5-7B, a multimodal LLM that works with llama.cpp's
+ recently-added support for image inputs.
+
+ llamafile \
+ -m llava-v1.5-7b-Q8_0.gguf \
+ --mmproj llava-v1.5-7b-mmproj-Q8_0.gguf \
+ --host 0.0.0.0
+
+ Here's an example of how to generate code for a libc function using the
+ llama.cpp command line interface, utilizing WizardCoder-Python-13B
+ weights:
+
+ llamafile \
+ -m wizardcoder-python-13b-v1.0.Q8_0.gguf \
+ --temp 0 -r '}\n' -r '```\n' \
+ -e -p '```c\nvoid *memcpy(void *dst, const void *src, size_t size) {\n'
+
+ Here's a similar example that instead utilizes Mistral-7B-Instruct
+ weights for prose composition:
+
+ llamafile \
+ -m mistral-7b-instruct-v0.2.Q5_K_M.gguf \
+ -p '[INST]Write a story about llamas[/INST]'
+
+ Here's an example of how llamafile can be used as an interactive chatbot
+ that lets you query knowledge contained in training data:
+
+ llamafile -m llama-65b-Q5_K.gguf -p '
+ The following is a conversation between a Researcher and their helpful AI
+ assistant Digital Athena which is a large language model trained on the
+ sum of human knowledge.
+ Researcher: Good morning.
+ Digital Athena: How can I help you today?
+ Researcher:' --interactive --color --batch_size 1024 --ctx_size 4096 \
+ --keep -1 --temp 0 --mirostat 2 --in-prefix ' ' --interactive-first \
+ --in-suffix 'Digital Athena:' --reverse-prompt 'Researcher:'
+
+ Here's an example of how you can use llamafile to summarize HTML URLs:
+
+ (
+ echo '[INST]Summarize the following text:'
+ links -codepage utf-8 \
+ -force-html \
+ -width 500 \
+ -dump https://www.poetryfoundation.org/poems/48860/the-raven |
+ sed 's/ */ /g'
+ echo '[/INST]'
+ ) | llamafile \
+ -m mistral-7b-instruct-v0.2.Q5_K_M.gguf \
+ -f /dev/stdin \
+ -c 0 \
+ --temp 0 \
+ -n 500 \
+ --silent-prompt 2>/dev/null
+
+ Here's how you can use llamafile to describe a jpg/png/gif/bmp image:
+
+ llamafile --temp 0 \
+ --image lemurs.jpg \
+ -m llava-v1.5-7b-Q4_K.gguf \
+ --mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \
+ -e -p '### User: What do you see?\n### Assistant: ' \
+ --silent-prompt 2>/dev/null
+
+ If you wanted to write a script to rename all your image files, you could
+ use the following command to generate a safe filename:
+
+ llamafile --temp 0 \
+ --image ~/Pictures/lemurs.jpg \
+ -m llava-v1.5-7b-Q4_K.gguf \
+ --mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \
+ --grammar 'root ::= [a-z]+ (" " [a-z]+)+' \
+ -e -p '### User: What do you see?\n### Assistant: ' \
+ --silent-prompt 2>/dev/null |
+ sed -e's/ /_/g' -e's/$/.jpg/'
+ a_baby_monkey_on_the_back_of_a_mother.jpg
+
+ Here's an example of how to make an API request to the OpenAI API
+ compatible completions endpoint when your llllaammaaffiillee is running in the
+ background in ----sseerrvveerr mode.
+
+ curl -s http://localhost:8080/v1/chat/completions \
+ -H "Content-Type: application/json" -d '{
+ "model": "gpt-3.5-turbo",
+ "stream": true,
+ "messages": [
+ {
+ "role": "system",
+ "content": "You are a poetic assistant."
+ },
+ {
+ "role": "user",
+ "content": "Compose a poem that explains FORTRAN."
+ }
+ ]
+ }' | python3 -c '
+ import json
+ import sys
+ json.dump(json.load(sys.stdin), sys.stdout, indent=2)
+ print()
+
+BBUUGGSS
+ Your HTTP server is great for local hosting, but it would need much more
+ work to be a production worthy component of a public-facing service. For
+ example, C++ exceptions caused by JSON parsing errors will make it abort
+ and print a backtrace.
+
+PPRROOTTIIPP
+ NVIDIA users need to pass the --nnggll _3_5 flag to enable GPU acceleration.
+ It's not enabled by default since it sometimes needs to be tuned for
+ system hardware and model architecture.
+
+SSEEEE AALLSSOO
+ llamafile-quantize(1), llamafile-perplexity(1), llava-quantize(1),
+ zipalign(1), unzip(1)
+
+Mozilla Ocho January 1, 2024 Mozilla Ocho
diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cpp
index 641b318b..09f03904 100644
--- a/llama.cpp/main/main.cpp
+++ b/llama.cpp/main/main.cpp
@@ -109,22 +109,27 @@ static bool has_argument(int argc, char ** argv, const char * arg) {
}
int main(int argc, char ** argv) {
+ llamafile_init();
+ llamafile_check_cpu();
+ ShowCrashReports();
+ LoadZipArgs(&argc, &argv);
if (has_argument(argc, argv, "--version")) {
printf("llamafile v" LLAMAFILE_VERSION_STRING "\n");
- exit(0);
+ return 0;
}
- if (has_argument(argc, argv, "--server") ||
- (!has_argument(argc, argv, "-p") &&
- !has_argument(argc, argv, "-f"))) {
- return server_cli(argc, argv);
+ if (has_argument(argc, argv, "--help")) {
+ llamafile_help("/zip/llama.cpp/main/main.1.asc");
}
- llamafile_init();
- llamafile_check_cpu();
- ShowCrashReports();
- LoadZipArgs(&argc, &argv);
+ if (!has_argument(argc, argv, "--cli") &&
+ (has_argument(argc, argv, "--server") ||
+ (!has_argument(argc, argv, "-p") &&
+ !has_argument(argc, argv, "-f") &&
+ !has_argument(argc, argv, "--random-prompt")))) {
+ return server_cli(argc, argv);
+ }
if (has_argument(argc, argv, "--mmproj")) {
return llava_cli(argc, argv);
diff --git a/llama.cpp/quantize/quantize.1 b/llama.cpp/quantize/quantize.1
index 103ec208..50068d0e 100644
--- a/llama.cpp/quantize/quantize.1
+++ b/llama.cpp/quantize/quantize.1
@@ -84,7 +84,6 @@ COPY Only copy tensors, no quantizing.
.El
.Sh SEE ALSO
.Xr llamafile 1 ,
-.Xr llamafile-server 1 ,
.Xr llamafile-perplexity 1 ,
.Xr llava-quantize 1 ,
.Xr zipalign 1 ,
diff --git a/llama.cpp/server/server.1 b/llama.cpp/server/server.1
deleted file mode 100644
index 69d18ca7..00000000
--- a/llama.cpp/server/server.1
+++ /dev/null
@@ -1,121 +0,0 @@
-.Dd December 5, 2023
-.Dt llamafile-server 1
-.Os
-.Sh NAME
-.Nm llamafile-server
-.Nd large language model server
-.Sh SYNOPSIS
-.Nm llamafile
-.Fl Fl server
-.Op flags...
-.Op Fl m Ar weights.gguf
-.Sh DESCRIPTION
-.Nm
-launches an HTTP server and browser tab with a web user interface for
-the llama.cpp software. The UI lets you chat with your LLM and upload
-images it can analyze. Furthermore, this server provides an OpenAI API
-compatible completions endpoint.
-.Pp
-.Nm llamafile-server
-is launched via the
-.Nm llamafile
-command when the
-.Fl Fl server
-flag is passed. The server flag is implied by default if no prompt is
-specified, i.e. neither the
-.Fl p
-or
-.Fl f
-flags are passed.
-.Sh OPTIONS
-The following options are available:
-.Bl -tag -width indent
-.It Fl h , Fl Fl help
-Show this help message and exit.
-.It Fl v , Fl Fl verbose
-Verbose output (default: disabled)
-.It Fl t Ar N , Fl Fl threads Ar N
-Number of threads to use during computation (default: nproc/2)
-.It Fl tb Ar N , Fl Fl threads-batch Ar N
-Number of threads to use during batch and prompt processing (default:
-same as
-.Fl Fl threads )
-.It Fl c Ar N , Fl Fl ctx-size Ar N
-Size of the prompt context (default: 512, 0 = loaded from model)
-.It Fl Fl rope-scaling Ar {none,linear,yarn}
-RoPE frequency scaling method, defaults to linear unless specified by the model
-.It Fl Fl rope-scale Ar N
-RoPE context scaling factor, expands context by a factor of N
-.It Fl Fl rope-freq-base Ar N
-RoPE base frequency, used by NTK-aware scaling (default: loaded from model)
-.It Fl Fl rope-freq-scale Ar N
-RoPE frequency scaling factor, expands context by a factor of 1/N
-.It Fl Fl yarn-orig-ctx Ar N
-YaRN: original context size of model (default: 0 = model training context size)
-.It Fl Fl yarn-ext-factor Ar N
-YaRN: extrapolation mix factor (default: 1.0, 0.0 = full interpolation)
-.It Fl Fl yarn-attn-factor Ar N
-YaRN: scale sqrt(t) or attention magnitude (default: 1.0)
-.It Fl Fl yarn-beta-slow Ar N
-YaRN: high correction dim or alpha (default: 1.0)
-.It Fl Fl yarn-beta-fast Ar N
-YaRN: low correction dim or beta (default: 32.0)
-.It Fl b Ar N , Fl Fl batch-size Ar N
-Batch size for prompt processing (default: 512)
-.It Fl Fl memory-f32
-Use f32 instead of f16 for memory key+value (default: disabled) Not recommended: doubles context memory required and no measurable increase in quality.
-.It Fl Fl mlock
-Force system to keep model in RAM rather than swapping or compressing.
-.It Fl Fl no-mmap
-Do not memory-map model (slower load but may reduce pageouts if not using mlock).
-.It Fl Fl numa
-Attempt optimizations that help on some NUMA systems if run without this previously, it is recommended to drop the system page cache before using this. See https://github.com/ggerganov/llama.cpp/issues/1437.
-.It Fl ngl Ar N , Fl Fl n-gpu-layers Ar N
-Number of layers to store in VRAM.
-.It Fl ts Ar SPLIT , Fl Fl tensor-split Ar SPLIT
-How to split tensors across multiple GPUs, comma-separated list of proportions, e.g. 3,1
-.It Fl mg Ar i , Fl Fl main-gpu Ar i
-The GPU to use for scratch and small tensors.
-.It Fl nommq , Fl Fl no-mul-mat-q
-Use cuBLAS instead of custom mul_mat_q CUDA kernels. Not recommended since this is both slower and uses more VRAM.
-.It Fl m Ar FNAME , Fl Fl model Ar FNAME
-Model path (default: models/7B/ggml-model-f16.gguf)
-.It Fl a Ar ALIAS , Fl Fl alias Ar ALIAS
-Set an alias for the model, will be added as `model` field in completion response.
-.It Fl Fl lora Ar FNAME
-Apply LoRA adapter (implies
-.Fl Fl no-mmap )
-.It Fl Fl lora-base Ar FNAME
-Optional model to use as a base for the layers modified by the LoRA adapter
-.It Fl Fl host Ar IPADDR
-IP address to listen (default: 127.0.0.1)
-.It Fl Fl port Ar PORT
-Port to listen (default (default: 8080)
-.It Fl Fl path Ar PUBLIC_PATH
-Path from which to serve static files (default /zip/llama.cpp/server/public)
-.It Fl to Ar N , Fl Fl timeout Ar N
-Server read/write timeout in seconds (default: 600)
-.It Fl Fl embedding
-Enable embedding vector output (default: disabled)
-.It Fl np Ar N , Fl Fl parallel Ar N
-Number of slots for process requests (default: 1)
-.It Fl cb , Fl Fl cont-batching
-Enable continuous batching (a.k.a dynamic batching) (default: disabled)
-.It Fl spf Ar FNAME , Fl Fl system-prompt-file Ar FNAME
-Set a file to load a system prompt (initial prompt of all slots), this is useful for chat applications.
-.It Fl Fl mmproj Ar MMPROJ_FILE
-Path to a multimodal projector file for LLaVA.
-.It Fl Fl log-disable
-Disables logging to a file.
-.It Fl Fl nobrowser
-Do not attempt to open a web browser tab at startup.
-.It Fl Fl unsecure
-Disables pledge() sandboxing on Linux and OpenBSD
-.El
-.Sh SEE ALSO
-.Xr llamafile 1 ,
-.Xr llamafile-quantize 1 ,
-.Xr llamafile-perplexity 1 ,
-.Xr llava-quantize 1 ,
-.Xr zipalign 1 ,
-.Xr unzip 1
diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cpp
index 87ea917d..4da15d94 100644
--- a/llama.cpp/server/server.cpp
+++ b/llama.cpp/server/server.cpp
@@ -2651,17 +2651,7 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_con
}
}
-int server_cli(int argc, char ** argv)
-{
- if (argc == 2 && !strcmp(argv[1], "--version")) {
- printf("llamafile v" LLAMAFILE_VERSION_STRING " server\n");
- exit(0);
- }
-
- llamafile_init();
- llamafile_check_cpu();
- LoadZipArgs(&argc, &argv);
- ShowCrashReports();
+int server_cli(int argc, char ** argv) {
// own arguments required by this example
gpt_params params;
diff --git a/llamafile/help.c b/llamafile/help.c
new file mode 100644
index 00000000..54c73010
--- /dev/null
+++ b/llamafile/help.c
@@ -0,0 +1,78 @@
+// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "llamafile.h"
+#include <fcntl.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+
+/**
+ * Displays man page.
+ *
+ * This function uses the system pagination program (less or more) when
+ * it's available. That'll be the case for all OSes, including Windows.
+ * When the standard output file descriptor isn't a teletypewriter, the
+ * man page will be formatted as plain text utf8 without any paginator.
+ *
+ * @param path should be a .txt or .asc file, which is allowed to use
+ * ascii combining characters (for bold and underscore text)
+ * @noreturn
+ */
+wontreturn void llamafile_help(const char *path) {
+
+ // slurp rendered `man llamafile` page
+ int fd;
+ fd = open(path, O_RDONLY);
+ if (fd == -1) {
+ perror(path);
+ exit(1);
+ }
+ int size;
+ size = lseek(fd, 0, SEEK_END);
+ if (size == -1) {
+ perror(path);
+ exit(1);
+ }
+ char *text = calloc(1, size + 1);
+ if (pread(fd, text, size, 0) != size) {
+ perror(path);
+ exit(1);
+ }
+ close(fd);
+
+ // strip ascii combining sequences for bold and underscore text
+ if (!isatty(1)) {
+ int j = 0;
+ for (int i = 0; i < size; ++i) {
+ switch (text[i]) {
+ case '\b':
+ --j;
+ break;
+ default:
+ text[j++] = text[i];
+ break;
+ }
+ }
+ text[j] = 0;
+ }
+
+ // diplay manual page in less or more if available
+ __paginate(1, text);
+ free(text);
+ exit(0);
+}
diff --git a/llamafile/llamafile.h b/llamafile/llamafile.h
index d51c63dd..c380ed52 100644
--- a/llamafile/llamafile.h
+++ b/llamafile/llamafile.h
@@ -2,9 +2,7 @@
#define LLAMAFILE_H_
#include <stdio.h>
#include <stdbool.h>
-#ifdef __cplusplus
-extern "C" {
-#endif
+COSMOPOLITAN_C_START_
struct llamafile;
struct llamafile *llamafile_open(const char *, const char *);
@@ -19,6 +17,7 @@ FILE *llamafile_fp(struct llamafile *);
void llamafile_init(void);
void llamafile_check_cpu(void);
+void llamafile_help(const char *) wontreturn;
const char *llamafile_get_tmp_dir(void);
bool llamafile_extract(const char *, const char *);
int llamafile_is_file_newer_than(const char *, const char *);
@@ -26,7 +25,5 @@ void llamafile_schlep(const void *, size_t);
void llamafile_get_app_dir(char *, size_t);
bool llamafile_launch_browser(const char *);
-#ifdef __cplusplus
-}
-#endif
+COSMOPOLITAN_C_END_
#endif /* LLAMAFILE_H_ */