Skip to content

Distribute and run LLMs with a single file.

Add more content to manual

Justine Tunney7ad00db0deae
diff --git a/Makefile b/Makefile
index 614ac39d..69543a1a 100644
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,7 @@ o/$(MODE)/: o/$(MODE)/llama.cpp o/$(MODE)/llamafile
# for installing to `make PREFIX=/usr/local`
.PHONY: install
install: llamafile/zipalign.1 \
- llamafile/llamafile.1 \
+ llama.cpp/main/main.1 \
llama.cpp/quantize/quantize.1 \
llama.cpp/perplexity/perplexity.1 \
llama.cpp/llava/llava-quantize.1 \
@@ -40,7 +40,7 @@ install: llamafile/zipalign.1 \
$(INSTALL) o/$(MODE)/llama.cpp/llava/llava-quantize $(PREFIX)/bin/llava-quantize
mkdir -p $(PREFIX)/share/man/man1
$(INSTALL) -m 0644 llamafile/zipalign.1 $(PREFIX)/share/man/man1/zipalign.1
- $(INSTALL) -m 0644 llamafile/llamafile.1 $(PREFIX)/share/man/man1/llamafile.1
+ $(INSTALL) -m 0644 llama.cpp/main/main.1 $(PREFIX)/share/man/man1/llamafile.1
$(INSTALL) -m 0644 llama.cpp/quantize/quantize.1 $(PREFIX)/share/man/man1/llamafile-quantize.1
$(INSTALL) -m 0644 llama.cpp/perplexity/perplexity.1 $(PREFIX)/share/man/man1/llamafile-perplexity.1
$(INSTALL) -m 0644 llama.cpp/llava/llava-quantize.1 $(PREFIX)/share/man/man1/llava-quantize.1
diff --git a/build/config.mk b/build/config.mk
index 9373e5e5..cd1db0ef 100644
--- a/build/config.mk
+++ b/build/config.mk
@@ -33,7 +33,7 @@ endif
endif
# make build more deterministic
-LC_ALL = C
+LC_ALL = C.UTF-8
SOURCE_DATE_EPOCH = 0
export MODE
export TMPDIR
diff --git a/build/rules.mk b/build/rules.mk
index c3352276..2fd99a91 100644
--- a/build/rules.mk
+++ b/build/rules.mk
@@ -25,7 +25,7 @@ o/$(MODE)/%: o/$(MODE)/%.o
.PRECIOUS: %.1.asc
%.1.asc: %.1
- -man $< >[email protected] && mv -f [email protected] $@
+ -MANWIDTH=80 MAN_KEEP_FORMATTING=1 man $< >[email protected] && mv -f [email protected] $@
o/$(MODE)/%.zip.o: % $(COSMOCC)
diff --git a/llama.cpp/main/BUILD.mk b/llama.cpp/main/BUILD.mk
index 13e63c72..090be82e 100644
--- a/llama.cpp/main/BUILD.mk
+++ b/llama.cpp/main/BUILD.mk
@@ -13,7 +13,7 @@ o/$(MODE)/llama.cpp/main/main: \
o/$(MODE)/llama.cpp/server/server.a \
o/$(MODE)/llama.cpp/llava/llava.a \
o/$(MODE)/llama.cpp/llama.cpp.a \
- o/$(MODE)/llamafile/llamafile.1.asc.zip.o \
+ o/$(MODE)/llama.cpp/main/main.1.asc.zip.o \
$(LLAMA_CPP_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)
.PHONY: o/$(MODE)/llama.cpp/main
diff --git a/llama.cpp/main/README.md b/llama.cpp/main/README.md
new file mode 100644
index 00000000..e381ea4c
--- /dev/null
+++ b/llama.cpp/main/README.md
@@ -0,0 +1,339 @@
+# llama.cpp/example/main
+
+This example program allows you to use various LLaMA language models in an easy and efficient way. It is specifically designed to work with the [llama.cpp](https://github.com/ggerganov/llama.cpp) project, which provides a plain C/C++ implementation with optional 4-bit quantization support for faster, lower memory inference, and is optimized for desktop CPUs. This program can be used to perform various inference tasks with LLaMA models, including generating text based on user-provided prompts and chat-like interactions with reverse prompts.
+
+## Table of Contents
+
+1. [Quick Start](#quick-start)
+2. [Common Options](#common-options)
+3. [Input Prompts](#input-prompts)
+4. [Interaction](#interaction)
+5. [Context Management](#context-management)
+6. [Generation Flags](#generation-flags)
+7. [Performance Tuning and Memory Options](#performance-tuning-and-memory-options)
+8. [Additional Options](#additional-options)
+
+## Quick Start
+
+To get started right away, run the following command, making sure to use the correct path for the model you have:
+
+#### Unix-based systems (Linux, macOS, etc.):
+
+```bash
+./main -m models/7B/ggml-model.bin --prompt "Once upon a time"
+```
+
+#### Windows:
+
+```powershell
+main.exe -m models\7B\ggml-model.bin --prompt "Once upon a time"
+```
+
+For an interactive experience, try this command:
+
+#### Unix-based systems (Linux, macOS, etc.):
+
+```bash
+./main -m models/7B/ggml-model.bin -n -1 --color -r "User:" --in-prefix " " -i -p \
+'User: Hi
+AI: Hello. I am an AI chatbot. Would you like to talk?
+User: Sure!
+AI: What would you like to talk about?
+User:'
+```
+
+#### Windows:
+
+```powershell
+main.exe -m models\7B\ggml-model.bin -n -1 --color -r "User:" --in-prefix " " -i -e -p "User: Hi\nAI: Hello. I am an AI chatbot. Would you like to talk?\nUser: Sure!\nAI: What would you like to talk about?\nUser:"
+```
+
+The following command generates "infinite" text from a starting prompt (you can use `Ctrl-C` to stop it):
+
+#### Unix-based systems (Linux, macOS, etc.):
+
+```bash
+./main -m models/7B/ggml-model.bin --ignore-eos -n -1 --random-prompt
+```
+
+#### Windows:
+
+```powershell
+main.exe -m models\7B\ggml-model.bin --ignore-eos -n -1 --random-prompt
+```
+
+## Common Options
+
+In this section, we cover the most commonly used options for running the `main` program with the LLaMA models:
+
+- `-m FNAME, --model FNAME`: Specify the path to the LLaMA model file (e.g., `models/7B/ggml-model.bin`).
+- `-i, --interactive`: Run the program in interactive mode, allowing you to provide input directly and receive real-time responses.
+- `-ins, --instruct`: Run the program in instruction mode, which is particularly useful when working with Alpaca models.
+- `-n N, --n-predict N`: Set the number of tokens to predict when generating text. Adjusting this value can influence the length of the generated text.
+- `-c N, --ctx-size N`: Set the size of the prompt context. The default is 512, but LLaMA models were built with a context of 2048, which will provide better results for longer input/inference.
+
+## Input Prompts
+
+The `main` program provides several ways to interact with the LLaMA models using input prompts:
+
+- `--prompt PROMPT`: Provide a prompt directly as a command-line option.
+- `--file FNAME`: Provide a file containing a prompt or multiple prompts.
+- `--interactive-first`: Run the program in interactive mode and wait for input right away. (More on this below.)
+- `--random-prompt`: Start with a randomized prompt.
+
+## Interaction
+
+The `main` program offers a seamless way to interact with LLaMA models, allowing users to engage in real-time conversations or provide instructions for specific tasks. The interactive mode can be triggered using various options, including `--interactive`, `--interactive-first`, and `--instruct`.
+
+In interactive mode, users can participate in text generation by injecting their input during the process. Users can press `Ctrl+C` at any time to interject and type their input, followed by pressing `Return` to submit it to the LLaMA model. To submit additional lines without finalizing input, users can end the current line with a backslash (`\`) and continue typing.
+
+### Interaction Options
+
+- `-i, --interactive`: Run the program in interactive mode, allowing users to engage in real-time conversations or provide specific instructions to the model.
+- `--interactive-first`: Run the program in interactive mode and immediately wait for user input before starting the text generation.
+- `-ins, --instruct`: Run the program in instruction mode, which is specifically designed to work with Alpaca models that excel in completing tasks based on user instructions.
+- `--color`: Enable colorized output to differentiate visually distinguishing between prompts, user input, and generated text.
+
+By understanding and utilizing these interaction options, you can create engaging and dynamic experiences with the LLaMA models, tailoring the text generation process to your specific needs.
+
+### Reverse Prompts
+
+Reverse prompts are a powerful way to create a chat-like experience with a LLaMA model by pausing the text generation when specific text strings are encountered:
+
+- `-r PROMPT, --reverse-prompt PROMPT`: Specify one or multiple reverse prompts to pause text generation and switch to interactive mode. For example, `-r "User:"` can be used to jump back into the conversation whenever it's the user's turn to speak. This helps create a more interactive and conversational experience. However, the reverse prompt doesn't work when it ends with a space.
+
+To overcome this limitation, you can use the `--in-prefix` flag to add a space or any other characters after the reverse prompt.
+
+### In-Prefix
+
+The `--in-prefix` flag is used to add a prefix to your input, primarily, this is used to insert a space after the reverse prompt. Here's an example of how to use the `--in-prefix` flag in conjunction with the `--reverse-prompt` flag:
+
+```sh
+./main -r "User:" --in-prefix " "
+```
+
+### In-Suffix
+
+The `--in-suffix` flag is used to add a suffix after your input. This is useful for adding an "Assistant:" prompt after the user's input. It's added after the new-line character (`\n`) that's automatically added to the end of the user's input. Here's an example of how to use the `--in-suffix` flag in conjunction with the `--reverse-prompt` flag:
+
+```sh
+./main -r "User:" --in-prefix " " --in-suffix "Assistant:"
+```
+
+### Instruction Mode
+
+Instruction mode is particularly useful when working with Alpaca models, which are designed to follow user instructions for specific tasks:
+
+- `-ins, --instruct`: Enable instruction mode to leverage the capabilities of Alpaca models in completing tasks based on user-provided instructions.
+
+Technical detail: the user's input is internally prefixed with the reverse prompt (or `### Instruction:` as the default), and followed by `### Response:` (except if you just press Return without any input, to keep generating a longer response).
+
+By understanding and utilizing these interaction options, you can create engaging and dynamic experiences with the LLaMA models, tailoring the text generation process to your specific needs.
+
+## Context Management
+
+During text generation, LLaMA models have a limited context size, which
+means they can only consider a certain number of tokens from the input
+and generated text. When the context fills up, the model resets
+internally, potentially losing some information from the beginning of
+the conversation or instructions. Context management options help
+maintain continuity and coherence in these situations.
+
+### Context Size
+
+The `--ctx-size` option allows you to set the size of the prompt context
+used by the LLaMA models during text generation. A larger context size
+helps the model to better comprehend and generate responses for longer
+input or conversations.
+
+- `-c N, --ctx-size N`: Set the size of the prompt context (default: 512). The LLaMA models were built with a context of 2048, which will yield the best results on longer input/inference. However, increasing the context size beyond 2048 may lead to unpredictable results.
+
+### Extended Context Size
+
+Some fine-tuned models have extended the context length by scaling RoPE.
+For example, if the original pre-trained model have a context length
+(max sequence length) of 4096 (4k) and the fine-tuned model have 32k.
+That is a scaling factor of 8, and should work by setting the above
+`--ctx-size` to 32768 (32k) and `--rope-scale` to 8.
+
+- `--rope-scale N`: Where N is the linear scaling factor used by the fine-tuned model.
+
+### Keep Prompt
+
+The `--keep` option allows users to retain the original prompt when the
+model runs out of context, ensuring a connection to the initial
+instruction or conversation topic is maintained.
+
+- `--keep N`: Specify the number of tokens from the initial prompt to
+ retain when the model resets its internal context. By default, this
+ value is set to 0 (meaning no tokens are kept). Use `-1` to retain
+ all tokens from the initial prompt.
+
+By utilizing context management options like `--ctx-size` and `--keep`, you can maintain a more coherent and consistent interaction with the LLaMA models, ensuring that the generated text remains relevant to the original prompt or conversation.
+
+## Generation Flags
+
+The following options allow you to control the text generation process
+and fine-tune the diversity, creativity, and quality of the generated
+text according to your needs. By adjusting these options and
+experimenting with different combinations of values, you can find the
+best settings for your specific use case.
+
+### Number of Tokens to Predict
+
+- `-n N, --n-predict N`: Set the number of tokens to predict when generating text (default: 128, -1 = infinity, -2 = until context filled)
+
+The `--n-predict` option controls the number of tokens the model generates in response to the input prompt. By adjusting this value, you can influence the length of the generated text. A higher value will result in longer text, while a lower value will produce shorter text.
+
+A value of -1 will enable infinite text generation, even though we have a finite context window. When the context window is full, some of the earlier tokens (half of the tokens after `--n-keep`) will be discarded. The context must then be re-evaluated before generation can resume. On large models and/or large context windows, this will result in significant pause in output.
+
+If the pause is undesirable, a value of -2 will stop generation immediately when the context is filled.
+
+It is important to note that the generated text may be shorter than the specified number of tokens if an End-of-Sequence (EOS) token or a reverse prompt is encountered. In interactive mode text generation will pause and control will be returned to the user. In non-interactive mode, the program will end. In both cases, the text generation may stop before reaching the specified `n-predict` value. If you want the model to keep going without ever producing End-of-Sequence on its own, you can use the `--ignore-eos` parameter.
+
+### Temperature
+
+- `--temp N`: Adjust the randomness of the generated text (default: 0.8).
+
+Temperature is a hyperparameter that controls the randomness of the generated text. It affects the probability distribution of the model's output tokens. A higher temperature (e.g., 1.5) makes the output more random and creative, while a lower temperature (e.g., 0.5) makes the output more focused, deterministic, and conservative. The default value is 0.8, which provides a balance between randomness and determinism. At the extreme, a temperature of 0 will always pick the most likely next token, leading to identical outputs in each run.
+
+Example usage: `--temp 0.5`
+
+### Repeat Penalty
+
+- `--repeat-penalty N`: Control the repetition of token sequences in the generated text (default: 1.1).
+- `--repeat-last-n N`: Last n tokens to consider for penalizing repetition (default: 64, 0 = disabled, -1 = ctx-size).
+- `--no-penalize-nl`: Disable penalization for newline tokens when applying the repeat penalty.
+
+The `repeat-penalty` option helps prevent the model from generating repetitive or monotonous text. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. The default value is 1.1.
+
+The `repeat-last-n` option controls the number of tokens in the history to consider for penalizing repetition. A larger value will look further back in the generated text to prevent repetitions, while a smaller value will only consider recent tokens. A value of 0 disables the penalty, and a value of -1 sets the number of tokens considered equal to the context size (`ctx-size`).
+
+Use the `--no-penalize-nl` option to disable newline penalization when applying the repeat penalty. This option is particularly useful for generating chat conversations, dialogues, code, poetry, or any text where newline tokens play a significant role in structure and formatting. Disabling newline penalization helps maintain the natural flow and intended formatting in these specific use cases.
+
+Example usage: `--repeat-penalty 1.15 --repeat-last-n 128 --no-penalize-nl`
+
+### Top-K Sampling
+
+- `--top-k N`: Limit the next token selection to the K most probable tokens (default: 40).
+
+Top-k sampling is a text generation method that selects the next token only from the top k most likely tokens predicted by the model. It helps reduce the risk of generating low-probability or nonsensical tokens, but it may also limit the diversity of the output. A higher value for top-k (e.g., 100) will consider more tokens and lead to more diverse text, while a lower value (e.g., 10) will focus on the most probable tokens and generate more conservative text. The default value is 40.
+
+Example usage: `--top-k 30`
+
+### Top-P Sampling
+
+- `--top-p N`: Limit the next token selection to a subset of tokens with a cumulative probability above a threshold P (default: 0.9).
+
+Top-p sampling, also known as nucleus sampling, is another text generation method that selects the next token from a subset of tokens that together have a cumulative probability of at least p. This method provides a balance between diversity and quality by considering both the probabilities of tokens and the number of tokens to sample from. A higher value for top-p (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. The default value is 0.9.
+
+Example usage: `--top-p 0.95`
+
+### Min P Sampling
+
+- `--min-p N`: Sets a minimum base probability threshold for token selection (default: 0.05).
+
+The Min-P sampling method was designed as an alternative to Top-P, and aims to ensure a balance of quality and variety. The parameter *p* represents the minimum probability for a token to be considered, relative to the probability of the most likely token. For example, with *p*=0.05 and the most likely token having a probability of 0.9, logits with a value less than 0.045 are filtered out.
+
+Example usage: `--min-p 0.05`
+
+### Tail Free Sampling (TFS)
+
+- `--tfs N`: Enable tail free sampling with parameter z (default: 1.0, 1.0 = disabled).
+
+Tail free sampling (TFS) is a text generation technique that aims to reduce the impact of less likely tokens, which may be less relevant, less coherent, or nonsensical, on the output. Similar to Top-P it tries to determine the bulk of the most likely tokens dynamically. But TFS filters out logits based on the second derivative of their probabilities. Adding tokens is stopped after the sum of the second derivatives reaches the parameter z. In short: TFS looks how quickly the probabilities of the tokens decrease and cuts off the tail of unlikely tokens using the parameter z. Typical values for z are in the range of 0.9 to 0.95. A value of 1.0 would include all tokens, and thus disables the effect of TFS.
+
+Example usage: `--tfs 0.95`
+
+### Locally Typical Sampling
+
+- `--typical N`: Enable locally typical sampling with parameter p (default: 1.0, 1.0 = disabled).
+
+Locally typical sampling promotes the generation of contextually coherent and diverse text by sampling tokens that are typical or expected based on the surrounding context. By setting the parameter p between 0 and 1, you can control the balance between producing text that is locally coherent and diverse. A value closer to 1 will promote more contextually coherent tokens, while a value closer to 0 will promote more diverse tokens. A value equal to 1 disables locally typical sampling.
+
+Example usage: `--typical 0.9`
+
+### Mirostat Sampling
+
+- `--mirostat N`: Enable Mirostat sampling, controlling perplexity during text generation (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0).
+- `--mirostat-lr N`: Set the Mirostat learning rate, parameter eta (default: 0.1).
+- `--mirostat-ent N`: Set the Mirostat target entropy, parameter tau (default: 5.0).
+
+Mirostat is an algorithm that actively maintains the quality of generated text within a desired range during text generation. It aims to strike a balance between coherence and diversity, avoiding low-quality output caused by excessive repetition (boredom traps) or incoherence (confusion traps).
+
+The `--mirostat-lr` option sets the Mirostat learning rate (eta). The learning rate influences how quickly the algorithm responds to feedback from the generated text. A lower learning rate will result in slower adjustments, while a higher learning rate will make the algorithm more responsive. The default value is `0.1`.
+
+The `--mirostat-ent` option sets the Mirostat target entropy (tau), which represents the desired perplexity value for the generated text. Adjusting the target entropy allows you to control the balance between coherence and diversity in the generated text. A lower value will result in more focused and coherent text, while a higher value will lead to more diverse and potentially less coherent text. The default value is `5.0`.
+
+Example usage: `--mirostat 2 --mirostat-lr 0.05 --mirostat-ent 3.0`
+
+### Logit Bias
+
+- `-l TOKEN_ID(+/-)BIAS, --logit-bias TOKEN_ID(+/-)BIAS`: Modify the likelihood of a token appearing in the generated text completion.
+
+The logit bias option allows you to manually adjust the likelihood of specific tokens appearing in the generated text. By providing a token ID and a positive or negative bias value, you can increase or decrease the probability of that token being generated.
+
+For example, use `--logit-bias 15043+1` to increase the likelihood of the token 'Hello', or `--logit-bias 15043-1` to decrease its likelihood. Using a value of negative infinity, `--logit-bias 15043-inf` ensures that the token `Hello` is never produced.
+
+A more practical use case might be to prevent the generation of `\code{begin}` and `\code{end}` by setting the `\` token (29905) to negative infinity with `-l 29905-inf`. (This is due to the prevalence of LaTeX codes that show up in LLaMA model inference.)
+
+Example usage: `--logit-bias 29905-inf`
+
+### RNG Seed
+
+- `-s SEED, --seed SEED`: Set the random number generator (RNG) seed (default: -1, -1 = random seed).
+
+The RNG seed is used to initialize the random number generator that influences the text generation process. By setting a specific seed value, you can obtain consistent and reproducible results across multiple runs with the same input and settings. This can be helpful for testing, debugging, or comparing the effects of different options on the generated text to see when they diverge. If the seed is set to a value less than 0, a random seed will be used, which will result in different outputs on each run.
+
+## Performance Tuning and Memory Options
+
+These options help improve the performance and memory usage of the LLaMA models. By adjusting these settings, you can fine-tune the model's behavior to better suit your system's capabilities and achieve optimal performance for your specific use case.
+
+### Number of Threads
+
+- `-t N, --threads N`: Set the number of threads to use during generation. For optimal performance, it is recommended to set this value to the number of physical CPU cores your system has (as opposed to the logical number of cores). Using the correct number of threads can greatly improve performance.
+- `-tb N, --threads-batch N`: Set the number of threads to use during batch and prompt processing. In some systems, it is beneficial to use a higher number of threads during batch processing than during generation. If not specified, the number of threads used for batch processing will be the same as the number of threads used for generation.
+
+### Mlock
+
+- `--mlock`: Lock the model in memory, preventing it from being swapped out when memory-mapped. This can improve performance but trades away some of the advantages of memory-mapping by requiring more RAM to run and potentially slowing down load times as the model loads into RAM.
+
+### No Memory Mapping
+
+- `--no-mmap`: Do not memory-map the model. By default, models are mapped into memory, which allows the system to load only the necessary parts of the model as needed. However, if the model is larger than your total amount of RAM or if your system is low on available memory, using mmap might increase the risk of pageouts, negatively impacting performance. Disabling mmap results in slower load times but may reduce pageouts if you're not using `--mlock`. Note that if the model is larger than the total amount of RAM, turning off mmap would prevent the model from loading at all.
+
+### NUMA support
+
+- `--numa`: Attempt optimizations that help on some systems with non-uniform memory access. This currently consists of pinning an equal proportion of the threads to the cores on each NUMA node, and disabling prefetch and readahead for mmap. The latter causes mapped pages to be faulted in on first access instead of all at once, and in combination with pinning threads to NUMA nodes, more of the pages end up on the NUMA node where they are used. Note that if the model is already in the system page cache, for example because of a previous run without this option, this will have little effect unless you drop the page cache first. This can be done by rebooting the system or on Linux by writing '3' to '/proc/sys/vm/drop_caches' as root.
+
+### Memory Float 32
+
+- `--memory-f32`: Use 32-bit floats instead of 16-bit floats for memory key+value. This doubles the context memory requirement and cached prompt file size but does not appear to increase generation quality in a measurable way. Not recommended.
+
+### Batch Size
+
+- `-b N, --batch-size N`: Set the batch size for prompt processing (default: 512). This large batch size benefits users who have BLAS installed and enabled it during the build. If you don't have BLAS enabled ("BLAS=0"), you can use a smaller number, such as 8, to see the prompt progress as it's evaluated in some situations.
+
+### Prompt Caching
+
+- `--prompt-cache FNAME`: Specify a file to cache the model state after the initial prompt. This can significantly speed up the startup time when you're using longer prompts. The file is created during the first run and is reused and updated in subsequent runs. **Note**: Restoring a cached prompt does not imply restoring the exact state of the session at the point it was saved. So even when specifying a specific seed, you are not guaranteed to get the same sequence of tokens as the original generation.
+
+### Grammars
+
+- `--grammar GRAMMAR`, `--grammar-file FILE`: Specify a grammar (defined inline or in a file) to constrain model output to a specific format. For example, you could force the model to output JSON or to speak only in emojis. See the [GBNF guide](../../grammars/README.md) for details on the syntax.
+
+### Quantization
+
+For information about 4-bit quantization, which can significantly improve performance and reduce memory usage, please refer to llama.cpp's primary [README](../../README.md#prepare-data--run).
+
+## Additional Options
+
+These options provide extra functionality and customization when running the LLaMA models:
+
+- `-h, --help`: Display a help message showing all available options and their default values. This is particularly useful for checking the latest options and default values, as they can change frequently, and the information in this document may become outdated.
+- `--verbose-prompt`: Print the prompt before generating text.
+- `-ngl N, --n-gpu-layers N`: When compiled with appropriate support (currently CLBlast or cuBLAS), this option allows offloading some layers to the GPU for computation. Generally results in increased performance.
+- `-mg i, --main-gpu i`: When using multiple GPUs this option controls which GPU is used for small tensors for which the overhead of splitting the computation across all GPUs is not worthwhile. The GPU in question will use slightly more VRAM to store a scratch buffer for temporary results. By default GPU 0 is used. Requires cuBLAS.
+- `-ts SPLIT, --tensor-split SPLIT`: When using multiple GPUs this option controls how large tensors should be split across all GPUs. `SPLIT` is a comma-separated list of non-negative values that assigns the proportion of data that each GPU should get in order. For example, "3,2" will assign 60% of the data to GPU 0 and 40% to GPU 1. By default the data is split in proportion to VRAM but this may not be optimal for performance. Requires cuBLAS.
+- `--lora FNAME`: Apply a LoRA (Low-Rank Adaptation) adapter to the model (implies --no-mmap). This allows you to adapt the pretrained model to specific tasks or domains.
+- `--lora-base FNAME`: Optional model to use as a base for the layers modified by the LoRA adapter. This flag is used in conjunction with the `--lora` flag, and specifies the base model for the adaptation.
diff --git a/llamafile/llamafile.1 b/llama.cpp/main/main.1
similarity index 85%
rename from llamafile/llamafile.1
rename to llama.cpp/main/main.1
index 25e5732f..ab8a21e7 100644
--- a/llamafile/llamafile.1
+++ b/llama.cpp/main/main.1
@@ -1,16 +1,3 @@
-.\" Copyright 2024 Mozilla Foundation
-.\"
-.\" Licensed under the Apache License, Version 2.0 (the "License");
-.\" you may not use this file except in compliance with the License.
-.\" You may obtain a copy of the License at
-.\"
-.\" http://www.apache.org/licenses/LICENSE-2.0
-.\"
-.\" Unless required by applicable law or agreed to in writing, software
-.\" distributed under the License is distributed on an "AS IS" BASIS,
-.\" WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-.\" See the License for the specific language governing permissions and
-.\" limitations under the License.
.Dd January 1, 2024
.Dt LLAMAFILE 1
.Os Mozilla Ocho
@@ -106,11 +93,28 @@ Prefix BOS to user inputs, preceding the
.Fl Fl in-prefix
string.
.It Fl Fl in-prefix Ar STRING
-String to prefix user inputs with.
+This flag is used to add a prefix to your input, primarily, this is used
+to insert a space after the reverse prompt. Here's an example of how to
+use the
+.Fl Fl in-prefix
+flag in conjunction with the
+.Fl Fl reverse-prompt
+flag:
+.Pp
+.Dl "./main -r \[dq]User:\[dq] --in-prefix \[dq] \[dq]"
.Pp
Default: empty
.It Fl Fl in-suffix Ar STRING
-String to suffix after user inputs with.
+This flag is used to add a suffix after your input. This is useful for
+adding an \[dq]Assistant:\[dq] prompt after the user's input. It's added
+after the new-line character (\[rs]n) that's automatically added to the
+end of the user's input. Here's an example of how to use the
+.Fl Fl in-suffix
+flag in conjunction with the
+.Fl Fl reverse-prompt
+flag:
+.Pp
+.Dl "./main -r \[dq]User:\[dq] --in-prefix \[dq] \[dq] --in-suffix \[dq]Assistant:\[dq]"
.Pp
Default: empty
.It Fl n Ar N , Fl Fl n-predict Ar N
@@ -125,11 +129,14 @@ Number of tokens to predict.
.Pp
Default: -1
.It Fl c Ar N , Fl Fl ctx-size Ar N
-Size of the prompt context.
+Set the size of the prompt context. A larger context size helps the
+model to better comprehend and generate responses for longer input or
+conversations. The LLaMA models were built with a context of 2048, which
+yields the best results on longer input / inference.
.Pp
.Bl -dash -compact
.It
-0 = loaded from model
+0 = loaded automatically from model
.El
.Pp
Default: 512
@@ -275,7 +282,19 @@ Default: 1.0
.It Fl Fl rope-scaling Ar {none,linear,yarn}
RoPE frequency scaling method, defaults to linear unless specified by the model
.It Fl Fl rope-scale Ar N
-RoPE context scaling factor, expands context by a factor of N
+RoPE context scaling factor, expands context by a factor of
+.Ar N
+where
+.Ar N
+is the linear scaling factor used by the fine-tuned model. Some
+fine-tuned models have extended the context length by scaling RoPE. For
+example, if the original pre-trained model have a context length (max
+sequence length) of 4096 (4k) and the fine-tuned model have 32k. That is
+a scaling factor of 8, and should work by setting the above
+.Fl Fl ctx-size
+to 32768 (32k) and
+.Fl Fl rope-scale
+to 8.
.It Fl Fl rope-freq-base Ar N
RoPE base frequency, used by NTK-aware scaling.
.Pp
@@ -327,11 +346,18 @@ Number of tasks to use when computing the HellaSwag score.
.Pp
Default: 400
.It Fl Fl keep Ar N
-Number of tokens to keep from the initial prompt.
+This flag allows users to retain the original prompt when the model runs
+out of context, ensuring a connection to the initial instruction or
+conversation topic is maintained, where
+.Ar N
+is the number of tokens from the initial prompt to retain when the model
+resets its internal context.
.Pp
.Bl -dash -compact
.It
--1 = all
+0 = no tokens are kept from initial prompt
+.It
+-1 = retain all tokens from initial prompt
.El
.Pp
Default: 0
@@ -522,17 +548,37 @@ URL with the image/jpeg MIME type. See also the
.Fl Fl mmproj
flag for supplying the vision model.
.It Fl i , Fl Fl interactive
-Run in interactive mode.
+Run the program in interactive mode, allowing users to engage in
+real-time conversations or provide specific instructions to the model.
.It Fl Fl interactive-first
-Run in interactive mode and wait for input right away.
+Run the program in interactive mode and immediately wait for user input
+before starting the text generation.
.It Fl ins , Fl Fl instruct
-Run in instruction mode (use with Alpaca models).
+Run the program in instruction mode, which is specifically designed to
+work with Alpaca models that excel in completing tasks based on user
+instructions.
+.Pp
+Technical details: The user's input is internally prefixed with the
+reverse prompt (or \[dq]### Instruction:\[dq] as the default), and
+followed by \[dq]### Response:\[dq] (except if you just press Return
+without any input, to keep generating a longer response).
+.Pp
+By understanding and utilizing these interaction options, you can create
+engaging and dynamic experiences with the LLaMA models, tailoring the
+text generation process to your specific needs.
.It Fl r Ar PROMPT , Fl Fl reverse-prompt Ar PROMPT
-Halt generation at
-.Ar PROMPT
-and return control in interactive mode (can be specified more than once for multiple prompts).
+Specify one or multiple reverse prompts to pause text generation and
+switch to interactive mode. For example,
+.Fl r Ar \[dq]User:\[dq]
+can be used to jump back into the conversation whenever it's the user's
+turn to speak. This helps create a more interactive and conversational
+experience. However, the reverse prompt doesn't work when it ends with a
+space. To overcome this limitation, you can use the
+.Fl Fl in-prefix
+flag to add a space or any other characters after the reverse prompt.
.It Fl Fl color
-Colorise output to distinguish prompt and user input from generations.
+Enable colorized output to differentiate visually distinguishing between
+prompts, user input, and generated text.
.It Fl Fl silent-prompt
Don't echo the prompt itself to standard output.
.It Fl Fl multiline-input
@@ -716,11 +762,6 @@ json.dump(json.load(sys.stdin), sys.stdout, indent=2)
print()
'
.Ed
-.Sh BUGS
-Your HTTP server is great for local hosting, but it would need much more
-work to be a production worthy component of a public-facing service. For
-example, C++ exceptions caused by JSON parsing errors will make it abort
-and print a backtrace.
.Sh PROTIP
The
.Fl ngl Ar 35
diff --git a/llamafile/llamafile.1.asc b/llama.cpp/main/main.1.asc
similarity index 80%
rename from llamafile/llamafile.1.asc
rename to llama.cpp/main/main.1.asc
index 9300e5a2..48dec637 100644
--- a/llamafile/llamafile.1.asc
+++ b/llama.cpp/main/main.1.asc
@@ -1,14 +1,13 @@
-LLAMAFILE(1) General Commands Manual LLAMAFILE(1)
+LLAMAFILE(1) BSD General Commands Manual LLAMAFILE(1)
NNAAMMEE
- llllaammaaffiillee - large language model runner
+ llllaammaaffiillee — large language model runner
SSYYNNOOPPSSIISS
llllaammaaffiillee [----sseerrvveerr] [flags...] --mm _m_o_d_e_l_._g_g_u_f [----mmmmpprroojj _v_i_s_i_o_n_._g_g_u_f]
llllaammaaffiillee [----ccllii] [flags...] --mm _m_o_d_e_l_._g_g_u_f --pp _p_r_o_m_p_t
- llllaammaaffiillee [----ccllii] [flags...]
- --mm _m_o_d_e_l_._g_g_u_f ----mmmmpprroojj _v_i_s_i_o_n_._g_g_u_f ----iimmaaggee _g_r_a_p_h_i_c_._p_n_g
- --pp _p_r_o_m_p_t
+ llllaammaaffiillee [----ccllii] [flags...] --mm _m_o_d_e_l_._g_g_u_f ----mmmmpprroojj _v_i_s_i_o_n_._g_g_u_f ----iimmaaggee
+ _g_r_a_p_h_i_c_._p_n_g --pp _p_r_o_m_p_t
DDEESSCCRRIIPPTTIIOONN
llllaammaaffiillee is a large language model tool. It has use cases such as:
@@ -32,8 +31,8 @@ OOPPTTIIOONNSS
----sseerrvveerr
Puts program in server mode. This will launch an HTTP server on a
- local port. This server has both a web UI and an OpenAI API
- compatible completions endpoint. When the server is run on a desk
+ local port. This server has both a web UI and an OpenAI API com‐
+ patible completions endpoint. When the server is run on a desk
system, a tab browser tab will be launched automatically that
displays the web UI. This ----sseerrvveerr flag is implied if no prompt
is specified, i.e. neither the --pp or --ff flags are passed.
@@ -68,12 +67,24 @@ OOPPTTIIOONNSS
Prefix BOS to user inputs, preceding the ----iinn--pprreeffiixx string.
----iinn--pprreeffiixx _S_T_R_I_N_G
- String to prefix user inputs with.
+ This flag is used to add a prefix to your input, primarily, this
+ is used to insert a space after the reverse prompt. Here's an ex‐
+ ample of how to use the ----iinn--pprreeffiixx flag in conjunction with the
+ ----rreevveerrssee--pprroommpptt flag:
+
+ ./main -r "User:" --in-prefix " "
Default: empty
----iinn--ssuuffffiixx _S_T_R_I_N_G
- String to suffix after user inputs with.
+ This flag is used to add a suffix after your input. This is use‐
+ ful for adding an "Assistant:" prompt after the user's input.
+ It's added after the new-line character (\n) that's automatically
+ added to the end of the user's input. Here's an example of how to
+ use the ----iinn--ssuuffffiixx flag in conjunction with the ----rreevveerrssee--pprroommpptt
+ flag:
+
+ ./main -r "User:" --in-prefix " " --in-suffix "Assistant:"
Default: empty
@@ -86,9 +97,13 @@ OOPPTTIIOONNSS
Default: -1
--cc _N, ----ccttxx--ssiizzee _N
- Size of the prompt context.
+ Set the size of the prompt context. A larger context size helps
+ the model to better comprehend and generate responses for longer
+ input or conversations. The LLaMA models were built with a con‐
+ text of 2048, which yields the best results on longer input / in‐
+ ference.
- -- 0 = loaded from model
+ -- 0 = loaded automatically from model
Default: 512
@@ -162,8 +177,8 @@ OOPPTTIIOONNSS
Default: 0.0
----mmiirroossttaatt _N
- Use Mirostat sampling. Top K, Nucleus, Tail Free and Locally
- Typical samplers are ignored if used..
+ Use Mirostat sampling. Top K, Nucleus, Tail Free and Locally Typ‐
+ ical samplers are ignored if used..
-- 0 = disabled
-- 1 = Mirostat
@@ -184,8 +199,8 @@ OOPPTTIIOONNSS
--ll _T_O_K_E_N___I_D_(_+_/_-_)_B_I_A_S, ----llooggiitt--bbiiaass _T_O_K_E_N___I_D_(_+_/_-_)_B_I_A_S
Modifies the likelihood of token appearing in the completion,
i.e. ----llooggiitt--bbiiaass _1_5_0_4_3_+_1 to increase likelihood of token
- _' _H_e_l_l_o_', or ----llooggiitt--bbiiaass _1_5_0_4_3_-_1 to decrease likelihood of token
- _' _H_e_l_l_o_'.
+ _'_ _H_e_l_l_o_', or ----llooggiitt--bbiiaass _1_5_0_4_3_-_1 to decrease likelihood of token
+ _'_ _H_e_l_l_o_'.
--mmdd _F_N_A_M_E, ----mmooddeell--ddrraafftt _F_N_A_M_E
Draft model for speculative decoding.
@@ -210,11 +225,18 @@ OOPPTTIIOONNSS
Default: 1.0
----rrooppee--ssccaalliinngg _{_n_o_n_e_,_l_i_n_e_a_r_,_y_a_r_n_}
- RoPE frequency scaling method, defaults to linear unless
- specified by the model
+ RoPE frequency scaling method, defaults to linear unless speci‐
+ fied by the model
----rrooppee--ssccaallee _N
- RoPE context scaling factor, expands context by a factor of N
+ RoPE context scaling factor, expands context by a factor of _N
+ where _N is the linear scaling factor used by the fine-tuned
+ model. Some fine-tuned models have extended the context length by
+ scaling RoPE. For example, if the original pre-trained model have
+ a context length (max sequence length) of 4096 (4k) and the fine-
+ tuned model have 32k. That is a scaling factor of 8, and should
+ work by setting the above ----ccttxx--ssiizzee to 32768 (32k) and
+ ----rrooppee--ssccaallee to 8.
----rrooppee--ffrreeqq--bbaassee _N
RoPE base frequency, used by NTK-aware scaling.
@@ -278,9 +300,14 @@ OOPPTTIIOONNSS
Default: 400
----kkeeeepp _N
- Number of tokens to keep from the initial prompt.
+ This flag allows users to retain the original prompt when the
+ model runs out of context, ensuring a connection to the initial
+ instruction or conversation topic is maintained, where _N is the
+ number of tokens from the initial prompt to retain when the model
+ resets its internal context.
- -- -1 = all
+ -- 0 = no tokens are kept from initial prompt
+ -- -1 = retain all tokens from initial prompt
Default: 0
@@ -312,16 +339,16 @@ OOPPTTIIOONNSS
Default: 0.1
----mmlloocckk
- Force system to keep model in RAM rather than swapping or
- compressing.
+ Force system to keep model in RAM rather than swapping or com‐
+ pressing.
----nnoo--mmmmaapp
Do not memory-map model (slower load but may reduce pageouts if
not using mlock).
- ----nnuummaa Attempt optimizations that help on some NUMA systems if run
- without this previously, it is recommended to drop the system
- page cache before using this. See
+ ----nnuummaa Attempt optimizations that help on some NUMA systems if run with‐
+ out this previously, it is recommended to drop the system page
+ cache before using this. See
https://github.com/ggerganov/llama.cpp/issues/1437.
----rreeccoommppiillee
@@ -332,20 +359,20 @@ OOPPTTIIOONNSS
If the appropriate DSO file already exists under _~_/_._l_l_a_m_a_f_i_l_e_/
then it'll be linked as-is without question. If a prebuilt DSO is
- present in the PKZIP content of the executable, then it'll be
- extracted and linked if possible. Otherwise, llllaammaaffiillee will skip
+ present in the PKZIP content of the executable, then it'll be ex‐
+ tracted and linked if possible. Otherwise, llllaammaaffiillee will skip
any attempt to compile GPU support and simply fall back to using
CPU inference.
----ggppuu _G_P_U
Specifies which brand of GPU should be used. Valid choices are:
- -- _A_U_T_O: Use any GPU if possible, otherwise fall back to CPU
- inference (default)
+ -- _A_U_T_O: Use any GPU if possible, otherwise fall back to CPU in‐
+ ference (default)
-- _A_P_P_L_E: Use Apple Metal GPU. This is only available on MacOS
- ARM64. If Metal could not be used for any reason, then a
- fatal error will be raised.
+ ARM64. If Metal could not be used for any reason, then a fa‐
+ tal error will be raised.
-- _A_M_D: Use AMD GPUs. The AMD HIP ROCm SDK should be installed
in which case we assume the HIP_PATH environment variable has
@@ -388,8 +415,8 @@ OOPPTTIIOONNSS
The GPU to use for scratch and small tensors.
--nnoommmmqq, ----nnoo--mmuull--mmaatt--qq
- Use cuBLAS instead of custom mul_mat_q CUDA kernels. Not
- recommended since this is both slower and uses more VRAM.
+ Use cuBLAS instead of custom mul_mat_q CUDA kernels. Not recom‐
+ mended since this is both slower and uses more VRAM.
----vveerrbboossee--pprroommpptt
Print prompt before generation.
@@ -439,18 +466,18 @@ CCLLII OOPPTTIIOONNSS
mode.
--ee, ----eessccaappee
- Process prompt escapes sequences (\n, \r, \t, \', \", \\)
+ Process prompt escapes sequences (\n, \r, \t, \´, \", \\)
--pp _S_T_R_I_N_G, ----pprroommpptt _S_T_R_I_N_G
- Prompt to start text generation. Your LLM works by auto-
- completing this text. For example:
+ Prompt to start text generation. Your LLM works by auto-complet‐
+ ing this text. For example:
llamafile -m model.gguf -p "four score and"
- Stands a pretty good chance of printing Lincoln's Gettysburg
- Address. Prompts can take on a structured format too. Depending
- on how your model was trained, it may specify in its docs an
- instruction notation. With some models that might be:
+ Stands a pretty good chance of printing Lincoln's Gettysburg Ad‐
+ dress. Prompts can take on a structured format too. Depending on
+ how your model was trained, it may specify in its docs an in‐
+ struction notation. With some models that might be:
llamafile -p "[INST]Summarize this: $(cat file)[/INST]"
@@ -490,28 +517,48 @@ CCLLII OOPPTTIIOONNSS
Start with a randomized prompt.
----iimmaaggee _I_M_A_G_E___F_I_L_E
- Path to an image file. This should be used with multimodal
- models. Alternatively, it's possible to embed an image directly
+ Path to an image file. This should be used with multimodal mod‐
+ els. Alternatively, it's possible to embed an image directly
into the prompt instead; in which case, it must be base64 encoded
into an HTML img tag URL with the image/jpeg MIME type. See also
the ----mmmmpprroojj flag for supplying the vision model.
--ii, ----iinntteerraaccttiivvee
- Run in interactive mode.
+ Run the program in interactive mode, allowing users to engage in
+ real-time conversations or provide specific instructions to the
+ model.
----iinntteerraaccttiivvee--ffiirrsstt
- Run in interactive mode and wait for input right away.
+ Run the program in interactive mode and immediately wait for user
+ input before starting the text generation.
--iinnss, ----iinnssttrruucctt
- Run in instruction mode (use with Alpaca models).
+ Run the program in instruction mode, which is specifically de‐
+ signed to work with Alpaca models that excel in completing tasks
+ based on user instructions.
+
+ Technical details: The user's input is internally prefixed with
+ the reverse prompt (or "### Instruction:" as the default), and
+ followed by "### Response:" (except if you just press Return
+ without any input, to keep generating a longer response).
+
+ By understanding and utilizing these interaction options, you can
+ create engaging and dynamic experiences with the LLaMA models,
+ tailoring the text generation process to your specific needs.
--rr _P_R_O_M_P_T, ----rreevveerrssee--pprroommpptt _P_R_O_M_P_T
- Halt generation at _P_R_O_M_P_T and return control in interactive mode
- (can be specified more than once for multiple prompts).
+ Specify one or multiple reverse prompts to pause text generation
+ and switch to interactive mode. For example, --rr _"_U_s_e_r_:_" can be
+ used to jump back into the conversation whenever it's the user's
+ turn to speak. This helps create a more interactive and conversa‐
+ tional experience. However, the reverse prompt doesn't work when
+ it ends with a space. To overcome this limitation, you can use
+ the ----iinn--pprreeffiixx flag to add a space or any other characters after
+ the reverse prompt.
----ccoolloorr
- Colorise output to distinguish prompt and user input from
- generations.
+ Enable colorized output to differentiate visually distinguishing
+ between prompts, user input, and generated text.
----ssiilleenntt--pprroommpptt
Don't echo the prompt itself to standard output.
@@ -673,9 +720,9 @@ EEXXAAMMPPLLEESS
sed -e's/ /_/g' -e's/$/.jpg/'
three_baby_lemurs_on_the_back_of_an_adult_lemur.jpg
- Here's an example of how to make an API request to the OpenAI API
- compatible completions endpoint when your llllaammaaffiillee is running in the
- background in ----sseerrvveerr mode.
+ Here's an example of how to make an API request to the OpenAI API compat‐
+ ible completions endpoint when your llllaammaaffiillee is running in the back‐
+ ground in ----sseerrvveerr mode.
curl -s http://localhost:8080/v1/chat/completions \
-H "Content-Type: application/json" -d '{
@@ -697,12 +744,6 @@ EEXXAAMMPPLLEESS
json.dump(json.load(sys.stdin), sys.stdout, indent=2)
print()
-BBUUGGSS
- Your HTTP server is great for local hosting, but it would need much more
- work to be a production worthy component of a public-facing service. For
- example, C++ exceptions caused by JSON parsing errors will make it abort
- and print a backtrace.
-
PPRROOTTIIPP
The --nnggll _3_5 flag needs to be passed in order to use GPUs made by NVIDIA
and AMD. It's not enabled by default since it sometimes needs to be
diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cpp
index cbce0494..739d2ce1 100644
--- a/llama.cpp/main/main.cpp
+++ b/llama.cpp/main/main.cpp
@@ -109,7 +109,7 @@ int main(int argc, char ** argv) {
if (llamafile_has(argv, "-h") ||
llamafile_has(argv, "-help") ||
llamafile_has(argv, "--help")) {
- llamafile_help("/zip/llamafile/llamafile.1.asc");
+ llamafile_help("/zip/llama.cpp/main/main.1.asc");
__builtin_unreachable();
}